Trim optimization: fallback on format mismatches

Manual testing: tested manually with pixel 4a

PiperOrigin-RevId: 590284361

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
 import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap;
 import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
+import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
@@ -423,7 +424,7 @@ public class TransformerEndToEndTest {
     }
     Transformer transformer = new Transformer.Builder(context).build();
     long clippingStartMs = 10_000;
-    long clippingEndMs = 11_000;
+    long clippingEndMs = 13_000;
     MediaItem mediaItem =
         new MediaItem.Builder()
             .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
@@ -442,6 +443,43 @@ public class TransformerEndToEndTest {
     assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
   }
 
+  @Test
+  public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
+      throws Exception {
+    String testId = "clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch";
+    if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
+        /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
+      return;
+    }
+    Transformer transformer =
+        new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
+    long clippingStartMs = 10_000;
+    long clippingEndMs = 13_000;
+    // The file is made artificially on computer software so phones will not have the encoder
+    // available to match the csd.
+    MediaItem mediaItem =
+        new MediaItem.Builder()
+            .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
+            .setClippingConfiguration(
+                new MediaItem.ClippingConfiguration.Builder()
+                    .setStartPositionMs(clippingStartMs)
+                    .setEndPositionMs(clippingEndMs)
+                    .build())
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, mediaItem);
+
+    assertThat(result.exportResult.optimizationResult)
+        .isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
+    assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
+  }
+
   @Test
   public void clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied()
       throws Exception {

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java

@@ -201,6 +201,11 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
    */
   @Override
   public DefaultCodec createForVideoEncoding(Format format) throws ExportException {
+    // TODO b/304476154 - Investigate how to match initialization data without running into
+    // mediaCodec errors under API 29.
+    if (SDK_INT < 29) {
+      format = format.buildUpon().setInitializationData(null).build();
+    }
     if (format.frameRate == Format.NO_VALUE || deviceNeedsDefaultFrameRateWorkaround()) {
       format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build();
     }

libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java

@@ -301,6 +301,7 @@ public final class ExportResult {
     OPTIMIZATION_SUCCEEDED,
     OPTIMIZATION_ABANDONED,
     OPTIMIZATION_FAILED_EXTRACTION_FAILED,
+    OPTIMIZATION_FAILED_FORMAT_MISMATCH
   })
   @interface OptimizationResult {}
 
@@ -322,6 +323,12 @@ public final class ExportResult {
    */
   public static final int OPTIMIZATION_FAILED_EXTRACTION_FAILED = 3;
 
+  /**
+   * The optimization failed because the format between the two parts of the media to be put
+   * together did not match. Normal export proceeded.
+   */
+  public static final int OPTIMIZATION_FAILED_FORMAT_MISMATCH = 4;
+
   /** The list of {@linkplain ProcessedInput processed inputs}. */
   public final ImmutableList<ProcessedInput> processedInputs;
 

libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java

@@ -292,6 +292,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
   }
 
+  /**
+   * Returns the {@link Format} of given {@code trackType} that was {@linkplain #addTrackFormat
+   * added}.
+   *
+   * @throws IllegalArgumentException If the {@code trackType} has not been {@linkplain
+   *     #addTrackFormat added}.
+   */
+  public Format getTrackFormat(@C.TrackType int trackType) {
+    checkArgument(contains(trackTypeToInfo, trackType));
+    return trackTypeToInfo.get(trackType).format;
+  }
+
   /**
    * Attempts to write a sample to the muxer.
    *

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -22,6 +22,7 @@ import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
+import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
 import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
 import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
 import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
@@ -925,7 +926,8 @@ public final class Transformer {
           composition,
           new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT),
           componentListener,
-          /* initialTimestampOffsetUs= */ 0);
+          /* initialTimestampOffsetUs= */ 0,
+          /* matchInitializationData= */ false);
     } else {
       processMediaBeforeFirstSyncSampleAfterTrimStartTime();
     }
@@ -1113,7 +1115,8 @@ public final class Transformer {
             componentListener,
             MuxerWrapper.MUXER_MODE_DEFAULT),
         componentListener,
-        /* initialTimestampOffsetUs= */ 0);
+        /* initialTimestampOffsetUs= */ 0,
+        /* matchInitializationData= */ false);
   }
 
   private void remuxProcessedVideo() {
@@ -1149,7 +1152,8 @@ public final class Transformer {
                     /* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs),
                 checkNotNull(remuxingMuxerWrapper),
                 componentListener,
-                /* initialTimestampOffsetUs= */ 0);
+                /* initialTimestampOffsetUs= */ 0,
+                /* matchInitializationData= */ false);
           }
 
           @Override
@@ -1177,7 +1181,8 @@ public final class Transformer {
         videoOnlyComposition,
         remuxingMuxerWrapper,
         componentListener,
-        /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs);
+        /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs,
+        /* matchInitializationData= */ false);
   }
 
   private void processAudio() {
@@ -1192,7 +1197,8 @@ public final class Transformer {
             componentListener,
             MuxerWrapper.MUXER_MODE_DEFAULT),
         componentListener,
-        /* initialTimestampOffsetUs= */ 0);
+        /* initialTimestampOffsetUs= */ 0,
+        /* matchInitializationData= */ false);
   }
 
   // TODO: b/308253384 - Move copy output logic into MuxerWrapper.
@@ -1297,7 +1303,8 @@ public final class Transformer {
                 trancodeComposition,
                 checkNotNull(remuxingMuxerWrapper),
                 componentListener,
-                /* initialTimestampOffsetUs= */ 0);
+                /* initialTimestampOffsetUs= */ 0,
+                /* matchInitializationData= */ true);
           }
 
           @Override
@@ -1311,8 +1318,13 @@ public final class Transformer {
 
   private void remuxRemainingMedia() {
     transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
-    // TODO: b/304476154 - check original file format against transcode file format here to fail
+    if (!doesFormatsMatch()) {
-    //  fast if necessary.
+      remuxingMuxerWrapper = null;
+      transformerInternal = null;
+      exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
+      processFullInput();
+      return;
+    }
     MediaItem firstMediaItem =
         checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
     long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
@@ -1332,7 +1344,21 @@ public final class Transformer {
         remuxingMuxerWrapper,
         componentListener,
         /* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs
-            - trimStartTimeUs);
+            - trimStartTimeUs,
+        /* matchInitializationData= */ false);
+  }
+
+  private boolean doesFormatsMatch() {
+    checkNotNull(mp4MetadataInfo);
+    boolean videoFormatMatches =
+        checkNotNull(remuxingMuxerWrapper)
+            .getTrackFormat(C.TRACK_TYPE_VIDEO)
+            .initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat));
+    boolean audioFormatMatches =
+        mp4MetadataInfo.audioFormat == null
+            || mp4MetadataInfo.audioFormat.initializationDataEquals(
+                checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
+    return videoFormatMatches && audioFormatMatches;
   }
 
   private boolean isMultiAsset() {
@@ -1350,7 +1376,8 @@ public final class Transformer {
       Composition composition,
       MuxerWrapper muxerWrapper,
       ComponentListener componentListener,
-      long initialTimestampOffsetUs) {
+      long initialTimestampOffsetUs,
+      boolean matchInitializationData) {
     checkArgument(composition.effects.audioProcessors.isEmpty());
     checkState(transformerInternal == null, "There is already an export in progress.");
     TransformationRequest transformationRequest = this.transformationRequest;
@@ -1387,7 +1414,7 @@ public final class Transformer {
             debugViewProvider,
             clock,
             initialTimestampOffsetUs,
-            /* matchInitializationData= */ trimOptimizationEnabled);
+            matchInitializationData);
     transformerInternal.start();
   }