diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 3806dfd895..03697e043b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects; import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED; +import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; @@ -423,7 +424,7 @@ public class TransformerEndToEndTest { } Transformer transformer = new Transformer.Builder(context).build(); long clippingStartMs = 10_000; - long clippingEndMs = 11_000; + long clippingEndMs = 13_000; MediaItem mediaItem = new MediaItem.Builder() .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) @@ -442,6 +443,43 @@ public class TransformerEndToEndTest { assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); } + @Test + public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch() + throws Exception { + String testId = "clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch"; + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) { + return; + } + Transformer transformer = + new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build(); + long clippingStartMs = 10_000; + long clippingEndMs = 13_000; + // The file is made artificially on computer software so phones will not have the encoder + // available to match the csd. + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(clippingStartMs) + .setEndPositionMs(clippingEndMs) + .build()) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, mediaItem); + + assertThat(result.exportResult.optimizationResult) + .isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH); + assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); + } + @Test public void clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied() throws Exception { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index adbc738120..9a9bfb4892 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -201,6 +201,11 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { */ @Override public DefaultCodec createForVideoEncoding(Format format) throws ExportException { + // TODO b/304476154 - Investigate how to match initialization data without running into + // mediaCodec errors under API 29. + if (SDK_INT < 29) { + format = format.buildUpon().setInitializationData(null).build(); + } if (format.frameRate == Format.NO_VALUE || deviceNeedsDefaultFrameRateWorkaround()) { format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java index c111826310..2afd2673b4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java @@ -301,6 +301,7 @@ public final class ExportResult { OPTIMIZATION_SUCCEEDED, OPTIMIZATION_ABANDONED, OPTIMIZATION_FAILED_EXTRACTION_FAILED, + OPTIMIZATION_FAILED_FORMAT_MISMATCH }) @interface OptimizationResult {} @@ -322,6 +323,12 @@ public final class ExportResult { */ public static final int OPTIMIZATION_FAILED_EXTRACTION_FAILED = 3; + /** + * The optimization failed because the format between the two parts of the media to be put + * together did not match. Normal export proceeded. + */ + public static final int OPTIMIZATION_FAILED_FORMAT_MISMATCH = 4; + /** The list of {@linkplain ProcessedInput processed inputs}. */ public final ImmutableList processedInputs; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index 8d32bae50b..bdb08d68a2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -292,6 +292,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } + /** + * Returns the {@link Format} of given {@code trackType} that was {@linkplain #addTrackFormat + * added}. + * + * @throws IllegalArgumentException If the {@code trackType} has not been {@linkplain + * #addTrackFormat added}. + */ + public Format getTrackFormat(@C.TrackType int trackType) { + checkArgument(contains(trackTypeToInfo, trackType)); + return trackTypeToInfo.get(trackType).format; + } + /** * Attempts to write a sample to the muxer. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 2e5711d262..e7a239ae9a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -22,6 +22,7 @@ import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED; +import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH; import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio; import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo; import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes; @@ -925,7 +926,8 @@ public final class Transformer { composition, new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } else { processMediaBeforeFirstSyncSampleAfterTrimStartTime(); } @@ -1113,7 +1115,8 @@ public final class Transformer { componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } private void remuxProcessedVideo() { @@ -1149,7 +1152,8 @@ public final class Transformer { /* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs), checkNotNull(remuxingMuxerWrapper), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } @Override @@ -1177,7 +1181,8 @@ public final class Transformer { videoOnlyComposition, remuxingMuxerWrapper, componentListener, - /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs); + /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs, + /* matchInitializationData= */ false); } private void processAudio() { @@ -1192,7 +1197,8 @@ public final class Transformer { componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ false); } // TODO: b/308253384 - Move copy output logic into MuxerWrapper. @@ -1297,7 +1303,8 @@ public final class Transformer { trancodeComposition, checkNotNull(remuxingMuxerWrapper), componentListener, - /* initialTimestampOffsetUs= */ 0); + /* initialTimestampOffsetUs= */ 0, + /* matchInitializationData= */ true); } @Override @@ -1311,8 +1318,13 @@ public final class Transformer { private void remuxRemainingMedia() { transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA; - // TODO: b/304476154 - check original file format against transcode file format here to fail - // fast if necessary. + if (!doesFormatsMatch()) { + remuxingMuxerWrapper = null; + transformerInternal = null; + exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH); + processFullInput(); + return; + } MediaItem firstMediaItem = checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem; long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs; @@ -1332,7 +1344,21 @@ public final class Transformer { remuxingMuxerWrapper, componentListener, /* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs - - trimStartTimeUs); + - trimStartTimeUs, + /* matchInitializationData= */ false); + } + + private boolean doesFormatsMatch() { + checkNotNull(mp4MetadataInfo); + boolean videoFormatMatches = + checkNotNull(remuxingMuxerWrapper) + .getTrackFormat(C.TRACK_TYPE_VIDEO) + .initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat)); + boolean audioFormatMatches = + mp4MetadataInfo.audioFormat == null + || mp4MetadataInfo.audioFormat.initializationDataEquals( + checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO)); + return videoFormatMatches && audioFormatMatches; } private boolean isMultiAsset() { @@ -1350,7 +1376,8 @@ public final class Transformer { Composition composition, MuxerWrapper muxerWrapper, ComponentListener componentListener, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean matchInitializationData) { checkArgument(composition.effects.audioProcessors.isEmpty()); checkState(transformerInternal == null, "There is already an export in progress."); TransformationRequest transformationRequest = this.transformationRequest; @@ -1387,7 +1414,7 @@ public final class Transformer { debugViewProvider, clock, initialTimestampOffsetUs, - /* matchInitializationData= */ trimOptimizationEnabled); + matchInitializationData); transformerInternal.start(); }