diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a6187d73f2..e6a549acf8 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -10,6 +10,7 @@ was made. * Add workaround for exception thrown due to `MediaMuxer` not supporting negative presentation timestamps before API 30. + * Relax trim optimization H.264 level checks. * Track Selection: * Extractors: * Audio: diff --git a/libraries/test_data/src/test/assets/media/mp4/pixel7_videoOnly_cleaned.mp4 b/libraries/test_data/src/test/assets/media/mp4/pixel7_videoOnly_cleaned.mp4 new file mode 100644 index 0000000000..abe429d5be Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/pixel7_videoOnly_cleaned.mp4 differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 0448a34fe4..d2f9108417 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -74,6 +74,9 @@ public final class AndroidTestUtil { public static final String MP4_TRIM_OPTIMIZATION_URI_STRING = "asset:///media/mp4/internal_emulator_transformer_output.mp4"; + public static final String MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING = + "asset:///media/mp4/pixel7_videoOnly_cleaned.mp4"; + public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4"; public static final Format MP4_ASSET_FORMAT = new Format.Builder() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java index 30f18fc488..591a2dce64 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java @@ -28,9 +28,13 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_URI_STRI import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; +import static androidx.media3.transformer.ExportResult.CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED; +import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; import android.content.Context; import android.net.Uri; @@ -382,4 +386,42 @@ public class ExportTest { .build() .run(testId, editedMediaItem); } + + @Test + public void clippedMedia_trimOptimizationEnabled_pixel7Plus_completesWithOptimizationApplied() + throws Exception { + String testId = + TAG + "_clippedMedia_trimOptimizationEnabled_pixel_completesWithOptimizationApplied"; + Context context = ApplicationProvider.getApplicationContext(); + // Devices with Tensor G2 & G3 chipsets should work, but Pixel 7a is flaky. + assumeTrue( + Ascii.toLowerCase(Util.MODEL).contains("pixel") + && (Ascii.toLowerCase(Util.MODEL).contains("7") + || Ascii.toLowerCase(Util.MODEL).contains("8") + || Ascii.toLowerCase(Util.MODEL).contains("fold") + || Ascii.toLowerCase(Util.MODEL).contains("tablet"))); + assumeFalse(Ascii.toLowerCase(Util.MODEL).contains("7a")); + Transformer transformer = + new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build(); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(500) + .setEndPositionMs(1200) + .build()) + .build(); + EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED); + assertThat(result.exportResult.durationMs).isAtMost(700); + assertThat(result.exportResult.videoConversionProcess) + .isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index aaa8324c25..e53aa2256e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -36,6 +36,7 @@ import androidx.media3.common.Format; import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; +import androidx.media3.container.NalUnitUtil; import androidx.media3.effect.DebugTraceUtil; import com.google.common.collect.ImmutableList; import java.io.File; @@ -44,6 +45,8 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; @@ -148,6 +151,72 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; abortScheduledExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME); } + /** + * Returns whether the initialization data of two video formats can be muxed together. + * + * @param existingVideoTrackFormat The starting video format to compare. + * @param newVideoTrackFormat The candidate format of the video bitstream to be appended after the + * existing format. + * @return {@code true} if both input formats can be muxed together in the same container. + */ + public static boolean isInitializationDataCompatible( + Format existingVideoTrackFormat, Format newVideoTrackFormat) { + if (existingVideoTrackFormat.initializationDataEquals(newVideoTrackFormat)) { + return true; + } + if (!Objects.equals(newVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264) + || !Objects.equals(existingVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)) { + return false; + } + // For H.264 we allow a different level number. This is not spec-compliant, but such videos are + // already common on Android. See, for example: {@link MediaFormat#KEY_LEVEL}. + // Android players are advised to support level mismatch between container + // and bitstream: see {@link android.media.MediaExtractor#getTrackFormat(int)}. + if (newVideoTrackFormat.initializationData.size() != 2 + || existingVideoTrackFormat.initializationData.size() != 2) { + return false; + } + // Check picture parameter sets match. + if (!Arrays.equals( + newVideoTrackFormat.initializationData.get(1), + existingVideoTrackFormat.initializationData.get(1))) { + return false; + } + // Allow level_idc to be lower in the new stream. + // Note: the SPS doesn't need to be unescaped because it's not possible to have two + // consecutive 0 bytes at/before level_idc. + byte[] newSps = newVideoTrackFormat.initializationData.get(0); + byte[] existingSps = existingVideoTrackFormat.initializationData.get(0); + // Skip 3 bytes: NAL unit type, profile, and reserved fields. + int spsLevelIndex = NalUnitUtil.NAL_START_CODE.length + 3; + if (spsLevelIndex >= newSps.length) { + return false; + } + if (newSps.length != existingSps.length) { + return false; + } + for (int i = 0; i < newSps.length; i++) { + if (i != spsLevelIndex && newSps[i] != existingSps[i]) { + return false; + } + } + for (int i = 0; i < NalUnitUtil.NAL_START_CODE.length; i++) { + if (newSps[i] != NalUnitUtil.NAL_START_CODE[i]) { + return false; + } + } + int nalUnitTypeMask = 0x1F; + if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask) + != NalUnitUtil.NAL_UNIT_TYPE_SPS) { + return false; + } + // Check that H.264 profile is non-zero. + if (newSps[NalUnitUtil.NAL_START_CODE.length + 1] == 0) { + return false; + } + return true; + } + /** * Changes {@link MuxerMode} to {@link #MUXER_MODE_APPEND}. * @@ -255,7 +324,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType)); checkArgument(existingFormat.width == format.width); checkArgument(existingFormat.height == format.height); - checkArgument(existingFormat.initializationDataEquals(format)); + checkArgument(isInitializationDataCompatible(existingFormat, format)); } else if (trackType == C.TRACK_TYPE_AUDIO) { checkState(contains(trackTypeToInfo, C.TRACK_TYPE_AUDIO)); TrackInfo audioTrackInfo = trackTypeToInfo.get(C.TRACK_TYPE_AUDIO); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 49572dd706..94af544ccd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -1508,10 +1508,10 @@ public final class Transformer { } private boolean doesFormatsMatch(Mp4Info mediaItemInfo, EditedMediaItem firstMediaItem) { + Format videoFormat = checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_VIDEO); boolean videoFormatMatches = - checkNotNull(remuxingMuxerWrapper) - .getTrackFormat(C.TRACK_TYPE_VIDEO) - .initializationDataEquals(checkNotNull(mediaItemInfo.videoFormat)); + MuxerWrapper.isInitializationDataCompatible( + videoFormat, checkNotNull(mediaItemInfo.videoFormat)); boolean audioFormatMatches = mediaItemInfo.audioFormat == null || firstMediaItem.removeAudio diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java index 87a6a935c3..49a5811530 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java @@ -17,6 +17,7 @@ package androidx.media3.transformer; import static androidx.media3.common.MimeTypes.AUDIO_AAC; import static androidx.media3.common.MimeTypes.VIDEO_H264; +import static androidx.media3.common.MimeTypes.VIDEO_H265; import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT; import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_MUX_PARTIAL; import static androidx.media3.transformer.TestUtil.getDumpFileName; @@ -335,6 +336,107 @@ public class MuxerWrapperTest { assertThat(muxerWrapper.isEnded()).isFalse(); } + @Test + public void isInitializationDataCompatible_h265_differentCSD_returnsFalse() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 0})) + .build(); + Format otherFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); + } + + @Test + public void isInitializationDataCompatible_h265_matchingCSD_returnsTrue() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) + .build(); + Format otherFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H265) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); + } + + @Test + public void isInitializationDataCompatible_h264_matchingCSD_returnsTrue() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) + .build(); + Format otherFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); + } + + @Test + public void isInitializationDataCompatible_h264_ignoresLevel_returnsTrue() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 40}, new byte[] {0, 0, 0, 1})) + .build(); + Format otherFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); + } + + @Test + public void isInitializationDataCompatible_h264_mismatchProfile_returnsFalse() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 110, 0, 41}, new byte[] {0, 0, 0, 1})) + .build(); + Format otherFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); + } + + @Test + public void isInitializationDataCompatible_h264_missingMimeType_returnsFalse() { + Format existingFormat = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 40}, new byte[] {0, 0, 0, 1})) + .build(); + Format otherFormat = + new Format.Builder() + .setInitializationData( + ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) + .build(); + + assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); + } + private static final class NoOpMuxerListenerImpl implements MuxerWrapper.Listener { @Override