sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-13 12:35:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow mismatching H.264/AVC level for trimming or resuming

Browse source 
MediaCodec docs already allude to potentially mismatching H.264 level
between container and bitstream. Relax the initialization data check to
reflect this.

PiperOrigin-RevId: 608942322
This commit is contained in:
Googler 2024-02-21 04:18:02 -08:00 committed by Copybara-Service
parent a82d7e7098
commit dca7bae416
7 changed files with 221 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
RELEASENOTES.md
View file

@ -10,6 +10,7 @@
was made.
* Add workaround for exception thrown due to `MediaMuxer` not supporting
negative presentation timestamps before API 30.
* Relax trim optimization H.264 level checks.
* Track Selection:
* Extractors:
* Audio:

BIN
libraries/test_data/src/test/assets/media/mp4/pixel7_videoOnly_cleaned.mp4 Normal file
View file

Binary file not shown.

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View file

@ -74,6 +74,9 @@ public final class AndroidTestUtil {
public static final String MP4_TRIM_OPTIMIZATION_URI_STRING =
"asset:///media/mp4/internal_emulator_transformer_output.mp4";
public static final String MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING =
"asset:///media/mp4/pixel7_videoOnly_cleaned.mp4";
public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4";
public static final Format MP4_ASSET_FORMAT =
new Format.Builder()

42
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java
View file

@ -28,9 +28,13 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_SEF_URI_STRI
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.ExportResult.CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import android.content.Context;
import android.net.Uri;
@ -382,4 +386,42 @@ public class ExportTest {
.build()
.run(testId, editedMediaItem);
}
@Test
public void clippedMedia_trimOptimizationEnabled_pixel7Plus_completesWithOptimizationApplied()
throws Exception {
String testId =
TAG + "_clippedMedia_trimOptimizationEnabled_pixel_completesWithOptimizationApplied";
Context context = ApplicationProvider.getApplicationContext();
// Devices with Tensor G2 & G3 chipsets should work, but Pixel 7a is flaky.
assumeTrue(
Ascii.toLowerCase(Util.MODEL).contains("pixel")
&& (Ascii.toLowerCase(Util.MODEL).contains("7")
|| Ascii.toLowerCase(Util.MODEL).contains("8")
|| Ascii.toLowerCase(Util.MODEL).contains("fold")
|| Ascii.toLowerCase(Util.MODEL).contains("tablet")));
assumeFalse(Ascii.toLowerCase(Util.MODEL).contains("7a"));
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING)
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(1200)
.build())
.build();
EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(700);
assertThat(result.exportResult.videoConversionProcess)
.isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED);
}
}

71
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View file

@ -36,6 +36,7 @@ import androidx.media3.common.Format;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util;
import androidx.media3.container.NalUnitUtil;
import androidx.media3.effect.DebugTraceUtil;
import com.google.common.collect.ImmutableList;
import java.io.File;
@ -44,6 +45,8 @@ import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
@ -148,6 +151,72 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
abortScheduledExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME);
}
/**
* Returns whether the initialization data of two video formats can be muxed together.
*
* @param existingVideoTrackFormat The starting video format to compare.
* @param newVideoTrackFormat The candidate format of the video bitstream to be appended after the
* existing format.
* @return {@code true} if both input formats can be muxed together in the same container.
*/
public static boolean isInitializationDataCompatible(
Format existingVideoTrackFormat, Format newVideoTrackFormat) {
if (existingVideoTrackFormat.initializationDataEquals(newVideoTrackFormat)) {
return true;
}
if (!Objects.equals(newVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)
|| !Objects.equals(existingVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)) {
return false;
}
// For H.264 we allow a different level number. This is not spec-compliant, but such videos are
// already common on Android. See, for example: {@link MediaFormat#KEY_LEVEL}.
// Android players are advised to support level mismatch between container
// and bitstream: see {@link android.media.MediaExtractor#getTrackFormat(int)}.
if (newVideoTrackFormat.initializationData.size() != 2
|| existingVideoTrackFormat.initializationData.size() != 2) {
return false;
}
// Check picture parameter sets match.
if (!Arrays.equals(
newVideoTrackFormat.initializationData.get(1),
existingVideoTrackFormat.initializationData.get(1))) {
return false;
}
// Allow level_idc to be lower in the new stream.
// Note: the SPS doesn't need to be unescaped because it's not possible to have two
// consecutive 0 bytes at/before level_idc.
byte[] newSps = newVideoTrackFormat.initializationData.get(0);
byte[] existingSps = existingVideoTrackFormat.initializationData.get(0);
// Skip 3 bytes: NAL unit type, profile, and reserved fields.
int spsLevelIndex = NalUnitUtil.NAL_START_CODE.length + 3;
if (spsLevelIndex >= newSps.length) {
return false;
}
if (newSps.length != existingSps.length) {
return false;
}
for (int i = 0; i < newSps.length; i++) {
if (i != spsLevelIndex && newSps[i] != existingSps[i]) {
return false;
}
}
for (int i = 0; i < NalUnitUtil.NAL_START_CODE.length; i++) {
if (newSps[i] != NalUnitUtil.NAL_START_CODE[i]) {
return false;
}
}
int nalUnitTypeMask = 0x1F;
if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask)
!= NalUnitUtil.NAL_UNIT_TYPE_SPS) {
return false;
}
// Check that H.264 profile is non-zero.
if (newSps[NalUnitUtil.NAL_START_CODE.length + 1] == 0) {
return false;
}
return true;
}
/**
* Changes {@link MuxerMode} to {@link #MUXER_MODE_APPEND}.
*
@ -255,7 +324,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType));
checkArgument(existingFormat.width == format.width);
checkArgument(existingFormat.height == format.height);
checkArgument(existingFormat.initializationDataEquals(format));
checkArgument(isInitializationDataCompatible(existingFormat, format));
} else if (trackType == C.TRACK_TYPE_AUDIO) {
checkState(contains(trackTypeToInfo, C.TRACK_TYPE_AUDIO));
TrackInfo audioTrackInfo = trackTypeToInfo.get(C.TRACK_TYPE_AUDIO);

6
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View file

@ -1508,10 +1508,10 @@ public final class Transformer {
}
private boolean doesFormatsMatch(Mp4Info mediaItemInfo, EditedMediaItem firstMediaItem) {
Format videoFormat = checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_VIDEO);
boolean videoFormatMatches =
checkNotNull(remuxingMuxerWrapper)
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(checkNotNull(mediaItemInfo.videoFormat));
MuxerWrapper.isInitializationDataCompatible(
videoFormat, checkNotNull(mediaItemInfo.videoFormat));
boolean audioFormatMatches =
mediaItemInfo.audioFormat == null
|| firstMediaItem.removeAudio

102
libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java
View file

@ -17,6 +17,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.AUDIO_AAC;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_MUX_PARTIAL;
import static androidx.media3.transformer.TestUtil.getDumpFileName;
@ -335,6 +336,107 @@ public class MuxerWrapperTest {
assertThat(muxerWrapper.isEnded()).isFalse();
}
@Test
public void isInitializationDataCompatible_h265_differentCSD_returnsFalse() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 0}))
.build();
Format otherFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
}
@Test
public void isInitializationDataCompatible_h265_matchingCSD_returnsTrue() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
Format otherFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
}
@Test
public void isInitializationDataCompatible_h264_matchingCSD_returnsTrue() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
Format otherFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
}
@Test
public void isInitializationDataCompatible_h264_ignoresLevel_returnsTrue() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 40}, new byte[] {0, 0, 0, 1}))
.build();
Format otherFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
}
@Test
public void isInitializationDataCompatible_h264_mismatchProfile_returnsFalse() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 110, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
Format otherFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
}
@Test
public void isInitializationDataCompatible_h264_missingMimeType_returnsFalse() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 40}, new byte[] {0, 0, 0, 1}))
.build();
Format otherFormat =
new Format.Builder()
.setInitializationData(
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
}
private static final class NoOpMuxerListenerImpl implements MuxerWrapper.Listener {
@Override