Pick max H.264 level when trim optimizing

Instead of always starting with the transcoded H.264 level, take the maximum
from transcoded and transmuxed levels

PiperOrigin-RevId: 610759438
This commit is contained in:
Googler 2024-02-27 08:24:41 -08:00 committed by Copybara-Service
parent c3aec4f19d
commit 6f28eeff31
10 changed files with 327 additions and 100 deletions

View file

@ -11,6 +11,7 @@ format video:
colorRange = 2 colorRange = 2
colorTransfer = 3 colorTransfer = 3
initializationData: initializationData:
data = length 28, hash D27A8A6F
data = length 4, hash E93C3 data = length 4, hash E93C3
sample: sample:
trackType = audio trackType = audio

View file

@ -45,6 +45,11 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.media3.effect.Presentation; import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.extractor.mp4.Mp4Extractor;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.test.utils.FakeExtractorOutput;
import androidx.media3.test.utils.FakeTrackOutput;
import androidx.media3.test.utils.TestUtil;
import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.AndroidTestUtil.ForceEncodeEncoderFactory; import androidx.media3.transformer.AndroidTestUtil.ForceEncodeEncoderFactory;
import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.DefaultEncoderFactory;
@ -387,8 +392,6 @@ public class ExportTest {
@Test @Test
public void clippedMedia_trimOptimizationEnabled_pixel7Plus_completesWithOptimizationApplied() public void clippedMedia_trimOptimizationEnabled_pixel7Plus_completesWithOptimizationApplied()
throws Exception { throws Exception {
String testId =
TAG + "_clippedMedia_trimOptimizationEnabled_pixel_completesWithOptimizationApplied";
Context context = ApplicationProvider.getApplicationContext(); Context context = ApplicationProvider.getApplicationContext();
// Devices with Tensor G2 & G3 chipsets should work, but Pixel 7a is flaky. // Devices with Tensor G2 & G3 chipsets should work, but Pixel 7a is flaky.
assumeTrue( assumeTrue(
@ -415,10 +418,19 @@ public class ExportTest {
new TransformerAndroidTestRunner.Builder(context, transformer) new TransformerAndroidTestRunner.Builder(context, transformer)
.build() .build()
.run(testId, editedMediaItem); .run(testId, editedMediaItem);
Mp4Extractor mp4Extractor = new Mp4Extractor(new DefaultSubtitleParserFactory());
FakeExtractorOutput fakeExtractorOutput =
TestUtil.extractAllSamplesFromFilePath(mp4Extractor, result.filePath);
FakeTrackOutput videoTrack = fakeExtractorOutput.trackOutputs.get(0);
byte[] sps = videoTrack.lastFormat.initializationData.get(0);
// Skip 7 bytes: NAL unit start code (4) and NAL unit type, profile, and reserved fields.
int spsLevelIndex = 7;
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED); assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(700); assertThat(result.exportResult.durationMs).isAtMost(700);
assertThat(result.exportResult.videoConversionProcess) assertThat(result.exportResult.videoConversionProcess)
.isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED); .isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED);
int higherVideoLevel = 41;
assertThat(sps[spsLevelIndex]).isEqualTo(higherVideoLevel);
} }
} }

View file

@ -185,6 +185,13 @@ public final class ExportException extends Exception {
*/ */
public static final int ERROR_CODE_MUXING_TIMEOUT = 7002; public static final int ERROR_CODE_MUXING_TIMEOUT = 7002;
/**
* Caused by mismatching formats in MuxerWrapper.
*
* @see MuxerWrapper.AppendTrackFormatException
*/
public static final int ERROR_CODE_MUXING_APPEND = 7003;
/* package */ static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE = /* package */ static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE =
new ImmutableBiMap.Builder<String, @ErrorCode Integer>() new ImmutableBiMap.Builder<String, @ErrorCode Integer>()
.put("ERROR_CODE_FAILED_RUNTIME_CHECK", ERROR_CODE_FAILED_RUNTIME_CHECK) .put("ERROR_CODE_FAILED_RUNTIME_CHECK", ERROR_CODE_FAILED_RUNTIME_CHECK)
@ -207,6 +214,7 @@ public final class ExportException extends Exception {
.put("ERROR_CODE_AUDIO_PROCESSING_FAILED", ERROR_CODE_AUDIO_PROCESSING_FAILED) .put("ERROR_CODE_AUDIO_PROCESSING_FAILED", ERROR_CODE_AUDIO_PROCESSING_FAILED)
.put("ERROR_CODE_MUXING_FAILED", ERROR_CODE_MUXING_FAILED) .put("ERROR_CODE_MUXING_FAILED", ERROR_CODE_MUXING_FAILED)
.put("ERROR_CODE_MUXING_TIMEOUT", ERROR_CODE_MUXING_TIMEOUT) .put("ERROR_CODE_MUXING_TIMEOUT", ERROR_CODE_MUXING_TIMEOUT)
.put("ERROR_CODE_MUXING_APPEND", ERROR_CODE_MUXING_APPEND)
.buildOrThrow(); .buildOrThrow();
/** Returns the name of a given {@code errorCode}. */ /** Returns the name of a given {@code errorCode}. */

View file

@ -16,6 +16,7 @@
package androidx.media3.transformer; package androidx.media3.transformer;
import static androidx.annotation.VisibleForTesting.PRIVATE;
import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkState;
@ -31,6 +32,7 @@ import android.util.SparseArray;
import androidx.annotation.IntDef; import androidx.annotation.IntDef;
import androidx.annotation.IntRange; import androidx.annotation.IntRange;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.Metadata; import androidx.media3.common.Metadata;
@ -46,6 +48,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target; import java.lang.annotation.Target;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledFuture;
@ -58,6 +61,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper can contain at most one video track and one audio track. * <p>This wrapper can contain at most one video track and one audio track.
*/ */
/* package */ final class MuxerWrapper { /* package */ final class MuxerWrapper {
/**
* Thrown when video formats fail to match between {@link #MUXER_MODE_MUX_PARTIAL} and {@link
* #MUXER_MODE_APPEND}.
*/
public static final class AppendTrackFormatException extends Exception {
/**
* Creates an instance.
*
* @param message See {@link #getMessage()}.
*/
public AppendTrackFormatException(String message) {
super(message);
}
}
/** Different modes for muxing. */ /** Different modes for muxing. */
@Documented @Documented
@Retention(RetentionPolicy.SOURCE) @Retention(RetentionPolicy.SOURCE)
@ -125,6 +144,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final boolean dropSamplesBeforeFirstVideoSample; private final boolean dropSamplesBeforeFirstVideoSample;
private final SparseArray<TrackInfo> trackTypeToInfo; private final SparseArray<TrackInfo> trackTypeToInfo;
private final ScheduledExecutorService abortScheduledExecutorService; private final ScheduledExecutorService abortScheduledExecutorService;
private final @MonotonicNonNull Format appendVideoFormat;
private boolean isReady; private boolean isReady;
private boolean isEnded; private boolean isEnded;
@ -145,6 +165,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* Creates an instance. * Creates an instance.
* *
* <p>{@code appendVideoFormat} must be non-{@code null} when using {@link
* #MUXER_MODE_MUX_PARTIAL}.
*
* @param outputPath The output file path to write the media data to. * @param outputPath The output file path to write the media data to.
* @param muxerFactory A {@link Muxer.Factory} to create a {@link Muxer}. * @param muxerFactory A {@link Muxer.Factory} to create a {@link Muxer}.
* @param listener A {@link MuxerWrapper.Listener}. * @param listener A {@link MuxerWrapper.Listener}.
@ -152,19 +175,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* {@link #MUXER_MODE_MUX_PARTIAL}. * {@link #MUXER_MODE_MUX_PARTIAL}.
* @param dropSamplesBeforeFirstVideoSample Whether to drop any non-video samples with * @param dropSamplesBeforeFirstVideoSample Whether to drop any non-video samples with
* presentation timestamps before the first video sample. * presentation timestamps before the first video sample.
* @param appendVideoFormat The format which will be used to write samples after transitioning
* from {@link #MUXER_MODE_MUX_PARTIAL} to {@link #MUXER_MODE_APPEND}.
*/ */
public MuxerWrapper( public MuxerWrapper(
String outputPath, String outputPath,
Muxer.Factory muxerFactory, Muxer.Factory muxerFactory,
Listener listener, Listener listener,
@MuxerMode int muxerMode, @MuxerMode int muxerMode,
boolean dropSamplesBeforeFirstVideoSample) { boolean dropSamplesBeforeFirstVideoSample,
@Nullable Format appendVideoFormat) {
this.outputPath = outputPath; this.outputPath = outputPath;
this.muxerFactory = muxerFactory; this.muxerFactory = muxerFactory;
this.listener = listener; this.listener = listener;
checkArgument(muxerMode == MUXER_MODE_DEFAULT || muxerMode == MUXER_MODE_MUX_PARTIAL); checkArgument(muxerMode == MUXER_MODE_DEFAULT || muxerMode == MUXER_MODE_MUX_PARTIAL);
this.muxerMode = muxerMode; this.muxerMode = muxerMode;
this.dropSamplesBeforeFirstVideoSample = dropSamplesBeforeFirstVideoSample; this.dropSamplesBeforeFirstVideoSample = dropSamplesBeforeFirstVideoSample;
checkArgument(
(muxerMode == MUXER_MODE_DEFAULT && appendVideoFormat == null)
|| (muxerMode == MUXER_MODE_MUX_PARTIAL && appendVideoFormat != null),
"appendVideoFormat must be present if and only if muxerMode is MUXER_MODE_MUX_PARTIAL.");
this.appendVideoFormat = appendVideoFormat;
trackTypeToInfo = new SparseArray<>(); trackTypeToInfo = new SparseArray<>();
previousTrackType = C.TRACK_TYPE_NONE; previousTrackType = C.TRACK_TYPE_NONE;
firstVideoPresentationTimeUs = C.TIME_UNSET; firstVideoPresentationTimeUs = C.TIME_UNSET;
@ -172,35 +203,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** /**
* Returns whether the initialization data of two video formats can be muxed together. * Returns initialization data that is strict enough for both bitstreams, or {@code null} if the
* same initialization data cannot represent both bitstreams.
* *
* @param existingVideoTrackFormat The starting video format to compare. * @param existingVideoTrackFormat The starting video format to compare.
* @param newVideoTrackFormat The candidate format of the video bitstream to be appended after the * @param newVideoTrackFormat The candidate format of the video bitstream to be appended after the
* existing format. * existing format.
* @return {@code true} if both input formats can be muxed together in the same container. * @return The initialization data that captures both input formats, or {@code null} if both
* formats cannot be represented by the same initialization data.
*/ */
public static boolean isInitializationDataCompatible( @Nullable
@VisibleForTesting(otherwise = PRIVATE)
public static List<byte[]> getMostComatibleInitializationData(
Format existingVideoTrackFormat, Format newVideoTrackFormat) { Format existingVideoTrackFormat, Format newVideoTrackFormat) {
if (existingVideoTrackFormat.initializationDataEquals(newVideoTrackFormat)) { if (existingVideoTrackFormat.initializationDataEquals(newVideoTrackFormat)) {
return true; return existingVideoTrackFormat.initializationData;
} }
if (!Objects.equals(newVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264) if (!Objects.equals(newVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)
|| !Objects.equals(existingVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)) { || !Objects.equals(existingVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)) {
return false; return null;
} }
// For H.264 we allow a different level number. This is not spec-compliant, but such videos are
// already common on Android. See, for example: {@link MediaFormat#KEY_LEVEL}.
// Android players are advised to support level mismatch between container
// and bitstream: see {@link android.media.MediaExtractor#getTrackFormat(int)}.
if (newVideoTrackFormat.initializationData.size() != 2 if (newVideoTrackFormat.initializationData.size() != 2
|| existingVideoTrackFormat.initializationData.size() != 2) { || existingVideoTrackFormat.initializationData.size() != 2) {
return false; return null;
} }
// Check picture parameter sets match. // Check picture parameter sets match.
if (!Arrays.equals( if (!Arrays.equals(
newVideoTrackFormat.initializationData.get(1), newVideoTrackFormat.initializationData.get(1),
existingVideoTrackFormat.initializationData.get(1))) { existingVideoTrackFormat.initializationData.get(1))) {
return false; return null;
} }
// Allow level_idc to be lower in the new stream. // Allow level_idc to be lower in the new stream.
// Note: the SPS doesn't need to be unescaped because it's not possible to have two // Note: the SPS doesn't need to be unescaped because it's not possible to have two
@ -210,31 +241,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Skip 3 bytes: NAL unit type, profile, and reserved fields. // Skip 3 bytes: NAL unit type, profile, and reserved fields.
int spsLevelIndex = NalUnitUtil.NAL_START_CODE.length + 3; int spsLevelIndex = NalUnitUtil.NAL_START_CODE.length + 3;
if (spsLevelIndex >= newSps.length) { if (spsLevelIndex >= newSps.length) {
return false; return null;
} }
if (newSps.length != existingSps.length) { if (newSps.length != existingSps.length) {
return false; return null;
} }
for (int i = 0; i < newSps.length; i++) { for (int i = 0; i < newSps.length; i++) {
if (i != spsLevelIndex && newSps[i] != existingSps[i]) { if (i != spsLevelIndex && newSps[i] != existingSps[i]) {
return false; return null;
} }
} }
for (int i = 0; i < NalUnitUtil.NAL_START_CODE.length; i++) { for (int i = 0; i < NalUnitUtil.NAL_START_CODE.length; i++) {
if (newSps[i] != NalUnitUtil.NAL_START_CODE[i]) { if (newSps[i] != NalUnitUtil.NAL_START_CODE[i]) {
return false; return null;
} }
} }
int nalUnitTypeMask = 0x1F; int nalUnitTypeMask = 0x1F;
if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask) if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask)
!= NalUnitUtil.NAL_UNIT_TYPE_SPS) { != NalUnitUtil.NAL_UNIT_TYPE_SPS) {
return false; return null;
} }
// Check that H.264 profile is non-zero. // Check that H.264 profile is non-zero.
if (newSps[NalUnitUtil.NAL_START_CODE.length + 1] == 0) { if (newSps[NalUnitUtil.NAL_START_CODE.length + 1] == 0) {
return false; return null;
} }
return true; return existingSps[spsLevelIndex] >= newSps[spsLevelIndex]
? existingVideoTrackFormat.initializationData
: newVideoTrackFormat.initializationData;
} }
/** /**
@ -317,6 +350,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param format The {@link Format} to be added. In {@link #MUXER_MODE_APPEND} mode, the added * @param format The {@link Format} to be added. In {@link #MUXER_MODE_APPEND} mode, the added
* {@link Format} must match the existing {@link Format} set when the muxer was in {@link * {@link Format} must match the existing {@link Format} set when the muxer was in {@link
* #MUXER_MODE_MUX_PARTIAL} mode. * #MUXER_MODE_MUX_PARTIAL} mode.
* @throws AppendTrackFormatException If the existing {@link Format} does not match the newly
* added {@link Format} in {@link #MUXER_MODE_APPEND}.
* @throws IllegalArgumentException If the format is unsupported or if it does not match the * @throws IllegalArgumentException If the format is unsupported or if it does not match the
* existing format in {@link #MUXER_MODE_APPEND} mode. * existing format in {@link #MUXER_MODE_APPEND} mode.
* @throws IllegalStateException If the number of formats added exceeds the {@linkplain * @throws IllegalStateException If the number of formats added exceeds the {@linkplain
@ -325,7 +360,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @throws Muxer.MuxerException If the underlying {@link Muxer} encounters a problem while adding * @throws Muxer.MuxerException If the underlying {@link Muxer} encounters a problem while adding
* the track. * the track.
*/ */
public void addTrackFormat(Format format) throws Muxer.MuxerException { public void addTrackFormat(Format format)
throws AppendTrackFormatException, Muxer.MuxerException {
@Nullable String sampleMimeType = format.sampleMimeType; @Nullable String sampleMimeType = format.sampleMimeType;
@C.TrackType int trackType = MimeTypes.getTrackType(sampleMimeType); @C.TrackType int trackType = MimeTypes.getTrackType(sampleMimeType);
checkArgument( checkArgument(
@ -341,19 +377,57 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// format but these fields can be ignored. // format but these fields can be ignored.
// TODO: b/308180225 - Compare Format.colorInfo as well. // TODO: b/308180225 - Compare Format.colorInfo as well.
Format existingFormat = videoTrackInfo.format; Format existingFormat = videoTrackInfo.format;
checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType)); if (!areEqual(existingFormat.sampleMimeType, format.sampleMimeType)) {
checkArgument(existingFormat.width == format.width); throw new AppendTrackFormatException(
checkArgument(existingFormat.height == format.height); "Video format mismatch - sampleMimeType: "
checkArgument(isInitializationDataCompatible(existingFormat, format)); + existingFormat.sampleMimeType
+ " != "
+ format.sampleMimeType);
}
if (existingFormat.width != format.width) {
throw new AppendTrackFormatException(
"Video format mismatch - width: " + existingFormat.width + " != " + format.width);
}
if (existingFormat.height != format.height) {
throw new AppendTrackFormatException(
"Video format mismatch - height: " + existingFormat.height + " != " + format.height);
}
// The initialization data of the existing format is already compatible with
// appendVideoFormat.
if (!format.initializationDataEquals(checkNotNull(appendVideoFormat))) {
throw new AppendTrackFormatException(
"The initialization data of the newly added track format doesn't match"
+ " appendVideoFormat.");
}
} else if (trackType == C.TRACK_TYPE_AUDIO) { } else if (trackType == C.TRACK_TYPE_AUDIO) {
checkState(contains(trackTypeToInfo, C.TRACK_TYPE_AUDIO)); checkState(contains(trackTypeToInfo, C.TRACK_TYPE_AUDIO));
TrackInfo audioTrackInfo = trackTypeToInfo.get(C.TRACK_TYPE_AUDIO); TrackInfo audioTrackInfo = trackTypeToInfo.get(C.TRACK_TYPE_AUDIO);
Format existingFormat = audioTrackInfo.format; Format existingFormat = audioTrackInfo.format;
checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType)); if (!areEqual(existingFormat.sampleMimeType, format.sampleMimeType)) {
checkArgument(existingFormat.channelCount == format.channelCount); throw new AppendTrackFormatException(
checkArgument(existingFormat.sampleRate == format.sampleRate); "Audio format mismatch - sampleMimeType: "
checkArgument(existingFormat.initializationDataEquals(format)); + existingFormat.sampleMimeType
+ " != "
+ format.sampleMimeType);
}
if (existingFormat.channelCount != format.channelCount) {
throw new AppendTrackFormatException(
"Audio format mismatch - channelCount: "
+ existingFormat.channelCount
+ " != "
+ format.channelCount);
}
if (existingFormat.sampleRate != format.sampleRate) {
throw new AppendTrackFormatException(
"Audio format mismatch - sampleRate: "
+ existingFormat.sampleRate
+ " != "
+ format.sampleRate);
}
if (!existingFormat.initializationDataEquals(format)) {
throw new AppendTrackFormatException("Audio format mismatch - initializationData.");
}
} }
resetAbortTimer(); resetAbortTimer();
return; return;
@ -365,15 +439,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkState( checkState(
!contains(trackTypeToInfo, trackType), "There is already a track of type " + trackType); !contains(trackTypeToInfo, trackType), "There is already a track of type " + trackType);
ensureMuxerInitialized();
if (trackType == C.TRACK_TYPE_VIDEO) { if (trackType == C.TRACK_TYPE_VIDEO) {
format = format =
format format
.buildUpon() .buildUpon()
.setRotationDegrees((format.rotationDegrees + additionalRotationDegrees) % 360) .setRotationDegrees((format.rotationDegrees + additionalRotationDegrees) % 360)
.build(); .build();
if (muxerMode == MUXER_MODE_MUX_PARTIAL) {
List<byte[]> mostCompatibleInitializationData =
getMostComatibleInitializationData(format, checkNotNull(appendVideoFormat));
if (mostCompatibleInitializationData == null) {
throw new AppendTrackFormatException("Switching to MUXER_MODE_APPEND will fail.");
}
format = format.buildUpon().setInitializationData(mostCompatibleInitializationData).build();
}
} }
ensureMuxerInitialized();
TrackInfo trackInfo = new TrackInfo(format, muxer.addTrack(format)); TrackInfo trackInfo = new TrackInfo(format, muxer.addTrack(format));
trackTypeToInfo.put(trackType, trackInfo); trackTypeToInfo.put(trackType, trackInfo);

View file

@ -111,6 +111,8 @@ import java.util.List;
muxerWrapper.addTrackFormat(inputFormat); muxerWrapper.addTrackFormat(inputFormat);
} catch (Muxer.MuxerException e) { } catch (Muxer.MuxerException e) {
throw ExportException.createForMuxer(e, ExportException.ERROR_CODE_MUXING_FAILED); throw ExportException.createForMuxer(e, ExportException.ERROR_CODE_MUXING_FAILED);
} catch (MuxerWrapper.AppendTrackFormatException e) {
throw ExportException.createForMuxer(e, ExportException.ERROR_CODE_MUXING_APPEND);
} }
muxerWrapperTrackAdded = true; muxerWrapperTrackAdded = true;
} }

View file

@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.extractor.AacUtil.AAC_LC_AUDIO_SAMPLE_COUNT; import static androidx.media3.extractor.AacUtil.AAC_LC_AUDIO_SAMPLE_COUNT;
import static androidx.media3.transformer.ExportException.ERROR_CODE_MUXING_APPEND;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_OTHER; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_OTHER;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_TRIM_AND_TRANSCODING_TRANSFORMATION_REQUESTED; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_TRIM_AND_TRANSCODING_TRANSFORMATION_REQUESTED;
@ -1004,7 +1005,8 @@ public final class Transformer {
muxerFactory, muxerFactory,
componentListener, componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT, MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ fileStartsOnVideoFrameEnabled), /* dropSamplesBeforeFirstVideoSample= */ fileStartsOnVideoFrameEnabled,
/* appendVideoFormat= */ null),
componentListener, componentListener,
/* initialTimestampOffsetUs= */ 0, /* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false); /* useDefaultAssetLoaderFactory= */ false);
@ -1123,7 +1125,7 @@ public final class Transformer {
return PROGRESS_STATE_UNAVAILABLE; return PROGRESS_STATE_UNAVAILABLE;
} }
if (transformerState != TRANSFORMER_STATE_PROCESS_FULL_INPUT) { if (isExportTrimOptimization()) {
return getTrimOptimizationProgress(progressHolder); return getTrimOptimizationProgress(progressHolder);
} }
@ -1139,6 +1141,11 @@ public final class Transformer {
|| transformerState == TRANSFORMER_STATE_COPY_OUTPUT; || transformerState == TRANSFORMER_STATE_COPY_OUTPUT;
} }
private boolean isExportTrimOptimization() {
return transformerState == TRANSFORMER_STATE_PROCESS_MEDIA_START
|| transformerState == TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
}
private @ProgressState int getTrimOptimizationProgress(ProgressHolder progressHolder) { private @ProgressState int getTrimOptimizationProgress(ProgressHolder progressHolder) {
if (mediaItemInfo == null) { if (mediaItemInfo == null) {
return PROGRESS_STATE_WAITING_FOR_AVAILABILITY; return PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
@ -1248,7 +1255,8 @@ public final class Transformer {
muxerFactory, muxerFactory,
componentListener, componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT, MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false), /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null),
componentListener, componentListener,
/* initialTimestampOffsetUs= */ 0, /* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false); /* useDefaultAssetLoaderFactory= */ false);
@ -1280,7 +1288,8 @@ public final class Transformer {
muxerFactory, muxerFactory,
componentListener, componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL, MuxerWrapper.MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ resumeMetadata.videoFormat);
startInternal( startInternal(
TransmuxTranscodeHelper.createVideoOnlyComposition( TransmuxTranscodeHelper.createVideoOnlyComposition(
@ -1324,15 +1333,19 @@ public final class Transformer {
private void processAudio() { private void processAudio() {
transformerState = TRANSFORMER_STATE_PROCESS_AUDIO; transformerState = TRANSFORMER_STATE_PROCESS_AUDIO;
startInternal( MuxerWrapper muxerWrapper =
TransmuxTranscodeHelper.createAudioTranscodeAndVideoTransmuxComposition(
checkNotNull(composition), checkNotNull(outputFilePath)),
new MuxerWrapper( new MuxerWrapper(
checkNotNull(oldFilePath), checkNotNull(oldFilePath),
muxerFactory, muxerFactory,
componentListener, componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT, MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false), /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
startInternal(
TransmuxTranscodeHelper.createAudioTranscodeAndVideoTransmuxComposition(
checkNotNull(composition), checkNotNull(outputFilePath)),
muxerWrapper,
componentListener, componentListener,
/* initialTimestampOffsetUs= */ 0, /* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false); /* useDefaultAssetLoaderFactory= */ false);
@ -1421,7 +1434,8 @@ public final class Transformer {
muxerFactory, muxerFactory,
componentListener, componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL, MuxerWrapper.MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
mp4Info.videoFormat);
if (shouldTranscodeVideo( if (shouldTranscodeVideo(
checkNotNull(mp4Info.videoFormat), checkNotNull(mp4Info.videoFormat),
composition, composition,
@ -1478,14 +1492,6 @@ public final class Transformer {
EditedMediaItem firstEditedMediaItem = EditedMediaItem firstEditedMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0); checkNotNull(composition).sequences.get(0).editedMediaItems.get(0);
Mp4Info mediaItemInfo = checkNotNull(this.mediaItemInfo); Mp4Info mediaItemInfo = checkNotNull(this.mediaItemInfo);
if (!doesFormatsMatch(mediaItemInfo, firstEditedMediaItem)) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.reset();
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
long trimStartTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs; long trimStartTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.endPositionUs; long trimEndTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.endPositionUs;
Composition transmuxComposition = Composition transmuxComposition =
@ -1507,19 +1513,6 @@ public final class Transformer {
/* useDefaultAssetLoaderFactory= */ false); /* useDefaultAssetLoaderFactory= */ false);
} }
private boolean doesFormatsMatch(Mp4Info mediaItemInfo, EditedMediaItem firstMediaItem) {
Format videoFormat = checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_VIDEO);
boolean videoFormatMatches =
MuxerWrapper.isInitializationDataCompatible(
videoFormat, checkNotNull(mediaItemInfo.videoFormat));
boolean audioFormatMatches =
mediaItemInfo.audioFormat == null
|| firstMediaItem.removeAudio
|| mediaItemInfo.audioFormat.initializationDataEquals(
checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
return videoFormatMatches && audioFormatMatches;
}
private boolean isMultiAsset() { private boolean isMultiAsset() {
return checkNotNull(composition).sequences.size() > 1 return checkNotNull(composition).sequences.size() > 1
|| composition.sequences.get(0).editedMediaItems.size() > 1; || composition.sequences.get(0).editedMediaItems.size() > 1;
@ -1636,6 +1629,16 @@ public final class Transformer {
@Nullable String audioEncoderName, @Nullable String audioEncoderName,
@Nullable String videoEncoderName, @Nullable String videoEncoderName,
ExportException exportException) { ExportException exportException) {
if (exportException.errorCode == ERROR_CODE_MUXING_APPEND
&& (isExportTrimOptimization() || isExportResumed())) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.reset();
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
exportResultBuilder.addProcessedInputs(processedInputs); exportResultBuilder.addProcessedInputs(processedInputs);
// When an export is resumed, the audio and video encoder name (if any) can comes from // When an export is resumed, the audio and video encoder name (if any) can comes from

View file

@ -21,6 +21,7 @@ import android.content.Context;
import android.util.Pair; import android.util.Pair;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem; import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
@ -51,11 +52,16 @@ import java.util.List;
*/ */
public final ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo; public final ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo;
/** The video {@link Format} or {@code null} if there is no video track. */
@Nullable public final Format videoFormat;
public ResumeMetadata( public ResumeMetadata(
long lastSyncSampleTimestampUs, long lastSyncSampleTimestampUs,
ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo) { ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo,
@Nullable Format videoFormat) {
this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs; this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs;
this.firstMediaItemIndexAndOffsetInfo = firstMediaItemIndexAndOffsetInfo; this.firstMediaItemIndexAndOffsetInfo = firstMediaItemIndexAndOffsetInfo;
this.videoFormat = videoFormat;
} }
} }
@ -268,8 +274,8 @@ import java.util.List;
if (resumeMetadataSettableFuture.isCancelled()) { if (resumeMetadataSettableFuture.isCancelled()) {
return; return;
} }
long lastSyncSampleTimestampUs = Mp4Info mp4Info = Mp4Info.create(context, filePath);
Mp4Info.create(context, filePath).lastSyncSampleTimestampUs; long lastSyncSampleTimestampUs = mp4Info.lastSyncSampleTimestampUs;
ImmutableList.Builder<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfoBuilder = ImmutableList.Builder<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfoBuilder =
new ImmutableList.Builder<>(); new ImmutableList.Builder<>();
@ -301,7 +307,9 @@ import java.util.List;
} }
resumeMetadataSettableFuture.set( resumeMetadataSettableFuture.set(
new ResumeMetadata( new ResumeMetadata(
lastSyncSampleTimestampUs, firstMediaItemIndexAndOffsetInfoBuilder.build())); lastSyncSampleTimestampUs,
firstMediaItemIndexAndOffsetInfoBuilder.build(),
mp4Info.videoFormat));
} catch (Exception ex) { } catch (Exception ex) {
resumeMetadataSettableFuture.setException(ex); resumeMetadataSettableFuture.setException(ex);
} }

View file

@ -68,7 +68,8 @@ public final class EncodedSampleExporterTest {
new InAppMuxer.Factory.Builder().build(), new InAppMuxer.Factory.Builder().build(),
mock(MuxerWrapper.Listener.class), mock(MuxerWrapper.Listener.class),
MuxerWrapper.MUXER_MODE_DEFAULT, MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false), /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null),
fallbackListener, fallbackListener,
/* initialTimestampOffsetUs= */ 0); /* initialTimestampOffsetUs= */ 0);
} }

View file

@ -18,6 +18,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.AUDIO_AAC; import static androidx.media3.common.MimeTypes.AUDIO_AAC;
import static androidx.media3.common.MimeTypes.VIDEO_H264; import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265; import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.test.utils.TestUtil.createByteArray;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT; import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_MUX_PARTIAL; import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_MUX_PARTIAL;
import static androidx.media3.transformer.TestUtil.getDumpFileName; import static androidx.media3.transformer.TestUtil.getDumpFileName;
@ -34,6 +35,8 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import org.junit.After; import org.junit.After;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@ -44,12 +47,16 @@ import org.junit.runner.RunWith;
/** Unit tests for {@link MuxerWrapper}. */ /** Unit tests for {@link MuxerWrapper}. */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public class MuxerWrapperTest { public class MuxerWrapperTest {
private static final byte[] SPS_TEST_DATA =
createByteArray(
0x00, 0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x0A,
0x90, 0x91, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB);
private static final Format FAKE_VIDEO_TRACK_FORMAT = private static final Format FAKE_VIDEO_TRACK_FORMAT =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H264) .setSampleMimeType(VIDEO_H264)
.setWidth(1080) .setWidth(1080)
.setHeight(720) .setHeight(720)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) .setInitializationData(ImmutableList.of(SPS_TEST_DATA, new byte[] {1, 2, 3, 4}))
.setColorInfo(ColorInfo.SDR_BT709_LIMITED) .setColorInfo(ColorInfo.SDR_BT709_LIMITED)
.build(); .build();
private static final Format FAKE_AUDIO_TRACK_FORMAT = private static final Format FAKE_AUDIO_TRACK_FORMAT =
@ -83,7 +90,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
muxerWrapper.setAdditionalRotationDegrees(90); muxerWrapper.setAdditionalRotationDegrees(90);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.setAdditionalRotationDegrees(180); muxerWrapper.setAdditionalRotationDegrees(180);
@ -100,7 +108,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
muxerWrapper.setAdditionalRotationDegrees(90); muxerWrapper.setAdditionalRotationDegrees(90);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.setAdditionalRotationDegrees(180); muxerWrapper.setAdditionalRotationDegrees(180);
@ -117,11 +126,27 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThrows(IllegalStateException.class, muxerWrapper::changeToAppendMode); assertThrows(IllegalStateException.class, muxerWrapper::changeToAppendMode);
} }
@Test
public void constructor_withAppendVideoFormatMissingInPartialMode_throws() {
assertThrows(
IllegalArgumentException.class,
() ->
muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null));
}
@Test @Test
public void addTrackFormat_withSameVideoFormatInAppendMode_doesNotThrow() throws Exception { public void addTrackFormat_withSameVideoFormatInAppendMode_doesNotThrow() throws Exception {
muxerWrapper = muxerWrapper =
@ -130,7 +155,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -152,7 +178,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -167,13 +194,15 @@ public class MuxerWrapperTest {
@Test @Test
public void addTrackFormat_withDifferentVideoFormatInAppendMode_throws() throws Exception { public void addTrackFormat_withDifferentVideoFormatInAppendMode_throws() throws Exception {
Format differentVideoFormat = FAKE_VIDEO_TRACK_FORMAT.buildUpon().setHeight(5000).build();
muxerWrapper = muxerWrapper =
new MuxerWrapper( new MuxerWrapper(
temporaryFolder.newFile().getPath(), temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ differentVideoFormat);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -182,10 +211,47 @@ public class MuxerWrapperTest {
muxerWrapper.release(MuxerWrapper.MUXER_RELEASE_REASON_COMPLETED); muxerWrapper.release(MuxerWrapper.MUXER_RELEASE_REASON_COMPLETED);
muxerWrapper.changeToAppendMode(); muxerWrapper.changeToAppendMode();
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
Format differentVideoFormat = FAKE_VIDEO_TRACK_FORMAT.buildUpon().setHeight(5000).build();
assertThrows( assertThrows(
IllegalArgumentException.class, () -> muxerWrapper.addTrackFormat(differentVideoFormat)); MuxerWrapper.AppendTrackFormatException.class,
() -> muxerWrapper.addTrackFormat(differentVideoFormat));
}
@Test
public void addTrackFormat_withCompatibleVideoFormatInAppendMode_savesTheMostCompatibleInitData()
throws Exception {
byte[] newSpsTestData = Arrays.copyOf(SPS_TEST_DATA, SPS_TEST_DATA.length);
int spsLevelIndex = 7;
byte lowSpsLevel = 11;
newSpsTestData[spsLevelIndex] = lowSpsLevel;
Format differentVideoFormat =
FAKE_VIDEO_TRACK_FORMAT
.buildUpon()
.setInitializationData(
ImmutableList.of(newSpsTestData, FAKE_VIDEO_TRACK_FORMAT.initializationData.get(1)))
.build();
muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ differentVideoFormat);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
C.TRACK_TYPE_VIDEO, FAKE_SAMPLE, /* isKeyFrame= */ true, /* presentationTimeUs= */ 0);
muxerWrapper.endTrack(C.TRACK_TYPE_VIDEO);
muxerWrapper.changeToAppendMode();
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(differentVideoFormat);
assertThat(
muxerWrapper
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(FAKE_VIDEO_TRACK_FORMAT))
.isTrue();
} }
@Test @Test
@ -196,7 +262,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -208,7 +275,8 @@ public class MuxerWrapperTest {
Format differentAudioFormat = FAKE_AUDIO_TRACK_FORMAT.buildUpon().setSampleRate(48000).build(); Format differentAudioFormat = FAKE_AUDIO_TRACK_FORMAT.buildUpon().setSampleRate(48000).build();
assertThrows( assertThrows(
IllegalArgumentException.class, () -> muxerWrapper.addTrackFormat(differentAudioFormat)); MuxerWrapper.AppendTrackFormatException.class,
() -> muxerWrapper.addTrackFormat(differentAudioFormat));
} }
@Test @Test
@ -221,7 +289,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ true); /* dropSamplesBeforeFirstVideoSample= */ true,
/* appendVideoFormat= */ null);
muxerWrapper.setTrackCount(2); muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -248,7 +317,8 @@ public class MuxerWrapperTest {
muxerFactory, muxerFactory,
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ true); /* dropSamplesBeforeFirstVideoSample= */ true,
/* appendVideoFormat= */ null);
muxerWrapper.setTrackCount(2); muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -286,7 +356,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -306,7 +377,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(2); muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -334,7 +406,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -349,7 +422,7 @@ public class MuxerWrapperTest {
} }
@Test @Test
public void isInitializationDataCompatible_h265_differentCSD_returnsFalse() { public void getMostCompatibleFormat_h265_differentCSD_returnsNull() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H265) .setSampleMimeType(VIDEO_H265)
@ -361,11 +434,12 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
} }
@Test @Test
public void isInitializationDataCompatible_h265_matchingCSD_returnsTrue() { public void getMostCompatibleFormat_h265_matchingCSD_returnsFormat() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H265) .setSampleMimeType(VIDEO_H265)
@ -377,11 +451,19 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(1);
Byte[] expectedInitializationData = new Byte[] {1, 2, 3, 4};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationData)
.inOrder();
} }
@Test @Test
public void isInitializationDataCompatible_h264_matchingCSD_returnsTrue() { public void getMostCompatibleFormat_h264_matchingCSD_returnsFormat() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H264) .setSampleMimeType(VIDEO_H264)
@ -393,11 +475,19 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4})) .setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(1);
Byte[] expectedInitializationData = new Byte[] {1, 2, 3, 4};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationData)
.inOrder();
} }
@Test @Test
public void isInitializationDataCompatible_h264_ignoresLevel_returnsTrue() { public void getMostCompatibleFormat_h264_differentLevel_returnsFormat() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H264) .setSampleMimeType(VIDEO_H264)
@ -411,11 +501,24 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue(); List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(2);
Byte[] expectedInitializationDataSps = new Byte[] {0, 0, 0, 1, 103, 100, 0, 41};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationDataSps)
.inOrder();
Byte[] expectedInitializationDataPps = new Byte[] {0, 0, 0, 1};
assertThat(initializationData.get(1))
.asList()
.containsExactlyElementsIn(expectedInitializationDataPps)
.inOrder();
} }
@Test @Test
public void isInitializationDataCompatible_h264_mismatchProfile_returnsFalse() { public void getMostCompatibleFormat_h264_mismatchProfile_returnsNull() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H264) .setSampleMimeType(VIDEO_H264)
@ -429,11 +532,12 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
} }
@Test @Test
public void isInitializationDataCompatible_h264_missingMimeType_returnsFalse() { public void getMostCompatibleFormat_h264_missingMimeType_returnsNull() {
Format existingFormat = Format existingFormat =
new Format.Builder() new Format.Builder()
.setSampleMimeType(VIDEO_H264) .setSampleMimeType(VIDEO_H264)
@ -446,7 +550,8 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1})) ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build(); .build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse(); assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
} }
@Test @Test
@ -458,7 +563,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -483,7 +589,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(
@ -507,7 +614,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL, MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1); muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT); muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample( muxerWrapper.writeSample(

View file

@ -70,7 +70,8 @@ public final class TransformerUtilTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThat( assertThat(
shouldTranscodeVideo( shouldTranscodeVideo(
@ -102,7 +103,8 @@ public final class TransformerUtilTest {
new DefaultMuxer.Factory(), new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(), new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT, MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false); /* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThat( assertThat(
shouldTranscodeVideo( shouldTranscodeVideo(