sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-24 14:37:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Pick max H.264 level when trim optimizing

Browse source 
Instead of always starting with the transcoded H.264 level, take the maximum
from transcoded and transmuxed levels

PiperOrigin-RevId: 610759438
This commit is contained in:
Googler 2024-02-27 08:24:41 -08:00 committed by Copybara-Service
parent c3aec4f19d
commit 6f28eeff31
10 changed files with 327 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
libraries/test_data/src/test/assets/transformerdumps/testspecificdumps/writeSample_dropSamplesBeforeFirstVideoSampleEnabled_dropsAudioSamplesTimedBeforeFirstVideoSample/original.dump
View file

@ -11,6 +11,7 @@ format video:
colorRange = 2
colorTransfer = 3
initializationData:
data = length 28, hash D27A8A6F
data = length 4, hash E93C3
sample:
trackType = audio

16
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ExportTest.java
View file

@ -45,6 +45,11 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.extractor.mp4.Mp4Extractor;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.test.utils.FakeExtractorOutput;
import androidx.media3.test.utils.FakeTrackOutput;
import androidx.media3.test.utils.TestUtil;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.AndroidTestUtil.ForceEncodeEncoderFactory;
import androidx.media3.transformer.DefaultEncoderFactory;
@ -387,8 +392,6 @@ public class ExportTest {
@Test
public void clippedMedia_trimOptimizationEnabled_pixel7Plus_completesWithOptimizationApplied()
throws Exception {
String testId =
TAG + "_clippedMedia_trimOptimizationEnabled_pixel_completesWithOptimizationApplied";
Context context = ApplicationProvider.getApplicationContext();
// Devices with Tensor G2 & G3 chipsets should work, but Pixel 7a is flaky.
assumeTrue(
@ -415,10 +418,19 @@ public class ExportTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
Mp4Extractor mp4Extractor = new Mp4Extractor(new DefaultSubtitleParserFactory());
FakeExtractorOutput fakeExtractorOutput =
TestUtil.extractAllSamplesFromFilePath(mp4Extractor, result.filePath);
FakeTrackOutput videoTrack = fakeExtractorOutput.trackOutputs.get(0);
byte[] sps = videoTrack.lastFormat.initializationData.get(0);
// Skip 7 bytes: NAL unit start code (4) and NAL unit type, profile, and reserved fields.
int spsLevelIndex = 7;
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(700);
assertThat(result.exportResult.videoConversionProcess)
.isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED);
int higherVideoLevel = 41;
assertThat(sps[spsLevelIndex]).isEqualTo(higherVideoLevel);
}
}

8
libraries/transformer/src/main/java/androidx/media3/transformer/ExportException.java
View file

@ -185,6 +185,13 @@ public final class ExportException extends Exception {
*/
public static final int ERROR_CODE_MUXING_TIMEOUT = 7002;
/**
* Caused by mismatching formats in MuxerWrapper.
*
* @see MuxerWrapper.AppendTrackFormatException
*/
public static final int ERROR_CODE_MUXING_APPEND = 7003;
/* package */ static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE =
new ImmutableBiMap.Builder<String, @ErrorCode Integer>()
.put("ERROR_CODE_FAILED_RUNTIME_CHECK", ERROR_CODE_FAILED_RUNTIME_CHECK)
@ -207,6 +214,7 @@ public final class ExportException extends Exception {
.put("ERROR_CODE_AUDIO_PROCESSING_FAILED", ERROR_CODE_AUDIO_PROCESSING_FAILED)
.put("ERROR_CODE_MUXING_FAILED", ERROR_CODE_MUXING_FAILED)
.put("ERROR_CODE_MUXING_TIMEOUT", ERROR_CODE_MUXING_TIMEOUT)
.put("ERROR_CODE_MUXING_APPEND", ERROR_CODE_MUXING_APPEND)
.buildOrThrow();
/** Returns the name of a given {@code errorCode}. */

142
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View file

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.annotation.VisibleForTesting.PRIVATE;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
@ -31,6 +32,7 @@ import android.util.SparseArray;
import androidx.annotation.IntDef;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Metadata;
@ -46,6 +48,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
@ -58,6 +61,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper can contain at most one video track and one audio track.
*/
/* package */ final class MuxerWrapper {
/**
* Thrown when video formats fail to match between {@link #MUXER_MODE_MUX_PARTIAL} and {@link
* #MUXER_MODE_APPEND}.
*/
public static final class AppendTrackFormatException extends Exception {
/**
* Creates an instance.
*
* @param message See {@link #getMessage()}.
*/
public AppendTrackFormatException(String message) {
super(message);
}
}
/** Different modes for muxing. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@ -125,6 +144,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final boolean dropSamplesBeforeFirstVideoSample;
private final SparseArray<TrackInfo> trackTypeToInfo;
private final ScheduledExecutorService abortScheduledExecutorService;
private final @MonotonicNonNull Format appendVideoFormat;
private boolean isReady;
private boolean isEnded;
@ -145,6 +165,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Creates an instance.
*
* <p>{@code appendVideoFormat} must be non-{@code null} when using {@link
* #MUXER_MODE_MUX_PARTIAL}.
*
* @param outputPath The output file path to write the media data to.
* @param muxerFactory A {@link Muxer.Factory} to create a {@link Muxer}.
* @param listener A {@link MuxerWrapper.Listener}.
@ -152,19 +175,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* {@link #MUXER_MODE_MUX_PARTIAL}.
* @param dropSamplesBeforeFirstVideoSample Whether to drop any non-video samples with
* presentation timestamps before the first video sample.
* @param appendVideoFormat The format which will be used to write samples after transitioning
* from {@link #MUXER_MODE_MUX_PARTIAL} to {@link #MUXER_MODE_APPEND}.
*/
public MuxerWrapper(
String outputPath,
Muxer.Factory muxerFactory,
Listener listener,
@MuxerMode int muxerMode,
boolean dropSamplesBeforeFirstVideoSample) {
boolean dropSamplesBeforeFirstVideoSample,
@Nullable Format appendVideoFormat) {
this.outputPath = outputPath;
this.muxerFactory = muxerFactory;
this.listener = listener;
checkArgument(muxerMode == MUXER_MODE_DEFAULT || muxerMode == MUXER_MODE_MUX_PARTIAL);
this.muxerMode = muxerMode;
this.dropSamplesBeforeFirstVideoSample = dropSamplesBeforeFirstVideoSample;
checkArgument(
(muxerMode == MUXER_MODE_DEFAULT && appendVideoFormat == null)
|| (muxerMode == MUXER_MODE_MUX_PARTIAL && appendVideoFormat != null),
"appendVideoFormat must be present if and only if muxerMode is MUXER_MODE_MUX_PARTIAL.");
this.appendVideoFormat = appendVideoFormat;
trackTypeToInfo = new SparseArray<>();
previousTrackType = C.TRACK_TYPE_NONE;
firstVideoPresentationTimeUs = C.TIME_UNSET;
@ -172,35 +203,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
/**
* Returns whether the initialization data of two video formats can be muxed together.
* Returns initialization data that is strict enough for both bitstreams, or {@code null} if the
* same initialization data cannot represent both bitstreams.
*
* @param existingVideoTrackFormat The starting video format to compare.
* @param newVideoTrackFormat The candidate format of the video bitstream to be appended after the
* existing format.
* @return {@code true} if both input formats can be muxed together in the same container.
* @return The initialization data that captures both input formats, or {@code null} if both
* formats cannot be represented by the same initialization data.
*/
public static boolean isInitializationDataCompatible(
@Nullable
@VisibleForTesting(otherwise = PRIVATE)
public static List<byte[]> getMostComatibleInitializationData(
Format existingVideoTrackFormat, Format newVideoTrackFormat) {
if (existingVideoTrackFormat.initializationDataEquals(newVideoTrackFormat)) {
return true;
return existingVideoTrackFormat.initializationData;
}
if (!Objects.equals(newVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)
|| !Objects.equals(existingVideoTrackFormat.sampleMimeType, MimeTypes.VIDEO_H264)) {
return false;
return null;
}
// For H.264 we allow a different level number. This is not spec-compliant, but such videos are
// already common on Android. See, for example: {@link MediaFormat#KEY_LEVEL}.
// Android players are advised to support level mismatch between container
// and bitstream: see {@link android.media.MediaExtractor#getTrackFormat(int)}.
if (newVideoTrackFormat.initializationData.size() != 2
|| existingVideoTrackFormat.initializationData.size() != 2) {
return false;
return null;
}
// Check picture parameter sets match.
if (!Arrays.equals(
newVideoTrackFormat.initializationData.get(1),
existingVideoTrackFormat.initializationData.get(1))) {
return false;
return null;
}
// Allow level_idc to be lower in the new stream.
// Note: the SPS doesn't need to be unescaped because it's not possible to have two
@ -210,31 +241,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Skip 3 bytes: NAL unit type, profile, and reserved fields.
int spsLevelIndex = NalUnitUtil.NAL_START_CODE.length + 3;
if (spsLevelIndex >= newSps.length) {
return false;
return null;
}
if (newSps.length != existingSps.length) {
return false;
return null;
}
for (int i = 0; i < newSps.length; i++) {
if (i != spsLevelIndex && newSps[i] != existingSps[i]) {
return false;
return null;
}
}
for (int i = 0; i < NalUnitUtil.NAL_START_CODE.length; i++) {
if (newSps[i] != NalUnitUtil.NAL_START_CODE[i]) {
return false;
return null;
}
}
int nalUnitTypeMask = 0x1F;
if ((newSps[NalUnitUtil.NAL_START_CODE.length] & nalUnitTypeMask)
!= NalUnitUtil.NAL_UNIT_TYPE_SPS) {
return false;
return null;
}
// Check that H.264 profile is non-zero.
if (newSps[NalUnitUtil.NAL_START_CODE.length + 1] == 0) {
return false;
return null;
}
return true;
return existingSps[spsLevelIndex] >= newSps[spsLevelIndex]
? existingVideoTrackFormat.initializationData
: newVideoTrackFormat.initializationData;
}
/**
@ -317,6 +350,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param format The {@link Format} to be added. In {@link #MUXER_MODE_APPEND} mode, the added
* {@link Format} must match the existing {@link Format} set when the muxer was in {@link
* #MUXER_MODE_MUX_PARTIAL} mode.
* @throws AppendTrackFormatException If the existing {@link Format} does not match the newly
* added {@link Format} in {@link #MUXER_MODE_APPEND}.
* @throws IllegalArgumentException If the format is unsupported or if it does not match the
* existing format in {@link #MUXER_MODE_APPEND} mode.
* @throws IllegalStateException If the number of formats added exceeds the {@linkplain
@ -325,7 +360,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @throws Muxer.MuxerException If the underlying {@link Muxer} encounters a problem while adding
* the track.
*/
public void addTrackFormat(Format format) throws Muxer.MuxerException {
public void addTrackFormat(Format format)
throws AppendTrackFormatException, Muxer.MuxerException {
@Nullable String sampleMimeType = format.sampleMimeType;
@C.TrackType int trackType = MimeTypes.getTrackType(sampleMimeType);
checkArgument(
@ -341,19 +377,57 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// format but these fields can be ignored.
// TODO: b/308180225 - Compare Format.colorInfo as well.
Format existingFormat = videoTrackInfo.format;
checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType));
checkArgument(existingFormat.width == format.width);
checkArgument(existingFormat.height == format.height);
checkArgument(isInitializationDataCompatible(existingFormat, format));
if (!areEqual(existingFormat.sampleMimeType, format.sampleMimeType)) {
throw new AppendTrackFormatException(
"Video format mismatch - sampleMimeType: "
+ existingFormat.sampleMimeType
+ " != "
+ format.sampleMimeType);
}
if (existingFormat.width != format.width) {
throw new AppendTrackFormatException(
"Video format mismatch - width: " + existingFormat.width + " != " + format.width);
}
if (existingFormat.height != format.height) {
throw new AppendTrackFormatException(
"Video format mismatch - height: " + existingFormat.height + " != " + format.height);
}
// The initialization data of the existing format is already compatible with
// appendVideoFormat.
if (!format.initializationDataEquals(checkNotNull(appendVideoFormat))) {
throw new AppendTrackFormatException(
"The initialization data of the newly added track format doesn't match"
+ " appendVideoFormat.");
}
} else if (trackType == C.TRACK_TYPE_AUDIO) {
checkState(contains(trackTypeToInfo, C.TRACK_TYPE_AUDIO));
TrackInfo audioTrackInfo = trackTypeToInfo.get(C.TRACK_TYPE_AUDIO);
Format existingFormat = audioTrackInfo.format;
checkArgument(areEqual(existingFormat.sampleMimeType, format.sampleMimeType));
checkArgument(existingFormat.channelCount == format.channelCount);
checkArgument(existingFormat.sampleRate == format.sampleRate);
checkArgument(existingFormat.initializationDataEquals(format));
if (!areEqual(existingFormat.sampleMimeType, format.sampleMimeType)) {
throw new AppendTrackFormatException(
"Audio format mismatch - sampleMimeType: "
+ existingFormat.sampleMimeType
+ " != "
+ format.sampleMimeType);
}
if (existingFormat.channelCount != format.channelCount) {
throw new AppendTrackFormatException(
"Audio format mismatch - channelCount: "
+ existingFormat.channelCount
+ " != "
+ format.channelCount);
}
if (existingFormat.sampleRate != format.sampleRate) {
throw new AppendTrackFormatException(
"Audio format mismatch - sampleRate: "
+ existingFormat.sampleRate
+ " != "
+ format.sampleRate);
}
if (!existingFormat.initializationDataEquals(format)) {
throw new AppendTrackFormatException("Audio format mismatch - initializationData.");
}
}
resetAbortTimer();
return;
@ -365,15 +439,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkState(
!contains(trackTypeToInfo, trackType), "There is already a track of type " + trackType);
ensureMuxerInitialized();
if (trackType == C.TRACK_TYPE_VIDEO) {
format =
format
.buildUpon()
.setRotationDegrees((format.rotationDegrees + additionalRotationDegrees) % 360)
.build();
if (muxerMode == MUXER_MODE_MUX_PARTIAL) {
List<byte[]> mostCompatibleInitializationData =
getMostComatibleInitializationData(format, checkNotNull(appendVideoFormat));
if (mostCompatibleInitializationData == null) {
throw new AppendTrackFormatException("Switching to MUXER_MODE_APPEND will fail.");
}
format = format.buildUpon().setInitializationData(mostCompatibleInitializationData).build();
}
}
ensureMuxerInitialized();
TrackInfo trackInfo = new TrackInfo(format, muxer.addTrack(format));
trackTypeToInfo.put(trackType, trackInfo);

2
libraries/transformer/src/main/java/androidx/media3/transformer/SampleExporter.java
View file

@ -111,6 +111,8 @@ import java.util.List;
muxerWrapper.addTrackFormat(inputFormat);
} catch (Muxer.MuxerException e) {
throw ExportException.createForMuxer(e, ExportException.ERROR_CODE_MUXING_FAILED);
} catch (MuxerWrapper.AppendTrackFormatException e) {
throw ExportException.createForMuxer(e, ExportException.ERROR_CODE_MUXING_APPEND);
}
muxerWrapperTrackAdded = true;
}

63
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View file

@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.extractor.AacUtil.AAC_LC_AUDIO_SAMPLE_COUNT;
import static androidx.media3.transformer.ExportException.ERROR_CODE_MUXING_APPEND;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_OTHER;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_TRIM_AND_TRANSCODING_TRANSFORMATION_REQUESTED;
@ -1004,7 +1005,8 @@ public final class Transformer {
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ fileStartsOnVideoFrameEnabled),
/* dropSamplesBeforeFirstVideoSample= */ fileStartsOnVideoFrameEnabled,
/* appendVideoFormat= */ null),
componentListener,
/* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false);
@ -1123,7 +1125,7 @@ public final class Transformer {
return PROGRESS_STATE_UNAVAILABLE;
}
if (transformerState != TRANSFORMER_STATE_PROCESS_FULL_INPUT) {
if (isExportTrimOptimization()) {
return getTrimOptimizationProgress(progressHolder);
}
@ -1139,6 +1141,11 @@ public final class Transformer {
|| transformerState == TRANSFORMER_STATE_COPY_OUTPUT;
}
private boolean isExportTrimOptimization() {
return transformerState == TRANSFORMER_STATE_PROCESS_MEDIA_START
|| transformerState == TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
}
private @ProgressState int getTrimOptimizationProgress(ProgressHolder progressHolder) {
if (mediaItemInfo == null) {
return PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
@ -1248,7 +1255,8 @@ public final class Transformer {
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false),
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null),
componentListener,
/* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false);
@ -1280,7 +1288,8 @@ public final class Transformer {
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ resumeMetadata.videoFormat);
startInternal(
TransmuxTranscodeHelper.createVideoOnlyComposition(
@ -1324,15 +1333,19 @@ public final class Transformer {
private void processAudio() {
transformerState = TRANSFORMER_STATE_PROCESS_AUDIO;
startInternal(
TransmuxTranscodeHelper.createAudioTranscodeAndVideoTransmuxComposition(
checkNotNull(composition), checkNotNull(outputFilePath)),
MuxerWrapper muxerWrapper =
new MuxerWrapper(
checkNotNull(oldFilePath),
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false),
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
startInternal(
TransmuxTranscodeHelper.createAudioTranscodeAndVideoTransmuxComposition(
checkNotNull(composition), checkNotNull(outputFilePath)),
muxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ 0,
/* useDefaultAssetLoaderFactory= */ false);
@ -1421,7 +1434,8 @@ public final class Transformer {
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
mp4Info.videoFormat);
if (shouldTranscodeVideo(
checkNotNull(mp4Info.videoFormat),
composition,
@ -1478,14 +1492,6 @@ public final class Transformer {
EditedMediaItem firstEditedMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0);
Mp4Info mediaItemInfo = checkNotNull(this.mediaItemInfo);
if (!doesFormatsMatch(mediaItemInfo, firstEditedMediaItem)) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.reset();
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
long trimStartTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.endPositionUs;
Composition transmuxComposition =
@ -1507,19 +1513,6 @@ public final class Transformer {
/* useDefaultAssetLoaderFactory= */ false);
}
private boolean doesFormatsMatch(Mp4Info mediaItemInfo, EditedMediaItem firstMediaItem) {
Format videoFormat = checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_VIDEO);
boolean videoFormatMatches =
MuxerWrapper.isInitializationDataCompatible(
videoFormat, checkNotNull(mediaItemInfo.videoFormat));
boolean audioFormatMatches =
mediaItemInfo.audioFormat == null
|| firstMediaItem.removeAudio
|| mediaItemInfo.audioFormat.initializationDataEquals(
checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
return videoFormatMatches && audioFormatMatches;
}
private boolean isMultiAsset() {
return checkNotNull(composition).sequences.size() > 1
|| composition.sequences.get(0).editedMediaItems.size() > 1;
@ -1636,6 +1629,16 @@ public final class Transformer {
@Nullable String audioEncoderName,
@Nullable String videoEncoderName,
ExportException exportException) {
if (exportException.errorCode == ERROR_CODE_MUXING_APPEND
&& (isExportTrimOptimization() || isExportResumed())) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.reset();
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
exportResultBuilder.addProcessedInputs(processedInputs);
// When an export is resumed, the audio and video encoder name (if any) can comes from

16
libraries/transformer/src/main/java/androidx/media3/transformer/TransmuxTranscodeHelper.java
View file

@ -21,6 +21,7 @@ import android.content.Context;
import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
@ -51,11 +52,16 @@ import java.util.List;
*/
public final ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo;
/** The video {@link Format} or {@code null} if there is no video track. */
@Nullable public final Format videoFormat;
public ResumeMetadata(
long lastSyncSampleTimestampUs,
ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo) {
ImmutableList<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfo,
@Nullable Format videoFormat) {
this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs;
this.firstMediaItemIndexAndOffsetInfo = firstMediaItemIndexAndOffsetInfo;
this.videoFormat = videoFormat;
}
}
@ -268,8 +274,8 @@ import java.util.List;
if (resumeMetadataSettableFuture.isCancelled()) {
return;
}
long lastSyncSampleTimestampUs =
Mp4Info.create(context, filePath).lastSyncSampleTimestampUs;
Mp4Info mp4Info = Mp4Info.create(context, filePath);
long lastSyncSampleTimestampUs = mp4Info.lastSyncSampleTimestampUs;
ImmutableList.Builder<Pair<Integer, Long>> firstMediaItemIndexAndOffsetInfoBuilder =
new ImmutableList.Builder<>();
@ -301,7 +307,9 @@ import java.util.List;
}
resumeMetadataSettableFuture.set(
new ResumeMetadata(
lastSyncSampleTimestampUs, firstMediaItemIndexAndOffsetInfoBuilder.build()));
lastSyncSampleTimestampUs,
firstMediaItemIndexAndOffsetInfoBuilder.build(),
mp4Info.videoFormat));
} catch (Exception ex) {
resumeMetadataSettableFuture.setException(ex);
}

3
libraries/transformer/src/test/java/androidx/media3/transformer/EncodedSampleExporterTest.java
View file

@ -68,7 +68,8 @@ public final class EncodedSampleExporterTest {
new InAppMuxer.Factory.Builder().build(),
mock(MuxerWrapper.Listener.class),
MuxerWrapper.MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false),
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null),
fallbackListener,
/* initialTimestampOffsetUs= */ 0);
}

170
libraries/transformer/src/test/java/androidx/media3/transformer/MuxerWrapperTest.java
View file

@ -18,6 +18,7 @@ package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.AUDIO_AAC;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.test.utils.TestUtil.createByteArray;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_MUX_PARTIAL;
import static androidx.media3.transformer.TestUtil.getDumpFileName;
@ -34,6 +35,8 @@ import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
@ -44,12 +47,16 @@ import org.junit.runner.RunWith;
/** Unit tests for {@link MuxerWrapper}. */
@RunWith(AndroidJUnit4.class)
public class MuxerWrapperTest {
private static final byte[] SPS_TEST_DATA =
createByteArray(
0x00, 0x00, 0x00, 0x01, 0x67, 0x4D, 0x40, 0x16, 0xEC, 0xA0, 0x50, 0x17, 0xFC, 0xB8, 0x0A,
0x90, 0x91, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0F, 0x47, 0x8B, 0x16, 0xCB);
private static final Format FAKE_VIDEO_TRACK_FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setWidth(1080)
.setHeight(720)
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.setInitializationData(ImmutableList.of(SPS_TEST_DATA, new byte[] {1, 2, 3, 4}))
.setColorInfo(ColorInfo.SDR_BT709_LIMITED)
.build();
private static final Format FAKE_AUDIO_TRACK_FORMAT =
@ -83,7 +90,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
muxerWrapper.setAdditionalRotationDegrees(90);
muxerWrapper.setTrackCount(1);
muxerWrapper.setAdditionalRotationDegrees(180);
@ -100,7 +108,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
muxerWrapper.setAdditionalRotationDegrees(90);
muxerWrapper.setTrackCount(1);
muxerWrapper.setAdditionalRotationDegrees(180);
@ -117,11 +126,27 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThrows(IllegalStateException.class, muxerWrapper::changeToAppendMode);
}
@Test
public void constructor_withAppendVideoFormatMissingInPartialMode_throws() {
assertThrows(
IllegalArgumentException.class,
() ->
muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null));
}
@Test
public void addTrackFormat_withSameVideoFormatInAppendMode_doesNotThrow() throws Exception {
muxerWrapper =
@ -130,7 +155,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -152,7 +178,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -167,13 +194,15 @@ public class MuxerWrapperTest {
@Test
public void addTrackFormat_withDifferentVideoFormatInAppendMode_throws() throws Exception {
Format differentVideoFormat = FAKE_VIDEO_TRACK_FORMAT.buildUpon().setHeight(5000).build();
muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ differentVideoFormat);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -182,10 +211,47 @@ public class MuxerWrapperTest {
muxerWrapper.release(MuxerWrapper.MUXER_RELEASE_REASON_COMPLETED);
muxerWrapper.changeToAppendMode();
muxerWrapper.setTrackCount(1);
Format differentVideoFormat = FAKE_VIDEO_TRACK_FORMAT.buildUpon().setHeight(5000).build();
assertThrows(
IllegalArgumentException.class, () -> muxerWrapper.addTrackFormat(differentVideoFormat));
MuxerWrapper.AppendTrackFormatException.class,
() -> muxerWrapper.addTrackFormat(differentVideoFormat));
}
@Test
public void addTrackFormat_withCompatibleVideoFormatInAppendMode_savesTheMostCompatibleInitData()
throws Exception {
byte[] newSpsTestData = Arrays.copyOf(SPS_TEST_DATA, SPS_TEST_DATA.length);
int spsLevelIndex = 7;
byte lowSpsLevel = 11;
newSpsTestData[spsLevelIndex] = lowSpsLevel;
Format differentVideoFormat =
FAKE_VIDEO_TRACK_FORMAT
.buildUpon()
.setInitializationData(
ImmutableList.of(newSpsTestData, FAKE_VIDEO_TRACK_FORMAT.initializationData.get(1)))
.build();
muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ differentVideoFormat);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
C.TRACK_TYPE_VIDEO, FAKE_SAMPLE, /* isKeyFrame= */ true, /* presentationTimeUs= */ 0);
muxerWrapper.endTrack(C.TRACK_TYPE_VIDEO);
muxerWrapper.changeToAppendMode();
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(differentVideoFormat);
assertThat(
muxerWrapper
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(FAKE_VIDEO_TRACK_FORMAT))
.isTrue();
}
@Test
@ -196,7 +262,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -208,7 +275,8 @@ public class MuxerWrapperTest {
Format differentAudioFormat = FAKE_AUDIO_TRACK_FORMAT.buildUpon().setSampleRate(48000).build();
assertThrows(
IllegalArgumentException.class, () -> muxerWrapper.addTrackFormat(differentAudioFormat));
MuxerWrapper.AppendTrackFormatException.class,
() -> muxerWrapper.addTrackFormat(differentAudioFormat));
}
@Test
@ -221,7 +289,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ true);
/* dropSamplesBeforeFirstVideoSample= */ true,
/* appendVideoFormat= */ null);
muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -248,7 +317,8 @@ public class MuxerWrapperTest {
muxerFactory,
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ true);
/* dropSamplesBeforeFirstVideoSample= */ true,
/* appendVideoFormat= */ null);
muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_AUDIO_TRACK_FORMAT);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -286,7 +356,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -306,7 +377,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(2);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
@ -334,7 +406,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -349,7 +422,7 @@ public class MuxerWrapperTest {
}
@Test
public void isInitializationDataCompatible_h265_differentCSD_returnsFalse() {
public void getMostCompatibleFormat_h265_differentCSD_returnsNull() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
@ -361,11 +434,12 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
}
@Test
public void isInitializationDataCompatible_h265_matchingCSD_returnsTrue() {
public void getMostCompatibleFormat_h265_matchingCSD_returnsFormat() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
@ -377,11 +451,19 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(1);
Byte[] expectedInitializationData = new Byte[] {1, 2, 3, 4};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationData)
.inOrder();
}
@Test
public void isInitializationDataCompatible_h264_matchingCSD_returnsTrue() {
public void getMostCompatibleFormat_h264_matchingCSD_returnsFormat() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
@ -393,11 +475,19 @@ public class MuxerWrapperTest {
.setInitializationData(ImmutableList.of(new byte[] {1, 2, 3, 4}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(1);
Byte[] expectedInitializationData = new Byte[] {1, 2, 3, 4};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationData)
.inOrder();
}
@Test
public void isInitializationDataCompatible_h264_ignoresLevel_returnsTrue() {
public void getMostCompatibleFormat_h264_differentLevel_returnsFormat() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
@ -411,11 +501,24 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isTrue();
List<byte[]> initializationData =
MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat);
assertThat(initializationData).hasSize(2);
Byte[] expectedInitializationDataSps = new Byte[] {0, 0, 0, 1, 103, 100, 0, 41};
assertThat(initializationData.get(0))
.asList()
.containsExactlyElementsIn(expectedInitializationDataSps)
.inOrder();
Byte[] expectedInitializationDataPps = new Byte[] {0, 0, 0, 1};
assertThat(initializationData.get(1))
.asList()
.containsExactlyElementsIn(expectedInitializationDataPps)
.inOrder();
}
@Test
public void isInitializationDataCompatible_h264_mismatchProfile_returnsFalse() {
public void getMostCompatibleFormat_h264_mismatchProfile_returnsNull() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
@ -429,11 +532,12 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
}
@Test
public void isInitializationDataCompatible_h264_missingMimeType_returnsFalse() {
public void getMostCompatibleFormat_h264_missingMimeType_returnsNull() {
Format existingFormat =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
@ -446,7 +550,8 @@ public class MuxerWrapperTest {
ImmutableList.of(new byte[] {0, 0, 0, 1, 103, 100, 0, 41}, new byte[] {0, 0, 0, 1}))
.build();
assertThat(MuxerWrapper.isInitializationDataCompatible(existingFormat, otherFormat)).isFalse();
assertThat(MuxerWrapper.getMostComatibleInitializationData(existingFormat, otherFormat))
.isNull();
}
@Test
@ -458,7 +563,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -483,7 +589,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(
@ -507,7 +614,8 @@ public class MuxerWrapperTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_MUX_PARTIAL,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.setTrackCount(1);
muxerWrapper.addTrackFormat(FAKE_VIDEO_TRACK_FORMAT);
muxerWrapper.writeSample(

6
libraries/transformer/src/test/java/androidx/media3/transformer/TransformerUtilTest.java
View file

@ -70,7 +70,8 @@ public final class TransformerUtilTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThat(
shouldTranscodeVideo(
@ -102,7 +103,8 @@ public final class TransformerUtilTest {
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
/* dropSamplesBeforeFirstVideoSample= */ false,
/* appendVideoFormat= */ null);
assertThat(
shouldTranscodeVideo(