sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-10 12:05:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Trim optimization: fallback on format mismatches

Browse source 
Manual testing: tested manually with pixel 4a

PiperOrigin-RevId: 590284361
This commit is contained in:
tofunmi 2023-12-12 11:51:30 -08:00 committed by Copybara-Service
parent ee1147ffe1
commit 37def3679f
5 changed files with 101 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View file

@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
@ -423,7 +424,7 @@ public class TransformerEndToEndTest {
}
Transformer transformer = new Transformer.Builder(context).build();
long clippingStartMs = 10_000;
long clippingEndMs = 11_000;
long clippingEndMs = 13_000;
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
@ -442,6 +443,43 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}
@Test
public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
throws Exception {
String testId = "clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
return;
}
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
long clippingStartMs = 10_000;
long clippingEndMs = 13_000;
// The file is made artificially on computer software so phones will not have the encoder
// available to match the csd.
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(clippingStartMs)
.setEndPositionMs(clippingEndMs)
.build())
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);
assertThat(result.exportResult.optimizationResult)
.isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}
@Test
public void clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied()
throws Exception {

5
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View file

@ -201,6 +201,11 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
*/
@Override
public DefaultCodec createForVideoEncoding(Format format) throws ExportException {
// TODO b/304476154 - Investigate how to match initialization data without running into
// mediaCodec errors under API 29.
if (SDK_INT < 29) {
format = format.buildUpon().setInitializationData(null).build();
}
if (format.frameRate == Format.NO_VALUE || deviceNeedsDefaultFrameRateWorkaround()) {
format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build();
}

7
libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java
View file

@ -301,6 +301,7 @@ public final class ExportResult {
OPTIMIZATION_SUCCEEDED,
OPTIMIZATION_ABANDONED,
OPTIMIZATION_FAILED_EXTRACTION_FAILED,
OPTIMIZATION_FAILED_FORMAT_MISMATCH
})
@interface OptimizationResult {}
@ -322,6 +323,12 @@ public final class ExportResult {
*/
public static final int OPTIMIZATION_FAILED_EXTRACTION_FAILED = 3;
/**
* The optimization failed because the format between the two parts of the media to be put
* together did not match. Normal export proceeded.
*/
public static final int OPTIMIZATION_FAILED_FORMAT_MISMATCH = 4;
/** The list of {@linkplain ProcessedInput processed inputs}. */
public final ImmutableList<ProcessedInput> processedInputs;

12
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View file

@ -292,6 +292,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
}
/**
* Returns the {@link Format} of given {@code trackType} that was {@linkplain #addTrackFormat
* added}.
*
* @throws IllegalArgumentException If the {@code trackType} has not been {@linkplain
* #addTrackFormat added}.
*/
public Format getTrackFormat(@C.TrackType int trackType) {
checkArgument(contains(trackTypeToInfo, trackType));
return trackTypeToInfo.get(trackType).format;
}
/**
* Attempts to write a sample to the muxer.
*

49
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View file

@ -22,6 +22,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
@ -925,7 +926,8 @@ public final class Transformer {
composition,
new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
} else {
processMediaBeforeFirstSyncSampleAfterTrimStartTime();
}
@ -1113,7 +1115,8 @@ public final class Transformer {
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}
private void remuxProcessedVideo() {
@ -1149,7 +1152,8 @@ public final class Transformer {
/* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs),
checkNotNull(remuxingMuxerWrapper),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}
@Override
@ -1177,7 +1181,8 @@ public final class Transformer {
videoOnlyComposition,
remuxingMuxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs);
/* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs,
/* matchInitializationData= */ false);
}
private void processAudio() {
@ -1192,7 +1197,8 @@ public final class Transformer {
componentListener,
MuxerWrapper.MUXER_MODE_DEFAULT),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ false);
}
// TODO: b/308253384 - Move copy output logic into MuxerWrapper.
@ -1297,7 +1303,8 @@ public final class Transformer {
trancodeComposition,
checkNotNull(remuxingMuxerWrapper),
componentListener,
/* initialTimestampOffsetUs= */ 0);
/* initialTimestampOffsetUs= */ 0,
/* matchInitializationData= */ true);
}
@Override
@ -1311,8 +1318,13 @@ public final class Transformer {
private void remuxRemainingMedia() {
transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
// TODO: b/304476154 - check original file format against transcode file format here to fail
// fast if necessary.
if (!doesFormatsMatch()) {
remuxingMuxerWrapper = null;
transformerInternal = null;
exportResultBuilder.setOptimizationResult(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
processFullInput();
return;
}
MediaItem firstMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
@ -1332,7 +1344,21 @@ public final class Transformer {
remuxingMuxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs
- trimStartTimeUs);
- trimStartTimeUs,
/* matchInitializationData= */ false);
}
private boolean doesFormatsMatch() {
checkNotNull(mp4MetadataInfo);
boolean videoFormatMatches =
checkNotNull(remuxingMuxerWrapper)
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat));
boolean audioFormatMatches =
mp4MetadataInfo.audioFormat == null
|| mp4MetadataInfo.audioFormat.initializationDataEquals(
checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
return videoFormatMatches && audioFormatMatches;
}
private boolean isMultiAsset() {
@ -1350,7 +1376,8 @@ public final class Transformer {
Composition composition,
MuxerWrapper muxerWrapper,
ComponentListener componentListener,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean matchInitializationData) {
checkArgument(composition.effects.audioProcessors.isEmpty());
checkState(transformerInternal == null, "There is already an export in progress.");
TransformationRequest transformationRequest = this.transformationRequest;
@ -1387,7 +1414,7 @@ public final class Transformer {
debugViewProvider,
clock,
initialTimestampOffsetUs,
/* matchInitializationData= */ trimOptimizationEnabled);
matchInitializationData);
transformerInternal.start();
}