From a652c90483259fed121e61013ec2a96402782600 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 19 Jan 2022 18:32:59 +0000 Subject: [PATCH] Communicate sample MIME type changes to FallbackListener. We may fall back to a different sample MIME type because a) the sample MIME type inferred from the input is not supported by the muxer or b) no encoders are available for the the requested sample MIME type. PiperOrigin-RevId: 422849036 --- .../transformer/AudioSamplePipeline.java | 53 ++++++++++++------- .../PassthroughSamplePipeline.java | 6 ++- .../media3/transformer/Transformer.java | 12 ++++- .../transformer/TransformerAudioRenderer.java | 10 ++-- .../transformer/TransformerBaseRenderer.java | 6 ++- .../transformer/TransformerVideoRenderer.java | 7 ++- .../transformer/VideoSamplePipeline.java | 42 ++++++++++----- .../media3/transformer/TransformerTest.java | 1 + 8 files changed, 98 insertions(+), 39 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java index fe605a5351..b3533b8703 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java @@ -25,11 +25,13 @@ import android.media.MediaCodec.BufferInfo; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.audio.AudioProcessor; import androidx.media3.exoplayer.audio.AudioProcessor.AudioFormat; import androidx.media3.exoplayer.audio.SonicAudioProcessor; import java.nio.ByteBuffer; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. @@ -39,13 +41,12 @@ import java.nio.ByteBuffer; private static final String TAG = "AudioSamplePipeline"; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; - private final TransformationRequest transformationRequest; - private final Codec decoder; private final DecoderInputBuffer decoderInputBuffer; private final SonicAudioProcessor sonicAudioProcessor; private final SpeedProvider speedProvider; + private final boolean flattenForSlowMotion; private final Codec encoder; private final AudioFormat encoderInputAudioFormat; @@ -63,9 +64,9 @@ import java.nio.ByteBuffer; Format inputFormat, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) throws TransformationException { - this.transformationRequest = transformationRequest; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = @@ -75,6 +76,7 @@ import java.nio.ByteBuffer; this.decoder = decoderFactory.createForAudioDecoding(inputFormat); + this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion; sonicAudioProcessor = new SonicAudioProcessor(); sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; speedProvider = new SegmentSpeedProvider(inputFormat); @@ -86,7 +88,7 @@ import java.nio.ByteBuffer; // The decoder uses ENCODING_PCM_16BIT by default. // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers C.ENCODING_PCM_16BIT); - if (transformationRequest.flattenForSlowMotion) { + if (flattenForSlowMotion) { try { encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat); } catch (AudioProcessor.UnhandledAudioFormatException impossible) { @@ -96,19 +98,23 @@ import java.nio.ByteBuffer; sonicAudioProcessor.setPitch(currentSpeed); sonicAudioProcessor.flush(); } - - encoder = - encoderFactory.createForAudioEncoding( - new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) - .setSampleRate(encoderInputAudioFormat.sampleRate) - .setChannelCount(encoderInputAudioFormat.channelCount) - .setAverageBitrate(DEFAULT_ENCODER_BITRATE) - .build()); this.encoderInputAudioFormat = encoderInputAudioFormat; + + Format requestedOutputFormat = + new Format.Builder() + .setSampleMimeType( + transformationRequest.audioMimeType == null + ? inputFormat.sampleMimeType + : transformationRequest.audioMimeType) + .setSampleRate(encoderInputAudioFormat.sampleRate) + .setChannelCount(encoderInputAudioFormat.channelCount) + .setAverageBitrate(DEFAULT_ENCODER_BITRATE) + .build(); + encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat); + + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); } @Override @@ -293,7 +299,7 @@ import java.nio.ByteBuffer; } private boolean isSpeedChanging(BufferInfo bufferInfo) { - if (!transformationRequest.flattenForSlowMotion) { + if (!flattenForSlowMotion) { return false; } float newSpeed = speedProvider.getSpeed(bufferInfo.presentationTimeUs); @@ -325,4 +331,15 @@ import java.nio.ByteBuffer; } nextEncoderInputBufferTimeUs += bufferDurationUs; } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update bitrate and other params once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index b6e827beb7..f2387ace36 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -28,10 +28,14 @@ import androidx.media3.decoder.DecoderInputBuffer; private boolean hasPendingBuffer; - public PassthroughSamplePipeline(Format format) { + public PassthroughSamplePipeline( + Format format, + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; + fallbackListener.onTransformationRequestFinalized(transformationRequest); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index c1f145bff4..eab488fc01 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -705,6 +705,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + new FallbackListener(mediaItem, listeners, transformationRequest), debugViewProvider)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) @@ -807,6 +808,7 @@ public final class Transformer { private final TransformationRequest transformationRequest; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; + private final FallbackListener fallbackListener; private final Transformer.DebugViewProvider debugViewProvider; public TransformerRenderersFactory( @@ -817,6 +819,7 @@ public final class Transformer { TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { this.context = context; this.muxerWrapper = muxerWrapper; @@ -825,6 +828,7 @@ public final class Transformer { this.transformationRequest = transformationRequest; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; + this.fallbackListener = fallbackListener; this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -842,7 +846,12 @@ public final class Transformer { if (!removeAudio) { renderers[index] = new TransformerAudioRenderer( - muxerWrapper, mediaClock, transformationRequest, encoderFactory, decoderFactory); + muxerWrapper, + mediaClock, + transformationRequest, + encoderFactory, + decoderFactory, + fallbackListener); index++; } if (!removeVideo) { @@ -854,6 +863,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); index++; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index 35d9c3b87d..2ac47e6f30 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -41,8 +41,9 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; TransformerMediaClock mediaClock, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) { - super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest); + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) { + super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; decoderInputBuffer = @@ -78,11 +79,12 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new AudioSamplePipeline( - inputFormat, transformationRequest, encoderFactory, decoderFactory); + inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener); } return true; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java index 9e39456dd0..94b398dcc1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java @@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected final MuxerWrapper muxerWrapper; protected final TransformerMediaClock mediaClock; protected final TransformationRequest transformationRequest; + protected final FallbackListener fallbackListener; protected boolean isRendererStarted; protected boolean muxerWrapperTrackAdded; @@ -50,11 +51,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int trackType, MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - TransformationRequest transformationRequest) { + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { super(trackType); this.muxerWrapper = muxerWrapper; this.mediaClock = mediaClock; this.transformationRequest = transformationRequest; + this.fallbackListener = fallbackListener; } /** @@ -112,6 +115,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { muxerWrapper.registerTrack(); + fallbackListener.registerTrack(); mediaClock.updateTimeForTrackType(getTrackType(), 0L); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index 6b067bfec1..ae8fef409a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -48,8 +48,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { - super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest); + super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.context = context; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; @@ -87,7 +88,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new VideoSamplePipeline( @@ -96,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); } if (transformationRequest.flattenForSlowMotion) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 40fed2fbf3..e3e7568d56 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -27,8 +27,10 @@ import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. @@ -54,6 +56,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { decoderInputBuffer = @@ -63,7 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Scale width and height to desired transformationRequest.outputHeight, preserving aspect // ratio. - // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. + // TODO(b/209781577): Think about which edge length should be set for portrait videos. float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height; int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; @@ -102,17 +105,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // postRotate in a later vertex shader. transformationMatrix.postRotate(outputRotationDegrees); - encoder = - encoderFactory.createForVideoEncoding( - new Format.Builder() - .setWidth(outputWidth) - .setHeight(outputHeight) - .setRotationDegrees(0) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .build()); + Format requestedOutputFormat = + new Format.Builder() + .setWidth(outputWidth) + .setHeight(outputHeight) + .setRotationDegrees(0) + .setSampleMimeType( + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : inputFormat.sampleMimeType) + .build(); + encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat); + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); + if (inputFormat.height != outputHeight || inputFormat.width != outputWidth || !transformationMatrix.isIdentity()) { @@ -261,4 +268,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; decoder.release(); encoder.release(); } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update resolution, bitrate etc. once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setVideoMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java index fffa5ba9f6..3df816852b 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java @@ -65,6 +65,7 @@ import org.robolectric.shadows.ShadowMediaCodec; /** Unit test for {@link Transformer}. */ @RunWith(AndroidJUnit4.class) public final class TransformerTest { + // TODO(b/214973843): Disable fallback for all tests that aren't specifically testing fallback. private static final String URI_PREFIX = "asset:///media/"; private static final String FILE_VIDEO_ONLY = "mp4/sample_18byte_nclx_colr.mp4";