From 18280723be60903945088df4e3c80a25eb55a6e6 Mon Sep 17 00:00:00 2001 From: claincly Date: Tue, 29 Nov 2022 17:35:55 +0000 Subject: [PATCH] Move audio MIME type fallback away to ATSP PiperOrigin-RevId: 491660842 --- .../transformer/AndroidTestUtil.java | 6 +-- .../TransformerAndroidTestRunner.java | 6 +-- .../transformer/TransformerEndToEndTest.java | 6 +-- .../AudioTranscodingSamplePipeline.java | 43 ++++++++++++++++--- .../transformer/BaseSamplePipeline.java | 11 +++++ .../android/exoplayer2/transformer/Codec.java | 19 ++++---- .../transformer/DefaultEncoderFactory.java | 15 +------ .../exoplayer2/transformer/EncoderUtil.java | 4 +- .../VideoTranscodingSamplePipeline.java | 8 +--- .../transformer/TransformerEndToEndTest.java | 21 ++++++--- .../transformer/VideoEncoderWrapperTest.java | 3 +- 11 files changed, 81 insertions(+), 61 deletions(-) diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java index 21d3a5473c..b1d53acfc9 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.List; import org.json.JSONException; import org.json.JSONObject; @@ -472,9 +471,8 @@ public final class AndroidTestUtil { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - return encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + return encoderFactory.createForAudioEncoding(format); } @Override diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java index f6edebc7eb..8f54f7989a 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java @@ -35,7 +35,6 @@ import com.google.common.base.Ascii; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; @@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - Codec audioEncoder = encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + Codec audioEncoder = encoderFactory.createForAudioEncoding(format); audioEncoderName = audioEncoder.getName(); return audioEncoder; } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index 55ce1bfb30..4f15e6b894 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -26,7 +26,6 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.MediaItem; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -140,9 +139,8 @@ public class TransformerEndToEndTest { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - return encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + return encoderFactory.createForAudioEncoding(format); } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java index 24db2537c6..b6aada7114 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java @@ -31,6 +31,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.List; import org.checkerframework.dataflow.qual.Pure; /** @@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure; audioProcessingPipeline.flush(); + String requestedMimeType = + transformationRequest.audioMimeType != null + ? transformationRequest.audioMimeType + : checkNotNull(inputFormat.sampleMimeType); Format requestedOutputFormat = new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) + .setSampleMimeType(requestedMimeType) .setSampleRate(encoderInputAudioFormat.sampleRate) .setChannelCount(encoderInputAudioFormat.channelCount) .setAverageBitrate(DEFAULT_ENCODER_BITRATE) .build(); + + ImmutableList muxerSupportedMimeTypes = + muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO); + + // TODO(b/259570024): investigate overhauling fallback. + @Nullable + String supportedMimeType = + selectEncoderAndMuxerSupportedMimeType(requestedMimeType, muxerSupportedMimeTypes); + if (supportedMimeType == null) { + throw createNoSupportedMimeTypeException(requestedOutputFormat); + } + encoder = encoderFactory.createForAudioEncoding( - requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO)); - + requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build()); + checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType)); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); @@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure; encoder.queueInputBuffer(encoderInputBuffer); } + @Nullable + private static String selectEncoderAndMuxerSupportedMimeType( + String requestedMimeType, List muxerSupportedMimeTypes) { + if (!EncoderUtil.getSupportedEncoders(requestedMimeType).isEmpty()) { + return requestedMimeType; + } else { + // No encoder supports the requested MIME type. + for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) { + String mimeType = muxerSupportedMimeTypes.get(i); + if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + return mimeType; + } + } + } + return null; + } + @Pure private static TransformationRequest createFallbackTransformationRequest( TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java index 170c1d6a82..ae1f4bdc09 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/BaseSamplePipeline.java @@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; : null; } + protected static TransformationException createNoSupportedMimeTypeException( + Format requestedEncoderFormat) { + return TransformationException.createForCodec( + new IllegalArgumentException("No MIME type is supported by both encoder and muxer."), + MimeTypes.isVideo(requestedEncoderFormat.sampleMimeType), + /* isDecoder= */ false, + requestedEncoderFormat, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + } + @Nullable @Override public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java index 9b5d209283..804194c260 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java @@ -22,9 +22,7 @@ import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.util.MimeTypes; import java.nio.ByteBuffer; -import java.util.List; /** * Provides a layer of abstraction for interacting with decoders and encoders. @@ -70,19 +68,18 @@ public interface Codec { /** * Returns a {@link Codec} for audio encoding. * - *

This method must validate that the {@link Codec} is configured to produce one of the - * {@code allowedMimeTypes}. The {@linkplain Format#sampleMimeType sample MIME type} given in - * {@code format} is not necessarily allowed. + *

The caller should ensure the {@linkplain Format#sampleMimeType MIME type} is supported on + * the device before calling this method. * * @param format The {@link Format} (of the output data) used to determine the underlying - * encoder and its configuration values. - * @param allowedMimeTypes The non-empty list of allowed output sample {@linkplain MimeTypes - * MIME types}. - * @return A {@link Codec} for audio encoding. + * encoder and its configuration values. {@link Format#sampleMimeType}, {@link + * Format#sampleRate}, {@link Format#channelCount} and {@link Format#bitrate} are set to + * those of the desired output video format. + * @return A {@link Codec} for encoding audio to the requested {@link Format#sampleMimeType MIME + * type}. * @throws TransformationException If no suitable {@link Codec} can be created. */ - Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException; + Codec createForAudioEncoding(Format format) throws TransformationException; /** * Returns a {@link Codec} for video encoding. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java index 4dbcb397f5..f6f3b62cbd 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java @@ -172,23 +172,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { + public DefaultCodec createForAudioEncoding(Format format) throws TransformationException { // TODO(b/210591626) Add encoder selection for audio. - checkArgument(!allowedMimeTypes.isEmpty()); checkNotNull(format.sampleMimeType); - if (!allowedMimeTypes.contains(format.sampleMimeType)) { - if (enableFallback) { - // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder - // capabilities limitations. - format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build(); - } else { - throw createTransformationException(format); - } - } MediaFormat mediaFormat = MediaFormat.createAudioFormat( - checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); + format.sampleMimeType, format.sampleRate, format.channelCount); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate); @Nullable diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java index dc1126b7ec..ee422166cd 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java @@ -444,9 +444,7 @@ public final class EncoderUtil { } String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes(); for (String mimeType : supportedMimeTypes) { - if (MimeTypes.isVideo(mimeType)) { - encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo); - } + encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo); } } return encoderInfosBuilder.build(); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 9dfd6f0589..79f78b429d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure; selectEncoderAndMuxerSupportedMimeType( requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo); if (supportedMimeType == null) { - throw TransformationException.createForCodec( - new IllegalArgumentException("No MIME type is supported by both encoder and muxer."), - /* isVideo= */ true, - /* isDecoder= */ false, - requestedEncoderFormat, - /* mediaCodecName= */ null, - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + throw createNoSupportedMimeTypeException(requestedEncoderFormat); } encoder = diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index 0cada82975..0a16ff5b61 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -453,18 +453,25 @@ public final class TransformerEndToEndTest { } @Test - public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError() + public void startTransformation_withAudioMuxerFormatUnsupported_completesSuccessfully() throws Exception { - Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); + // Test succeeds because MIME type fallback is mandatory. + Transformer.Listener mockListener = mock(Transformer.Listener.class); + TransformationRequest originalTransformationRequest = + new TransformationRequest.Builder().build(); + TransformationRequest fallbackTransformationRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).addListener(mockListener).build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); transformer.startTransformation(mediaItem, outputPath); - TransformationException exception = TransformerTestRunner.runUntilError(transformer); + TransformerTestRunner.runUntilCompleted(transformer); - assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); - assertThat(exception).hasMessageThat().contains("audio"); - assertThat(exception.errorCode) - .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + DumpFileAsserts.assertOutput( + context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback")); + verify(mockListener) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); } @Test diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java index 9fc8a231ff..f187aa370c 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java @@ -32,7 +32,6 @@ import com.google.android.exoplayer2.util.ListenerSet; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.SurfaceInfo; import com.google.common.collect.ImmutableList; -import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) { + public Codec createForAudioEncoding(Format format) { throw new UnsupportedOperationException(); }