From c12b1768a6b7b563e3a523ac908190f13970a69a Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 20 Dec 2024 05:47:18 -0800 Subject: [PATCH] Add sample rate fallback to DefaultEncoderFactory After this change if a sample rate is requested that is not supported by the available encoders and enableFallback is true, DefaultEncoderFactory will fall back to the encoder with the closest matching sample rate. In the case when an encoding profile is included in requestedAudioEncoderSettings, an encoder matching that profile will continue to be preferenced. PiperOrigin-RevId: 708295869 --- .../transformer/ConfigurationActivity.java | 9 ++- .../demo/transformer/TransformerActivity.java | 8 +- .../src/main/res/values/arrays.xml | 1 + .../transformer/AudioSampleExporter.java | 1 + .../transformer/DefaultEncoderFactory.java | 78 ++++++++++++++++--- .../media3/transformer/EncoderUtil.java | 32 ++++++++ .../DefaultEncoderFactoryTest.java | 69 +++++++++++++--- 7 files changed, 171 insertions(+), 27 deletions(-) diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 288f5449f7..e297d09b9a 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -120,10 +120,11 @@ public final class ConfigurationActivity extends AppCompatActivity { // Audio effect selections. public static final int HIGH_PITCHED_INDEX = 0; - public static final int SAMPLE_RATE_INDEX = 1; - public static final int SKIP_SILENCE_INDEX = 2; - public static final int CHANNEL_MIXING_INDEX = 3; - public static final int VOLUME_SCALING_INDEX = 4; + public static final int SAMPLE_RATE_48K_INDEX = 1; + public static final int SAMPLE_RATE_96K_INDEX = 2; + public static final int SKIP_SILENCE_INDEX = 3; + public static final int CHANNEL_MIXING_INDEX = 4; + public static final int VOLUME_SCALING_INDEX = 5; // Color filter options. public static final int COLOR_FILTER_GRAYSCALE = 0; diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 5f5c5c6203..8512f224c6 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -393,14 +393,18 @@ public final class TransformerActivity extends AppCompatActivity { ImmutableList.Builder processors = new ImmutableList.Builder<>(); if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX] - || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) { + || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX] + || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) { SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX]) { sonicAudioProcessor.setPitch(2f); } - if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) { + if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX]) { sonicAudioProcessor.setOutputSampleRateHz(48_000); } + if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) { + sonicAudioProcessor.setOutputSampleRateHz(96_000); + } processors.add(sonicAudioProcessor); } diff --git a/demos/transformer/src/main/res/values/arrays.xml b/demos/transformer/src/main/res/values/arrays.xml index b9df860749..9b6e07cb8f 100644 --- a/demos/transformer/src/main/res/values/arrays.xml +++ b/demos/transformer/src/main/res/values/arrays.xml @@ -35,6 +35,7 @@ High pitched Sample rate of 48000Hz + Sample rate of 96000Hz Skip silence Mix channels into mono Scale volume to 50% diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index dcffcbbae5..44b0f36b5b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -88,6 +88,7 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO))) .build()); + // TODO: b/324056144 - Fallback when sample rate is unsupported by encoder encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index 35b3165937..c137f43613 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -33,7 +33,6 @@ import android.content.Context; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.os.Build; -import android.util.Pair; import android.util.Size; import androidx.annotation.IntRange; import androidx.annotation.Nullable; @@ -206,13 +205,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } MediaCodecInfo selectedEncoder = mediaCodecInfos.get(0); - + boolean encoderSelectedForRequestedProfile = false; if (requestedAudioEncoderSettings.profile != AudioEncoderSettings.NO_VALUE) { for (int i = 0; i < mediaCodecInfos.size(); i++) { MediaCodecInfo encoderInfo = mediaCodecInfos.get(i); if (EncoderUtil.findSupportedEncodingProfiles(encoderInfo, format.sampleMimeType) .contains(requestedAudioEncoderSettings.profile)) { selectedEncoder = encoderInfo; + encoderSelectedForRequestedProfile = true; if (format.sampleMimeType.equals(MimeTypes.AUDIO_AAC)) { mediaFormat.setInteger( MediaFormat.KEY_AAC_PROFILE, requestedAudioEncoderSettings.profile); @@ -223,7 +223,16 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } } } - + if (!encoderSelectedForRequestedProfile && enableFallback) { + @Nullable + EncoderQueryResult encoderQueryResult = + findAudioEncoderWithClosestSupportedFormat(format, mediaCodecInfos); + if (encoderQueryResult != null) { + selectedEncoder = encoderQueryResult.encoder; + format = encoderQueryResult.supportedFormat; + mediaFormat = createMediaFormatFromFormat(format); + } + } if (requestedAudioEncoderSettings.bitrate != AudioEncoderSettings.NO_VALUE) { mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, requestedAudioEncoderSettings.bitrate); } @@ -261,7 +270,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { @Nullable VideoEncoderQueryResult encoderAndClosestFormatSupport = - findEncoderWithClosestSupportedFormat( + findVideoEncoderWithClosestSupportedFormat( format, requestedVideoEncoderSettings, videoEncoderSelector, enableFallback); if (encoderAndClosestFormatSupport == null) { @@ -402,15 +411,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } /** - * Finds an {@linkplain MediaCodecInfo encoder} that supports a format closest to the requested - * format. + * Finds a video {@linkplain MediaCodecInfo encoder} that supports a format closest to the + * requested format. * - *

Returns the {@linkplain MediaCodecInfo encoder} and the supported {@link Format} in a {@link - * Pair}, or {@code null} if none is found. + *

Returns a {@link VideoEncoderQueryResult}, or {@code null} if no encoder is found. */ @RequiresNonNull("#1.sampleMimeType") @Nullable - private static VideoEncoderQueryResult findEncoderWithClosestSupportedFormat( + private static VideoEncoderQueryResult findVideoEncoderWithClosestSupportedFormat( Format requestedFormat, VideoEncoderSettings videoEncoderSettings, EncoderSelector encoderSelector, @@ -576,17 +584,63 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { : Integer.MAX_VALUE); // Drops encoder. } - private static final class VideoEncoderQueryResult { + /** + * Finds an audio {@linkplain MediaCodecInfo encoder} that supports a format closest to the + * requested format. + * + *

Returns a {@link EncoderQueryResult}, or {@code null} if no encoder is found. + */ + @RequiresNonNull("#1.sampleMimeType") + @Nullable + private static EncoderQueryResult findAudioEncoderWithClosestSupportedFormat( + Format requestedFormat, ImmutableList filteredEncoderInfos) { + String mimeType = checkNotNull(requestedFormat.sampleMimeType); + if (filteredEncoderInfos.isEmpty()) { + return null; + } + MediaCodecInfo filteredEncoderInfo = + filterEncodersBySampleRate(filteredEncoderInfos, mimeType, requestedFormat.sampleRate) + .get(0); + int sampleRate = + EncoderUtil.getClosestSupportedSampleRate( + filteredEncoderInfo, mimeType, requestedFormat.sampleRate); + Format encoderFormat = requestedFormat.buildUpon().setSampleRate(sampleRate).build(); + return new EncoderQueryResult(filteredEncoderInfo, encoderFormat); + } + + /** + * Returns a list of {@linkplain MediaCodecInfo encoders} that support the requested sample rate + * most closely. + */ + private static ImmutableList filterEncodersBySampleRate( + List encoders, String mimeType, int requestedSampleRate) { + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> { + int closestSupportedSampleRate = + EncoderUtil.getClosestSupportedSampleRate(encoderInfo, mimeType, requestedSampleRate); + return Math.abs(closestSupportedSampleRate - requestedSampleRate); + }); + } + + private static class EncoderQueryResult { public final MediaCodecInfo encoder; public final Format supportedFormat; + + public EncoderQueryResult(MediaCodecInfo encoder, Format supportedFormat) { + this.encoder = encoder; + this.supportedFormat = supportedFormat; + } + } + + private static final class VideoEncoderQueryResult extends EncoderQueryResult { public final VideoEncoderSettings supportedEncoderSettings; public VideoEncoderQueryResult( MediaCodecInfo encoder, Format supportedFormat, VideoEncoderSettings supportedEncoderSettings) { - this.encoder = encoder; - this.supportedFormat = supportedFormat; + super(encoder, supportedFormat); this.supportedEncoderSettings = supportedEncoderSettings; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 2f8a17576c..e54a9770b3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -384,6 +384,38 @@ public final class EncoderUtil { Ints.asList(encoderInfo.getCapabilitiesForType(mimeType).colorFormats)); } + /** + * Returns the sample rate supported by the provided {@linkplain MediaCodecInfo encoder} that is + * closest to the provided sample rate. + */ + public static int getClosestSupportedSampleRate( + MediaCodecInfo encoderInfo, String mimeType, int requestedSampleRate) { + MediaCodecInfo.AudioCapabilities audioCapabilities = + encoderInfo.getCapabilitiesForType(mimeType).getAudioCapabilities(); + @Nullable int[] supportedSampleRates = audioCapabilities.getSupportedSampleRates(); + int closestSampleRate = Integer.MAX_VALUE; + if (supportedSampleRates != null) { + // The codec supports only discrete values. + for (int supportedSampleRate : supportedSampleRates) { + if (Math.abs(supportedSampleRate - requestedSampleRate) + < Math.abs(closestSampleRate - requestedSampleRate)) { + closestSampleRate = supportedSampleRate; + } + } + return closestSampleRate; + } else { + Range[] ranges = audioCapabilities.getSupportedSampleRateRanges(); + for (Range range : ranges) { + int supportedSampleRate = range.clamp(requestedSampleRate); + if (Math.abs(supportedSampleRate - requestedSampleRate) + < Math.abs(closestSampleRate - requestedSampleRate)) { + closestSampleRate = supportedSampleRate; + } + } + } + return closestSampleRate; + } + /** Checks if a {@linkplain MediaCodecInfo codec} is hardware-accelerated. */ public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) { // TODO(b/214964116): Merge into MediaCodecUtil. diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java index 733bea1da0..b5febb1f1a 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java @@ -45,6 +45,7 @@ public class DefaultEncoderFactoryTest { @Before public void setUp() { createShadowH264Encoder(); + createShadowAacEncoder(); } @After @@ -66,24 +67,40 @@ public class DefaultEncoderFactoryTest { createShadowVideoEncoder(avcFormat, profileLevel, "test.transformer.avc.encoder"); } + private static void createShadowAacEncoder() { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC); + MediaCodecInfo.CodecCapabilities capabilities = + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(format) + .setIsEncoder(true) + .build(); + createShadowEncoder("test.transformer.aac.encoder", capabilities); + } + private static void createShadowVideoEncoder( MediaFormat supportedFormat, MediaCodecInfo.CodecProfileLevel supportedProfileLevel, String name) { + MediaCodecInfo.CodecCapabilities capabilities = + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(supportedFormat) + .setIsEncoder(true) + .setColorFormats( + new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}) + .setProfileLevels(new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel}) + .build(); + createShadowEncoder(name, capabilities); + } + + private static void createShadowEncoder( + String name, MediaCodecInfo.CodecCapabilities... capabilities) { // ShadowMediaCodecList is static. The added encoders will be visible for every test. ShadowMediaCodecList.addCodec( MediaCodecInfoBuilder.newBuilder() .setName(name) .setIsEncoder(true) - .setCapabilities( - MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() - .setMediaFormat(supportedFormat) - .setIsEncoder(true) - .setColorFormats( - new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}) - .setProfileLevels( - new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel}) - .build()) + .setCapabilities(capabilities) .build()); } @@ -247,6 +264,36 @@ public class DefaultEncoderFactoryTest { .createForVideoEncoding(requestedVideoFormat)); } + @Test + public void createForAudioEncoding_unsupportedSampleRateWithFallback() throws Exception { + Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000); + + Format actualAudioFormat = + new DefaultEncoderFactory.Builder(context) + .setEnableFallback(true) + .build() + .createForAudioEncoding(requestedAudioFormat) + .getConfigurationFormat(); + + assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC); + assertThat(actualAudioFormat.sampleRate).isEqualTo(96_000); + } + + @Test + public void createForAudioEncoding_unsupportedSampleRateWithoutFallback() throws Exception { + Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000); + + Format actualAudioFormat = + new DefaultEncoderFactory.Builder(context) + .setEnableFallback(false) + .build() + .createForAudioEncoding(requestedAudioFormat) + .getConfigurationFormat(); + + assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC); + assertThat(actualAudioFormat.sampleRate).isEqualTo(192_000); + } + private static Format createVideoFormat(String mimeType, int width, int height, int frameRate) { return new Format.Builder() .setWidth(width) @@ -256,4 +303,8 @@ public class DefaultEncoderFactoryTest { .setSampleMimeType(mimeType) .build(); } + + private static Format createAudioFormat(String mimeType, int sampleRate) { + return new Format.Builder().setSampleRate(sampleRate).setSampleMimeType(mimeType).build(); + } }