diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 288f5449f7..e297d09b9a 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -120,10 +120,11 @@ public final class ConfigurationActivity extends AppCompatActivity { // Audio effect selections. public static final int HIGH_PITCHED_INDEX = 0; - public static final int SAMPLE_RATE_INDEX = 1; - public static final int SKIP_SILENCE_INDEX = 2; - public static final int CHANNEL_MIXING_INDEX = 3; - public static final int VOLUME_SCALING_INDEX = 4; + public static final int SAMPLE_RATE_48K_INDEX = 1; + public static final int SAMPLE_RATE_96K_INDEX = 2; + public static final int SKIP_SILENCE_INDEX = 3; + public static final int CHANNEL_MIXING_INDEX = 4; + public static final int VOLUME_SCALING_INDEX = 5; // Color filter options. public static final int COLOR_FILTER_GRAYSCALE = 0; diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 5f5c5c6203..8512f224c6 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -393,14 +393,18 @@ public final class TransformerActivity extends AppCompatActivity { ImmutableList.Builder processors = new ImmutableList.Builder<>(); if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX] - || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) { + || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX] + || selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) { SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX]) { sonicAudioProcessor.setPitch(2f); } - if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) { + if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX]) { sonicAudioProcessor.setOutputSampleRateHz(48_000); } + if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) { + sonicAudioProcessor.setOutputSampleRateHz(96_000); + } processors.add(sonicAudioProcessor); } diff --git a/demos/transformer/src/main/res/values/arrays.xml b/demos/transformer/src/main/res/values/arrays.xml index b9df860749..9b6e07cb8f 100644 --- a/demos/transformer/src/main/res/values/arrays.xml +++ b/demos/transformer/src/main/res/values/arrays.xml @@ -35,6 +35,7 @@ High pitched Sample rate of 48000Hz + Sample rate of 96000Hz Skip silence Mix channels into mono Scale volume to 50% diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index dcffcbbae5..44b0f36b5b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -88,6 +88,7 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO))) .build()); + // TODO: b/324056144 - Fallback when sample rate is unsupported by encoder encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index 35b3165937..c137f43613 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -33,7 +33,6 @@ import android.content.Context; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.os.Build; -import android.util.Pair; import android.util.Size; import androidx.annotation.IntRange; import androidx.annotation.Nullable; @@ -206,13 +205,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } MediaCodecInfo selectedEncoder = mediaCodecInfos.get(0); - + boolean encoderSelectedForRequestedProfile = false; if (requestedAudioEncoderSettings.profile != AudioEncoderSettings.NO_VALUE) { for (int i = 0; i < mediaCodecInfos.size(); i++) { MediaCodecInfo encoderInfo = mediaCodecInfos.get(i); if (EncoderUtil.findSupportedEncodingProfiles(encoderInfo, format.sampleMimeType) .contains(requestedAudioEncoderSettings.profile)) { selectedEncoder = encoderInfo; + encoderSelectedForRequestedProfile = true; if (format.sampleMimeType.equals(MimeTypes.AUDIO_AAC)) { mediaFormat.setInteger( MediaFormat.KEY_AAC_PROFILE, requestedAudioEncoderSettings.profile); @@ -223,7 +223,16 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } } } - + if (!encoderSelectedForRequestedProfile && enableFallback) { + @Nullable + EncoderQueryResult encoderQueryResult = + findAudioEncoderWithClosestSupportedFormat(format, mediaCodecInfos); + if (encoderQueryResult != null) { + selectedEncoder = encoderQueryResult.encoder; + format = encoderQueryResult.supportedFormat; + mediaFormat = createMediaFormatFromFormat(format); + } + } if (requestedAudioEncoderSettings.bitrate != AudioEncoderSettings.NO_VALUE) { mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, requestedAudioEncoderSettings.bitrate); } @@ -261,7 +270,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { @Nullable VideoEncoderQueryResult encoderAndClosestFormatSupport = - findEncoderWithClosestSupportedFormat( + findVideoEncoderWithClosestSupportedFormat( format, requestedVideoEncoderSettings, videoEncoderSelector, enableFallback); if (encoderAndClosestFormatSupport == null) { @@ -402,15 +411,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } /** - * Finds an {@linkplain MediaCodecInfo encoder} that supports a format closest to the requested - * format. + * Finds a video {@linkplain MediaCodecInfo encoder} that supports a format closest to the + * requested format. * - *

Returns the {@linkplain MediaCodecInfo encoder} and the supported {@link Format} in a {@link - * Pair}, or {@code null} if none is found. + *

Returns a {@link VideoEncoderQueryResult}, or {@code null} if no encoder is found. */ @RequiresNonNull("#1.sampleMimeType") @Nullable - private static VideoEncoderQueryResult findEncoderWithClosestSupportedFormat( + private static VideoEncoderQueryResult findVideoEncoderWithClosestSupportedFormat( Format requestedFormat, VideoEncoderSettings videoEncoderSettings, EncoderSelector encoderSelector, @@ -576,17 +584,63 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { : Integer.MAX_VALUE); // Drops encoder. } - private static final class VideoEncoderQueryResult { + /** + * Finds an audio {@linkplain MediaCodecInfo encoder} that supports a format closest to the + * requested format. + * + *

Returns a {@link EncoderQueryResult}, or {@code null} if no encoder is found. + */ + @RequiresNonNull("#1.sampleMimeType") + @Nullable + private static EncoderQueryResult findAudioEncoderWithClosestSupportedFormat( + Format requestedFormat, ImmutableList filteredEncoderInfos) { + String mimeType = checkNotNull(requestedFormat.sampleMimeType); + if (filteredEncoderInfos.isEmpty()) { + return null; + } + MediaCodecInfo filteredEncoderInfo = + filterEncodersBySampleRate(filteredEncoderInfos, mimeType, requestedFormat.sampleRate) + .get(0); + int sampleRate = + EncoderUtil.getClosestSupportedSampleRate( + filteredEncoderInfo, mimeType, requestedFormat.sampleRate); + Format encoderFormat = requestedFormat.buildUpon().setSampleRate(sampleRate).build(); + return new EncoderQueryResult(filteredEncoderInfo, encoderFormat); + } + + /** + * Returns a list of {@linkplain MediaCodecInfo encoders} that support the requested sample rate + * most closely. + */ + private static ImmutableList filterEncodersBySampleRate( + List encoders, String mimeType, int requestedSampleRate) { + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> { + int closestSupportedSampleRate = + EncoderUtil.getClosestSupportedSampleRate(encoderInfo, mimeType, requestedSampleRate); + return Math.abs(closestSupportedSampleRate - requestedSampleRate); + }); + } + + private static class EncoderQueryResult { public final MediaCodecInfo encoder; public final Format supportedFormat; + + public EncoderQueryResult(MediaCodecInfo encoder, Format supportedFormat) { + this.encoder = encoder; + this.supportedFormat = supportedFormat; + } + } + + private static final class VideoEncoderQueryResult extends EncoderQueryResult { public final VideoEncoderSettings supportedEncoderSettings; public VideoEncoderQueryResult( MediaCodecInfo encoder, Format supportedFormat, VideoEncoderSettings supportedEncoderSettings) { - this.encoder = encoder; - this.supportedFormat = supportedFormat; + super(encoder, supportedFormat); this.supportedEncoderSettings = supportedEncoderSettings; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 2f8a17576c..e54a9770b3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -384,6 +384,38 @@ public final class EncoderUtil { Ints.asList(encoderInfo.getCapabilitiesForType(mimeType).colorFormats)); } + /** + * Returns the sample rate supported by the provided {@linkplain MediaCodecInfo encoder} that is + * closest to the provided sample rate. + */ + public static int getClosestSupportedSampleRate( + MediaCodecInfo encoderInfo, String mimeType, int requestedSampleRate) { + MediaCodecInfo.AudioCapabilities audioCapabilities = + encoderInfo.getCapabilitiesForType(mimeType).getAudioCapabilities(); + @Nullable int[] supportedSampleRates = audioCapabilities.getSupportedSampleRates(); + int closestSampleRate = Integer.MAX_VALUE; + if (supportedSampleRates != null) { + // The codec supports only discrete values. + for (int supportedSampleRate : supportedSampleRates) { + if (Math.abs(supportedSampleRate - requestedSampleRate) + < Math.abs(closestSampleRate - requestedSampleRate)) { + closestSampleRate = supportedSampleRate; + } + } + return closestSampleRate; + } else { + Range[] ranges = audioCapabilities.getSupportedSampleRateRanges(); + for (Range range : ranges) { + int supportedSampleRate = range.clamp(requestedSampleRate); + if (Math.abs(supportedSampleRate - requestedSampleRate) + < Math.abs(closestSampleRate - requestedSampleRate)) { + closestSampleRate = supportedSampleRate; + } + } + } + return closestSampleRate; + } + /** Checks if a {@linkplain MediaCodecInfo codec} is hardware-accelerated. */ public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) { // TODO(b/214964116): Merge into MediaCodecUtil. diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java index 733bea1da0..b5febb1f1a 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java @@ -45,6 +45,7 @@ public class DefaultEncoderFactoryTest { @Before public void setUp() { createShadowH264Encoder(); + createShadowAacEncoder(); } @After @@ -66,24 +67,40 @@ public class DefaultEncoderFactoryTest { createShadowVideoEncoder(avcFormat, profileLevel, "test.transformer.avc.encoder"); } + private static void createShadowAacEncoder() { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC); + MediaCodecInfo.CodecCapabilities capabilities = + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(format) + .setIsEncoder(true) + .build(); + createShadowEncoder("test.transformer.aac.encoder", capabilities); + } + private static void createShadowVideoEncoder( MediaFormat supportedFormat, MediaCodecInfo.CodecProfileLevel supportedProfileLevel, String name) { + MediaCodecInfo.CodecCapabilities capabilities = + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(supportedFormat) + .setIsEncoder(true) + .setColorFormats( + new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}) + .setProfileLevels(new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel}) + .build(); + createShadowEncoder(name, capabilities); + } + + private static void createShadowEncoder( + String name, MediaCodecInfo.CodecCapabilities... capabilities) { // ShadowMediaCodecList is static. The added encoders will be visible for every test. ShadowMediaCodecList.addCodec( MediaCodecInfoBuilder.newBuilder() .setName(name) .setIsEncoder(true) - .setCapabilities( - MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() - .setMediaFormat(supportedFormat) - .setIsEncoder(true) - .setColorFormats( - new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}) - .setProfileLevels( - new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel}) - .build()) + .setCapabilities(capabilities) .build()); } @@ -247,6 +264,36 @@ public class DefaultEncoderFactoryTest { .createForVideoEncoding(requestedVideoFormat)); } + @Test + public void createForAudioEncoding_unsupportedSampleRateWithFallback() throws Exception { + Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000); + + Format actualAudioFormat = + new DefaultEncoderFactory.Builder(context) + .setEnableFallback(true) + .build() + .createForAudioEncoding(requestedAudioFormat) + .getConfigurationFormat(); + + assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC); + assertThat(actualAudioFormat.sampleRate).isEqualTo(96_000); + } + + @Test + public void createForAudioEncoding_unsupportedSampleRateWithoutFallback() throws Exception { + Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000); + + Format actualAudioFormat = + new DefaultEncoderFactory.Builder(context) + .setEnableFallback(false) + .build() + .createForAudioEncoding(requestedAudioFormat) + .getConfigurationFormat(); + + assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC); + assertThat(actualAudioFormat.sampleRate).isEqualTo(192_000); + } + private static Format createVideoFormat(String mimeType, int width, int height, int frameRate) { return new Format.Builder() .setWidth(width) @@ -256,4 +303,8 @@ public class DefaultEncoderFactoryTest { .setSampleMimeType(mimeType) .build(); } + + private static Format createAudioFormat(String mimeType, int sampleRate) { + return new Format.Builder().setSampleRate(sampleRate).setSampleMimeType(mimeType).build(); + } }