From 5c9c0e207366a3bbe613c3b16691ba925f7c71be Mon Sep 17 00:00:00 2001 From: olly Date: Mon, 13 Jul 2020 14:41:45 +0100 Subject: [PATCH] Improve handling of floating point audio - DefaultAudioSink always supports floating point input. Make it advertise this fact. - Remove the ability to enable/disable floating point output in FfmpegAudioRenderer, since this ability is now also provided on DefaultAudioSink. - Let FfmpegAudioRenderer query the sink to determine whether it will output floating point PCM directly or resample it to 16-bit PCM. PiperOrigin-RevId: 320945360 --- .../ext/ffmpeg/FfmpegAudioRenderer.java | 73 +++++++++---------- .../android/exoplayer2/audio/AudioSink.java | 40 +++++++++- .../audio/DecoderAudioRenderer.java | 12 +++ .../exoplayer2/audio/DefaultAudioSink.java | 33 ++++++--- .../exoplayer2/audio/ForwardingAudioSink.java | 6 ++ .../audio/DefaultAudioSinkTest.java | 38 +++++++++- 6 files changed, 146 insertions(+), 56 deletions(-) diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java index f94733f9c4..a29aeb68ae 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java @@ -15,6 +15,10 @@ */ package com.google.android.exoplayer2.ext.ffmpeg; +import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; +import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING; +import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_UNSUPPORTED; + import android.os.Handler; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; @@ -22,6 +26,7 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.AudioSink; +import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport; import com.google.android.exoplayer2.audio.DecoderAudioRenderer; import com.google.android.exoplayer2.audio.DefaultAudioSink; import com.google.android.exoplayer2.drm.ExoMediaCrypto; @@ -41,8 +46,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { /** The default input buffer size. */ private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6; - private final boolean enableFloatOutput; - private @MonotonicNonNull FfmpegAudioDecoder decoder; public FfmpegAudioRenderer() { @@ -64,8 +67,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { this( eventHandler, eventListener, - new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors), - /* enableFloatOutput= */ false); + new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors)); } /** @@ -75,21 +77,15 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { * null if delivery of events is not required. * @param eventListener A listener of events. May be null if delivery of events is not required. * @param audioSink The sink to which audio will be output. - * @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the - * device/build and if the input format may have bit depth higher than 16-bit. When using - * 32-bit float output, any audio processing will be disabled, including playback speed/pitch - * adjustment. */ public FfmpegAudioRenderer( @Nullable Handler eventHandler, @Nullable AudioRendererEventListener eventListener, - AudioSink audioSink, - boolean enableFloatOutput) { + AudioSink audioSink) { super( eventHandler, eventListener, audioSink); - this.enableFloatOutput = enableFloatOutput; } @Override @@ -103,7 +99,9 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { String mimeType = Assertions.checkNotNull(format.sampleMimeType); if (!FfmpegLibrary.isAvailable() || !MimeTypes.isAudio(mimeType)) { return FORMAT_UNSUPPORTED_TYPE; - } else if (!FfmpegLibrary.supportsFormat(mimeType) || !isOutputSupported(format)) { + } else if (!FfmpegLibrary.supportsFormat(mimeType) + || (!sinkSupportsFormat(format, C.ENCODING_PCM_16BIT) + && !sinkSupportsFormat(format, C.ENCODING_PCM_FLOAT))) { return FORMAT_UNSUPPORTED_SUBTYPE; } else if (format.drmInitData != null && format.exoMediaCryptoType == null) { return FORMAT_UNSUPPORTED_DRM; @@ -126,7 +124,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; decoder = new FfmpegAudioDecoder( - format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldUseFloatOutput(format)); + format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldOutputFloat(format)); TraceUtil.endSection(); return decoder; } @@ -142,31 +140,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { .build(); } - private boolean isOutputSupported(Format inputFormat) { - return shouldUseFloatOutput(inputFormat) - || sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT); - } - - private boolean shouldUseFloatOutput(Format inputFormat) { - Assertions.checkNotNull(inputFormat.sampleMimeType); - if (!enableFloatOutput || !sinkSupportsFormat(inputFormat, C.ENCODING_PCM_FLOAT)) { - return false; - } - switch (inputFormat.sampleMimeType) { - case MimeTypes.AUDIO_RAW: - // For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit. - return inputFormat.encoding == C.ENCODING_PCM_24BIT - || inputFormat.encoding == C.ENCODING_PCM_32BIT - || inputFormat.encoding == C.ENCODING_PCM_FLOAT; - case MimeTypes.AUDIO_AC3: - // AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding. - return false; - default: - // For all other formats, assume that it's worth using 32-bit float encoding. - return true; - } - } - /** * Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output * from the decoder for the given input format and requested output encoding. @@ -175,4 +148,28 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { return sinkSupportsFormat( Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate)); } + + private boolean shouldOutputFloat(Format inputFormat) { + if (!sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT)) { + // We have no choice because the sink doesn't support 16-bit integer PCM. + return true; + } + + @SinkFormatSupport + int formatSupport = + getSinkFormatSupport( + Util.getPcmFormat( + C.ENCODING_PCM_FLOAT, inputFormat.channelCount, inputFormat.sampleRate)); + switch (formatSupport) { + case SINK_FORMAT_SUPPORTED_DIRECTLY: + // AC-3 is always 16-bit, so there's no point using floating point. Assume that it's worth + // using for all other formats. + return !MimeTypes.AUDIO_AC3.equals(inputFormat.sampleMimeType); + case SINK_FORMAT_UNSUPPORTED: + case SINK_FORMAT_SUPPORTED_WITH_TRANSCODING: + default: + // Always prefer 16-bit PCM if the sink does not provide direct support for floating point. + return false; + } + } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java index 241ddaebac..2de8dcf8a6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java @@ -16,10 +16,14 @@ package com.google.android.exoplayer2.audio; import android.media.AudioTrack; +import androidx.annotation.IntDef; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.PlaybackParameters; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.nio.ByteBuffer; /** @@ -172,8 +176,29 @@ public interface AudioSink { } /** - * Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set. + * The level of support the sink provides for a format. One of {@link + * #SINK_FORMAT_SUPPORTED_DIRECTLY}, {@link #SINK_FORMAT_SUPPORTED_WITH_TRANSCODING} or {@link + * #SINK_FORMAT_UNSUPPORTED}. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @IntDef({ + SINK_FORMAT_SUPPORTED_DIRECTLY, + SINK_FORMAT_SUPPORTED_WITH_TRANSCODING, + SINK_FORMAT_UNSUPPORTED + }) + @interface SinkFormatSupport {} + /** The sink supports the format directly, without the need for internal transcoding. */ + int SINK_FORMAT_SUPPORTED_DIRECTLY = 2; + /** + * The sink supports the format, but needs to transcode it internally to do so. Internal + * transcoding may result in lower quality and higher CPU load in some cases. + */ + int SINK_FORMAT_SUPPORTED_WITH_TRANSCODING = 1; + /** The sink does not support the format. */ + int SINK_FORMAT_UNSUPPORTED = 0; + + /** Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set. */ long CURRENT_POSITION_NOT_SET = Long.MIN_VALUE; /** @@ -192,8 +217,17 @@ public interface AudioSink { boolean supportsFormat(Format format); /** - * Returns the playback position in the stream starting at zero, in microseconds, or - * {@link #CURRENT_POSITION_NOT_SET} if it is not yet available. + * Returns the level of support that the sink provides for a given {@link Format}. + * + * @param format The format. + * @return The level of support provided. + */ + @SinkFormatSupport + int getFormatSupport(Format format); + + /** + * Returns the playback position in the stream starting at zero, in microseconds, or {@link + * #CURRENT_POSITION_NOT_SET} if it is not yet available. * * @param sourceEnded Specify {@code true} if no more input buffers will be provided. * @return The playback position relative to the start of playback, in microseconds. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java index 0654fa8e27..d3f5dff113 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java @@ -29,6 +29,7 @@ import com.google.android.exoplayer2.FormatHolder; import com.google.android.exoplayer2.PlayerMessage.Target; import com.google.android.exoplayer2.RendererCapabilities; import com.google.android.exoplayer2.audio.AudioRendererEventListener.EventDispatcher; +import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport; import com.google.android.exoplayer2.decoder.Decoder; import com.google.android.exoplayer2.decoder.DecoderCounters; import com.google.android.exoplayer2.decoder.DecoderException; @@ -219,6 +220,17 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media return audioSink.supportsFormat(format); } + /** + * Returns the level of support that the renderer's {@link AudioSink} provides for a given {@link + * Format}. + * + * @see AudioSink#getFormatSupport(Format) (Format) + */ + @SinkFormatSupport + protected final int getSinkFormatSupport(Format format) { + return audioSink.getFormatSupport(format); + } + @Override public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { if (outputStreamEnded) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index dda218fbcc..bba93fe45a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -380,7 +380,7 @@ public final class DefaultAudioSink implements AudioSink { boolean enableOffload) { this.audioCapabilities = audioCapabilities; this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain); - this.enableFloatOutput = enableFloatOutput; + this.enableFloatOutput = Util.SDK_INT >= 21 && enableFloatOutput; this.enableOffload = Util.SDK_INT >= 29 && enableOffload; releasingConditionVariable = new ConditionVariable(true); audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener()); @@ -420,15 +420,23 @@ public final class DefaultAudioSink implements AudioSink { @Override public boolean supportsFormat(Format format) { + return getFormatSupport(format) != SINK_FORMAT_UNSUPPORTED; + } + + @Override + @SinkFormatSupport + public int getFormatSupport(Format format) { if (format.encoding == C.ENCODING_INVALID) { - return false; + return SINK_FORMAT_UNSUPPORTED; } if (Util.isEncodingLinearPcm(format.encoding)) { - // AudioTrack supports 16-bit integer PCM output in all platform API versions, and float - // output from platform API version 21 only. Other integer PCM encodings are resampled by this - // sink to 16-bit PCM. We assume that the audio framework will downsample any number of - // channels to the output device's required number of channels. - return format.encoding != C.ENCODING_PCM_FLOAT || Util.SDK_INT >= 21; + if (format.encoding == C.ENCODING_PCM_16BIT + || (enableFloatOutput && format.encoding == C.ENCODING_PCM_FLOAT)) { + return SINK_FORMAT_SUPPORTED_DIRECTLY; + } + // We can resample all linear PCM encodings to 16-bit integer PCM, which AudioTrack is + // guaranteed to support. + return SINK_FORMAT_SUPPORTED_WITH_TRANSCODING; } if (enableOffload && isOffloadedPlaybackSupported( @@ -438,9 +446,12 @@ public final class DefaultAudioSink implements AudioSink { audioAttributes, format.encoderDelay, format.encoderPadding)) { - return true; + return SINK_FORMAT_SUPPORTED_DIRECTLY; } - return isPassthroughPlaybackSupported(format); + if (isPassthroughPlaybackSupported(format)) { + return SINK_FORMAT_SUPPORTED_DIRECTLY; + } + return SINK_FORMAT_UNSUPPORTED; } @Override @@ -471,9 +482,7 @@ public final class DefaultAudioSink implements AudioSink { int channelCount = inputFormat.channelCount; @C.Encoding int encoding = inputFormat.encoding; boolean useFloatOutput = - enableFloatOutput - && Util.isEncodingHighResolutionPcm(inputFormat.encoding) - && supportsFormat(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build()); + enableFloatOutput && Util.isEncodingHighResolutionPcm(inputFormat.encoding); AudioProcessor[] availableAudioProcessors = useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; if (processingEnabled) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java index 36459c6f89..43ed7a9a4c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java @@ -39,6 +39,12 @@ public class ForwardingAudioSink implements AudioSink { return sink.supportsFormat(format); } + @Override + @SinkFormatSupport + public int getFormatSupport(Format format) { + return sink.getFormatSupport(format); + } + @Override public long getCurrentPositionUs(boolean sourceEnded) { return sink.getCurrentPositionUs(sourceEnded); diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java index 1266bd2584..6587614b50 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.audio; +import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; +import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING; import static com.google.common.truth.Truth.assertThat; import static org.robolectric.annotation.Config.OLDEST_SDK; import static org.robolectric.annotation.Config.TARGET_SDK; @@ -204,16 +206,46 @@ public final class DefaultAudioSinkTest { .isEqualTo(8 * C.MICROS_PER_SECOND); } + @Test + public void floatPcmNeedsTranscodingIfFloatOutputDisabled() { + defaultAudioSink = + new DefaultAudioSink( + AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, + new AudioProcessor[0], + /* enableFloatOutput= */ false); + Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); + assertThat(defaultAudioSink.getFormatSupport(floatFormat)) + .isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING); + } + @Config(minSdk = OLDEST_SDK, maxSdk = 20) @Test - public void doesNotSupportFloatPcmBeforeApi21() { + public void floatPcmNeedsTranscodingIfFloatOutputEnabledBeforeApi21() { + defaultAudioSink = + new DefaultAudioSink( + AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, + new AudioProcessor[0], + /* enableFloatOutput= */ true); Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); - assertThat(defaultAudioSink.supportsFormat(floatFormat)).isFalse(); + assertThat(defaultAudioSink.getFormatSupport(floatFormat)) + .isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING); } @Config(minSdk = 21, maxSdk = TARGET_SDK) @Test - public void supportsFloatPcmFromApi21() { + public void floatOutputSupportedIfFloatOutputEnabledFromApi21() { + defaultAudioSink = + new DefaultAudioSink( + AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, + new AudioProcessor[0], + /* enableFloatOutput= */ true); + Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); + assertThat(defaultAudioSink.getFormatSupport(floatFormat)) + .isEqualTo(SINK_FORMAT_SUPPORTED_DIRECTLY); + } + + @Test + public void supportsFloatPcm() { Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build(); assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue(); }