sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-31 10:25:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve handling of floating point audio

Browse source 
- DefaultAudioSink always supports floating point input. Make it
  advertise this fact.
- Remove the ability to enable/disable floating point output in
  FfmpegAudioRenderer, since this ability is now also provided on
  DefaultAudioSink.
- Let FfmpegAudioRenderer query the sink to determine whether it
  will output floating point PCM directly or resample it to 16-bit
  PCM.

PiperOrigin-RevId: 320945360
This commit is contained in:
olly 2020-07-13 14:41:45 +01:00 committed by Ian Baker
parent 21e56f571d
commit 5c9c0e2073
6 changed files with 146 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java
View file

@ -15,6 +15,10 @@
*/
package com.google.android.exoplayer2.ext.ffmpeg;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_UNSUPPORTED;
import android.os.Handler;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
@ -22,6 +26,7 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.audio.AudioProcessor;
import com.google.android.exoplayer2.audio.AudioRendererEventListener;
import com.google.android.exoplayer2.audio.AudioSink;
import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport;
import com.google.android.exoplayer2.audio.DecoderAudioRenderer;
import com.google.android.exoplayer2.audio.DefaultAudioSink;
import com.google.android.exoplayer2.drm.ExoMediaCrypto;
@ -41,8 +46,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
/** The default input buffer size. */
private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6;
private final boolean enableFloatOutput;
private @MonotonicNonNull FfmpegAudioDecoder decoder;
public FfmpegAudioRenderer() {
@ -64,8 +67,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
this(
eventHandler,
eventListener,
new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors),
/* enableFloatOutput= */ false);
new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors));
}
/**
@ -75,21 +77,15 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
* null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param audioSink The sink to which audio will be output.
* @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the
* device/build and if the input format may have bit depth higher than 16-bit. When using
* 32-bit float output, any audio processing will be disabled, including playback speed/pitch
* adjustment.
*/
public FfmpegAudioRenderer(
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioSink audioSink,
boolean enableFloatOutput) {
AudioSink audioSink) {
super(
eventHandler,
eventListener,
audioSink);
this.enableFloatOutput = enableFloatOutput;
}
@Override
@ -103,7 +99,9 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
String mimeType = Assertions.checkNotNull(format.sampleMimeType);
if (!FfmpegLibrary.isAvailable() || !MimeTypes.isAudio(mimeType)) {
return FORMAT_UNSUPPORTED_TYPE;
} else if (!FfmpegLibrary.supportsFormat(mimeType) || !isOutputSupported(format)) {
} else if (!FfmpegLibrary.supportsFormat(mimeType)
|| (!sinkSupportsFormat(format, C.ENCODING_PCM_16BIT)
&& !sinkSupportsFormat(format, C.ENCODING_PCM_FLOAT))) {
return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (format.drmInitData != null && format.exoMediaCryptoType == null) {
return FORMAT_UNSUPPORTED_DRM;
@ -126,7 +124,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
decoder =
new FfmpegAudioDecoder(
format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldUseFloatOutput(format));
format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldOutputFloat(format));
TraceUtil.endSection();
return decoder;
}
@ -142,31 +140,6 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
.build();
}
private boolean isOutputSupported(Format inputFormat) {
return shouldUseFloatOutput(inputFormat)
|| sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT);
}
private boolean shouldUseFloatOutput(Format inputFormat) {
Assertions.checkNotNull(inputFormat.sampleMimeType);
if (!enableFloatOutput || !sinkSupportsFormat(inputFormat, C.ENCODING_PCM_FLOAT)) {
return false;
}
switch (inputFormat.sampleMimeType) {
case MimeTypes.AUDIO_RAW:
// For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit.
return inputFormat.encoding == C.ENCODING_PCM_24BIT
|| inputFormat.encoding == C.ENCODING_PCM_32BIT
|| inputFormat.encoding == C.ENCODING_PCM_FLOAT;
case MimeTypes.AUDIO_AC3:
// AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding.
return false;
default:
// For all other formats, assume that it's worth using 32-bit float encoding.
return true;
}
}
/**
* Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output
* from the decoder for the given input format and requested output encoding.
@ -175,4 +148,28 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
return sinkSupportsFormat(
Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate));
}
private boolean shouldOutputFloat(Format inputFormat) {
if (!sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT)) {
// We have no choice because the sink doesn't support 16-bit integer PCM.
return true;
}
@SinkFormatSupport
int formatSupport =
getSinkFormatSupport(
Util.getPcmFormat(
C.ENCODING_PCM_FLOAT, inputFormat.channelCount, inputFormat.sampleRate));
switch (formatSupport) {
case SINK_FORMAT_SUPPORTED_DIRECTLY:
// AC-3 is always 16-bit, so there's no point using floating point. Assume that it's worth
// using for all other formats.
return !MimeTypes.AUDIO_AC3.equals(inputFormat.sampleMimeType);
case SINK_FORMAT_UNSUPPORTED:
case SINK_FORMAT_SUPPORTED_WITH_TRANSCODING:
default:
// Always prefer 16-bit PCM if the sink does not provide direct support for floating point.
return false;
}
}
}

40
library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java
View file

@ -16,10 +16,14 @@
package com.google.android.exoplayer2.audio;
import android.media.AudioTrack;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.PlaybackParameters;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.nio.ByteBuffer;
/**
@ -172,8 +176,29 @@ public interface AudioSink {
}
/**
* Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set.
* The level of support the sink provides for a format. One of {@link
* #SINK_FORMAT_SUPPORTED_DIRECTLY}, {@link #SINK_FORMAT_SUPPORTED_WITH_TRANSCODING} or {@link
* #SINK_FORMAT_UNSUPPORTED}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
SINK_FORMAT_SUPPORTED_DIRECTLY,
SINK_FORMAT_SUPPORTED_WITH_TRANSCODING,
SINK_FORMAT_UNSUPPORTED
})
@interface SinkFormatSupport {}
/** The sink supports the format directly, without the need for internal transcoding. */
int SINK_FORMAT_SUPPORTED_DIRECTLY = 2;
/**
* The sink supports the format, but needs to transcode it internally to do so. Internal
* transcoding may result in lower quality and higher CPU load in some cases.
*/
int SINK_FORMAT_SUPPORTED_WITH_TRANSCODING = 1;
/** The sink does not support the format. */
int SINK_FORMAT_UNSUPPORTED = 0;
/** Returned by {@link #getCurrentPositionUs(boolean)} when the position is not set. */
long CURRENT_POSITION_NOT_SET = Long.MIN_VALUE;
/**
@ -192,8 +217,17 @@ public interface AudioSink {
boolean supportsFormat(Format format);
/**
* Returns the playback position in the stream starting at zero, in microseconds, or
* {@link #CURRENT_POSITION_NOT_SET} if it is not yet available.
* Returns the level of support that the sink provides for a given {@link Format}.
*
* @param format The format.
* @return The level of support provided.
*/
@SinkFormatSupport
int getFormatSupport(Format format);
/**
* Returns the playback position in the stream starting at zero, in microseconds, or {@link
* #CURRENT_POSITION_NOT_SET} if it is not yet available.
*
* @param sourceEnded Specify {@code true} if no more input buffers will be provided.
* @return The playback position relative to the start of playback, in microseconds.

12
library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java
View file

@ -29,6 +29,7 @@ import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.PlayerMessage.Target;
import com.google.android.exoplayer2.RendererCapabilities;
import com.google.android.exoplayer2.audio.AudioRendererEventListener.EventDispatcher;
import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport;
import com.google.android.exoplayer2.decoder.Decoder;
import com.google.android.exoplayer2.decoder.DecoderCounters;
import com.google.android.exoplayer2.decoder.DecoderException;
@ -219,6 +220,17 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media
return audioSink.supportsFormat(format);
}
/**
* Returns the level of support that the renderer's {@link AudioSink} provides for a given {@link
* Format}.
*
* @see AudioSink#getFormatSupport(Format) (Format)
*/
@SinkFormatSupport
protected final int getSinkFormatSupport(Format format) {
return audioSink.getFormatSupport(format);
}
@Override
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
if (outputStreamEnded) {

33
library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java
View file

@ -380,7 +380,7 @@ public final class DefaultAudioSink implements AudioSink {
boolean enableOffload) {
this.audioCapabilities = audioCapabilities;
this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain);
this.enableFloatOutput = enableFloatOutput;
this.enableFloatOutput = Util.SDK_INT >= 21 && enableFloatOutput;
this.enableOffload = Util.SDK_INT >= 29 && enableOffload;
releasingConditionVariable = new ConditionVariable(true);
audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener());
@ -420,15 +420,23 @@ public final class DefaultAudioSink implements AudioSink {
@Override
public boolean supportsFormat(Format format) {
return getFormatSupport(format) != SINK_FORMAT_UNSUPPORTED;
}
@Override
@SinkFormatSupport
public int getFormatSupport(Format format) {
if (format.encoding == C.ENCODING_INVALID) {
return false;
return SINK_FORMAT_UNSUPPORTED;
}
if (Util.isEncodingLinearPcm(format.encoding)) {
// AudioTrack supports 16-bit integer PCM output in all platform API versions, and float
// output from platform API version 21 only. Other integer PCM encodings are resampled by this
// sink to 16-bit PCM. We assume that the audio framework will downsample any number of
// channels to the output device's required number of channels.
return format.encoding != C.ENCODING_PCM_FLOAT || Util.SDK_INT >= 21;
if (format.encoding == C.ENCODING_PCM_16BIT
|| (enableFloatOutput && format.encoding == C.ENCODING_PCM_FLOAT)) {
return SINK_FORMAT_SUPPORTED_DIRECTLY;
}
// We can resample all linear PCM encodings to 16-bit integer PCM, which AudioTrack is
// guaranteed to support.
return SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
}
if (enableOffload
&& isOffloadedPlaybackSupported(
@ -438,9 +446,12 @@ public final class DefaultAudioSink implements AudioSink {
audioAttributes,
format.encoderDelay,
format.encoderPadding)) {
return true;
return SINK_FORMAT_SUPPORTED_DIRECTLY;
}
return isPassthroughPlaybackSupported(format);
if (isPassthroughPlaybackSupported(format)) {
return SINK_FORMAT_SUPPORTED_DIRECTLY;
}
return SINK_FORMAT_UNSUPPORTED;
}
@Override
@ -471,9 +482,7 @@ public final class DefaultAudioSink implements AudioSink {
int channelCount = inputFormat.channelCount;
@C.Encoding int encoding = inputFormat.encoding;
boolean useFloatOutput =
enableFloatOutput
&& Util.isEncodingHighResolutionPcm(inputFormat.encoding)
&& supportsFormat(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build());
enableFloatOutput && Util.isEncodingHighResolutionPcm(inputFormat.encoding);
AudioProcessor[] availableAudioProcessors =
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) {

6
library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java
View file

@ -39,6 +39,12 @@ public class ForwardingAudioSink implements AudioSink {
return sink.supportsFormat(format);
}
@Override
@SinkFormatSupport
public int getFormatSupport(Format format) {
return sink.getFormatSupport(format);
}
@Override
public long getCurrentPositionUs(boolean sourceEnded) {
return sink.getCurrentPositionUs(sourceEnded);

38
library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java
View file

@ -15,6 +15,8 @@
*/
package com.google.android.exoplayer2.audio;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
import static com.google.common.truth.Truth.assertThat;
import static org.robolectric.annotation.Config.OLDEST_SDK;
import static org.robolectric.annotation.Config.TARGET_SDK;
@ -204,16 +206,46 @@ public final class DefaultAudioSinkTest {
.isEqualTo(8 * C.MICROS_PER_SECOND);
}
@Test
public void floatPcmNeedsTranscodingIfFloatOutputDisabled() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ false);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING);
}
@Config(minSdk = OLDEST_SDK, maxSdk = 20)
@Test
public void doesNotSupportFloatPcmBeforeApi21() {
public void floatPcmNeedsTranscodingIfFloatOutputEnabledBeforeApi21() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ true);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isFalse();
assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_WITH_TRANSCODING);
}
@Config(minSdk = 21, maxSdk = TARGET_SDK)
@Test
public void supportsFloatPcmFromApi21() {
public void floatOutputSupportedIfFloatOutputEnabledFromApi21() {
defaultAudioSink =
new DefaultAudioSink(
AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES,
new AudioProcessor[0],
/* enableFloatOutput= */ true);
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.getFormatSupport(floatFormat))
.isEqualTo(SINK_FORMAT_SUPPORTED_DIRECTLY);
}
@Test
public void supportsFloatPcm() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue();
}