From a038346ecc6f3d8a30b0a955fb25f8092919adad Mon Sep 17 00:00:00 2001 From: olly Date: Wed, 25 Mar 2020 12:50:03 +0000 Subject: [PATCH] Resample float audio to 16-bit by default to enable audio processing This is less confusing than having audio processing functionality (e.g., playback speed adjustment) just "not work" for some pieces of media. If this change is merged, I will update #6749 to also track making DefaultAudioSink intelligently enable/disable float output depending on how the audio processors are configured. Issue: #7134 PiperOrigin-RevId: 302871568 --- RELEASENOTES.md | 16 +++-- .../exoplayer2/audio/DefaultAudioSink.java | 45 ++++++------ .../audio/FloatResamplingAudioProcessor.java | 70 ++++++++++++------- .../google/android/exoplayer2/util/Util.java | 10 +-- 4 files changed, 81 insertions(+), 60 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6ae74edb12..dca9188320 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -8,15 +8,21 @@ ([#6885](https://github.com/google/ExoPlayer/issues/6885)). * Allow missing hours in SubRip (.srt) timecodes ([#7122](https://github.com/google/ExoPlayer/issues/7122)). -* WAV: Fix failure to play WAV files that contain trailing non-media bytes - ([#7129](https://github.com/google/ExoPlayer/issues/7129)) -* UI: Add an option to set whether to use the orientation sensor for rotation - in spherical playbacks - ([#6761](https://github.com/google/ExoPlayer/issues/6761)). +* Audio: + * Enable playback speed adjustment and silence skipping for floating point PCM + audio, via resampling to 16-bit integer PCM. To output the original floating + point audio without adjustment, pass `enableFloatOutput=true` to the + `DefaultAudioSink` constructor + ([#7134](https://github.com/google/ExoPlayer/issues/7134)). + * Fix failure to play WAV files that contain trailing non-media bytes + ([#7129](https://github.com/google/ExoPlayer/issues/7129)) * DASH: * Update the manifest URI to avoid repeated HTTP redirects ([#6907](https://github.com/google/ExoPlayer/issues/6907)). * Parse period `AssetIdentifier` elements. +* UI: Add an option to set whether to use the orientation sensor for rotation + in spherical playbacks + ([#6761](https://github.com/google/ExoPlayer/issues/6761)). * FFmpeg extension: Add support for x86_64. ### 2.11.3 (2020-02-19) ### diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index 1824303a57..ba31c118e7 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -238,7 +238,7 @@ public final class DefaultAudioSink implements AudioSink { @Nullable private final AudioCapabilities audioCapabilities; private final AudioProcessorChain audioProcessorChain; - private final boolean enableConvertHighResIntPcmToFloat; + private final boolean enableFloatOutput; private final ChannelMappingAudioProcessor channelMappingAudioProcessor; private final TrimmingAudioProcessor trimmingAudioProcessor; private final AudioProcessor[] toIntPcmAvailableAudioProcessors; @@ -299,7 +299,7 @@ public final class DefaultAudioSink implements AudioSink { */ public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) { - this(audioCapabilities, audioProcessors, /* enableConvertHighResIntPcmToFloat= */ false); + this(audioCapabilities, audioProcessors, /* enableFloatOutput= */ false); } /** @@ -309,19 +309,16 @@ public final class DefaultAudioSink implements AudioSink { * default capabilities (no encoded audio passthrough support) should be assumed. * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before * output. May be empty. - * @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution - * integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer - * audio processing (for example, speed and pitch adjustment) will not be available when float - * output is in use. + * @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float + * output will be used if the input is 32-bit float, and also if the input is high resolution + * (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not + * be available when float output is in use. */ public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors, - boolean enableConvertHighResIntPcmToFloat) { - this( - audioCapabilities, - new DefaultAudioProcessorChain(audioProcessors), - enableConvertHighResIntPcmToFloat); + boolean enableFloatOutput) { + this(audioCapabilities, new DefaultAudioProcessorChain(audioProcessors), enableFloatOutput); } /** @@ -332,18 +329,18 @@ public final class DefaultAudioSink implements AudioSink { * default capabilities (no encoded audio passthrough support) should be assumed. * @param audioProcessorChain An {@link AudioProcessorChain} which is used to apply playback * parameters adjustments. The instance passed in must not be reused in other sinks. - * @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution - * integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer - * audio processing (for example, speed and pitch adjustment) will not be available when float - * output is in use. + * @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float + * output will be used if the input is 32-bit float, and also if the input is high resolution + * (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not + * be available when float output is in use. */ public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessorChain audioProcessorChain, - boolean enableConvertHighResIntPcmToFloat) { + boolean enableFloatOutput) { this.audioCapabilities = audioCapabilities; this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain); - this.enableConvertHighResIntPcmToFloat = enableConvertHighResIntPcmToFloat; + this.enableFloatOutput = enableFloatOutput; releasingConditionVariable = new ConditionVariable(true); audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener()); channelMappingAudioProcessor = new ChannelMappingAudioProcessor(); @@ -422,18 +419,16 @@ public final class DefaultAudioSink implements AudioSink { } boolean isInputPcm = Util.isEncodingLinearPcm(inputEncoding); - boolean processingEnabled = isInputPcm && inputEncoding != C.ENCODING_PCM_FLOAT; + boolean processingEnabled = isInputPcm; int sampleRate = inputSampleRate; int channelCount = inputChannelCount; @C.Encoding int encoding = inputEncoding; - boolean shouldConvertHighResIntPcmToFloat = - enableConvertHighResIntPcmToFloat + boolean useFloatOutput = + enableFloatOutput && supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT) - && Util.isEncodingHighResolutionIntegerPcm(inputEncoding); + && Util.isEncodingHighResolutionPcm(inputEncoding); AudioProcessor[] availableAudioProcessors = - shouldConvertHighResIntPcmToFloat - ? toFloatPcmAvailableAudioProcessors - : toIntPcmAvailableAudioProcessors; + useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors; if (processingEnabled) { trimmingAudioProcessor.setTrimFrameCount(trimStartFrames, trimEndFrames); channelMappingAudioProcessor.setChannelMap(outputChannels); @@ -463,7 +458,7 @@ public final class DefaultAudioSink implements AudioSink { isInputPcm ? Util.getPcmFrameSize(inputEncoding, inputChannelCount) : C.LENGTH_UNSET; int outputPcmFrameSize = isInputPcm ? Util.getPcmFrameSize(encoding, channelCount) : C.LENGTH_UNSET; - boolean canApplyPlaybackParameters = processingEnabled && !shouldConvertHighResIntPcmToFloat; + boolean canApplyPlaybackParameters = processingEnabled && !useFloatOutput; Configuration pendingConfiguration = new Configuration( isInputPcm, diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java index a75e675e6e..ca6b4f3f13 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java @@ -16,13 +16,19 @@ package com.google.android.exoplayer2.audio; import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.Util; import java.nio.ByteBuffer; /** - * An {@link AudioProcessor} that converts 24-bit and 32-bit integer PCM audio to 32-bit float PCM - * audio. + * An {@link AudioProcessor} that converts high resolution PCM audio to 32-bit float. The following + * encodings are supported as input: + * + * */ /* package */ final class FloatResamplingAudioProcessor extends BaseAudioProcessor { @@ -32,10 +38,11 @@ import java.nio.ByteBuffer; @Override public AudioFormat onConfigure(AudioFormat inputAudioFormat) throws UnhandledAudioFormatException { - if (!Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)) { + @C.PcmEncoding int encoding = inputAudioFormat.encoding; + if (!Util.isEncodingHighResolutionPcm(encoding)) { throw new UnhandledAudioFormatException(inputAudioFormat); } - return Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding) + return encoding != C.ENCODING_PCM_FLOAT ? new AudioFormat( inputAudioFormat.sampleRate, inputAudioFormat.channelCount, C.ENCODING_PCM_FLOAT) : AudioFormat.NOT_SET; @@ -43,31 +50,42 @@ import java.nio.ByteBuffer; @Override public void queueInput(ByteBuffer inputBuffer) { - Assertions.checkState(Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)); - boolean isInput32Bit = inputAudioFormat.encoding == C.ENCODING_PCM_32BIT; int position = inputBuffer.position(); int limit = inputBuffer.limit(); int size = limit - position; - int resampledSize = isInput32Bit ? size : (size / 3) * 4; - ByteBuffer buffer = replaceOutputBuffer(resampledSize); - if (isInput32Bit) { - for (int i = position; i < limit; i += 4) { - int pcm32BitInteger = - (inputBuffer.get(i) & 0xFF) - | ((inputBuffer.get(i + 1) & 0xFF) << 8) - | ((inputBuffer.get(i + 2) & 0xFF) << 16) - | ((inputBuffer.get(i + 3) & 0xFF) << 24); - writePcm32BitFloat(pcm32BitInteger, buffer); - } - } else { // Input is 24-bit PCM. - for (int i = position; i < limit; i += 3) { - int pcm32BitInteger = - ((inputBuffer.get(i) & 0xFF) << 8) - | ((inputBuffer.get(i + 1) & 0xFF) << 16) - | ((inputBuffer.get(i + 2) & 0xFF) << 24); - writePcm32BitFloat(pcm32BitInteger, buffer); - } + ByteBuffer buffer; + switch (inputAudioFormat.encoding) { + case C.ENCODING_PCM_24BIT: + buffer = replaceOutputBuffer((size / 3) * 4); + for (int i = position; i < limit; i += 3) { + int pcm32BitInteger = + ((inputBuffer.get(i) & 0xFF) << 8) + | ((inputBuffer.get(i + 1) & 0xFF) << 16) + | ((inputBuffer.get(i + 2) & 0xFF) << 24); + writePcm32BitFloat(pcm32BitInteger, buffer); + } + break; + case C.ENCODING_PCM_32BIT: + buffer = replaceOutputBuffer(size); + for (int i = position; i < limit; i += 4) { + int pcm32BitInteger = + (inputBuffer.get(i) & 0xFF) + | ((inputBuffer.get(i + 1) & 0xFF) << 8) + | ((inputBuffer.get(i + 2) & 0xFF) << 16) + | ((inputBuffer.get(i + 3) & 0xFF) << 24); + writePcm32BitFloat(pcm32BitInteger, buffer); + } + break; + case C.ENCODING_PCM_8BIT: + case C.ENCODING_PCM_16BIT: + case C.ENCODING_PCM_16BIT_BIG_ENDIAN: + case C.ENCODING_PCM_FLOAT: + case C.ENCODING_INVALID: + case Format.NO_VALUE: + default: + // Never happens. + throw new IllegalStateException(); } inputBuffer.position(inputBuffer.limit()); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java index 5518eeaf36..ea43ee7bb3 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -1360,13 +1360,15 @@ public final class Util { } /** - * Returns whether {@code encoding} is high resolution (> 16-bit) integer PCM. + * Returns whether {@code encoding} is high resolution (> 16-bit) PCM. * * @param encoding The encoding of the audio data. - * @return Whether the encoding is high resolution integer PCM. + * @return Whether the encoding is high resolution PCM. */ - public static boolean isEncodingHighResolutionIntegerPcm(@C.PcmEncoding int encoding) { - return encoding == C.ENCODING_PCM_24BIT || encoding == C.ENCODING_PCM_32BIT; + public static boolean isEncodingHighResolutionPcm(@C.PcmEncoding int encoding) { + return encoding == C.ENCODING_PCM_24BIT + || encoding == C.ENCODING_PCM_32BIT + || encoding == C.ENCODING_PCM_FLOAT; } /**