From ea01489c8b62974c71a7e5ae66fad278c5e121d7 Mon Sep 17 00:00:00 2001 From: claincly Date: Mon, 3 Aug 2020 20:42:24 +0100 Subject: [PATCH] Added float output mode for Opus extension The working of libOpus is different from ffmpeg. With ffmpeg, the decoder can be configured to output floating point PCM. While in libOpus, floating samples are acquired by calling a different function. This is the reason the new JNI functions and the logic in OpusDecoder/LibopusAudioRenderer is added to support float output. PiperOrigin-RevId: 324661603 --- RELEASENOTES.md | 2 + .../ext/opus/LibopusAudioRenderer.java | 15 +++++- .../exoplayer2/ext/opus/OpusDecoder.java | 49 ++++++++++++------- extensions/opus/src/main/jni/opus_jni.cc | 29 ++++++++--- 4 files changed, 68 insertions(+), 27 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 17e8649509..f27019121c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -199,6 +199,8 @@ Codec2 MP3 decoder having lower timestamps on the output side. * Propagate gapless audio metadata without the need to recreate the audio decoders. + * Add floating point PCM output capability in `MediaCodecAudioRenderer`, + and `LibopusAudioRenderer`. * DASH: * Enable support for embedded CEA-708. * Add support for load cancelation when discarding upstream diff --git a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java index ff4385542b..0678a107c4 100644 --- a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java +++ b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java @@ -22,6 +22,7 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.AudioSink; +import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport; import com.google.android.exoplayer2.audio.DecoderAudioRenderer; import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.util.MimeTypes; @@ -39,6 +40,7 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { private int channelCount; private int sampleRate; + private boolean outputFloat; public LibopusAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -102,6 +104,12 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { protected OpusDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) throws OpusDecoderException { TraceUtil.beginSection("createOpusDecoder"); + @SinkFormatSupport + int formatSupport = + getSinkFormatSupport( + Util.getPcmFormat(C.ENCODING_PCM_FLOAT, format.channelCount, format.sampleRate)); + outputFloat = formatSupport == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; + int initialInputBufferSize = format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; OpusDecoder decoder = @@ -110,15 +118,18 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer { NUM_BUFFERS, initialInputBufferSize, format.initializationData, - mediaCrypto); + mediaCrypto, + outputFloat); channelCount = decoder.getChannelCount(); sampleRate = decoder.getSampleRate(); + TraceUtil.endSection(); return decoder; } @Override protected Format getOutputFormat() { - return Util.getPcmFormat(C.ENCODING_PCM_16BIT, channelCount, sampleRate); + @C.PcmEncoding int pcmEncoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; + return Util.getPcmFormat(pcmEncoding, channelCount, sampleRate); } } diff --git a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java index e082ab46b1..4d9bedcfbe 100644 --- a/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java +++ b/extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java @@ -29,11 +29,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.List; -/** - * Opus decoder. - */ -/* package */ final class OpusDecoder extends - SimpleDecoder { +/** Opus decoder. */ +/* package */ final class OpusDecoder + extends SimpleDecoder { private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840; @@ -52,6 +50,7 @@ import java.util.List; private final long nativeDecoderContext; private int skipSamples; + private final boolean outputFloat; /** * Creates an Opus decoder. @@ -64,6 +63,7 @@ import java.util.List; * the encoder delay and seek pre roll values in nanoseconds, encoded as longs. * @param exoMediaCrypto The {@link ExoMediaCrypto} object required for decoding encrypted * content. Maybe null and can be ignored if decoder does not handle encrypted content. + * @param outputFloat Forces the decoder to output float PCM samples when set * @throws OpusDecoderException Thrown if an exception occurs when initializing the decoder. */ public OpusDecoder( @@ -71,7 +71,8 @@ import java.util.List; int numOutputBuffers, int initialInputBufferSize, List initializationData, - @Nullable ExoMediaCrypto exoMediaCrypto) + @Nullable ExoMediaCrypto exoMediaCrypto, + boolean outputFloat) throws OpusDecoderException { super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); if (!OpusLibrary.isAvailable()) { @@ -127,12 +128,17 @@ import java.util.List; headerSkipSamples = preskip; headerSeekPreRollSamples = DEFAULT_SEEK_PRE_ROLL_SAMPLES; } - nativeDecoderContext = opusInit(SAMPLE_RATE, channelCount, numStreams, numCoupled, gain, - streamMap); + nativeDecoderContext = + opusInit(SAMPLE_RATE, channelCount, numStreams, numCoupled, gain, streamMap); if (nativeDecoderContext == 0) { throw new OpusDecoderException("Failed to initialize decoder"); } setInitialInputBufferSize(initialInputBufferSize); + + this.outputFloat = outputFloat; + if (outputFloat) { + opusSetFloatOutput(); + } } @Override @@ -192,8 +198,8 @@ import java.util.List; if (result < 0) { if (result == DRM_ERROR) { String message = "Drm error: " + opusGetErrorMessage(nativeDecoderContext); - DecryptionException cause = new DecryptionException( - opusGetErrorCode(nativeDecoderContext), message); + DecryptionException cause = + new DecryptionException(opusGetErrorCode(nativeDecoderContext), message); return new OpusDecoderException(message, cause); } else { return new OpusDecoderException("Decode error: " + opusGetErrorMessage(result)); @@ -224,16 +230,12 @@ import java.util.List; opusClose(nativeDecoderContext); } - /** - * Returns the channel count of output audio. - */ + /** Returns the channel count of output audio. */ public int getChannelCount() { return channelCount; } - /** - * Returns the sample rate of output audio. - */ + /** Returns the sample rate of output audio. */ public int getSampleRate() { return SAMPLE_RATE; } @@ -252,9 +254,14 @@ import java.util.List; return (short) readUnsignedLittleEndian16(input, offset); } - private native long opusInit(int sampleRate, int channelCount, int numStreams, int numCoupled, - int gain, byte[] streamMap); - private native int opusDecode(long decoder, long timeUs, ByteBuffer inputBuffer, int inputSize, + private native long opusInit( + int sampleRate, int channelCount, int numStreams, int numCoupled, int gain, byte[] streamMap); + + private native int opusDecode( + long decoder, + long timeUs, + ByteBuffer inputBuffer, + int inputSize, SimpleOutputBuffer outputBuffer); private native int opusSecureDecode( @@ -273,8 +280,12 @@ import java.util.List; @Nullable int[] numBytesOfEncryptedData); private native void opusClose(long decoder); + private native void opusReset(long decoder); + private native int opusGetErrorCode(long decoder); + private native String opusGetErrorMessage(long decoder); + private native void opusSetFloatOutput(); } diff --git a/extensions/opus/src/main/jni/opus_jni.cc b/extensions/opus/src/main/jni/opus_jni.cc index 9042e4cb89..a2515be7f6 100644 --- a/extensions/opus/src/main/jni/opus_jni.cc +++ b/extensions/opus/src/main/jni/opus_jni.cc @@ -58,10 +58,12 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) { return JNI_VERSION_1_6; } -static const int kBytesPerSample = 2; // opus fixed point uses 16 bit samples. +static const int kBytesPerIntPcmSample = 2; +static const int kBytesPerFloatSample = 4; static const int kMaxOpusOutputPacketSizeSamples = 960 * 6; static int channelCount; static int errorCode; +static bool outputFloat = false; DECODER_FUNC(jlong, opusInit, jint sampleRate, jint channelCount, jint numStreams, jint numCoupled, jint gain, jbyteArray jStreamMap) { @@ -99,8 +101,10 @@ DECODER_FUNC(jint, opusDecode, jlong jDecoder, jlong jTimeUs, reinterpret_cast( env->GetDirectBufferAddress(jInputBuffer)); + const int byteSizePerSample = outputFloat ? + kBytesPerFloatSample : kBytesPerIntPcmSample; const jint outputSize = - kMaxOpusOutputPacketSizeSamples * kBytesPerSample * channelCount; + kMaxOpusOutputPacketSizeSamples * byteSizePerSample * channelCount; env->CallObjectMethod(jOutputBuffer, outputBufferInit, jTimeUs, outputSize); if (env->ExceptionCheck()) { @@ -114,14 +118,23 @@ DECODER_FUNC(jint, opusDecode, jlong jDecoder, jlong jTimeUs, return -1; } - int16_t* outputBufferData = reinterpret_cast( - env->GetDirectBufferAddress(jOutputBufferData)); - int sampleCount = opus_multistream_decode(decoder, inputBuffer, inputSize, + int sampleCount; + if (outputFloat) { + float* outputBufferData = reinterpret_cast( + env->GetDirectBufferAddress(jOutputBufferData)); + sampleCount = opus_multistream_decode_float(decoder, inputBuffer, inputSize, outputBufferData, kMaxOpusOutputPacketSizeSamples, 0); + } else { + int16_t* outputBufferData = reinterpret_cast( + env->GetDirectBufferAddress(jOutputBufferData)); + sampleCount = opus_multistream_decode(decoder, inputBuffer, inputSize, + outputBufferData, kMaxOpusOutputPacketSizeSamples, 0); + } + // record error code errorCode = (sampleCount < 0) ? sampleCount : 0; return (sampleCount < 0) ? sampleCount - : sampleCount * kBytesPerSample * channelCount; + : sampleCount * byteSizePerSample * channelCount; } DECODER_FUNC(jint, opusSecureDecode, jlong jDecoder, jlong jTimeUs, @@ -154,6 +167,10 @@ DECODER_FUNC(jint, opusGetErrorCode, jlong jContext) { return errorCode; } +DECODER_FUNC(void, opusSetFloatOutput) { + outputFloat = true; +} + LIBRARY_FUNC(jstring, opusIsSecureDecodeSupported) { // Doesn't support return 0;