Added float output mode for Opus extension

The working of libOpus is different from ffmpeg. With ffmpeg, the decoder can
be configured to output floating point PCM. While in libOpus, floating samples
are acquired by calling a different function. This is the reason the new JNI
functions and the logic in OpusDecoder/LibopusAudioRenderer is added to
support float output.

PiperOrigin-RevId: 324661603

RELEASENOTES.md

@@ -199,6 +199,8 @@
         Codec2 MP3 decoder having lower timestamps on the output side.
     *   Propagate gapless audio metadata without the need to recreate the audio
         decoders.
+    *   Add floating point PCM output capability in `MediaCodecAudioRenderer`,
+        and `LibopusAudioRenderer`.
 *   DASH:
     *   Enable support for embedded CEA-708.
     *   Add support for load cancelation when discarding upstream

extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java

@@ -22,6 +22,7 @@ import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.audio.AudioProcessor;
 import com.google.android.exoplayer2.audio.AudioRendererEventListener;
 import com.google.android.exoplayer2.audio.AudioSink;
+import com.google.android.exoplayer2.audio.AudioSink.SinkFormatSupport;
 import com.google.android.exoplayer2.audio.DecoderAudioRenderer;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
 import com.google.android.exoplayer2.util.MimeTypes;
@@ -39,6 +40,7 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
 
   private int channelCount;
   private int sampleRate;
+  private boolean outputFloat;
 
   public LibopusAudioRenderer() {
     this(/* eventHandler= */ null, /* eventListener= */ null);
@@ -102,6 +104,12 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
   protected OpusDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto)
       throws OpusDecoderException {
     TraceUtil.beginSection("createOpusDecoder");
+    @SinkFormatSupport
+    int formatSupport =
+        getSinkFormatSupport(
+            Util.getPcmFormat(C.ENCODING_PCM_FLOAT, format.channelCount, format.sampleRate));
+    outputFloat = formatSupport == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
+
     int initialInputBufferSize =
         format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
     OpusDecoder decoder =
@@ -110,15 +118,18 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
             NUM_BUFFERS,
             initialInputBufferSize,
             format.initializationData,
-            mediaCrypto);
+            mediaCrypto,
+            outputFloat);
     channelCount = decoder.getChannelCount();
     sampleRate = decoder.getSampleRate();
+
     TraceUtil.endSection();
     return decoder;
   }
 
   @Override
   protected Format getOutputFormat() {
-    return Util.getPcmFormat(C.ENCODING_PCM_16BIT, channelCount, sampleRate);
+    @C.PcmEncoding int pcmEncoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT;
+    return Util.getPcmFormat(pcmEncoding, channelCount, sampleRate);
   }
 }

extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/OpusDecoder.java

@@ -29,11 +29,9 @@ import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.List;
 
-/**
- * Opus decoder.
- */
-/* package */ final class OpusDecoder extends
-    SimpleDecoder<DecoderInputBuffer, SimpleOutputBuffer, OpusDecoderException> {
+/** Opus decoder. */
+/* package */ final class OpusDecoder
+    extends SimpleDecoder<DecoderInputBuffer, SimpleOutputBuffer, OpusDecoderException> {
 
   private static final int DEFAULT_SEEK_PRE_ROLL_SAMPLES = 3840;
 
@@ -52,6 +50,7 @@ import java.util.List;
   private final long nativeDecoderContext;
 
   private int skipSamples;
+  private final boolean outputFloat;
 
   /**
    * Creates an Opus decoder.
@@ -64,6 +63,7 @@ import java.util.List;
    *     the encoder delay and seek pre roll values in nanoseconds, encoded as longs.
    * @param exoMediaCrypto The {@link ExoMediaCrypto} object required for decoding encrypted
    *     content. Maybe null and can be ignored if decoder does not handle encrypted content.
+   * @param outputFloat Forces the decoder to output float PCM samples when set
    * @throws OpusDecoderException Thrown if an exception occurs when initializing the decoder.
    */
   public OpusDecoder(
@@ -71,7 +71,8 @@ import java.util.List;
       int numOutputBuffers,
       int initialInputBufferSize,
       List<byte[]> initializationData,
-      @Nullable ExoMediaCrypto exoMediaCrypto)
+      @Nullable ExoMediaCrypto exoMediaCrypto,
+      boolean outputFloat)
       throws OpusDecoderException {
     super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]);
     if (!OpusLibrary.isAvailable()) {
@@ -127,12 +128,17 @@ import java.util.List;
       headerSkipSamples = preskip;
       headerSeekPreRollSamples = DEFAULT_SEEK_PRE_ROLL_SAMPLES;
     }
-    nativeDecoderContext = opusInit(SAMPLE_RATE, channelCount, numStreams, numCoupled, gain,
-        streamMap);
+    nativeDecoderContext =
+        opusInit(SAMPLE_RATE, channelCount, numStreams, numCoupled, gain, streamMap);
     if (nativeDecoderContext == 0) {
       throw new OpusDecoderException("Failed to initialize decoder");
     }
     setInitialInputBufferSize(initialInputBufferSize);
+
+    this.outputFloat = outputFloat;
+    if (outputFloat) {
+      opusSetFloatOutput();
+    }
   }
 
   @Override
@@ -192,8 +198,8 @@ import java.util.List;
     if (result < 0) {
       if (result == DRM_ERROR) {
         String message = "Drm error: " + opusGetErrorMessage(nativeDecoderContext);
-        DecryptionException cause = new DecryptionException(
-            opusGetErrorCode(nativeDecoderContext), message);
+        DecryptionException cause =
+            new DecryptionException(opusGetErrorCode(nativeDecoderContext), message);
         return new OpusDecoderException(message, cause);
       } else {
         return new OpusDecoderException("Decode error: " + opusGetErrorMessage(result));
@@ -224,16 +230,12 @@ import java.util.List;
     opusClose(nativeDecoderContext);
   }
 
-  /**
-   * Returns the channel count of output audio.
-   */
+  /** Returns the channel count of output audio. */
   public int getChannelCount() {
     return channelCount;
   }
 
-  /**
-   * Returns the sample rate of output audio.
-   */
+  /** Returns the sample rate of output audio. */
   public int getSampleRate() {
     return SAMPLE_RATE;
   }
@@ -252,9 +254,14 @@ import java.util.List;
     return (short) readUnsignedLittleEndian16(input, offset);
   }
 
-  private native long opusInit(int sampleRate, int channelCount, int numStreams, int numCoupled,
-      int gain, byte[] streamMap);
-  private native int opusDecode(long decoder, long timeUs, ByteBuffer inputBuffer, int inputSize,
+  private native long opusInit(
+      int sampleRate, int channelCount, int numStreams, int numCoupled, int gain, byte[] streamMap);
+
+  private native int opusDecode(
+      long decoder,
+      long timeUs,
+      ByteBuffer inputBuffer,
+      int inputSize,
       SimpleOutputBuffer outputBuffer);
 
   private native int opusSecureDecode(
@@ -273,8 +280,12 @@ import java.util.List;
       @Nullable int[] numBytesOfEncryptedData);
 
   private native void opusClose(long decoder);
+
   private native void opusReset(long decoder);
+
   private native int opusGetErrorCode(long decoder);
+
   private native String opusGetErrorMessage(long decoder);
 
+  private native void opusSetFloatOutput();
 }

extensions/opus/src/main/jni/opus_jni.cc

@@ -58,10 +58,12 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
   return JNI_VERSION_1_6;
 }
 
-static const int kBytesPerSample = 2;  // opus fixed point uses 16 bit samples.
+static const int kBytesPerIntPcmSample = 2;
+static const int kBytesPerFloatSample = 4;
 static const int kMaxOpusOutputPacketSizeSamples = 960 * 6;
 static int channelCount;
 static int errorCode;
+static bool outputFloat = false;
 
 DECODER_FUNC(jlong, opusInit, jint sampleRate, jint channelCount,
      jint numStreams, jint numCoupled, jint gain, jbyteArray jStreamMap) {
@@ -99,8 +101,10 @@ DECODER_FUNC(jint, opusDecode, jlong jDecoder, jlong jTimeUs,
       reinterpret_cast<const uint8_t*>(
           env->GetDirectBufferAddress(jInputBuffer));
 
+  const int byteSizePerSample = outputFloat ?
+      kBytesPerFloatSample : kBytesPerIntPcmSample;
   const jint outputSize =
-      kMaxOpusOutputPacketSizeSamples * kBytesPerSample * channelCount;
+      kMaxOpusOutputPacketSizeSamples * byteSizePerSample * channelCount;
 
   env->CallObjectMethod(jOutputBuffer, outputBufferInit, jTimeUs, outputSize);
   if (env->ExceptionCheck()) {
@@ -114,14 +118,23 @@ DECODER_FUNC(jint, opusDecode, jlong jDecoder, jlong jTimeUs,
     return -1;
   }
 
-  int16_t* outputBufferData = reinterpret_cast<int16_t*>(
-      env->GetDirectBufferAddress(jOutputBufferData));
-  int sampleCount = opus_multistream_decode(decoder, inputBuffer, inputSize,
+  int sampleCount;
+  if (outputFloat) {
+    float* outputBufferData = reinterpret_cast<float*>(
+        env->GetDirectBufferAddress(jOutputBufferData));
+    sampleCount = opus_multistream_decode_float(decoder, inputBuffer, inputSize,
       outputBufferData, kMaxOpusOutputPacketSizeSamples, 0);
+  } else {
+    int16_t* outputBufferData = reinterpret_cast<int16_t*>(
+        env->GetDirectBufferAddress(jOutputBufferData));
+    sampleCount = opus_multistream_decode(decoder, inputBuffer, inputSize,
+      outputBufferData, kMaxOpusOutputPacketSizeSamples, 0);
+  }
+
   // record error code
   errorCode = (sampleCount < 0) ? sampleCount : 0;
   return (sampleCount < 0) ? sampleCount
-      : sampleCount * kBytesPerSample * channelCount;
+      : sampleCount * byteSizePerSample * channelCount;
 }
 
 DECODER_FUNC(jint, opusSecureDecode, jlong jDecoder, jlong jTimeUs,
@@ -154,6 +167,10 @@ DECODER_FUNC(jint, opusGetErrorCode, jlong jContext) {
   return errorCode;
 }
 
+DECODER_FUNC(void, opusSetFloatOutput) {
+  outputFloat = true;
+}
+
 LIBRARY_FUNC(jstring, opusIsSecureDecodeSupported) {
   // Doesn't support
   return 0;