Synchronize codec interaction with buffer queueing

Add experimental method to synchronize MediaCodec interactions
with asynchronous queueing. When the feature is enabled, interactions
such as MediaCodec.setOutputSurface() triggered by the
MediaCodecRenderer will wait until all input buffers pending queueing
are first submitted to the MediaCodec.

PiperOrigin-RevId: 341423837

library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java

@@ -92,6 +92,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
   private boolean enableDecoderFallback;
   private MediaCodecSelector mediaCodecSelector;
   private boolean enableAsyncQueueing;
+  private boolean enableSynchronizeCodecInteractionsWithQueueing;
   private boolean enableFloatOutput;
   private boolean enableAudioTrackPlaybackParams;
   private boolean enableOffload;
@@ -155,11 +156,26 @@ public class DefaultRenderersFactory implements RenderersFactory {
    * @param enabled Whether asynchronous queueing is enabled.
    * @return This factory, for convenience.
    */
-  public DefaultRenderersFactory experimentalEnableAsynchronousBufferQueueing(boolean enabled) {
+  public DefaultRenderersFactory experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) {
     enableAsyncQueueing = enabled;
     return this;
   }
 
+  /**
+   * Enable synchronizing codec interactions with asynchronous buffer queueing.
+   *
+   * <p>This method is experimental, and will be renamed or removed in a future release.
+   *
+   * @param enabled Whether codec interactions will be synchronized with asynchronous buffer
+   *     queueing.
+   * @return This factory, for convenience.
+   */
+  public DefaultRenderersFactory experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
+      boolean enabled) {
+    enableSynchronizeCodecInteractionsWithQueueing = enabled;
+    return this;
+  }
+
   /**
    * Sets whether to enable fallback to lower-priority decoders if decoder initialization fails.
    * This may result in using a decoder that is less efficient or slower than the primary decoder.
@@ -336,7 +352,9 @@ public class DefaultRenderersFactory implements RenderersFactory {
             eventHandler,
             eventListener,
             MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
-    videoRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing);
+    videoRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing);
+    videoRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
+        enableSynchronizeCodecInteractionsWithQueueing);
     out.add(videoRenderer);
 
     if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
@@ -461,7 +479,9 @@ public class DefaultRenderersFactory implements RenderersFactory {
             eventHandler,
             eventListener,
             audioSink);
-    audioRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing);
+    audioRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing);
+    audioRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
+        enableSynchronizeCodecInteractionsWithQueueing);
     out.add(audioRenderer);
 
     if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java

@@ -56,6 +56,7 @@ import java.nio.ByteBuffer;
   private final MediaCodec codec;
   private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback;
   private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer;
+  private final boolean synchronizeCodecInteractionsWithQueueing;
   private boolean codecReleased;
   @State private int state;
 
@@ -65,20 +66,30 @@ import java.nio.ByteBuffer;
    * @param codec The {@link MediaCodec} to wrap.
    * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for
    *     labelling the internal thread accordingly.
+   * @param synchronizeCodecInteractionsWithQueueing Whether the adapter should synchronize {@link
+   *     MediaCodec} interactions with asynchronous buffer queueing. When {@code true}, codec
+   *     interactions will wait until all input buffers pending queueing wil be submitted to the
+   *     {@link MediaCodec}.
    */
-  /* package */ AsynchronousMediaCodecAdapter(MediaCodec codec, int trackType) {
+  /* package */ AsynchronousMediaCodecAdapter(
+      MediaCodec codec, int trackType, boolean synchronizeCodecInteractionsWithQueueing) {
     this(
         codec,
         new HandlerThread(createCallbackThreadLabel(trackType)),
-        new HandlerThread(createQueueingThreadLabel(trackType)));
+        new HandlerThread(createQueueingThreadLabel(trackType)),
+        synchronizeCodecInteractionsWithQueueing);
   }
 
   @VisibleForTesting
   /* package */ AsynchronousMediaCodecAdapter(
-      MediaCodec codec, HandlerThread callbackThread, HandlerThread enqueueingThread) {
+      MediaCodec codec,
+      HandlerThread callbackThread,
+      HandlerThread enqueueingThread,
+      boolean synchronizeCodecInteractionsWithQueueing) {
     this.codec = codec;
     this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread);
     this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread);
+    this.synchronizeCodecInteractionsWithQueueing = synchronizeCodecInteractionsWithQueueing;
     this.state = STATE_CREATED;
   }
 
@@ -181,6 +192,7 @@ import java.nio.ByteBuffer;
 
   @Override
   public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) {
+    maybeBlockOnQueueing();
     codec.setOnFrameRenderedListener(
         (codec, presentationTimeUs, nanoTime) ->
             listener.onFrameRendered(
@@ -190,16 +202,19 @@ import java.nio.ByteBuffer;
 
   @Override
   public void setOutputSurface(Surface surface) {
+    maybeBlockOnQueueing();
     codec.setOutputSurface(surface);
   }
 
   @Override
   public void setParameters(Bundle params) {
+    maybeBlockOnQueueing();
     codec.setParameters(params);
   }
 
   @Override
   public void setVideoScalingMode(@VideoScalingMode int scalingMode) {
+    maybeBlockOnQueueing();
     codec.setVideoScalingMode(scalingMode);
   }
 
@@ -213,6 +228,19 @@ import java.nio.ByteBuffer;
     asynchronousMediaCodecCallback.onOutputFormatChanged(codec, format);
   }
 
+  private void maybeBlockOnQueueing() {
+    if (synchronizeCodecInteractionsWithQueueing) {
+      try {
+        bufferEnqueuer.waitUntilQueueingComplete();
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        // The playback thread should not be interrupted. Raising this as an
+        // IllegalStateException.
+        throw new IllegalStateException(e);
+      }
+    }
+  }
+
   private static String createCallbackThreadLabel(int trackType) {
     return createThreadLabel(trackType, /* prefix= */ "ExoPlayer:MediaCodecAsyncAdapter:");
   }

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java

@@ -17,6 +17,7 @@
 package com.google.android.exoplayer2.mediacodec;
 
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.android.exoplayer2.util.Util.castNonNull;
 
 import android.media.MediaCodec;
 import android.os.Handler;
@@ -46,7 +47,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
 
   private static final int MSG_QUEUE_INPUT_BUFFER = 0;
   private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1;
-  private static final int MSG_FLUSH = 2;
+  private static final int MSG_OPEN_CV = 2;
 
   @GuardedBy("MESSAGE_PARAMS_INSTANCE_POOL")
   private static final ArrayDeque<MessageParams> MESSAGE_PARAMS_INSTANCE_POOL = new ArrayDeque<>();
@@ -110,8 +111,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
     maybeThrowException();
     MessageParams messageParams = getMessageParams();
     messageParams.setQueueParams(index, offset, size, presentationTimeUs, flags);
-    Message message =
-        Util.castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams);
+    Message message = castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams);
     message.sendToTarget();
   }
 
@@ -131,7 +131,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
     messageParams.setQueueParams(index, offset, /* size= */ 0, presentationTimeUs, flags);
     copy(info, messageParams.cryptoInfo);
     Message message =
-        Util.castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams);
+        castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams);
     message.sendToTarget();
   }
 
@@ -158,6 +158,11 @@ class AsynchronousMediaCodecBufferEnqueuer {
     started = false;
   }
 
+  /** Blocks the current thread until all input buffers pending queueing are submitted. */
+  public void waitUntilQueueingComplete() throws InterruptedException {
+    blockUntilHandlerThreadIsIdle();
+  }
+
   private void maybeThrowException() {
     @Nullable RuntimeException exception = pendingRuntimeException.getAndSet(null);
     if (exception != null) {
@@ -170,15 +175,19 @@ class AsynchronousMediaCodecBufferEnqueuer {
    * blocks until the {@link #handlerThread} is idle.
    */
   private void flushHandlerThread() throws InterruptedException {
-    Handler handler = Util.castNonNull(this.handler);
+    Handler handler = castNonNull(this.handler);
     handler.removeCallbacksAndMessages(null);
-    conditionVariable.close();
-    handler.obtainMessage(MSG_FLUSH).sendToTarget();
-    conditionVariable.block();
+    blockUntilHandlerThreadIsIdle();
     // Check if any exceptions happened during the last queueing action.
     maybeThrowException();
   }
 
+  private void blockUntilHandlerThreadIsIdle() throws InterruptedException {
+    conditionVariable.close();
+    castNonNull(handler).obtainMessage(MSG_OPEN_CV).sendToTarget();
+    conditionVariable.block();
+  }
+
   // Called from the handler thread
 
   @VisibleForTesting
@@ -203,7 +212,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
             params.presentationTimeUs,
             params.flags);
         break;
-      case MSG_FLUSH:
+      case MSG_OPEN_CV:
         conditionVariable.open();
         break;
       default:

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -348,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private boolean waitingForFirstSampleInFormat;
   private boolean pendingOutputEndOfStream;
   private boolean enableAsynchronousBufferQueueing;
+  private boolean enableSynchronizeCodecInteractionsWithQueueing;
   @Nullable private ExoPlaybackException pendingPlaybackException;
   protected DecoderCounters decoderCounters;
   private long outputStreamStartPositionUs;
@@ -412,10 +413,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * <p>This method is experimental, and will be renamed or removed in a future release. It should
    * only be called before the renderer is used.
    */
-  public void experimentalEnableAsynchronousBufferQueueing(boolean enabled) {
+  public void experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) {
     enableAsynchronousBufferQueueing = enabled;
   }
 
+  /**
+   * Enable synchronizing codec interactions with asynchronous buffer queueing.
+   *
+   * <p>When enabled, codec interactions will wait until all input buffers pending for asynchronous
+   * queueing are submitted to the {@link MediaCodec} first. This method is effective only if {@link
+   * #experimentalSetAsynchronousBufferQueueingEnabled asynchronous buffer queueing} is enabled.
+   *
+   * <p>This method is experimental, and will be renamed or removed in a future release. It should
+   * only be called before the renderer is used.
+   */
+  public void experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(boolean enabled) {
+    enableSynchronizeCodecInteractionsWithQueueing = enabled;
+  }
+
   @Override
   @AdaptiveSupport
   public final int supportsMixedMimeTypeAdaptation() {
@@ -1051,7 +1066,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       TraceUtil.beginSection("createCodec:" + codecName);
       MediaCodec codec = MediaCodec.createByCodecName(codecName);
       if (enableAsynchronousBufferQueueing && Util.SDK_INT >= 23) {
-        codecAdapter = new AsynchronousMediaCodecAdapter(codec, getTrackType());
+        codecAdapter =
+            new AsynchronousMediaCodecAdapter(
+                codec, getTrackType(), enableSynchronizeCodecInteractionsWithQueueing);
       } else {
         codecAdapter = new SynchronousMediaCodecAdapter(codec);
       }

library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java

@@ -46,7 +46,12 @@ public class AsynchronousMediaCodecAdapterTest {
     codec = MediaCodec.createByCodecName("h264");
     callbackThread = new HandlerThread("TestCallbackThread");
     queueingThread = new HandlerThread("TestQueueingThread");
-    adapter = new AsynchronousMediaCodecAdapter(codec, callbackThread, queueingThread);
+    adapter =
+        new AsynchronousMediaCodecAdapter(
+            codec,
+            callbackThread,
+            queueingThread,
+            /* synchronizeCodecInteractionsWithQueueing= */ false);
     bufferInfo = new MediaCodec.BufferInfo();
   }
 

library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java

@@ -219,6 +219,28 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
     assertThrows(IllegalStateException.class, () -> enqueuer.shutdown());
   }
 
+  private static CryptoInfo createCryptoInfo() {
+    CryptoInfo info = new CryptoInfo();
+    int numSubSamples = 5;
+    int[] numBytesOfClearData = new int[] {0, 1, 2, 3};
+    int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7};
+    byte[] key = new byte[] {0, 1, 2, 3};
+    byte[] iv = new byte[] {4, 5, 6, 7};
+    @C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC;
+    int encryptedBlocks = 16;
+    int clearBlocks = 8;
+    info.set(
+        numSubSamples,
+        numBytesOfClearData,
+        numBytesOfEncryptedData,
+        key,
+        iv,
+        mode,
+        encryptedBlocks,
+        clearBlocks);
+    return info;
+  }
+
   private static class TestHandlerThread extends HandlerThread {
     private boolean started;
     private boolean quit;
@@ -247,26 +269,4 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
       return super.quit();
     }
   }
-
-  private static CryptoInfo createCryptoInfo() {
-    CryptoInfo info = new CryptoInfo();
-    int numSubSamples = 5;
-    int[] numBytesOfClearData = new int[] {0, 1, 2, 3};
-    int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7};
-    byte[] key = new byte[] {0, 1, 2, 3};
-    byte[] iv = new byte[] {4, 5, 6, 7};
-    @C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC;
-    int encryptedBlocks = 16;
-    int clearBlocks = 8;
-    info.set(
-        numSubSamples,
-        numBytesOfClearData,
-        numBytesOfEncryptedData,
-        key,
-        iv,
-        mode,
-        encryptedBlocks,
-        clearBlocks);
-    return info;
-  }
 }