From 9473fda0568d94082a3f32197772de7248d2f353 Mon Sep 17 00:00:00 2001 From: christosts Date: Mon, 9 Nov 2020 17:40:32 +0000 Subject: [PATCH] Synchronize codec interaction with buffer queueing Add experimental method to synchronize MediaCodec interactions with asynchronous queueing. When the feature is enabled, interactions such as MediaCodec.setOutputSurface() triggered by the MediaCodecRenderer will wait until all input buffers pending queueing are first submitted to the MediaCodec. PiperOrigin-RevId: 341423837 --- .../exoplayer2/DefaultRenderersFactory.java | 26 +++++++++-- .../AsynchronousMediaCodecAdapter.java | 34 ++++++++++++-- .../AsynchronousMediaCodecBufferEnqueuer.java | 27 ++++++++---- .../mediacodec/MediaCodecRenderer.java | 21 ++++++++- .../AsynchronousMediaCodecAdapterTest.java | 7 ++- ...nchronousMediaCodecBufferEnqueuerTest.java | 44 +++++++++---------- 6 files changed, 119 insertions(+), 40 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java index 5d130442b3..ad58917160 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java @@ -92,6 +92,7 @@ public class DefaultRenderersFactory implements RenderersFactory { private boolean enableDecoderFallback; private MediaCodecSelector mediaCodecSelector; private boolean enableAsyncQueueing; + private boolean enableSynchronizeCodecInteractionsWithQueueing; private boolean enableFloatOutput; private boolean enableAudioTrackPlaybackParams; private boolean enableOffload; @@ -155,11 +156,26 @@ public class DefaultRenderersFactory implements RenderersFactory { * @param enabled Whether asynchronous queueing is enabled. * @return This factory, for convenience. */ - public DefaultRenderersFactory experimentalEnableAsynchronousBufferQueueing(boolean enabled) { + public DefaultRenderersFactory experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) { enableAsyncQueueing = enabled; return this; } + /** + * Enable synchronizing codec interactions with asynchronous buffer queueing. + * + *

This method is experimental, and will be renamed or removed in a future release. + * + * @param enabled Whether codec interactions will be synchronized with asynchronous buffer + * queueing. + * @return This factory, for convenience. + */ + public DefaultRenderersFactory experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled( + boolean enabled) { + enableSynchronizeCodecInteractionsWithQueueing = enabled; + return this; + } + /** * Sets whether to enable fallback to lower-priority decoders if decoder initialization fails. * This may result in using a decoder that is less efficient or slower than the primary decoder. @@ -336,7 +352,9 @@ public class DefaultRenderersFactory implements RenderersFactory { eventHandler, eventListener, MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); - videoRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing); + videoRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing); + videoRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled( + enableSynchronizeCodecInteractionsWithQueueing); out.add(videoRenderer); if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { @@ -461,7 +479,9 @@ public class DefaultRenderersFactory implements RenderersFactory { eventHandler, eventListener, audioSink); - audioRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing); + audioRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing); + audioRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled( + enableSynchronizeCodecInteractionsWithQueueing); out.add(audioRenderer); if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java index 2deb5fa2bf..54ad57cafe 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java @@ -56,6 +56,7 @@ import java.nio.ByteBuffer; private final MediaCodec codec; private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback; private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer; + private final boolean synchronizeCodecInteractionsWithQueueing; private boolean codecReleased; @State private int state; @@ -65,20 +66,30 @@ import java.nio.ByteBuffer; * @param codec The {@link MediaCodec} to wrap. * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for * labelling the internal thread accordingly. + * @param synchronizeCodecInteractionsWithQueueing Whether the adapter should synchronize {@link + * MediaCodec} interactions with asynchronous buffer queueing. When {@code true}, codec + * interactions will wait until all input buffers pending queueing wil be submitted to the + * {@link MediaCodec}. */ - /* package */ AsynchronousMediaCodecAdapter(MediaCodec codec, int trackType) { + /* package */ AsynchronousMediaCodecAdapter( + MediaCodec codec, int trackType, boolean synchronizeCodecInteractionsWithQueueing) { this( codec, new HandlerThread(createCallbackThreadLabel(trackType)), - new HandlerThread(createQueueingThreadLabel(trackType))); + new HandlerThread(createQueueingThreadLabel(trackType)), + synchronizeCodecInteractionsWithQueueing); } @VisibleForTesting /* package */ AsynchronousMediaCodecAdapter( - MediaCodec codec, HandlerThread callbackThread, HandlerThread enqueueingThread) { + MediaCodec codec, + HandlerThread callbackThread, + HandlerThread enqueueingThread, + boolean synchronizeCodecInteractionsWithQueueing) { this.codec = codec; this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread); this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread); + this.synchronizeCodecInteractionsWithQueueing = synchronizeCodecInteractionsWithQueueing; this.state = STATE_CREATED; } @@ -181,6 +192,7 @@ import java.nio.ByteBuffer; @Override public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) { + maybeBlockOnQueueing(); codec.setOnFrameRenderedListener( (codec, presentationTimeUs, nanoTime) -> listener.onFrameRendered( @@ -190,16 +202,19 @@ import java.nio.ByteBuffer; @Override public void setOutputSurface(Surface surface) { + maybeBlockOnQueueing(); codec.setOutputSurface(surface); } @Override public void setParameters(Bundle params) { + maybeBlockOnQueueing(); codec.setParameters(params); } @Override public void setVideoScalingMode(@VideoScalingMode int scalingMode) { + maybeBlockOnQueueing(); codec.setVideoScalingMode(scalingMode); } @@ -213,6 +228,19 @@ import java.nio.ByteBuffer; asynchronousMediaCodecCallback.onOutputFormatChanged(codec, format); } + private void maybeBlockOnQueueing() { + if (synchronizeCodecInteractionsWithQueueing) { + try { + bufferEnqueuer.waitUntilQueueingComplete(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // The playback thread should not be interrupted. Raising this as an + // IllegalStateException. + throw new IllegalStateException(e); + } + } + } + private static String createCallbackThreadLabel(int trackType) { return createThreadLabel(trackType, /* prefix= */ "ExoPlayer:MediaCodecAsyncAdapter:"); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java index 10d59d347c..79bb981955 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java @@ -17,6 +17,7 @@ package com.google.android.exoplayer2.mediacodec; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Util.castNonNull; import android.media.MediaCodec; import android.os.Handler; @@ -46,7 +47,7 @@ class AsynchronousMediaCodecBufferEnqueuer { private static final int MSG_QUEUE_INPUT_BUFFER = 0; private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1; - private static final int MSG_FLUSH = 2; + private static final int MSG_OPEN_CV = 2; @GuardedBy("MESSAGE_PARAMS_INSTANCE_POOL") private static final ArrayDeque MESSAGE_PARAMS_INSTANCE_POOL = new ArrayDeque<>(); @@ -110,8 +111,7 @@ class AsynchronousMediaCodecBufferEnqueuer { maybeThrowException(); MessageParams messageParams = getMessageParams(); messageParams.setQueueParams(index, offset, size, presentationTimeUs, flags); - Message message = - Util.castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams); + Message message = castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams); message.sendToTarget(); } @@ -131,7 +131,7 @@ class AsynchronousMediaCodecBufferEnqueuer { messageParams.setQueueParams(index, offset, /* size= */ 0, presentationTimeUs, flags); copy(info, messageParams.cryptoInfo); Message message = - Util.castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams); + castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams); message.sendToTarget(); } @@ -158,6 +158,11 @@ class AsynchronousMediaCodecBufferEnqueuer { started = false; } + /** Blocks the current thread until all input buffers pending queueing are submitted. */ + public void waitUntilQueueingComplete() throws InterruptedException { + blockUntilHandlerThreadIsIdle(); + } + private void maybeThrowException() { @Nullable RuntimeException exception = pendingRuntimeException.getAndSet(null); if (exception != null) { @@ -170,15 +175,19 @@ class AsynchronousMediaCodecBufferEnqueuer { * blocks until the {@link #handlerThread} is idle. */ private void flushHandlerThread() throws InterruptedException { - Handler handler = Util.castNonNull(this.handler); + Handler handler = castNonNull(this.handler); handler.removeCallbacksAndMessages(null); - conditionVariable.close(); - handler.obtainMessage(MSG_FLUSH).sendToTarget(); - conditionVariable.block(); + blockUntilHandlerThreadIsIdle(); // Check if any exceptions happened during the last queueing action. maybeThrowException(); } + private void blockUntilHandlerThreadIsIdle() throws InterruptedException { + conditionVariable.close(); + castNonNull(handler).obtainMessage(MSG_OPEN_CV).sendToTarget(); + conditionVariable.block(); + } + // Called from the handler thread @VisibleForTesting @@ -203,7 +212,7 @@ class AsynchronousMediaCodecBufferEnqueuer { params.presentationTimeUs, params.flags); break; - case MSG_FLUSH: + case MSG_OPEN_CV: conditionVariable.open(); break; default: diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 834923e0da..63069b5320 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -348,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private boolean waitingForFirstSampleInFormat; private boolean pendingOutputEndOfStream; private boolean enableAsynchronousBufferQueueing; + private boolean enableSynchronizeCodecInteractionsWithQueueing; @Nullable private ExoPlaybackException pendingPlaybackException; protected DecoderCounters decoderCounters; private long outputStreamStartPositionUs; @@ -412,10 +413,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer { *

This method is experimental, and will be renamed or removed in a future release. It should * only be called before the renderer is used. */ - public void experimentalEnableAsynchronousBufferQueueing(boolean enabled) { + public void experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) { enableAsynchronousBufferQueueing = enabled; } + /** + * Enable synchronizing codec interactions with asynchronous buffer queueing. + * + *

When enabled, codec interactions will wait until all input buffers pending for asynchronous + * queueing are submitted to the {@link MediaCodec} first. This method is effective only if {@link + * #experimentalSetAsynchronousBufferQueueingEnabled asynchronous buffer queueing} is enabled. + * + *

This method is experimental, and will be renamed or removed in a future release. It should + * only be called before the renderer is used. + */ + public void experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(boolean enabled) { + enableSynchronizeCodecInteractionsWithQueueing = enabled; + } + @Override @AdaptiveSupport public final int supportsMixedMimeTypeAdaptation() { @@ -1051,7 +1066,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer { TraceUtil.beginSection("createCodec:" + codecName); MediaCodec codec = MediaCodec.createByCodecName(codecName); if (enableAsynchronousBufferQueueing && Util.SDK_INT >= 23) { - codecAdapter = new AsynchronousMediaCodecAdapter(codec, getTrackType()); + codecAdapter = + new AsynchronousMediaCodecAdapter( + codec, getTrackType(), enableSynchronizeCodecInteractionsWithQueueing); } else { codecAdapter = new SynchronousMediaCodecAdapter(codec); } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java index 60e9c8b77f..8874d5ec7c 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java @@ -46,7 +46,12 @@ public class AsynchronousMediaCodecAdapterTest { codec = MediaCodec.createByCodecName("h264"); callbackThread = new HandlerThread("TestCallbackThread"); queueingThread = new HandlerThread("TestQueueingThread"); - adapter = new AsynchronousMediaCodecAdapter(codec, callbackThread, queueingThread); + adapter = + new AsynchronousMediaCodecAdapter( + codec, + callbackThread, + queueingThread, + /* synchronizeCodecInteractionsWithQueueing= */ false); bufferInfo = new MediaCodec.BufferInfo(); } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java index 9e2c715b31..f3a08df819 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java @@ -219,6 +219,28 @@ public class AsynchronousMediaCodecBufferEnqueuerTest { assertThrows(IllegalStateException.class, () -> enqueuer.shutdown()); } + private static CryptoInfo createCryptoInfo() { + CryptoInfo info = new CryptoInfo(); + int numSubSamples = 5; + int[] numBytesOfClearData = new int[] {0, 1, 2, 3}; + int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7}; + byte[] key = new byte[] {0, 1, 2, 3}; + byte[] iv = new byte[] {4, 5, 6, 7}; + @C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC; + int encryptedBlocks = 16; + int clearBlocks = 8; + info.set( + numSubSamples, + numBytesOfClearData, + numBytesOfEncryptedData, + key, + iv, + mode, + encryptedBlocks, + clearBlocks); + return info; + } + private static class TestHandlerThread extends HandlerThread { private boolean started; private boolean quit; @@ -247,26 +269,4 @@ public class AsynchronousMediaCodecBufferEnqueuerTest { return super.quit(); } } - - private static CryptoInfo createCryptoInfo() { - CryptoInfo info = new CryptoInfo(); - int numSubSamples = 5; - int[] numBytesOfClearData = new int[] {0, 1, 2, 3}; - int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7}; - byte[] key = new byte[] {0, 1, 2, 3}; - byte[] iv = new byte[] {4, 5, 6, 7}; - @C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC; - int encryptedBlocks = 16; - int clearBlocks = 8; - info.set( - numSubSamples, - numBytesOfClearData, - numBytesOfEncryptedData, - key, - iv, - mode, - encryptedBlocks, - clearBlocks); - return info; - } }