From bc02643df0a1b148df62387cd00ed6e25c1d93f2 Mon Sep 17 00:00:00 2001 From: christosts Date: Tue, 11 Feb 2020 10:25:30 +0000 Subject: [PATCH] Async queuing on MultiLockAsyncMediaCodecAdapter Add support for asynchronous input buffer queueing in the MultiLockAsyncMediaCodecAdapter. PiperOrigin-RevId: 294397811 --- .../mediacodec/MediaCodecRenderer.java | 64 +++++++++++------ .../MultiLockAsyncMediaCodecAdapter.java | 69 +++++++++++++++---- .../MultiLockAsyncMediaCodecAdapterTest.java | 5 +- 3 files changed, 101 insertions(+), 37 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index ae0c8dc3c0..6eec87e407 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -83,7 +83,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { OPERATION_MODE_ASYNCHRONOUS_PLAYBACK_THREAD, OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD, OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK, - OPERATION_MODE_ASYNCHRONOUS_QUEUEING + OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_ASYNCHRONOUS_QUEUEING, + OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK_ASYNCHRONOUS_QUEUEING }) public @interface MediaCodecOperationMode {} @@ -91,24 +92,30 @@ public abstract class MediaCodecRenderer extends BaseRenderer { public static final int OPERATION_MODE_SYNCHRONOUS = 0; /** * Operates the {@link MediaCodec} in asynchronous mode and routes {@link MediaCodec.Callback} - * callbacks to the playback Thread. + * callbacks to the playback thread. */ public static final int OPERATION_MODE_ASYNCHRONOUS_PLAYBACK_THREAD = 1; /** * Operates the {@link MediaCodec} in asynchronous mode and routes {@link MediaCodec.Callback} - * callbacks to a dedicated Thread. + * callbacks to a dedicated thread. */ public static final int OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD = 2; /** * Operates the {@link MediaCodec} in asynchronous mode and routes {@link MediaCodec.Callback} - * callbacks to a dedicated Thread. Uses granular locking for input and output buffers. + * callbacks to a dedicated thread. Uses granular locking for input and output buffers. */ public static final int OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK = 3; /** - * Operates the {@link MediaCodec} in asynchronous mode, routes {@link MediaCodec.Callback} - * callbacks to a dedicated Thread, and offloads queueing to another Thread. + * Same as {@link #OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD}, and offloads queueing to another + * thread. */ - public static final int OPERATION_MODE_ASYNCHRONOUS_QUEUEING = 4; + public static final int OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_ASYNCHRONOUS_QUEUEING = 4; + /** + * Same as {@link #OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK}, and offloads queueing + * to another thread. + */ + public static final int + OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK_ASYNCHRONOUS_QUEUEING = 5; /** Thrown when a failure occurs instantiating a decoder. */ public static class DecoderInitializationException extends Exception { @@ -493,21 +500,27 @@ public abstract class MediaCodecRenderer extends BaseRenderer { * * @param mode The mode of the MediaCodec. The supported modes are: * * By default, the operation mode is set to {@link * MediaCodecRenderer#OPERATION_MODE_SYNCHRONOUS}. @@ -1027,11 +1040,18 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } else if (mediaCodecOperationMode == OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK && Util.SDK_INT >= 23) { codecAdapter = new MultiLockAsyncMediaCodecAdapter(codec, getTrackType()); - } else if (mediaCodecOperationMode == OPERATION_MODE_ASYNCHRONOUS_QUEUEING + } else if (mediaCodecOperationMode + == OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_ASYNCHRONOUS_QUEUEING && Util.SDK_INT >= 23) { codecAdapter = new DedicatedThreadAsyncMediaCodecAdapter( codec, /* enableAsynchronousQueueing= */ true, getTrackType()); + } else if (mediaCodecOperationMode + == OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK_ASYNCHRONOUS_QUEUEING + && Util.SDK_INT >= 23) { + codecAdapter = + new MultiLockAsyncMediaCodecAdapter( + codec, /* enableAsynchronousQueueing= */ true, getTrackType()); } else { codecAdapter = new SynchronousMediaCodecAdapter(codec); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapter.java index 9c46011b26..8cb7b796c8 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapter.java @@ -35,7 +35,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in asynchronous mode - * and routes {@link MediaCodec.Callback} callbacks on a dedicated Thread that is managed + * and routes {@link MediaCodec.Callback} callbacks on a dedicated thread that is managed * internally. * *

The main difference of this class compared to the {@link @@ -43,8 +43,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * locking. The {@link DedicatedThreadAsyncMediaCodecAdapter} uses a single lock to synchronize * access, whereas this class uses a different lock to access the available input and available * output buffer indexes returned from the {@link MediaCodec}. This class assumes that the {@link - * MediaCodecAdapter} methods will be accessed by the Playback Thread and the {@link - * MediaCodec.Callback} methods will be accessed by the internal Thread. This class is + * MediaCodecAdapter} methods will be accessed by the playback thread and the {@link + * MediaCodec.Callback} methods will be accessed by the internal thread. This class is * NOT generally thread-safe in the sense that its public methods cannot be called * by any thread. */ @@ -86,20 +86,54 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Nullable private IllegalStateException codecException; - @GuardedBy("objectStateLock") - private @State int state; - private final HandlerThread handlerThread; private @MonotonicNonNull Handler handler; private Runnable codecStartRunnable; + private final MediaCodecInputBufferEnqueuer bufferEnqueuer; - /** Creates a new instance that wraps the specified {@link MediaCodec}. */ + @GuardedBy("objectStateLock") + @State + private int state; + + /** + * Creates a new instance that wraps the specified {@link MediaCodec}. An instance created with + * this constructor will queue input buffers synchronously. + * + * @param codec The {@link MediaCodec} to wrap. + * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for + * labelling the internal thread accordingly. + */ /* package */ MultiLockAsyncMediaCodecAdapter(MediaCodec codec, int trackType) { - this(codec, new HandlerThread(createThreadLabel(trackType))); + this( + codec, + /* enableAsynchronousQueueing= */ false, + trackType, + new HandlerThread(createThreadLabel(trackType))); + } + + /** + * Creates a new instance that wraps the specified {@link MediaCodec}. + * + * @param codec The {@link MediaCodec} to wrap. + * @param enableAsynchronousQueueing Whether input buffers will be queued asynchronously. + * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for + * labelling the internal thread accordingly. + */ + /* package */ MultiLockAsyncMediaCodecAdapter( + MediaCodec codec, boolean enableAsynchronousQueueing, int trackType) { + this( + codec, + enableAsynchronousQueueing, + trackType, + new HandlerThread(createThreadLabel(trackType))); } @VisibleForTesting - /* package */ MultiLockAsyncMediaCodecAdapter(MediaCodec codec, HandlerThread handlerThread) { + /* package */ MultiLockAsyncMediaCodecAdapter( + MediaCodec codec, + boolean enableAsynchronousQueueing, + int trackType, + HandlerThread handlerThread) { this.codec = codec; inputBufferLock = new Object(); outputBufferLock = new Object(); @@ -109,9 +143,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; bufferInfos = new ArrayDeque<>(); formats = new ArrayDeque<>(); codecException = null; - state = STATE_CREATED; this.handlerThread = handlerThread; codecStartRunnable = codec::start; + if (enableAsynchronousQueueing) { + bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, trackType); + } else { + bufferEnqueuer = new SynchronousMediaCodecBufferEnqueuer(codec); + } + state = STATE_CREATED; } @Override @@ -120,6 +159,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; handlerThread.start(); handler = new Handler(handlerThread.getLooper()); codec.setCallback(this, handler); + bufferEnqueuer.start(); codecStartRunnable.run(); state = STATE_STARTED; } @@ -165,7 +205,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int index, int offset, int size, long presentationTimeUs, int flags) { // This method does not need to be synchronized because it is not interacting with // MediaCodec.Callback and dequeueing buffers operations. - codec.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + bufferEnqueuer.queueInputBuffer(index, offset, size, presentationTimeUs, flags); } @Override @@ -173,13 +213,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) { // This method does not need to be synchronized because it is not interacting with // MediaCodec.Callback and dequeueing buffers operations. - codec.queueSecureInputBuffer( - index, offset, info.getFrameworkCryptoInfo(), presentationTimeUs, flags); + bufferEnqueuer.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); } @Override public void flush() { synchronized (objectStateLock) { + bufferEnqueuer.flush(); codec.flush(); pendingFlush++; Util.castNonNull(handler).post(this::onFlushComplete); @@ -190,6 +230,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void shutdown() { synchronized (objectStateLock) { if (state == STATE_STARTED) { + bufferEnqueuer.shutdown(); handlerThread.quit(); } state = STATE_SHUT_DOWN; @@ -246,7 +287,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - // Called by the internal Thread. + // Called by the internal thread. @Override public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) { diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapterTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapterTest.java index 1232d8bf3e..1910d902d0 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapterTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MultiLockAsyncMediaCodecAdapterTest.java @@ -27,6 +27,7 @@ import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -49,7 +50,9 @@ public class MultiLockAsyncMediaCodecAdapterTest { public void setUp() throws IOException { codec = MediaCodec.createByCodecName("h264"); handlerThread = new TestHandlerThread("TestHandlerThread"); - adapter = new MultiLockAsyncMediaCodecAdapter(codec, handlerThread); + adapter = + new MultiLockAsyncMediaCodecAdapter( + codec, /* enableAsynchronousQueueing= */ false, C.TRACK_TYPE_VIDEO, handlerThread); adapter.setCodecStartRunnable(() -> {}); bufferInfo = new MediaCodec.BufferInfo(); }