From ee37b453dd6eb5f4196ada8233fa0c86e54a6f34 Mon Sep 17 00:00:00 2001 From: kimvde Date: Fri, 6 Jan 2023 07:51:19 +0000 Subject: [PATCH] Split ExoPlayerAssetLoaderRenderer Split ExoPlayerAssetLoaderRenderer into audio and video renderers. PiperOrigin-RevId: 500102256 --- .../ExoAssetLoaderAudioRenderer.java | 97 ++++++++ ...r.java => ExoAssetLoaderBaseRenderer.java} | 213 ++++-------------- .../ExoAssetLoaderVideoRenderer.java | 149 ++++++++++++ .../transformer/ExoPlayerAssetLoader.java | 15 +- 4 files changed, 289 insertions(+), 185 deletions(-) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java rename library/transformer/src/main/java/com/google/android/exoplayer2/transformer/{ExoPlayerAssetLoaderRenderer.java => ExoAssetLoaderBaseRenderer.java} (55%) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java new file mode 100644 index 0000000000..2ed4d5323a --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java @@ -0,0 +1,97 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + +import android.media.MediaCodec; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/* package */ final class ExoAssetLoaderAudioRenderer extends ExoAssetLoaderBaseRenderer { + + private static final String TAG = "ExoAssetLoaderAudioRenderer"; + + private final Codec.DecoderFactory decoderFactory; + + @Nullable private ByteBuffer pendingDecoderOutputBuffer; + + public ExoAssetLoaderAudioRenderer( + Codec.DecoderFactory decoderFactory, + TransformerMediaClock mediaClock, + AssetLoader.Listener assetLoaderListener) { + super(C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener); + this.decoderFactory = decoderFactory; + } + + @Override + public String getName() { + return TAG; + } + + @Override + protected void initDecoder(Format inputFormat) throws TransformationException { + decoder = decoderFactory.createForAudioDecoding(inputFormat); + } + + /** + * Attempts to get decoded audio data and pass it to the sample consumer. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + * @throws TransformationException If an error occurs in the decoder. + */ + @Override + @RequiresNonNull("sampleConsumer") + protected boolean feedConsumerFromDecoder() throws TransformationException { + @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer(); + if (sampleConsumerInputBuffer == null) { + return false; + } + + Codec decoder = checkNotNull(this.decoder); + if (pendingDecoderOutputBuffer != null) { + if (pendingDecoderOutputBuffer.hasRemaining()) { + return false; + } else { + decoder.releaseOutputBuffer(/* render= */ false); + pendingDecoderOutputBuffer = null; + } + } + + if (decoder.isEnded()) { + sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); + sampleConsumer.queueInputBuffer(); + isEnded = true; + return false; + } + + pendingDecoderOutputBuffer = decoder.getOutputBuffer(); + if (pendingDecoderOutputBuffer == null) { + return false; + } + + sampleConsumerInputBuffer.data = pendingDecoderOutputBuffer; + MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo()); + sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs; + sampleConsumerInputBuffer.setFlags(bufferInfo.flags); + sampleConsumer.queueInputBuffer(); + return true; + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderBaseRenderer.java similarity index 55% rename from library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java rename to library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderBaseRenderer.java index 1dfad7722f..87129cb3a3 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderBaseRenderer.java @@ -22,7 +22,6 @@ import static com.google.android.exoplayer2.transformer.AssetLoader.SUPPORTED_OU import static com.google.android.exoplayer2.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; -import android.media.MediaCodec; import androidx.annotation.Nullable; import com.google.android.exoplayer2.BaseRenderer; import com.google.android.exoplayer2.C; @@ -33,53 +32,32 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; import com.google.android.exoplayer2.util.MediaClock; import com.google.android.exoplayer2.util.MimeTypes; -import com.google.android.exoplayer2.video.ColorInfo; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -/* package */ final class ExoPlayerAssetLoaderRenderer extends BaseRenderer { +/* package */ abstract class ExoAssetLoaderBaseRenderer extends BaseRenderer { - private static final String TAG = "ExoPlayerAssetLoaderRenderer"; + protected long streamOffsetUs; + protected @MonotonicNonNull SampleConsumer sampleConsumer; + protected @MonotonicNonNull Codec decoder; + protected boolean isEnded; - private final boolean flattenForSlowMotion; - private final Codec.DecoderFactory decoderFactory; private final TransformerMediaClock mediaClock; private final AssetLoader.Listener assetLoaderListener; private final DecoderInputBuffer decoderInputBuffer; - private final List decodeOnlyPresentationTimestamps; private boolean isTransformationRunning; private long streamStartPositionUs; - private long streamOffsetUs; - private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; - private @MonotonicNonNull Codec decoder; - @Nullable private ByteBuffer pendingDecoderOutputBuffer; - private int maxDecoderPendingFrameCount; - private @MonotonicNonNull SampleConsumer sampleConsumer; - private boolean isEnded; - public ExoPlayerAssetLoaderRenderer( - int trackType, - boolean flattenForSlowMotion, - Codec.DecoderFactory decoderFactory, + public ExoAssetLoaderBaseRenderer( + @C.TrackType int trackType, TransformerMediaClock mediaClock, AssetLoader.Listener assetLoaderListener) { super(trackType); - this.flattenForSlowMotion = flattenForSlowMotion; - this.decoderFactory = decoderFactory; this.mediaClock = mediaClock; this.assetLoaderListener = assetLoaderListener; decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); - decodeOnlyPresentationTimestamps = new ArrayList<>(); - } - - @Override - public String getName() { - return TAG; } /** @@ -119,13 +97,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } if (sampleConsumer.expectsDecodedData()) { - if (getTrackType() == C.TRACK_TYPE_AUDIO) { - while (feedConsumerAudioFromDecoder() || feedDecoderFromInput()) {} - } else if (getTrackType() == C.TRACK_TYPE_VIDEO) { - while (feedConsumerVideoFromDecoder() || feedDecoderFromInput()) {} - } else { - throw new IllegalStateException(); - } + while (feedConsumerFromDecoder() || feedDecoderFromInput()) {} } else { while (feedConsumerFromInput()) {} } @@ -163,6 +135,35 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } + /** Called when the {@link Format} of the samples fed to the renderer is known. */ + protected void onInputFormatRead(Format inputFormat) {} + + /** Initializes {@link #decoder} with an appropriate {@linkplain Codec decoder}. */ + @RequiresNonNull("sampleConsumer") + protected abstract void initDecoder(Format inputFormat) throws TransformationException; + + /** + * Preprocesses an encoded {@linkplain DecoderInputBuffer input buffer} and returns whether it + * should be dropped. + * + *

The input buffer is cleared if it should be dropped. + */ + protected boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) { + return false; + } + + /** Called before a {@link DecoderInputBuffer} is queued to the decoder. */ + protected void onDecoderInputReady(DecoderInputBuffer inputBuffer) {} + + /** + * Attempts to get decoded data and pass it to the sample consumer. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + * @throws TransformationException If an error occurs in the decoder. + */ + @RequiresNonNull("sampleConsumer") + protected abstract boolean feedConsumerFromDecoder() throws TransformationException; + @EnsuresNonNullIf(expression = "sampleConsumer", result = true) private boolean ensureConfigured() throws TransformationException { if (sampleConsumer != null) { @@ -181,107 +182,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; sampleConsumer = assetLoaderListener.onTrackAdded( inputFormat, supportedOutputTypes, streamStartPositionUs, streamOffsetUs); - if (getTrackType() == C.TRACK_TYPE_VIDEO && flattenForSlowMotion) { - sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); - } + onInputFormatRead(inputFormat); if (sampleConsumer.expectsDecodedData()) { - if (getTrackType() == C.TRACK_TYPE_AUDIO) { - decoder = decoderFactory.createForAudioDecoding(inputFormat); - } else if (getTrackType() == C.TRACK_TYPE_VIDEO) { - boolean isDecoderToneMappingRequired = - ColorInfo.isTransferHdr(inputFormat.colorInfo) - && !ColorInfo.isTransferHdr(sampleConsumer.getExpectedColorInfo()); - decoder = - decoderFactory.createForVideoDecoding( - inputFormat, - checkNotNull(sampleConsumer.getInputSurface()), - isDecoderToneMappingRequired); - maxDecoderPendingFrameCount = decoder.getMaxPendingFrameCount(); - } else { - throw new IllegalStateException(); - } + initDecoder(inputFormat); } return true; } - /** - * Attempts to get decoded audio data and pass it to the sample consumer. - * - * @return Whether it may be possible to read more data immediately by calling this method again. - * @throws TransformationException If an error occurs in the decoder. - */ - @RequiresNonNull("sampleConsumer") - private boolean feedConsumerAudioFromDecoder() throws TransformationException { - @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer(); - if (sampleConsumerInputBuffer == null) { - return false; - } - - Codec decoder = checkNotNull(this.decoder); - if (pendingDecoderOutputBuffer != null) { - if (pendingDecoderOutputBuffer.hasRemaining()) { - return false; - } else { - decoder.releaseOutputBuffer(/* render= */ false); - pendingDecoderOutputBuffer = null; - } - } - - if (decoder.isEnded()) { - sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); - sampleConsumer.queueInputBuffer(); - isEnded = true; - return false; - } - - pendingDecoderOutputBuffer = decoder.getOutputBuffer(); - if (pendingDecoderOutputBuffer == null) { - return false; - } - - sampleConsumerInputBuffer.data = pendingDecoderOutputBuffer; - MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo()); - sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs; - sampleConsumerInputBuffer.setFlags(bufferInfo.flags); - sampleConsumer.queueInputBuffer(); - return true; - } - - /** - * Attempts to get decoded video data and pass it to the sample consumer. - * - * @return Whether it may be possible to read more data immediately by calling this method again. - * @throws TransformationException If an error occurs in the decoder. - */ - @RequiresNonNull("sampleConsumer") - private boolean feedConsumerVideoFromDecoder() throws TransformationException { - Codec decoder = checkNotNull(this.decoder); - if (decoder.isEnded()) { - sampleConsumer.signalEndOfVideoInput(); - isEnded = true; - return false; - } - - @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); - if (decoderOutputBufferInfo == null) { - return false; - } - - if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { - decoder.releaseOutputBuffer(/* render= */ false); - return true; - } - - if (maxDecoderPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT - && sampleConsumer.getPendingVideoFrameCount() == maxDecoderPendingFrameCount) { - return false; - } - - sampleConsumer.registerVideoFrame(); - decoder.releaseOutputBuffer(/* render= */ true); - return true; - } - /** * Attempts to read input data and pass it to the decoder. * @@ -302,9 +209,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return true; } - if (decoderInputBuffer.isDecodeOnly()) { - decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); - } + onDecoderInputReady(decoderInputBuffer); decoder.queueInputBuffer(decoderInputBuffer); return true; } @@ -359,42 +264,4 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } } - - /** - * Preprocesses an encoded {@linkplain DecoderInputBuffer input buffer} and returns whether it - * should be dropped. - * - *

The input buffer is cleared if it should be dropped. - */ - private boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) { - ByteBuffer inputBytes = checkNotNull(inputBuffer.data); - - if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) { - return false; - } - - long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; - boolean shouldDropInputBuffer = - sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs); - if (shouldDropInputBuffer) { - inputBytes.clear(); - } else { - inputBuffer.timeUs = - streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs(); - } - return shouldDropInputBuffer; - } - - private boolean isDecodeOnlyBuffer(long presentationTimeUs) { - // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would - // box presentationTimeUs, creating a Long object that would need to be garbage collected. - int size = decodeOnlyPresentationTimestamps.size(); - for (int i = 0; i < size; i++) { - if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { - decodeOnlyPresentationTimestamps.remove(i); - return true; - } - } - return false; - } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java new file mode 100644 index 0000000000..5065262124 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java @@ -0,0 +1,149 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + +import android.media.MediaCodec; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.video.ColorInfo; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/* package */ final class ExoAssetLoaderVideoRenderer extends ExoAssetLoaderBaseRenderer { + + private static final String TAG = "ExoAssetLoaderVideoRenderer"; + + private final boolean flattenForSlowMotion; + private final Codec.DecoderFactory decoderFactory; + private final List decodeOnlyPresentationTimestamps; + + private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; + private int maxDecoderPendingFrameCount; + + public ExoAssetLoaderVideoRenderer( + boolean flattenForSlowMotion, + Codec.DecoderFactory decoderFactory, + TransformerMediaClock mediaClock, + AssetLoader.Listener assetLoaderListener) { + super(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener); + this.flattenForSlowMotion = flattenForSlowMotion; + this.decoderFactory = decoderFactory; + decodeOnlyPresentationTimestamps = new ArrayList<>(); + } + + @Override + public String getName() { + return TAG; + } + + @Override + protected void onInputFormatRead(Format inputFormat) { + if (flattenForSlowMotion) { + sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat); + } + } + + @Override + @RequiresNonNull("sampleConsumer") + protected void initDecoder(Format inputFormat) throws TransformationException { + boolean isDecoderToneMappingRequired = + ColorInfo.isTransferHdr(inputFormat.colorInfo) + && !ColorInfo.isTransferHdr(sampleConsumer.getExpectedColorInfo()); + decoder = + decoderFactory.createForVideoDecoding( + inputFormat, + checkNotNull(sampleConsumer.getInputSurface()), + isDecoderToneMappingRequired); + maxDecoderPendingFrameCount = decoder.getMaxPendingFrameCount(); + } + + @Override + protected boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) { + ByteBuffer inputBytes = checkNotNull(inputBuffer.data); + + if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) { + return false; + } + + long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs; + boolean shouldDropInputBuffer = + sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs); + if (shouldDropInputBuffer) { + inputBytes.clear(); + } else { + inputBuffer.timeUs = + streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs(); + } + return shouldDropInputBuffer; + } + + @Override + protected void onDecoderInputReady(DecoderInputBuffer inputBuffer) { + if (inputBuffer.isDecodeOnly()) { + decodeOnlyPresentationTimestamps.add(inputBuffer.timeUs); + } + } + + @Override + @RequiresNonNull("sampleConsumer") + protected boolean feedConsumerFromDecoder() throws TransformationException { + Codec decoder = checkNotNull(this.decoder); + if (decoder.isEnded()) { + sampleConsumer.signalEndOfVideoInput(); + isEnded = true; + return false; + } + + @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); + if (decoderOutputBufferInfo == null) { + return false; + } + + if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { + decoder.releaseOutputBuffer(/* render= */ false); + return true; + } + + if (maxDecoderPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT + && sampleConsumer.getPendingVideoFrameCount() == maxDecoderPendingFrameCount) { + return false; + } + + sampleConsumer.registerVideoFrame(); + decoder.releaseOutputBuffer(/* render= */ true); + return true; + } + + private boolean isDecodeOnlyBuffer(long presentationTimeUs) { + // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would + // box presentationTimeUs, creating a Long object that would need to be garbage collected. + int size = decodeOnlyPresentationTimestamps.size(); + for (int i = 0; i < size; i++) { + if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { + decodeOnlyPresentationTimestamps.remove(i); + return true; + } + } + return false; + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java index a534f84de3..6dd0247529 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java @@ -291,22 +291,13 @@ public final class ExoPlayerAssetLoader implements AssetLoader { int index = 0; if (!removeAudio) { renderers[index] = - new ExoPlayerAssetLoaderRenderer( - C.TRACK_TYPE_AUDIO, - /* flattenForSlowMotion= */ false, - decoderFactory, - mediaClock, - assetLoaderListener); + new ExoAssetLoaderAudioRenderer(decoderFactory, mediaClock, assetLoaderListener); index++; } if (!removeVideo) { renderers[index] = - new ExoPlayerAssetLoaderRenderer( - C.TRACK_TYPE_VIDEO, - flattenForSlowMotion, - decoderFactory, - mediaClock, - assetLoaderListener); + new ExoAssetLoaderVideoRenderer( + flattenForSlowMotion, decoderFactory, mediaClock, assetLoaderListener); index++; } return renderers;