From db6f043cb1eaf2081ae56d0d5f898d05dc463040 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 14 Aug 2023 09:39:53 +0000 Subject: [PATCH] Rollback of https://github.com/androidx/media/commit/f5a6ecdda1cbff67de320615e1eccf4be31b4db0 PiperOrigin-RevId: 556718678 --- .../video/CompositingVideoSinkProvider.java | 643 ------------- .../video/MediaCodecVideoRenderer.java | 846 ++++++++++++++---- .../media3/exoplayer/video/VideoSink.java | 42 +- .../CompositingVideoSinkProviderTest.java | 169 ---- 4 files changed, 659 insertions(+), 1041 deletions(-) delete mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java delete mode 100644 libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java deleted file mode 100644 index cdfc279988..0000000000 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ /dev/null @@ -1,643 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.exoplayer.video; - -import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - -import android.content.Context; -import android.graphics.Bitmap; -import android.os.Handler; -import android.util.Pair; -import android.view.Surface; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.Format; -import androidx.media3.common.FrameInfo; -import androidx.media3.common.MimeTypes; -import androidx.media3.common.SurfaceInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.VideoSize; -import androidx.media3.common.util.Size; -import androidx.media3.common.util.TimedValueQueue; -import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executor; -import org.checkerframework.checker.initialization.qual.Initialized; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** Handles composition of video sinks. */ -@UnstableApi -/* package */ final class CompositingVideoSinkProvider { - - private final Context context; - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - private final VideoSink.RenderControl renderControl; - - @Nullable private VideoSinkImpl videoSinkImpl; - @Nullable private List videoEffects; - @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; - private boolean released; - - /** Creates a new instance. */ - public CompositingVideoSinkProvider( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - VideoSink.RenderControl renderControl) { - this.context = context; - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - this.renderControl = renderControl; - } - - /** - * Initializes the provider for video frame processing. Can be called up to one time and only - * after video effects are {@linkplain #setVideoEffects(List) set}. - * - * @param sourceFormat The format of the compressed video. - * @throws VideoSink.VideoSinkException If enabling the provider failed. - */ - public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException { - checkState(!released && videoSinkImpl == null); - checkStateNotNull(videoEffects); - - try { - videoSinkImpl = - new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat); - if (videoFrameMetadataListener != null) { - videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener); - } - } catch (VideoFrameProcessingException e) { - throw new VideoSink.VideoSinkException(e, sourceFormat); - } - } - - /** Returns whether this provider is initialized for frame processing. */ - public boolean isInitialized() { - return videoSinkImpl != null; - } - - /** Releases the sink provider. */ - public void release() { - if (released) { - return; - } - - if (videoSinkImpl != null) { - videoSinkImpl.release(); - videoSinkImpl = null; - } - released = true; - } - - /** Returns a {@link VideoSink} to forward video frames for processing. */ - public VideoSink getSink() { - return checkStateNotNull(videoSinkImpl); - } - - /** Sets video effects on this provider. */ - public void setVideoEffects(List videoEffects) { - this.videoEffects = videoEffects; - if (isInitialized()) { - checkStateNotNull(videoSinkImpl).setVideoEffects(videoEffects); - } - } - - /** - * Sets the offset, in microseconds, that is added to the video frames presentation timestamps - * from the player. - * - *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. - */ - public void setStreamOffsetUs(long streamOffsetUs) { - checkStateNotNull(videoSinkImpl).setStreamOffsetUs(streamOffsetUs); - } - - /** - * Sets the output surface info. - * - *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. - */ - public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { - checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution); - } - - /** - * Clears the set output surface info. - * - *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. - */ - public void clearOutputSurfaceInfo() { - checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo(); - } - - /** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */ - public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) { - this.videoFrameMetadataListener = videoFrameMetadataListener; - if (isInitialized()) { - checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener); - } - } - - private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener { - - private final Context context; - private final RenderControl renderControl; - private final VideoFrameProcessor videoFrameProcessor; - // TODO b/293447478 - Use a queue for primitive longs to avoid the cost of boxing to Long. - private final ArrayDeque processedFramesBufferTimestampsUs; - private final TimedValueQueue streamOffsets; - private final TimedValueQueue videoSizeChanges; - private final Handler handler; - private final int videoFrameProcessorMaxPendingFrameCount; - private final ArrayList videoEffects; - @Nullable private final Effect rotationEffect; - - private VideoSink.@MonotonicNonNull Listener listener; - private @MonotonicNonNull Executor listenerExecutor; - @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; - @Nullable private Format inputFormat; - @Nullable private Pair currentSurfaceAndSize; - - /** - * Whether the last frame of the current stream is decoded and registered to {@link - * VideoFrameProcessor}. - */ - private boolean registeredLastFrame; - - /** - * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}. - */ - private boolean processedLastFrame; - - /** Whether the last frame of the current stream is released to the output {@link Surface}. */ - private boolean releasedLastFrame; - - private long lastCodecBufferPresentationTimestampUs; - private VideoSize processedFrameSize; - private VideoSize reportedVideoSize; - private boolean pendingVideoSizeChange; - private boolean renderedFirstFrame; - private long inputStreamOffsetUs; - private boolean pendingInputStreamOffsetChange; - private long outputStreamOffsetUs; - private float playbackSpeed; - - // TODO b/292111083 - Remove the field and trigger the callback on every video size change. - private boolean onVideoSizeChangedCalled; - - /** Creates a new instance. */ - public VideoSinkImpl( - Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, - RenderControl renderControl, - Format sourceFormat) - throws VideoFrameProcessingException { - this.context = context; - this.renderControl = renderControl; - processedFramesBufferTimestampsUs = new ArrayDeque<>(); - streamOffsets = new TimedValueQueue<>(); - videoSizeChanges = new TimedValueQueue<>(); - // TODO b/226330223 - Investigate increasing frame count when frame dropping is - // allowed. - // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed - // reduces decoder timeouts, and consider restoring. - videoFrameProcessorMaxPendingFrameCount = - Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; - processedFrameSize = VideoSize.UNKNOWN; - reportedVideoSize = VideoSize.UNKNOWN; - playbackSpeed = 1f; - - // Playback thread handler. - handler = Util.createHandlerForCurrentLooper(); - - Pair inputAndOutputColorInfos = - experimentalGetVideoFrameProcessorColorConfiguration(sourceFormat.colorInfo); - - @SuppressWarnings("nullness:assignment") - @Initialized - VideoSinkImpl thisRef = this; - videoFrameProcessor = - videoFrameProcessorFactory.create( - context, - DebugViewProvider.NONE, - inputAndOutputColorInfos.first, - inputAndOutputColorInfos.second, - /* renderFramesAutomatically= */ false, - /* listenerExecutor= */ handler::post, - thisRef); - if (currentSurfaceAndSize != null) { - Size outputSurfaceSize = currentSurfaceAndSize.second; - videoFrameProcessor.setOutputSurfaceInfo( - new SurfaceInfo( - currentSurfaceAndSize.first, - outputSurfaceSize.getWidth(), - outputSurfaceSize.getHeight())); - } - videoEffects = new ArrayList<>(); - // MediaCodec applies rotation after API 21 - rotationEffect = - Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0 - ? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees) - : null; - } - - // VideoSink impl - - @Override - public void flush() { - videoFrameProcessor.flush(); - processedFramesBufferTimestampsUs.clear(); - streamOffsets.clear(); - handler.removeCallbacksAndMessages(/* token= */ null); - renderedFirstFrame = false; - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; - } - } - - @Override - public boolean isReady() { - return renderedFirstFrame; - } - - @Override - public boolean isEnded() { - return releasedLastFrame; - } - - @Override - public void registerInputStream(@InputType int inputType, Format format) { - if (inputType != INPUT_TYPE_SURFACE) { - throw new UnsupportedOperationException("Unsupported input type " + inputType); - } - this.inputFormat = format; - maybeRegisterInputStream(); - - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; - } - } - - @Override - public void setListener(Listener listener, Executor executor) { - if (Util.areEqual(this.listener, listener)) { - checkState(Util.areEqual(listenerExecutor, executor)); - return; - } - this.listener = listener; - this.listenerExecutor = executor; - } - - @Override - public boolean isFrameDropAllowedOnInput() { - return Util.isFrameDropAllowedOnSurfaceInput(context); - } - - @Override - public Surface getInputSurface() { - return videoFrameProcessor.getInputSurface(); - } - - @Override - public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) { - checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); - if (videoFrameProcessor.getPendingInputFrameCount() - >= videoFrameProcessorMaxPendingFrameCount) { - return C.TIME_UNSET; - } - videoFrameProcessor.registerInputFrame(); - // The sink takes in frames with monotonically increasing, non-offset frame - // timestamps. That is, with two ten-second long videos, the first frame of the second video - // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the - // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include - // the duration of the first video. Thus this correction is need to correct for the different - // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. - long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs; - if (pendingInputStreamOffsetChange) { - streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs); - pendingInputStreamOffsetChange = false; - } - if (isLastFrame) { - registeredLastFrame = true; - lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs; - } - return bufferPresentationTimeUs * 1000; - } - - @Override - public boolean queueBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { - throw new UnsupportedOperationException(); - } - - @Override - public void render(long positionUs, long elapsedRealtimeUs) { - while (!processedFramesBufferTimestampsUs.isEmpty()) { - long bufferPresentationTimeUs = checkNotNull(processedFramesBufferTimestampsUs.peek()); - // check whether this buffer comes with a new stream offset. - if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) { - renderedFirstFrame = false; - } - long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; - boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1; - long frameRenderTimeNs = - renderControl.getFrameRenderTimeNs( - bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed); - if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) { - return; - } else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) { - // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush - // VideoFrameProcessor input frames in this case. - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - continue; - } - renderControl.onNextFrame(bufferPresentationTimeUs); - if (videoFrameMetadataListener != null) { - videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - framePresentationTimeUs, - frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY - ? System.nanoTime() - : frameRenderTimeNs, - checkNotNull(inputFormat), - /* mediaFormat= */ null); - } - releaseProcessedFrameInternal( - frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY - ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY - : frameRenderTimeNs, - isLastFrame); - - maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); - } - } - - @Override - public void setPlaybackSpeed(float speed) { - checkArgument(speed >= 0.0); - this.playbackSpeed = speed; - } - - // VideoFrameProcessor.Listener impl - - @Override - public void onOutputSizeChanged(int width, int height) { - VideoSize newVideoSize = new VideoSize(width, height); - if (!processedFrameSize.equals(newVideoSize)) { - processedFrameSize = newVideoSize; - pendingVideoSizeChange = true; - } - } - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (pendingVideoSizeChange) { - videoSizeChanges.add(presentationTimeUs, processedFrameSize); - pendingVideoSizeChange = false; - } - if (registeredLastFrame) { - checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); - } - processedFramesBufferTimestampsUs.add(presentationTimeUs); - // TODO b/257464707 - Support extensively modified media. - if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { - processedLastFrame = true; - } - } - - @Override - public void onError(VideoFrameProcessingException exception) { - if (listener == null || listenerExecutor == null) { - return; - } - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onError( - /* videoSink= */ this, - new VideoSink.VideoSinkException( - exception, - new Format.Builder() - .setSampleMimeType(MimeTypes.VIDEO_RAW) - .setWidth(processedFrameSize.width) - .setHeight(processedFrameSize.height) - .build())); - } - }); - } - - @Override - public void onEnded() { - throw new IllegalStateException(); - } - - // Other methods - - public void release() { - videoFrameProcessor.release(); - handler.removeCallbacksAndMessages(/* token= */ null); - streamOffsets.clear(); - processedFramesBufferTimestampsUs.clear(); - renderedFirstFrame = false; - } - - /** Sets the {@linkplain Effect video effects}. */ - public void setVideoEffects(List videoEffects) { - this.videoEffects.clear(); - this.videoEffects.addAll(videoEffects); - maybeRegisterInputStream(); - } - - public void setStreamOffsetUs(long streamOffsetUs) { - pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs; - inputStreamOffsetUs = streamOffsetUs; - } - - public void setVideoFrameMetadataListener( - VideoFrameMetadataListener videoFrameMetadataListener) { - this.videoFrameMetadataListener = videoFrameMetadataListener; - } - - private void maybeRegisterInputStream() { - if (inputFormat == null) { - return; - } - - ArrayList effects = new ArrayList<>(); - if (rotationEffect != null) { - effects.add(rotationEffect); - } - effects.addAll(videoEffects); - Format inputFormat = checkNotNull(this.inputFormat); - videoFrameProcessor.registerInputStream( - VideoFrameProcessor.INPUT_TYPE_SURFACE, - effects, - new FrameInfo.Builder(inputFormat.width, inputFormat.height) - .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) - .build()); - } - - /** - * Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output - * color} to configure the {@code VideoFrameProcessor}. - */ - private static Pair experimentalGetVideoFrameProcessorColorConfiguration( - @Nullable ColorInfo inputColorInfo) { - // TODO b/279163661 - Remove this method after VideoFrameProcessor supports texture ID - // input/output. - // explicit check for nullness - if (inputColorInfo == null || !ColorInfo.isTransferHdr(inputColorInfo)) { - return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED); - } - - if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { - // SurfaceView only supports BT2020 PQ input, converting HLG to PQ. - return Pair.create( - inputColorInfo, - inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build()); - } - - return Pair.create(inputColorInfo, inputColorInfo); - } - - /** - * Sets the output surface info. - * - * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs. - * @param outputResolution The {@link Size} of the output resolution. - */ - public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { - if (currentSurfaceAndSize != null - && currentSurfaceAndSize.first.equals(outputSurface) - && currentSurfaceAndSize.second.equals(outputResolution)) { - return; - } - renderedFirstFrame = - currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface); - currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); - videoFrameProcessor.setOutputSurfaceInfo( - new SurfaceInfo( - outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); - } - - /** Clears the set output surface info. */ - public void clearOutputSurfaceInfo() { - videoFrameProcessor.setOutputSurfaceInfo(null); - currentSurfaceAndSize = null; - renderedFirstFrame = false; - } - - private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) { - boolean updatedOffset = false; - @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs); - if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) { - outputStreamOffsetUs = newOutputStreamOffsetUs; - updatedOffset = true; - } - return updatedOffset; - } - - private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { - videoFrameProcessor.renderOutputFrame(releaseTimeNs); - processedFramesBufferTimestampsUs.remove(); - if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) { - renderControl.onFrameRendered(); - if (!renderedFirstFrame) { - if (listener != null) { - checkNotNull(listenerExecutor) - .execute(() -> checkNotNull(listener).onFirstFrameRendered(this)); - } - renderedFirstFrame = true; - } - } - if (isLastFrame) { - releasedLastFrame = true; - } - } - - private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) { - if (onVideoSizeChangedCalled || listener == null) { - return; - } - - @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs); - if (videoSize == null) { - return; - } - - if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) { - reportedVideoSize = videoSize; - checkNotNull(listenerExecutor) - .execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize)); - } - onVideoSizeChangedCalled = true; - } - - private static final class ScaleAndRotateAccessor { - private static @MonotonicNonNull Constructor - scaleAndRotateTransformationBuilderConstructor; - private static @MonotonicNonNull Method setRotationMethod; - private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod; - - public static Effect createRotationEffect(float rotationDegrees) { - try { - prepare(); - Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance(); - setRotationMethod.invoke(builder, rotationDegrees); - return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder)); - } catch (Exception e) { - throw new IllegalStateException(e); - } - } - - @EnsuresNonNull({ - "scaleAndRotateTransformationBuilderConstructor", - "setRotationMethod", - "buildScaleAndRotateTransformationMethod" - }) - private static void prepare() throws NoSuchMethodException, ClassNotFoundException { - if (scaleAndRotateTransformationBuilderConstructor == null - || setRotationMethod == null - || buildScaleAndRotateTransformationMethod == null) { - // TODO: b/284964524 - Add LINT and proguard checks for media3.effect reflection. - Class scaleAndRotateTransformationBuilderClass = - Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder"); - scaleAndRotateTransformationBuilderConstructor = - scaleAndRotateTransformationBuilderClass.getConstructor(); - setRotationMethod = - scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class); - buildScaleAndRotateTransformationMethod = - scaleAndRotateTransformationBuilderClass.getMethod("build"); - } - } - } - } -} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 17f7c7edaf..1957db36bc 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -18,6 +18,7 @@ package androidx.media3.exoplayer.video; import static android.view.Display.DEFAULT_DISPLAY; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED; @@ -52,8 +53,10 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.DrmInitData; import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.FrameInfo; import androidx.media3.common.MimeTypes; import androidx.media3.common.PlaybackException; +import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; @@ -84,10 +87,17 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatche import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.ArrayDeque; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; +import org.checkerframework.checker.initialization.qual.UnderInitialization; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Decodes and renders video using {@link MediaCodec}. @@ -113,7 +123,7 @@ import java.util.concurrent.Executor; * */ @UnstableApi -public class MediaCodecVideoRenderer extends MediaCodecRenderer implements VideoSink.RenderControl { +public class MediaCodecVideoRenderer extends MediaCodecRenderer { private static final String TAG = "MediaCodecVideoRenderer"; private static final String KEY_CROP_LEFT = "crop-left"; @@ -137,16 +147,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video /** The minimum input buffer size for HEVC. */ private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; - /** The maximum earliest time, in microseconds, to release a frame on the surface. */ - private static final long MAX_EARLY_US_THRESHOLD = 50_000; - private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; private final Context context; private final VideoFrameReleaseHelper frameReleaseHelper; - private final CompositingVideoSinkProvider videoSinkProvider; private final EventDispatcher eventDispatcher; + private final VideoFrameProcessorManager videoFrameProcessorManager; private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; @@ -154,6 +161,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video private CodecMaxValues codecMaxValues; private boolean codecNeedsSetOutputSurfaceWorkaround; private boolean codecHandlesHdr10PlusOutOfBandMetadata; + @Nullable private Surface displaySurface; @Nullable private PlaceholderSurface placeholderSurface; private boolean haveReportedFirstFrameRenderedForCurrentSurface; @@ -170,16 +178,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video private long totalVideoFrameProcessingOffsetUs; private int videoFrameProcessingOffsetCount; private long lastFrameReleaseTimeNs; + private VideoSize decodedVideoSize; @Nullable private VideoSize reportedVideoSize; - private boolean hasEffects; - private boolean hasInitializedPlayback; private boolean tunneling; private int tunnelingAudioSessionId; /* package */ @Nullable OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener; @Nullable private VideoFrameMetadataListener frameMetadataListener; - @Nullable private VideoSink videoSink; /** * @param context A context. @@ -396,9 +402,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video this.context = context.getApplicationContext(); frameReleaseHelper = new VideoFrameReleaseHelper(this.context); eventDispatcher = new EventDispatcher(eventHandler, eventListener); - videoSinkProvider = - new CompositingVideoSinkProvider( - context, videoFrameProcessorFactory, /* renderControl= */ this); + videoFrameProcessorManager = + new VideoFrameProcessorManager( + videoFrameProcessorFactory, frameReleaseHelper, /* renderer= */ this); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); joiningDeadlineMs = C.TIME_UNSET; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; @@ -521,49 +527,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video format); } - // RenderControl implementation - - @Override - public long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { - long earlyUs = - calculateEarlyTimeUs( - positionUs, - elapsedRealtimeUs, - presentationTimeUs, - getState() == STATE_STARTED, - playbackSpeed, - getClock()); - if (isBufferLate(earlyUs)) { - return VideoSink.RenderControl.RENDER_TIME_DROP; - } - if (shouldForceRender(positionUs, earlyUs)) { - return VideoSink.RenderControl.RENDER_TIME_IMMEDIATELY; - } - - if (getState() != STATE_STARTED - || positionUs == initialPositionUs - || earlyUs > MAX_EARLY_US_THRESHOLD) { - return VideoSink.RenderControl.RENDER_TIME_TRY_AGAIN_LATER; - } - // Compute the buffer's desired release time in nanoseconds. - long unadjustedFrameReleaseTimeNs = getClock().nanoTime() + (earlyUs * 1000); - // Apply a timestamp adjustment, if there is one. - return frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - } - - @Override - public void onNextFrame(long presentationTimeUs) { - frameReleaseHelper.onNextFrame(presentationTimeUs); - } - - @Override - public void onFrameRendered() { - lastRenderRealtimeUs = Util.msToUs(getClock().elapsedRealtime()); - } - - // Other methods - /** * Returns a list of decoders that can decode media in the specified format, in the priority order * specified by the {@link MediaCodecSelector}. Note that since the {@link MediaCodecSelector} @@ -653,14 +616,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Override protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { super.onPositionReset(positionUs, joining); - if (videoSink != null) { - videoSink.flush(); + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.flush(); } - - if (videoSinkProvider.isInitialized()) { - videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); - } - lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); frameReleaseHelper.onPositionReset(); lastBufferPresentationTimeUs = C.TIME_UNSET; @@ -675,13 +633,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Override public boolean isEnded() { - return super.isEnded() && (videoSink == null || videoSink.isEnded()); + boolean isEnded = super.isEnded(); + if (videoFrameProcessorManager.isEnabled()) { + isEnded &= videoFrameProcessorManager.releasedLastFrame(); + } + return isEnded; } @Override public boolean isReady() { if (super.isReady() - && (videoSink == null || videoSink.isReady()) + && (!videoFrameProcessorManager.isEnabled() || videoFrameProcessorManager.isReady()) && (firstFrameState == C.FIRST_FRAME_RENDERED || (placeholderSurface != null && displaySurface == placeholderSurface) || getCodec() == null @@ -743,21 +705,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video try { super.onReset(); } finally { - hasInitializedPlayback = false; + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.reset(); + } if (placeholderSurface != null) { releasePlaceholderSurface(); } } } - @Override - protected void onRelease() { - super.onRelease(); - if (videoSinkProvider.isInitialized()) { - videoSinkProvider.release(); - } - } - @Override public void handleMessage(@MessageType int messageType, @Nullable Object message) throws ExoPlaybackException { @@ -777,7 +733,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video break; case MSG_SET_VIDEO_FRAME_METADATA_LISTENER: frameMetadataListener = (VideoFrameMetadataListener) message; - videoSinkProvider.setVideoFrameMetadataListener(frameMetadataListener); break; case MSG_SET_AUDIO_SESSION_ID: int tunnelingAudioSessionId = (int) message; @@ -791,16 +746,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video case MSG_SET_VIDEO_EFFECTS: @SuppressWarnings("unchecked") List videoEffects = (List) checkNotNull(message); - videoSinkProvider.setVideoEffects(videoEffects); - hasEffects = true; + videoFrameProcessorManager.setVideoEffects(videoEffects); break; case MSG_SET_VIDEO_OUTPUT_RESOLUTION: Size outputResolution = (Size) checkNotNull(message); - if (videoSinkProvider.isInitialized() - && outputResolution.getWidth() != 0 + if (outputResolution.getWidth() != 0 && outputResolution.getHeight() != 0 && displaySurface != null) { - videoSinkProvider.setOutputSurfaceInfo(displaySurface, outputResolution); + videoFrameProcessorManager.setOutputSurfaceInfo(displaySurface, outputResolution); } break; case MSG_SET_AUDIO_ATTRIBUTES: @@ -839,7 +792,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @State int state = getState(); @Nullable MediaCodecAdapter codec = getCodec(); - if (codec != null && videoSinkProvider.isInitialized()) { + if (codec != null && !videoFrameProcessorManager.isEnabled()) { if (Util.SDK_INT >= 23 && displaySurface != null && !codecNeedsSetOutputSurfaceWorkaround) { setOutputSurfaceV23(codec, displaySurface); } else { @@ -856,16 +809,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video // Set joining deadline to report MediaCodecVideoRenderer is ready. setJoiningDeadlineMs(); } - // When effects previewing is enabled, set display surface and an unknown size. - if (videoSinkProvider.isInitialized()) { - videoSinkProvider.setOutputSurfaceInfo(displaySurface, Size.UNKNOWN); + // When VideoFrameProcessorManager is enabled, set VideoFrameProcessorManager's display + // surface and an unknown size. + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.setOutputSurfaceInfo(displaySurface, Size.UNKNOWN); } } else { // The display surface has been removed. clearReportedVideoSize(); lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); - if (videoSinkProvider.isInitialized()) { - videoSinkProvider.clearOutputSurfaceInfo(); + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.clearOutputSurfaceInfo(); } } } else if (displaySurface != null && displaySurface != placeholderSurface) { @@ -917,22 +871,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } displaySurface = placeholderSurface; } - maybeSetKeyAllowFrameDrop(mediaFormat); + + if (videoFrameProcessorManager.isEnabled()) { + mediaFormat = videoFrameProcessorManager.amendMediaFormatKeys(mediaFormat); + } + return MediaCodecAdapter.Configuration.createForVideoDecoding( codecInfo, mediaFormat, format, - videoSink != null ? videoSink.getInputSurface() : displaySurface, + videoFrameProcessorManager.isEnabled() + ? videoFrameProcessorManager.getInputSurface() + : displaySurface, crypto); } - @SuppressWarnings("InlinedApi") // VideoSink will check the API level - private void maybeSetKeyAllowFrameDrop(MediaFormat mediaFormat) { - if (videoSink != null && !videoSink.isFrameDropAllowedOnInput()) { - mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); - } - } - @Override protected DecoderReuseEvaluation canReuseCodec( MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { @@ -958,8 +911,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @Override public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { super.render(positionUs, elapsedRealtimeUs); - if (videoSink != null) { - videoSink.render(positionUs, elapsedRealtimeUs); + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.releaseProcessedFrames(positionUs, elapsedRealtimeUs); } } @@ -975,9 +928,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video throws ExoPlaybackException { super.setPlaybackSpeed(currentPlaybackSpeed, targetPlaybackSpeed); frameReleaseHelper.onPlaybackSpeed(currentPlaybackSpeed); - if (videoSink != null) { - videoSink.setPlaybackSpeed(currentPlaybackSpeed); - } } /** @@ -1069,51 +1019,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video @CallSuper @Override protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { - // We only enable effects preview on the first time a codec is initialized and if effects are - // already set. We do not enable effects mid-playback. For effects to be enabled after - // playback has started, the renderer needs to be reset first. - if (hasEffects && !hasInitializedPlayback && !videoSinkProvider.isInitialized()) { - try { - videoSinkProvider.initialize(format); - videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); - if (frameMetadataListener != null) { - videoSinkProvider.setVideoFrameMetadataListener(frameMetadataListener); - } - } catch (VideoSink.VideoSinkException e) { - throw createRendererException( - e, format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); - } + if (!videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.maybeEnable(format, getOutputStreamOffsetUs(), getClock()); } - - if (videoSink == null && videoSinkProvider.isInitialized()) { - videoSink = videoSinkProvider.getSink(); - videoSink.setListener( - new VideoSink.Listener() { - @Override - public void onFirstFrameRendered(VideoSink videoSink) { - maybeNotifyRenderedFirstFrame(); - } - - @Override - public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) { - maybeNotifyVideoSizeChanged(videoSize); - } - - @Override - public void onError( - VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) { - setPendingPlaybackException( - createRendererException( - videoSinkException, - videoSinkException.format, - PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } - }, - // Pass a direct executor since the callback handling involves posting on the app looper - // again, so there's no need to do two hops. - MoreExecutors.directExecutor()); - } - hasInitializedPlayback = true; } @Override @@ -1129,6 +1037,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video if (Util.SDK_INT >= 23 && tunneling) { tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec())); } + videoFrameProcessorManager.onCodecInitialized(name); } @Override @@ -1216,17 +1125,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video height = rotatedHeight; pixelWidthHeightRatio = 1 / pixelWidthHeightRatio; } - } else if (videoSink == null) { - // Neither the codec nor the video sink applies the rotation. + } else if (!videoFrameProcessorManager.isEnabled()) { + // Neither the codec nor the VideoFrameProcessor applies the rotation. unappliedRotationDegrees = format.rotationDegrees; } decodedVideoSize = new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio); frameReleaseHelper.onFormatChanged(format.frameRate); - if (videoSink != null) { - videoSink.registerInputStream( - /* inputType= */ VideoSink.INPUT_TYPE_SURFACE, + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.setInputFormat( format .buildUpon() .setWidth(width) @@ -1289,7 +1197,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) { - if (videoSink == null) { + if (!videoFrameProcessorManager.isEnabled()) { frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); } // else, update the frameReleaseHelper when releasing the processed frames. this.lastBufferPresentationTimeUs = bufferPresentationTimeUs; @@ -1303,15 +1211,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video return true; } + // Note: Use of double rather than float is intentional for accuracy in the calculations below. boolean isStarted = getState() == STATE_STARTED; + long elapsedRealtimeNowUs = msToUs(getClock().elapsedRealtime()); long earlyUs = calculateEarlyTimeUs( positionUs, elapsedRealtimeUs, + elapsedRealtimeNowUs, bufferPresentationTimeUs, - isStarted, - getPlaybackSpeed(), - getClock()); + isStarted); if (displaySurface == placeholderSurface) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. @@ -1323,21 +1232,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video return false; } - if (videoSink != null) { - videoSink.render(positionUs, elapsedRealtimeUs); - long releaseTimeNs = videoSink.registerInputFrame(presentationTimeUs, isLastBuffer); - if (releaseTimeNs == C.TIME_UNSET) { - return false; - } - renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); - return true; - } - boolean forceRenderOutputBuffer = shouldForceRender(positionUs, earlyUs); if (forceRenderOutputBuffer) { - long releaseTimeNs = getClock().nanoTime(); - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); + boolean notifyFrameMetaDataListener; + if (videoFrameProcessorManager.isEnabled()) { + notifyFrameMetaDataListener = false; + if (!videoFrameProcessorManager.maybeRegisterFrame( + format, presentationTimeUs, isLastBuffer)) { + return false; + } + } else { + notifyFrameMetaDataListener = true; + } + renderOutputBufferNow( + codec, format, bufferIndex, presentationTimeUs, notifyFrameMetaDataListener); updateVideoFrameProcessingOffsetCounters(earlyUs); return true; } @@ -1349,9 +1257,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video // Compute the buffer's desired release time in nanoseconds. long systemTimeNs = getClock().nanoTime(); long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000); + // Apply a timestamp adjustment, if there is one. long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; + if (!videoFrameProcessorManager.isEnabled()) { + earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; + } // else, use the unadjusted earlyUs in previewing use cases. + boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET; if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer) && maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped)) { @@ -1366,9 +1278,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video return true; } + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.releaseProcessedFrames(positionUs, elapsedRealtimeUs); + if (videoFrameProcessorManager.maybeRegisterFrame(format, presentationTimeUs, isLastBuffer)) { + renderOutputBufferNow( + codec, + format, + bufferIndex, + presentationTimeUs, + /* notifyFrameMetadataListener= */ false); + return true; + } + return false; + } + if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. - if (earlyUs < MAX_EARLY_US_THRESHOLD) { + if (earlyUs < 50000) { if (shouldSkipBuffersWithIdenticalReleaseTime() && adjustedReleaseTimeNs == lastFrameReleaseTimeNs) { // This frame should be displayed on the same vsync with the previous released frame. We @@ -1439,28 +1365,29 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video * iteration of the rendering loop. * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the * start of the current iteration of the rendering loop. + * @param elapsedRealtimeNowUs {@link SystemClock#elapsedRealtime()} in microseconds, measured + * before calling this method. * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds, * with {@linkplain #getOutputStreamOffsetUs() stream offset added}. * @param isStarted Whether the playback is in {@link #STATE_STARTED}. - * @param playbackSpeed The current playback speed. - * @param clock The {@link Clock} used by the renderer. * @return The calculated early time, in microseconds. */ - private static long calculateEarlyTimeUs( + private long calculateEarlyTimeUs( long positionUs, long elapsedRealtimeUs, + long elapsedRealtimeNowUs, long bufferPresentationTimeUs, - boolean isStarted, - float playbackSpeed, - Clock clock) { + boolean isStarted) { + // Note: Use of double rather than float is intentional for accuracy in the calculations below. + double playbackSpeed = getPlaybackSpeed(); + // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst // the renderer is started before the frame should be rendered. A negative value means that // we're already late. - // Note: Use of double rather than float is intentional for accuracy in the calculations below. - long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / (double) playbackSpeed); + long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / playbackSpeed); if (isStarted) { // Account for the elapsed time since the start of this iteration of the rendering loop. - earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; + earlyUs -= elapsedRealtimeNowUs - elapsedRealtimeUs; } return earlyUs; @@ -1501,9 +1428,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video protected void onProcessedStreamChange() { super.onProcessedStreamChange(); lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); - if (videoSinkProvider.isInitialized()) { - videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); - } } /** @@ -1614,8 +1538,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount); } flushOrReinitializeCodec(); - if (videoSink != null) { - videoSink.flush(); + if (videoFrameProcessorManager.isEnabled()) { + videoFrameProcessorManager.flush(); } return true; } @@ -1653,17 +1577,57 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video videoFrameProcessingOffsetCount++; } + /** + * Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output + * color} to configure the {@code VideoFrameProcessor}. + */ + protected Pair experimentalGetVideoFrameProcessorColorConfiguration( + @Nullable ColorInfo inputColorInfo) { + // TODO(b/279163661) Remove this method after VideoFrameProcessor supports texture ID + // input/output. + if (!ColorInfo.isTransferHdr(inputColorInfo)) { + return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED); + } + + if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { + // SurfaceView only supports BT2020 PQ input, converting HLG to PQ. + return Pair.create( + inputColorInfo, + inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build()); + } + + return Pair.create(inputColorInfo, inputColorInfo); + } + /** * Renders the output buffer with the specified index now. * * @param codec The codec that owns the output buffer. + * @param format The {@link Format} associated with the buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. - * @param releaseTimeNs The release timestamp that needs to be associated with this buffer, in - * nanoseconds. + * @param notifyFrameMetadataListener Whether to notify the {@link VideoFrameMetadataListener}. */ - private void renderOutputBuffer( - MediaCodecAdapter codec, int index, long presentationTimeUs, long releaseTimeNs) { + private void renderOutputBufferNow( + MediaCodecAdapter codec, + Format format, + int index, + long presentationTimeUs, + boolean notifyFrameMetadataListener) { + // In previewing mode, use the presentation time as release time so that the SurfaceTexture is + // accompanied by the rendered frame's presentation time. Setting a realtime based release time + // is only relevant when rendering to a SurfaceView (that is when not using VideoFrameProcessor) + // for better frame release. In previewing mode MediaCodec renders to VideoFrameProcessor's + // input surface, which is not a SurfaceView. + long releaseTimeNs = + videoFrameProcessorManager.isEnabled() + ? videoFrameProcessorManager.getCorrectedFramePresentationTimeUs( + presentationTimeUs, getOutputStreamOffsetUs()) + * 1000 + : getClock().nanoTime(); + if (notifyFrameMetadataListener) { + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + } if (Util.SDK_INT >= 21) { renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs); } else { @@ -1675,6 +1639,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video * Renders the output buffer with the specified index. This method is only called if the platform * API version of the device is less than 21. * + *

When video frame processing is {@linkplain VideoFrameProcessorManager#isEnabled()} enabled}, + * this method renders to {@link VideoFrameProcessorManager}'s {@linkplain + * VideoFrameProcessorManager#getInputSurface() input surface}. + * * @param codec The codec that owns the output buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. @@ -1685,7 +1653,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; - if (videoSink == null) { + if (!videoFrameProcessorManager.isEnabled()) { lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1696,6 +1664,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video * Renders the output buffer with the specified index. This method is only called if the platform * API version of the device is 21 or later. * + *

When video frame processing is {@linkplain VideoFrameProcessorManager#isEnabled()} enabled}, + * this method renders to {@link VideoFrameProcessorManager}'s {@linkplain + * VideoFrameProcessorManager#getInputSurface() input surface}. + * * @param codec The codec that owns the output buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. @@ -1709,7 +1681,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; - if (videoSink == null) { + if (!videoFrameProcessorManager.isEnabled()) { lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1754,7 +1726,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video } } - private void maybeNotifyRenderedFirstFrame() { + /* package */ void maybeNotifyRenderedFirstFrame() { if (firstFrameState != C.FIRST_FRAME_RENDERED) { firstFrameState = C.FIRST_FRAME_RENDERED; eventDispatcher.renderedFirstFrame(displaySurface); @@ -1966,6 +1938,504 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer implements Video return new MediaCodecVideoDecoderException(cause, codecInfo, displaySurface); } + /** Manages {@link VideoFrameProcessor} interactions. */ + private static final class VideoFrameProcessorManager { + + /** The threshold for releasing a processed frame. */ + private static final long EARLY_THRESHOLD_US = 50_000; + + private final VideoFrameReleaseHelper frameReleaseHelper; + // TODO(b/238302341) Consider removing the reference to the containing class and make this class + // non-static. + private final MediaCodecVideoRenderer renderer; + private final ArrayDeque processedFramesTimestampsUs; + private final ArrayDeque> pendingFrameFormats; + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + private @MonotonicNonNull Handler handler; + @Nullable private VideoFrameProcessor videoFrameProcessor; + @Nullable private CopyOnWriteArrayList videoEffects; + @Nullable private Format inputFormat; + + /** + * The current frame {@link Format} and the earliest presentationTimeUs that associates to it. + */ + private @MonotonicNonNull Pair currentFrameFormat; + + private @MonotonicNonNull Clock clock; + @Nullable private Pair currentSurfaceAndSize; + + private int videoFrameProcessorMaxPendingFrameCount; + private boolean canEnableFrameProcessing; + + /** + * Whether the last frame of the current stream is decoded and registered to {@link + * VideoFrameProcessor}. + */ + private boolean registeredLastFrame; + + /** + * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}. + */ + private boolean processedLastFrame; + + /** Whether the last frame of the current stream is released to the output {@link Surface}. */ + private boolean releasedLastFrame; + + private long lastCodecBufferPresentationTimestampUs; + private VideoSize processedFrameSize; + private boolean pendingOutputSizeChange; + + /** The presentation time, after which the listener should be notified about the size change. */ + private long pendingOutputSizeChangeNotificationTimeUs; + + private long initialStreamOffsetUs; + + /** Creates a new instance. */ + public VideoFrameProcessorManager( + VideoFrameProcessor.Factory videoFrameProcessorFactory, + VideoFrameReleaseHelper frameReleaseHelper, + @UnderInitialization MediaCodecVideoRenderer renderer) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + this.frameReleaseHelper = frameReleaseHelper; + this.renderer = renderer; + processedFramesTimestampsUs = new ArrayDeque<>(); + pendingFrameFormats = new ArrayDeque<>(); + videoFrameProcessorMaxPendingFrameCount = C.LENGTH_UNSET; + canEnableFrameProcessing = true; + lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; + processedFrameSize = VideoSize.UNKNOWN; + pendingOutputSizeChangeNotificationTimeUs = C.TIME_UNSET; + initialStreamOffsetUs = C.TIME_UNSET; + } + + /** Sets the {@linkplain Effect video effects}. */ + public void setVideoEffects(List videoEffects) { + if (this.videoEffects == null) { + this.videoEffects = new CopyOnWriteArrayList<>(videoEffects); + return; + } + this.videoEffects.clear(); + this.videoEffects.addAll(videoEffects); + } + + /** Returns whether video frame processing is enabled. */ + public boolean isEnabled() { + return videoFrameProcessor != null; + } + + /** Returns whether {@code VideoFrameProcessorManager} is ready to accept input frames. */ + public boolean isReady() { + return currentSurfaceAndSize == null || !currentSurfaceAndSize.second.equals(Size.UNKNOWN); + } + + /** + * Whether the {@link VideoFrameProcessor} has released the last frame in the current stream. + */ + public boolean releasedLastFrame() { + return releasedLastFrame; + } + + /** + * Flushes the {@link VideoFrameProcessor}. + * + *

Caller must ensure video frame processing {@linkplain #isEnabled() is enabled} before + * calling this method. + */ + public void flush() { + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.flush(); + processedFramesTimestampsUs.clear(); + handler.removeCallbacksAndMessages(/* token= */ null); + + if (registeredLastFrame) { + registeredLastFrame = false; + processedLastFrame = false; + releasedLastFrame = false; + } + } + + /** + * Tries to enable video frame processing. + * + *

Caller must ensure video frame processing {@linkplain #isEnabled() is not enabled} before + * calling this method. + * + * @param inputFormat The {@link Format} that is input into the {@link VideoFrameProcessor}. + * @return Whether video frame processing is enabled. + * @throws ExoPlaybackException When enabling the {@link VideoFrameProcessor} failed. + */ + @CanIgnoreReturnValue + public boolean maybeEnable(Format inputFormat, long initialStreamOffsetUs, Clock clock) + throws ExoPlaybackException { + checkState(!isEnabled()); + if (!canEnableFrameProcessing) { + return false; + } + if (videoEffects == null) { + canEnableFrameProcessing = false; + return false; + } + + // Playback thread handler. + handler = Util.createHandlerForCurrentLooper(); + this.clock = clock; + + Pair inputAndOutputColorInfos = + renderer.experimentalGetVideoFrameProcessorColorConfiguration(inputFormat.colorInfo); + try { + // TODO(b/243036513): Set rotation in setInputFormat() after supporting changing effects. + if (!codecAppliesRotation() && inputFormat.rotationDegrees != 0) { + // Insert as the first effect as if the decoder has applied the rotation. + videoEffects.add( + /* index= */ 0, + ScaleAndRotateAccessor.createRotationEffect(inputFormat.rotationDegrees)); + } + + videoFrameProcessor = + videoFrameProcessorFactory.create( + renderer.context, + DebugViewProvider.NONE, + inputAndOutputColorInfos.first, + inputAndOutputColorInfos.second, + /* renderFramesAutomatically= */ false, + /* listenerExecutor= */ handler::post, + new VideoFrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; + checkStateNotNull(inputFormat); + // TODO(b/264889146): Handle Effect that changes output size based on pts. + processedFrameSize = + new VideoSize( + width, + height, + // VideoFrameProcessor is configured to produce rotation free + // frames. + /* unappliedRotationDegrees= */ 0, + // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. + /* pixelWidthHeightRatio= */ 1.f); + pendingOutputSizeChange = true; + } + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (registeredLastFrame) { + checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); + } + processedFramesTimestampsUs.add(presentationTimeUs); + // TODO(b/257464707) Support extensively modified media. + if (registeredLastFrame + && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { + processedLastFrame = true; + } + if (pendingOutputSizeChange) { + // Report the size change on releasing this frame. + pendingOutputSizeChange = false; + pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; + } + } + + @Override + public void onError(VideoFrameProcessingException exception) { + renderer.setPendingPlaybackException( + renderer.createRendererException( + exception, + inputFormat, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } + + @Override + public void onEnded() { + throw new IllegalStateException(); + } + }); + + this.initialStreamOffsetUs = initialStreamOffsetUs; + } catch (Exception e) { + throw renderer.createRendererException( + e, inputFormat, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); + } + + if (currentSurfaceAndSize != null) { + Size outputSurfaceSize = currentSurfaceAndSize.second; + videoFrameProcessor.setOutputSurfaceInfo( + new SurfaceInfo( + currentSurfaceAndSize.first, + outputSurfaceSize.getWidth(), + outputSurfaceSize.getHeight())); + } + + return true; + } + + public long getCorrectedFramePresentationTimeUs( + long framePresentationTimeUs, long currentStreamOffsetUs) { + // VideoFrameProcessor takes in frames with monotonically increasing, non-offset frame + // timestamps. That is, with two ten-second long videos, the first frame of the second video + // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the + // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include + // the duration of the first video. Thus this correction is need to correct for the different + // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. + checkState(initialStreamOffsetUs != C.TIME_UNSET); + return framePresentationTimeUs + currentStreamOffsetUs - initialStreamOffsetUs; + } + + /** + * Returns the {@linkplain VideoFrameProcessor#getInputSurface input surface} of the {@link + * VideoFrameProcessor}. + * + *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before + * calling this method. + */ + public Surface getInputSurface() { + return checkNotNull(videoFrameProcessor).getInputSurface(); + } + + /** + * Sets the output surface info. + * + * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs. + * @param outputResolution The {@link Size} of the output resolution. + */ + public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { + if (currentSurfaceAndSize != null + && currentSurfaceAndSize.first.equals(outputSurface) + && currentSurfaceAndSize.second.equals(outputResolution)) { + return; + } + currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); + if (isEnabled()) { + checkNotNull(videoFrameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo( + outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); + } + } + + /** + * Clears the set output surface info. + * + *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before + * calling this method. + */ + public void clearOutputSurfaceInfo() { + checkNotNull(videoFrameProcessor).setOutputSurfaceInfo(null); + currentSurfaceAndSize = null; + } + + /** + * Sets the input surface info. + * + *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before + * calling this method. + */ + public void setInputFormat(Format inputFormat) { + checkNotNull(videoFrameProcessor) + .registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE, + checkNotNull(videoEffects), + new FrameInfo.Builder(inputFormat.width, inputFormat.height) + .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) + .build()); + this.inputFormat = inputFormat; + + if (registeredLastFrame) { + registeredLastFrame = false; + processedLastFrame = false; + releasedLastFrame = false; + } + } + + /** Sets the necessary {@link MediaFormat} keys for video frame processing. */ + @SuppressWarnings("InlinedApi") + public MediaFormat amendMediaFormatKeys(MediaFormat mediaFormat) { + if (Util.SDK_INT >= 29 + && renderer.context.getApplicationContext().getApplicationInfo().targetSdkVersion >= 29) { + mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); + } + return mediaFormat; + } + + /** + * Must be called when the codec is initialized. + * + *

Sets the {@code videoFrameProcessorMaxPendingFrameCount} based on the {@code codecName}. + */ + public void onCodecInitialized(String codecName) { + videoFrameProcessorMaxPendingFrameCount = + Util.getMaxPendingFramesCountForMediaCodecDecoders(renderer.context); + } + + /** + * Tries to {@linkplain VideoFrameProcessor#registerInputFrame register an input frame}. + * + *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before + * calling this method. + * + * @param format The {@link Format} associated with the frame. + * @param isLastBuffer Whether the buffer is the last from the decoder to register. + * @return Whether {@link MediaCodec} should render the frame to {@link VideoFrameProcessor}. + */ + public boolean maybeRegisterFrame( + Format format, long presentationTimestampUs, boolean isLastBuffer) { + checkStateNotNull(videoFrameProcessor); + checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); + + if (videoFrameProcessor.getPendingInputFrameCount() + < videoFrameProcessorMaxPendingFrameCount) { + videoFrameProcessor.registerInputFrame(); + + if (currentFrameFormat == null) { + currentFrameFormat = Pair.create(presentationTimestampUs, format); + } else if (!Util.areEqual(format, currentFrameFormat.second)) { + // TODO(b/258213806) Remove format comparison for better performance. + pendingFrameFormats.add(Pair.create(presentationTimestampUs, format)); + } + + if (isLastBuffer) { + registeredLastFrame = true; + lastCodecBufferPresentationTimestampUs = presentationTimestampUs; + } + return true; + } + return false; + } + + /** + * Releases the processed frames to the {@linkplain #setOutputSurfaceInfo output surface}. + * + *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before + * calling this method. + */ + public void releaseProcessedFrames(long positionUs, long elapsedRealtimeUs) { + checkStateNotNull(videoFrameProcessor); + while (!processedFramesTimestampsUs.isEmpty()) { + boolean isStarted = renderer.getState() == STATE_STARTED; + long framePresentationTimeUs = checkNotNull(processedFramesTimestampsUs.peek()); + long bufferPresentationTimeUs = framePresentationTimeUs + initialStreamOffsetUs; + long earlyUs = + renderer.calculateEarlyTimeUs( + positionUs, + elapsedRealtimeUs, + msToUs(clock.elapsedRealtime()), + bufferPresentationTimeUs, + isStarted); + + boolean isLastFrame = processedLastFrame && processedFramesTimestampsUs.size() == 1; + boolean shouldReleaseFrameImmediately = renderer.shouldForceRender(positionUs, earlyUs); + if (shouldReleaseFrameImmediately) { + releaseProcessedFrameInternal( + VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY, isLastFrame); + break; + } else if (!isStarted || positionUs == renderer.initialPositionUs) { + return; + } + + // Only release frames that are reasonably close to presentation. + // This way frameReleaseHelper.onNextFrame() is called only once for each frame. + if (earlyUs > EARLY_THRESHOLD_US) { + break; + } + + frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); + long systemNanoTime = checkNotNull(clock).nanoTime(); + long unadjustedFrameReleaseTimeNs = systemNanoTime + earlyUs * 1000; + long adjustedFrameReleaseTimeNs = + frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); + earlyUs = (adjustedFrameReleaseTimeNs - systemNanoTime) / 1000; + + // TODO(b/238302341) Handle very late buffers and drop to key frame. Need to flush + // VideoFrameProcessor input frames in this case. + if (renderer.shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastFrame)) { + releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); + continue; + } + + if (!pendingFrameFormats.isEmpty() + && bufferPresentationTimeUs > pendingFrameFormats.peek().first) { + currentFrameFormat = pendingFrameFormats.remove(); + } + renderer.notifyFrameMetadataListener( + framePresentationTimeUs, adjustedFrameReleaseTimeNs, currentFrameFormat.second); + if (pendingOutputSizeChangeNotificationTimeUs >= bufferPresentationTimeUs) { + pendingOutputSizeChangeNotificationTimeUs = C.TIME_UNSET; + renderer.maybeNotifyVideoSizeChanged(processedFrameSize); + } + releaseProcessedFrameInternal(adjustedFrameReleaseTimeNs, isLastFrame); + } + } + + /** + * Releases the resources. + * + *

Caller must ensure video frame processing {@linkplain #isEnabled() is not enabled} before + * calling this method. + */ + public void reset() { + checkNotNull(videoFrameProcessor).release(); + videoFrameProcessor = null; + if (handler != null) { + handler.removeCallbacksAndMessages(/* token= */ null); + } + if (videoEffects != null) { + videoEffects.clear(); + } + processedFramesTimestampsUs.clear(); + canEnableFrameProcessing = true; + } + + private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { + // VideoFrameProcessor renders to its output surface using + // VideoFrameProcessor.renderOutputFrame, to release the MediaCodecVideoRenderer frame. + checkStateNotNull(videoFrameProcessor); + videoFrameProcessor.renderOutputFrame(releaseTimeNs); + processedFramesTimestampsUs.remove(); + renderer.lastRenderRealtimeUs = msToUs(clock.elapsedRealtime()); + if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) { + renderer.maybeNotifyRenderedFirstFrame(); + } + if (isLastFrame) { + releasedLastFrame = true; + } + } + + private static final class ScaleAndRotateAccessor { + private static @MonotonicNonNull Constructor + scaleAndRotateTransformationBuilderConstructor; + private static @MonotonicNonNull Method setRotationMethod; + private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod; + + public static Effect createRotationEffect(float rotationDegrees) throws Exception { + prepare(); + Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance(); + setRotationMethod.invoke(builder, rotationDegrees); + return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder)); + } + + @EnsuresNonNull({ + "scaleAndRotateTransformationBuilderConstructor", + "setRotationMethod", + "buildScaleAndRotateTransformationMethod" + }) + private static void prepare() throws Exception { + if (scaleAndRotateTransformationBuilderConstructor == null + || setRotationMethod == null + || buildScaleAndRotateTransformationMethod == null) { + // TODO: b/284964524- Add LINT and proguard checks for media3.effect reflection. + Class scaleAndRotateTransformationBuilderClass = + Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder"); + scaleAndRotateTransformationBuilderConstructor = + scaleAndRotateTransformationBuilderClass.getConstructor(); + setRotationMethod = + scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class); + buildScaleAndRotateTransformationMethod = + scaleAndRotateTransformationBuilderClass.getMethod("build"); + } + } + } + } + /** * Delays reflection for loading a {@linkplain VideoFrameProcessor.Factory * DefaultVideoFrameProcessor} instance. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index d25a77feb5..11bbc7447f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -33,7 +33,7 @@ import java.util.concurrent.Executor; /** A sink that consumes decoded video frames. */ @UnstableApi -/* package */ interface VideoSink { +/*package */ interface VideoSink { /** Thrown by {@link VideoSink} implementations. */ final class VideoSinkException extends Exception { @@ -61,46 +61,6 @@ import java.util.concurrent.Executor; void onError(VideoSink videoSink, VideoSinkException videoSinkException); } - /** Controls the rendering of video frames. */ - interface RenderControl { - /** Signals a frame must be rendered immediately. */ - long RENDER_TIME_IMMEDIATELY = -1; - - /** Signals a frame must be dropped. */ - long RENDER_TIME_DROP = -2; - - /** Signals that a frame should not be rendered yet. */ - long RENDER_TIME_TRY_AGAIN_LATER = -3; - - /** - * Returns the render timestamp, in nanoseconds, associated with this video frames or one of the - * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not - * rendered yet. - * - * @param presentationTimeUs The presentation time of the video frame, in microseconds. - * @param positionUs The current playback position, in microseconds. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * taken approximately at the time the playback position was {@code positionUs}. - * @param playbackSpeed The current playback speed. - * @return The render timestamp, in nanoseconds, associated with this frame, or one of the - * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not - * rendered yet. - */ - long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed); - - /** - * Informs the rendering control that a video frame will be rendered. Call this method before - * rendering a frame. - * - * @param presentationTimeUs The frame's presentation time, in microseconds. - */ - void onNextFrame(long presentationTimeUs); - - /** Informs the rendering control that a video frame was rendered. */ - void onFrameRendered(); - } - /** * Specifies how the input frames are made available to the video sink. One of {@link * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java deleted file mode 100644 index b512583775..0000000000 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.exoplayer.video; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import android.content.Context; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Format; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.test.core.app.ApplicationProvider; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.common.collect.ImmutableList; -import java.util.concurrent.Executor; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; - -/** Unit test for {@link CompositingVideoSinkProvider}. */ -@RunWith(AndroidJUnit4.class) -public final class CompositingVideoSinkProviderTest { - - @Test - public void initialize() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - - provider.initialize(new Format.Builder().build()); - - assertThat(provider.isInitialized()).isTrue(); - } - - @Test - public void initialize_withoutEffects_throws() { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - - assertThrows( - IllegalStateException.class, - () -> provider.initialize(new Format.Builder().setWidth(640).setHeight(480).build())); - } - - @Test - public void initialize_calledTwice_throws() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - provider.initialize(new Format.Builder().build()); - - assertThrows( - IllegalStateException.class, () -> provider.initialize(new Format.Builder().build())); - } - - @Test - public void initialize_afterRelease_throws() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - Format format = new Format.Builder().build(); - - provider.initialize(format); - provider.release(); - - assertThrows(IllegalStateException.class, () -> provider.initialize(format)); - } - - @Test - public void registerInputStream_withInputTypeBitmap_throws() throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - provider.initialize(new Format.Builder().build()); - VideoSink videoSink = provider.getSink(); - - assertThrows( - UnsupportedOperationException.class, - () -> - videoSink.registerInputStream( - VideoSink.INPUT_TYPE_BITMAP, new Format.Builder().build())); - } - - @Test - public void setOutputStreamOffsetUs_frameReleaseTimesAreAdjusted() - throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - provider.initialize(new Format.Builder().build()); - VideoSink videoSink = provider.getSink(); - videoSink.registerInputStream( - VideoSink.INPUT_TYPE_SURFACE, new Format.Builder().setWidth(640).setHeight(480).build()); - - assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)).isEqualTo(0); - provider.setStreamOffsetUs(1_000); - assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)) - .isEqualTo(1_000_000); - provider.setStreamOffsetUs(2_000); - assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)) - .isEqualTo(2_000_000); - } - - @Test - public void setListener_calledTwiceWithDifferentExecutor_throws() - throws VideoSink.VideoSinkException { - CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); - provider.setVideoEffects(ImmutableList.of()); - provider.initialize(new Format.Builder().build()); - VideoSink videoSink = provider.getSink(); - VideoSink.Listener listener = Mockito.mock(VideoSink.Listener.class); - - videoSink.setListener(listener, /* executor= */ command -> {}); - - assertThrows( - IllegalStateException.class, - () -> videoSink.setListener(listener, /* executor= */ command -> {})); - } - - private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() { - VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory(); - VideoSink.RenderControl renderControl = new TestRenderControl(); - return new CompositingVideoSinkProvider( - ApplicationProvider.getApplicationContext(), factory, renderControl); - } - - private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory { - // Using a mock but we don't assert mock interactions. If needed to assert interactions, we - // should a fake instead. - private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class); - - @Override - public VideoFrameProcessor create( - Context context, - DebugViewProvider debugViewProvider, - ColorInfo inputColorInfo, - ColorInfo outputColorInfo, - boolean renderFramesAutomatically, - Executor listenerExecutor, - VideoFrameProcessor.Listener listener) - throws VideoFrameProcessingException { - return videoFrameProcessor; - } - } - - private static class TestRenderControl implements VideoSink.RenderControl { - - @Override - public long getFrameRenderTimeNs( - long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { - return presentationTimeUs; - } - - @Override - public void onNextFrame(long presentationTimeUs) {} - - @Override - public void onFrameRendered() {} - } -}