diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java index 4dbb5ef80c..48f513512c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java @@ -23,12 +23,18 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.Size; import androidx.media3.common.util.TimestampIterator; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.Renderer; +import java.util.ArrayDeque; import java.util.List; +import java.util.Queue; import java.util.concurrent.Executor; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * The default {@link VideoSink} implementation. This implementation renders video frames to an @@ -39,7 +45,7 @@ import java.util.concurrent.Executor; * * *

The {@linkplain #getInputSurface() input} and {@linkplain #setOutputSurfaceInfo(Surface, Size) @@ -48,19 +54,30 @@ import java.util.concurrent.Executor; /* package */ final class DefaultVideoSink implements VideoSink { private final VideoFrameReleaseControl videoFrameReleaseControl; + private final Clock clock; private final VideoFrameRenderControl videoFrameRenderControl; + private final Queue videoFrameHandlers; @Nullable private Surface outputSurface; private Format inputFormat; private long streamStartPositionUs; + private long bufferTimestampAdjustmentUs; + private Listener listener; + private Executor listenerExecutor; + private VideoFrameMetadataListener videoFrameMetadataListener; - public DefaultVideoSink( - VideoFrameReleaseControl videoFrameReleaseControl, - VideoFrameRenderControl videoFrameRenderControl) { + public DefaultVideoSink(VideoFrameReleaseControl videoFrameReleaseControl, Clock clock) { this.videoFrameReleaseControl = videoFrameReleaseControl; - this.videoFrameRenderControl = videoFrameRenderControl; + videoFrameReleaseControl.setClock(clock); + this.clock = clock; + videoFrameRenderControl = + new VideoFrameRenderControl(new FrameRendererImpl(), videoFrameReleaseControl); + videoFrameHandlers = new ArrayDeque<>(); inputFormat = new Format.Builder().build(); streamStartPositionUs = C.TIME_UNSET; + listener = Listener.NO_OP; + listenerExecutor = runnable -> {}; + videoFrameMetadataListener = (presentationTimeUs, releaseTimeNs, format, mediaFormat) -> {}; } @Override @@ -85,7 +102,8 @@ import java.util.concurrent.Executor; @Override public void setListener(Listener listener, Executor executor) { - throw new UnsupportedOperationException(); + this.listener = listener; + this.listenerExecutor = executor; } @Override @@ -104,6 +122,7 @@ import java.util.concurrent.Executor; videoFrameReleaseControl.reset(); } videoFrameRenderControl.flush(); + videoFrameHandlers.clear(); } @Override @@ -133,7 +152,7 @@ import java.util.concurrent.Executor; @Override public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) { - throw new UnsupportedOperationException(); + this.videoFrameMetadataListener = videoFrameMetadataListener; } @Override @@ -168,6 +187,7 @@ import java.util.concurrent.Executor; videoFrameRenderControl.onStreamStartPositionChanged(streamStartPositionUs); this.streamStartPositionUs = streamStartPositionUs; } + this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; } @Override @@ -206,7 +226,10 @@ import java.util.concurrent.Executor; @Override public boolean handleInputFrame( long framePresentationTimeUs, boolean isLastFrame, VideoFrameHandler videoFrameHandler) { - throw new UnsupportedOperationException(); + videoFrameHandlers.add(videoFrameHandler); + long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; + videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs); + return true; } /** @@ -245,4 +268,43 @@ import java.util.concurrent.Executor; @Override public void release() {} + + private final class FrameRendererImpl implements VideoFrameRenderControl.FrameRenderer { + + private @MonotonicNonNull Format outputFormat; + + @Override + public void onVideoSizeChanged(VideoSize videoSize) { + outputFormat = + new Format.Builder() + .setWidth(videoSize.width) + .setHeight(videoSize.height) + .setSampleMimeType(MimeTypes.VIDEO_RAW) + .build(); + listenerExecutor.execute(() -> listener.onVideoSizeChanged(DefaultVideoSink.this, videoSize)); + } + + @Override + public void renderFrame( + long renderTimeNs, long bufferPresentationTimeUs, boolean isFirstFrame) { + if (isFirstFrame && outputSurface != null) { + listenerExecutor.execute(() -> listener.onFirstFrameRendered(DefaultVideoSink.this)); + } + // TODO - b/292111083: outputFormat is initialized after the first frame is rendered because + // onVideoSizeChanged is announced after the first frame is available for rendering. + Format format = outputFormat == null ? new Format.Builder().build() : outputFormat; + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + /* presentationTimeUs= */ bufferPresentationTimeUs, + /* releaseTimeNs= */ clock.nanoTime(), + format, + /* mediaFormat= */ null); + videoFrameHandlers.remove().render(renderTimeNs); + } + + @Override + public void dropFrame() { + listenerExecutor.execute(() -> listener.onFrameDropped(DefaultVideoSink.this)); + videoFrameHandlers.remove().skip(); + } + } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 23ba850017..7df19ff8b2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -15,6 +15,7 @@ */ package androidx.media3.exoplayer.video; +import static androidx.media3.common.VideoFrameProcessor.DROP_OUTPUT_FRAME; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -36,7 +37,6 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.MimeTypes; import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -235,15 +235,14 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video */ private final TimedValueQueue streamStartPositionsUs; - private final VideoFrameRenderControl videoFrameRenderControl; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; private final List compositionEffects; private final VideoSink defaultVideoSink; + private final VideoSink.VideoFrameHandler videoFrameHandler; private final Clock clock; private final CopyOnWriteArraySet listeners; private Format videoGraphOutputFormat; - private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; private @MonotonicNonNull HandlerWrapper handler; private @MonotonicNonNull PreviewingVideoGraph videoGraph; private long outputStreamStartPositionUs; @@ -268,18 +267,26 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video context = builder.context; inputVideoSink = new InputVideoSink(context); streamStartPositionsUs = new TimedValueQueue<>(); - clock = builder.clock; - VideoFrameReleaseControl videoFrameReleaseControl = builder.videoFrameReleaseControl; - videoFrameReleaseControl.setClock(clock); - videoFrameRenderControl = - new VideoFrameRenderControl(new FrameRendererImpl(), videoFrameReleaseControl); previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory); compositionEffects = builder.compositionEffects; - defaultVideoSink = new DefaultVideoSink(videoFrameReleaseControl, videoFrameRenderControl); + clock = builder.clock; + defaultVideoSink = new DefaultVideoSink(builder.videoFrameReleaseControl, clock); + videoFrameHandler = + new VideoSink.VideoFrameHandler() { + @Override + public void render(long renderTimestampNs) { + checkStateNotNull(videoGraph).renderOutputFrame(renderTimestampNs); + } + + @Override + public void skip() { + checkStateNotNull(videoGraph).renderOutputFrame(DROP_OUTPUT_FRAME); + } + }; listeners = new CopyOnWriteArraySet<>(); - state = STATE_CREATED; + listeners.add(inputVideoSink); videoGraphOutputFormat = new Format.Builder().build(); - addListener(inputVideoSink); + state = STATE_CREATED; finalBufferPresentationTimeUs = C.TIME_UNSET; } @@ -334,11 +341,9 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video if (state == STATE_RELEASED) { return; } - if (handler != null) { handler.removeCallbacksAndMessages(/* token= */ null); } - if (videoGraph != null) { videoGraph.release(); } @@ -380,12 +385,14 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video if (newOutputStreamStartPositionUs != null && newOutputStreamStartPositionUs != outputStreamStartPositionUs) { defaultVideoSink.setStreamTimestampInfo( - newOutputStreamStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET); + newOutputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET); outputStreamStartPositionUs = newOutputStreamStartPositionUs; } - videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs); - if (finalBufferPresentationTimeUs != C.TIME_UNSET - && bufferPresentationTimeUs >= finalBufferPresentationTimeUs) { + boolean isLastFrame = + finalBufferPresentationTimeUs != C.TIME_UNSET + && bufferPresentationTimeUs >= finalBufferPresentationTimeUs; + defaultVideoSink.handleInputFrame(framePresentationTimeUs, isLastFrame, videoFrameHandler); + if (isLastFrame) { // TODO b/257464707 - Support extensively modified media. defaultVideoSink.signalEndOfCurrentInputStream(); hasSignaledEndOfCurrentInputStream = true; @@ -438,6 +445,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } + defaultVideoSink.setListener(new DefaultVideoSinkListener(), /* executor= */ handler::post); defaultVideoSink.initialize(sourceFormat); state = STATE_INITIALIZED; return videoGraph.getProcessor(/* inputIndex= */ 0); @@ -496,7 +504,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video long lastStartPositionUs = checkNotNull(streamStartPositionsUs.pollFirst()); // defaultVideoSink should use the latest startPositionUs if none is passed after flushing. defaultVideoSink.setStreamTimestampInfo( - lastStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET); + lastStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET); } finalBufferPresentationTimeUs = C.TIME_UNSET; hasSignaledEndOfCurrentInputStream = false; @@ -507,7 +515,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private void setVideoFrameMetadataListener( VideoFrameMetadataListener videoFrameMetadataListener) { - this.videoFrameMetadataListener = videoFrameMetadataListener; + defaultVideoSink.setVideoFrameMetadataListener(videoFrameMetadataListener); } private void setPlaybackSpeed(float speed) { @@ -516,6 +524,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private void setBufferTimestampAdjustment(long bufferTimestampAdjustmentUs) { this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; + defaultVideoSink.setStreamTimestampInfo( + outputStreamStartPositionUs, bufferTimestampAdjustmentUs, /* unused */ C.TIME_UNSET); } private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) { @@ -854,51 +864,35 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } } - private final class FrameRendererImpl implements VideoFrameRenderControl.FrameRenderer { - - private @MonotonicNonNull Format renderedFormat; + private final class DefaultVideoSinkListener implements VideoSink.Listener { @Override - public void onVideoSizeChanged(VideoSize videoSize) { - renderedFormat = - new Format.Builder() - .setWidth(videoSize.width) - .setHeight(videoSize.height) - .setSampleMimeType(MimeTypes.VIDEO_RAW) - .build(); + public void onFirstFrameRendered(VideoSink videoSink) { + for (PlaybackVideoGraphWrapper.Listener listener : listeners) { + listener.onFirstFrameRendered(PlaybackVideoGraphWrapper.this); + } + } + + @Override + public void onFrameDropped(VideoSink videoSink) { + for (PlaybackVideoGraphWrapper.Listener listener : listeners) { + listener.onFrameDropped(PlaybackVideoGraphWrapper.this); + } + } + + @Override + public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) { for (PlaybackVideoGraphWrapper.Listener listener : listeners) { listener.onVideoSizeChanged(PlaybackVideoGraphWrapper.this, videoSize); } } @Override - public void renderFrame( - long renderTimeNs, long bufferPresentationTimeUs, boolean isFirstFrame) { - if (isFirstFrame && currentSurfaceAndSize != null) { - for (PlaybackVideoGraphWrapper.Listener listener : listeners) { - listener.onFirstFrameRendered(PlaybackVideoGraphWrapper.this); - } - } - if (videoFrameMetadataListener != null) { - // TODO b/292111083 - renderedFormat is initialized after the first frame is rendered - // because onVideoSizeChanged is announced after the first frame is available for - // rendering. - Format format = renderedFormat == null ? new Format.Builder().build() : renderedFormat; - videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - /* presentationTimeUs= */ bufferPresentationTimeUs, - clock.nanoTime(), - format, - /* mediaFormat= */ null); - } - checkStateNotNull(videoGraph).renderOutputFrame(renderTimeNs); - } - - @Override - public void dropFrame() { + public void onError(VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) { for (PlaybackVideoGraphWrapper.Listener listener : listeners) { - listener.onFrameDropped(PlaybackVideoGraphWrapper.this); + listener.onError( + PlaybackVideoGraphWrapper.this, VideoFrameProcessingException.from(videoSinkException)); } - checkStateNotNull(videoGraph).renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME); } }