diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java index 3bf88db12f..7d532c1092 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.graphics.Bitmap; import android.view.Surface; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.util.Size; @@ -49,6 +50,7 @@ import java.util.concurrent.Executor; @Nullable private Surface outputSurface; private Format inputFormat; + private long streamStartPositionUs; public DefaultVideoSink( VideoFrameReleaseControl videoFrameReleaseControl, @@ -56,6 +58,7 @@ import java.util.concurrent.Executor; this.videoFrameReleaseControl = videoFrameReleaseControl; this.videoFrameRenderControl = videoFrameRenderControl; inputFormat = new Format.Builder().build(); + streamStartPositionUs = C.TIME_UNSET; } @Override @@ -149,7 +152,10 @@ import java.util.concurrent.Executor; @Override public void setStreamTimestampInfo( long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) { - throw new UnsupportedOperationException(); + if (streamStartPositionUs != this.streamStartPositionUs) { + videoFrameRenderControl.onStreamStartPositionChanged(streamStartPositionUs); + this.streamStartPositionUs = streamStartPositionUs; + } } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 7994b62ccf..f3510c3836 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -46,6 +46,7 @@ import androidx.media3.common.VideoSize; import androidx.media3.common.util.Clock; import androidx.media3.common.util.HandlerWrapper; import androidx.media3.common.util.Size; +import androidx.media3.common.util.TimedValueQueue; import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -228,6 +229,13 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private final Context context; private final InputVideoSink inputVideoSink; + + /** + * A queue of unprocessed input frame start positions. Each position is associated with the + * timestamp from which it should be applied. + */ + private final TimedValueQueue streamStartPositionsUs; + private final VideoFrameReleaseControl videoFrameReleaseControl; private final VideoFrameRenderControl videoFrameRenderControl; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; @@ -240,6 +248,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; private @MonotonicNonNull HandlerWrapper handler; private @MonotonicNonNull PreviewingVideoGraph videoGraph; + private long outputStreamStartPositionUs; @Nullable private Pair currentSurfaceAndSize; private int pendingFlushCount; private @State int state; @@ -254,6 +263,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private PlaybackVideoGraphWrapper(Builder builder) { context = builder.context; inputVideoSink = new InputVideoSink(context); + streamStartPositionsUs = new TimedValueQueue<>(); clock = builder.clock; videoFrameReleaseControl = builder.videoFrameReleaseControl; videoFrameReleaseControl.setClock(clock); @@ -355,8 +365,16 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } // The frame presentation time is relative to the start of the Composition and without the // renderer offset - videoFrameRenderControl.onFrameAvailableForRendering( - framePresentationTimeUs - bufferTimestampAdjustmentUs); + long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs; + Long newOutputStreamStartPositionUs = + streamStartPositionsUs.pollFloor(bufferPresentationTimeUs); + if (newOutputStreamStartPositionUs != null + && newOutputStreamStartPositionUs != outputStreamStartPositionUs) { + defaultVideoSink.setStreamTimestampInfo( + newOutputStreamStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET); + outputStreamStartPositionUs = newOutputStreamStartPositionUs; + } + videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs); } @Override @@ -454,6 +472,15 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video } pendingFlushCount++; defaultVideoSink.flush(resetPosition); + while (streamStartPositionsUs.size() > 1) { + streamStartPositionsUs.pollFirst(); + } + if (streamStartPositionsUs.size() == 1) { + long lastStartPositionUs = checkNotNull(streamStartPositionsUs.pollFirst()); + // defaultVideoSink should use the latest startPositionUs if none is passed after flushing. + defaultVideoSink.setStreamTimestampInfo( + lastStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET); + } // Handle pending video graph callbacks to ensure video size changes reach the video render // control. checkStateNotNull(handler).post(() -> pendingFlushCount--); @@ -472,12 +499,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; } - private void onStreamStartPositionChange( - long bufferPresentationTimeUs, long streamStartPositionUs) { - videoFrameRenderControl.onStreamStartPositionChanged( - bufferPresentationTimeUs, streamStartPositionUs); - } - private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) { if (inputColorInfo == null || !inputColorInfo.isDataSpaceValid()) { return ColorInfo.SDR_BT709_LIMITED; @@ -499,7 +520,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private long inputStreamStartPositionUs; private long inputBufferTimestampAdjustmentUs; private long lastResetPositionUs; - private boolean pendingInputStartPositionChange; /** The buffer presentation time, in microseconds, of the final frame in the stream. */ private long finalBufferPresentationTimeUs; @@ -662,9 +682,13 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public void setStreamTimestampInfo( long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) { - // Ors because this method could be called multiple times on a timestamp info change. - pendingInputStartPositionChange |= inputStreamStartPositionUs != streamStartPositionUs; inputStreamStartPositionUs = streamStartPositionUs; + // Input timestamps should always be positive because they are offset by ExoPlayer. Adding a + // position to the queue with timestamp 0 should therefore always apply it as long as it is + // the only position in the queue. + streamStartPositionsUs.add( + lastBufferPresentationTimeUs == C.TIME_UNSET ? 0 : lastBufferPresentationTimeUs + 1, + streamStartPositionUs); inputBufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; // The buffer timestamp adjustment is only allowed to change after a flush to make sure that // the buffer timestamps are increasing. We can update the buffer timestamp adjustment @@ -767,7 +791,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video return false; } - maybeSetStreamStartPosition(bufferPresentationTimeUs); lastBufferPresentationTimeUs = bufferPresentationTimeUs; if (isLastFrame) { finalBufferPresentationTimeUs = bufferPresentationTimeUs; @@ -792,15 +815,10 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video return false; } - // Create a copy of iterator because we need to take the next timestamp but we must not alter - // the state of the iterator. - TimestampIterator copyTimestampIterator = timestampIterator.copyOf(); - long bufferPresentationTimeUs = copyTimestampIterator.next(); // TimestampIterator generates frame time. long lastBufferPresentationTimeUs = - copyTimestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs; + timestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs; checkState(lastBufferPresentationTimeUs != C.TIME_UNSET); - maybeSetStreamStartPosition(bufferPresentationTimeUs); this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs; finalBufferPresentationTimeUs = lastBufferPresentationTimeUs; return true; @@ -823,14 +841,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video // Other methods - private void maybeSetStreamStartPosition(long bufferPresentationTimeUs) { - if (pendingInputStartPositionChange) { - PlaybackVideoGraphWrapper.this.onStreamStartPositionChange( - bufferPresentationTimeUs, inputStreamStartPositionUs); - pendingInputStartPositionChange = false; - } - } - /** * Attempt to register any pending input stream to the video graph input and returns {@code * true} if a pending stream was registered and/or there is no pending input stream waiting for diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java index e759f71b37..ae0e8ce732 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -68,7 +68,13 @@ import androidx.media3.exoplayer.ExoPlaybackException; */ private final TimedValueQueue videoSizes; + /** + * A queue of unprocessed input frame start positions. Each position is associated with the + * timestamp from which it should be applied. + */ private final TimedValueQueue streamStartPositionsUs; + + /** A queue of unprocessed input frame timestamps. */ private final LongArrayQueue presentationTimestampsUs; private long lastInputPresentationTimeUs; @@ -180,6 +186,12 @@ import androidx.media3.exoplayer.ExoPlaybackException; new VideoSize(width, height)); } + public void onStreamStartPositionChanged(long streamStartPositionUs) { + streamStartPositionsUs.add( + lastInputPresentationTimeUs == C.TIME_UNSET ? 0 : lastInputPresentationTimeUs + 1, + streamStartPositionUs); + } + /** * Called when a frame is available for rendering. * @@ -191,10 +203,6 @@ import androidx.media3.exoplayer.ExoPlaybackException; // TODO b/257464707 - Support extensively modified media. } - public void onStreamStartPositionChanged(long presentationTimeUs, long streamStartPositionUs) { - streamStartPositionsUs.add(presentationTimeUs, streamStartPositionUs); - } - private void dropFrame() { presentationTimestampsUs.remove(); frameRenderer.dropFrame(); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java index 07857b6b78..7ab727c642 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java @@ -121,8 +121,7 @@ public class VideoFrameRenderControlTest { videoFrameReleaseControl.onStarted(); videoFrameRenderControl.onVideoSizeChanged( /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); - videoFrameRenderControl.onStreamStartPositionChanged( - /* presentationTimeUs= */ 0, /* streamStartPositionUs= */ 10_000); + videoFrameRenderControl.onStreamStartPositionChanged(/* streamStartPositionUs= */ 10_000); videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0); videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); @@ -136,8 +135,7 @@ public class VideoFrameRenderControlTest { // 10 milliseconds pass clock.advanceTime(/* timeDiffMs= */ 10); - videoFrameRenderControl.onStreamStartPositionChanged( - /* presentationTimeUs= */ 10_000, /* streamStartPositionUs= */ 20_000); + videoFrameRenderControl.onStreamStartPositionChanged(/* streamStartPositionUs= */ 20_000); videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000); videoFrameRenderControl.render(/* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0);