diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 65081b0dfd..ae9d7f71af 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -488,13 +488,16 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi private final Context context; private final int videoFrameProcessorMaxPendingFrameCount; private final ArrayList videoEffects; - @Nullable private Effect rotationEffect; + private final VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo; private @MonotonicNonNull VideoFrameProcessor videoFrameProcessor; + @Nullable private Effect rotationEffect; @Nullable private Format inputFormat; private @InputType int inputType; + private long inputStreamStartPositionUs; private long inputStreamOffsetUs; private long inputBufferTimestampAdjustmentUs; + private long lastResetPositionUs; private boolean pendingInputStreamOffsetChange; /** The buffer presentation time, in microseconds, of the final frame in the stream. */ @@ -513,14 +516,13 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi /** Creates a new instance. */ public VideoSinkImpl(Context context) { this.context = context; - // TODO b/226330223 - Investigate increasing frame count when frame dropping is - // allowed. + // TODO b/226330223 - Investigate increasing frame count when frame dropping is allowed. // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - videoEffects = new ArrayList<>(); + frameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); finalBufferPresentationTimeUs = C.TIME_UNSET; lastBufferPresentationTimeUs = C.TIME_UNSET; listener = VideoSink.Listener.NO_OP; @@ -679,14 +681,19 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi } @Override - public void setStreamOffsetAndAdjustmentUs( - long streamOffsetUs, long bufferTimestampAdjustmentUs) { + public void setStreamTimestampInfo( + long streamStartPositionUs, + long streamOffsetUs, + long bufferTimestampAdjustmentUs, + long lastResetPositionUs) { // Ors because this method could be called multiple times on a stream offset change. pendingInputStreamOffsetChange |= inputStreamOffsetUs != streamOffsetUs || inputBufferTimestampAdjustmentUs != bufferTimestampAdjustmentUs; + inputStreamStartPositionUs = streamStartPositionUs; inputStreamOffsetUs = streamOffsetUs; inputBufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs; + this.lastResetPositionUs = lastResetPositionUs; } @Override @@ -711,9 +718,55 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi } @Override - public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) { + public boolean handleInputFrame( + long framePresentationTimeUs, + boolean isLastFrame, + long positionUs, + long elapsedRealtimeUs, + VideoFrameHandler videoFrameHandler) + throws VideoSinkException { checkState(isInitialized()); - checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); + + // The sink takes in frames with monotonically increasing, non-offset frame + // timestamps. That is, with two ten-second long videos, the first frame of the second video + // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the + // timestamp of the said frame would be 0s, but the streamOffset is incremented by 10s to + // include the duration of the first video. Thus this correction is needed to account for the + // different handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. + // + // inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to + // the start of a composition) to the buffer timestamp (that corresponds to the player + // position). + long bufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs; + // The frame release action should be retrieved for all frames (even the ones that will be + // skipped), because the release control estimates the content frame rate from frame + // timestamps and we want to have this information known as early as possible, especially + // during seeking. + @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction; + try { + frameReleaseAction = + videoFrameReleaseControl.getFrameReleaseAction( + bufferPresentationTimeUs, + positionUs, + elapsedRealtimeUs, + inputStreamStartPositionUs, + isLastFrame, + frameReleaseInfo); + } catch (ExoPlaybackException e) { + throw new VideoSinkException(e, checkStateNotNull(inputFormat)); + } + if (frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IGNORE) { + // The buffer is no longer valid and needs to be ignored. + return false; + } + + if (bufferPresentationTimeUs < lastResetPositionUs && !isLastFrame) { + videoFrameHandler.skip(); + return true; + } + + // Drain the sink to make room for a new input frame. + render(positionUs, elapsedRealtimeUs); // An input stream is fully decoded, wait until all of its frames are released before queueing // input frame from the next input stream. @@ -723,34 +776,27 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi maybeRegisterInputStream(); pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET; } else { - return C.TIME_UNSET; + return false; } } - if (checkStateNotNull(videoFrameProcessor).getPendingInputFrameCount() >= videoFrameProcessorMaxPendingFrameCount) { - return C.TIME_UNSET; + return false; } if (!checkStateNotNull(videoFrameProcessor).registerInputFrame()) { - return C.TIME_UNSET; + return false; } - // The sink takes in frames with monotonically increasing, non-offset frame - // timestamps. That is, with two ten-second long videos, the first frame of the second video - // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the - // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include - // the duration of the first video. Thus this correction is need to correct for the different - // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. - // - // inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to - // the start of a composition, to the buffer timestamp that is offset, and correspond to the - // player position). - long bufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs; + maybeSetStreamOffsetChange(bufferPresentationTimeUs); lastBufferPresentationTimeUs = bufferPresentationTimeUs; if (isLastFrame) { finalBufferPresentationTimeUs = bufferPresentationTimeUs; } - return framePresentationTimeUs * 1000; + // Use the frame presentation time as render time so that the SurfaceTexture is accompanied + // by this timestamp. Setting a realtime based release time is only relevant when rendering to + // a SurfaceView, but we render to a surface in this case. + videoFrameHandler.render(/* renderTimestampNs= */ framePresentationTimeUs * 1000); + return true; } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index e4a0cd27fd..cdf35f86f7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -737,8 +737,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer // Flush the video sink first to ensure it stops reading textures that will be owned by // MediaCodec once the codec is flushed. videoSink.flush(/* resetPosition= */ true); - videoSink.setStreamOffsetAndAdjustmentUs( - getOutputStreamOffsetUs(), getBufferTimestampAdjustmentUs()); + videoSink.setStreamTimestampInfo( + getOutputStreamStartPositionUs(), + getOutputStreamOffsetUs(), + getBufferTimestampAdjustmentUs(), + getLastResetPositionUs()); videoSinkNeedsRegisterInputStream = true; } super.onPositionReset(positionUs, joining); @@ -1404,6 +1407,34 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer long outputStreamOffsetUs = getOutputStreamOffsetUs(); long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; + if (videoSink != null) { + long framePresentationTimeUs = bufferPresentationTimeUs + getBufferTimestampAdjustmentUs(); + try { + return videoSink.handleInputFrame( + framePresentationTimeUs, + isLastBuffer, + positionUs, + elapsedRealtimeUs, + new VideoSink.VideoFrameHandler() { + @Override + public void render(long renderTimestampNs) { + renderOutputBuffer(codec, bufferIndex, presentationTimeUs, renderTimestampNs); + } + + @Override + public void skip() { + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); + } + }); + } catch (VideoSink.VideoSinkException e) { + throw createRendererException( + e, e.format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED); + } + } + + // The frame release action should be retrieved for all frames (even the ones that will be + // skipped), because the release control estimates the content frame rate from frame timestamps + // and we want to have this information known as early as possible, especially during seeking. @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction = videoFrameReleaseControl.getFrameReleaseAction( @@ -1419,18 +1450,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer return false; } - // Skip decode-only buffers, e.g. after seeking, immediately. This check must be performed after - // getting the release action from the video frame release control although not necessary. - // That's because the release control estimates the content frame rate from frame timestamps - // and we want to have this information known as early as possible, especially during seeking. + // Skip decode-only buffers, e.g. after seeking, immediately. if (isDecodeOnlyBuffer && !isLastBuffer) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } // We are not rendering on a surface, the renderer will wait until a surface is set. - // Opportunistically render to VideoSink if it is enabled. - if (displaySurface == placeholderSurface && videoSink == null) { + if (displaySurface == placeholderSurface) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. if (videoFrameReleaseInfo.getEarlyUs() < 30_000) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); @@ -1440,24 +1467,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer return false; } - if (videoSink != null) { - try { - videoSink.render(positionUs, elapsedRealtimeUs); - } catch (VideoSink.VideoSinkException e) { - throw createRendererException( - e, e.format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED); - } - - long releaseTimeNs = - videoSink.registerInputFrame( - bufferPresentationTimeUs + getBufferTimestampAdjustmentUs(), isLastBuffer); - if (releaseTimeNs == C.TIME_UNSET) { - return false; - } - renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); - return true; - } - switch (frameReleaseAction) { case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: long releaseTimeNs = getClock().nanoTime(); @@ -1567,8 +1576,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer protected void onProcessedStreamChange() { super.onProcessedStreamChange(); if (videoSink != null) { - videoSink.setStreamOffsetAndAdjustmentUs( - getOutputStreamOffsetUs(), getBufferTimestampAdjustmentUs()); + videoSink.setStreamTimestampInfo( + getOutputStreamStartPositionUs(), + getOutputStreamOffsetUs(), + getBufferTimestampAdjustmentUs(), + getLastResetPositionUs()); } else { videoFrameReleaseControl.onProcessedStreamChange(); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 2f555c415d..0cd3f6d7b7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -18,6 +18,7 @@ package androidx.media3.exoplayer.video; import static java.lang.annotation.ElementType.TYPE_USE; import android.graphics.Bitmap; +import android.os.SystemClock; import android.view.Surface; import androidx.annotation.FloatRange; import androidx.annotation.IntDef; @@ -94,6 +95,21 @@ public interface VideoSink { }; } + /** Handler for a video frame. */ + interface VideoFrameHandler { + + /** + * Renders the frame on the {@linkplain #getInputSurface() input surface}. + * + * @param renderTimestampNs The timestamp to associate with this frame when it is sent to the + * surface. + */ + void render(long renderTimestampNs); + + /** Skips the frame. */ + void skip(); + } + /** * Specifies how the input frames are made available to the video sink. One of {@link * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}. @@ -193,14 +209,21 @@ public interface VideoSink { void setPendingVideoEffects(List videoEffects); /** - * Sets the stream offset and buffer time adjustment. + * Sets information about the timestamps of the current input stream. * + * @param streamStartPositionUs The start position of the buffer presentation timestamps of the + * current stream, in microseconds. * @param streamOffsetUs The offset that is added to the buffer presentation timestamps by the * player, in microseconds. * @param bufferTimestampAdjustmentUs The timestamp adjustment to add to the buffer presentation * timestamps to convert them to frame presentation timestamps, in microseconds. + * @param lastResetPositionUs The renderer last reset position, in microseconds. */ - void setStreamOffsetAndAdjustmentUs(long streamOffsetUs, long bufferTimestampAdjustmentUs); + void setStreamTimestampInfo( + long streamStartPositionUs, + long streamOffsetUs, + long bufferTimestampAdjustmentUs, + long lastResetPositionUs); /** Sets the output surface info. */ void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution); @@ -236,19 +259,27 @@ public interface VideoSink { void registerInputStream(@InputType int inputType, Format format); /** - * Informs the video sink that a frame will be queued to its {@linkplain #getInputSurface() input - * surface}. + * Handles a video input frame. * *

Must be called after the corresponding stream is {@linkplain #registerInputStream(int, * Format) registered}. * * @param framePresentationTimeUs The frame's presentation time, in microseconds. * @param isLastFrame Whether this is the last frame of the video stream. - * @return A release timestamp, in nanoseconds, that should be associated when releasing this - * frame, or {@link C#TIME_UNSET} if the sink was not able to register the frame and the - * caller must try again later. + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, taken + * approximately at the time the playback position was {@code positionUs}. + * @param videoFrameHandler The {@link VideoFrameHandler} used to handle the input frame. + * @return Whether the frame was handled successfully. If {@code false}, the caller can try again + * later. */ - long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame); + boolean handleInputFrame( + long framePresentationTimeUs, + boolean isLastFrame, + long positionUs, + long elapsedRealtimeUs, + VideoFrameHandler videoFrameHandler) + throws VideoSinkException; /** * Provides an input {@link Bitmap} to the video sink. @@ -259,8 +290,8 @@ public interface VideoSink { * @param inputBitmap The {@link Bitmap} queued to the video sink. * @param timestampIterator The times within the current stream that the bitmap should be shown * at. The timestamps should be monotonically increasing. - * @return Whether the bitmap was queued successfully. A {@code false} value indicates the caller - * must try again later. + * @return Whether the bitmap was queued successfully. If {@code false}, the caller can try again + * later. */ boolean queueBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java index 0e1f7ed36b..a3ccfb94a0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java @@ -302,6 +302,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private @MonotonicNonNull EditedMediaItem editedMediaItem; @Nullable private ExoPlaybackException pendingExoPlaybackException; private boolean inputStreamPendingRegistration; + private long streamStartPositionUs; private long streamOffsetUs; private boolean mayRenderStartOfStream; private long offsetToCompositionTimeUs; @@ -314,6 +315,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider); videoSink = compositingVideoSinkProvider.getSink(); videoEffects = ImmutableList.of(); + streamStartPositionUs = C.TIME_UNSET; streamOffsetUs = C.TIME_UNSET; } @@ -399,6 +401,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throws ExoPlaybackException { checkState(getTimeline().getWindowCount() == 1); super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + streamStartPositionUs = startPositionUs; streamOffsetUs = offsetUs; int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); editedMediaItem = @@ -425,10 +428,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; protected boolean processOutputBuffer( long positionUs, long elapsedRealtimeUs, Bitmap outputImage, long timeUs) { if (inputStreamPendingRegistration) { + checkState(streamStartPositionUs != C.TIME_UNSET); checkState(streamOffsetUs != C.TIME_UNSET); videoSink.setPendingVideoEffects(videoEffects); - videoSink.setStreamOffsetAndAdjustmentUs( - streamOffsetUs, /* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs); + videoSink.setStreamTimestampInfo( + streamStartPositionUs, + streamOffsetUs, + /* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs, + getLastResetPositionUs()); videoSink.registerInputStream( VideoSink.INPUT_TYPE_BITMAP, new Format.Builder()