Call getFrameReleaseAction from VideoSink when enabled

VideoSink.registerInputFrame is now called for every input frame (not
only the ones that should be rendered to the input surface) because it's
the VideoSink that decides whether it wants the frame to be rendered.

PiperOrigin-RevId: 651049851

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -488,13 +488,16 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
     private final Context context;
     private final int videoFrameProcessorMaxPendingFrameCount;
     private final ArrayList<Effect> videoEffects;
-    @Nullable private Effect rotationEffect;
+    private final VideoFrameReleaseControl.FrameReleaseInfo frameReleaseInfo;
 
     private @MonotonicNonNull VideoFrameProcessor videoFrameProcessor;
+    @Nullable private Effect rotationEffect;
     @Nullable private Format inputFormat;
     private @InputType int inputType;
+    private long inputStreamStartPositionUs;
     private long inputStreamOffsetUs;
     private long inputBufferTimestampAdjustmentUs;
+    private long lastResetPositionUs;
     private boolean pendingInputStreamOffsetChange;
 
     /** The buffer presentation time, in microseconds, of the final frame in the stream. */
@@ -513,14 +516,13 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
     /** Creates a new instance. */
     public VideoSinkImpl(Context context) {
       this.context = context;
-      // TODO b/226330223 - Investigate increasing frame count when frame dropping is
+      // TODO b/226330223 - Investigate increasing frame count when frame dropping is allowed.
-      //  allowed.
       // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed
       //  reduces decoder timeouts, and consider restoring.
       videoFrameProcessorMaxPendingFrameCount =
           Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
-
       videoEffects = new ArrayList<>();
+      frameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
       finalBufferPresentationTimeUs = C.TIME_UNSET;
       lastBufferPresentationTimeUs = C.TIME_UNSET;
       listener = VideoSink.Listener.NO_OP;
@@ -679,14 +681,19 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
     }
 
     @Override
-    public void setStreamOffsetAndAdjustmentUs(
+    public void setStreamTimestampInfo(
-        long streamOffsetUs, long bufferTimestampAdjustmentUs) {
+        long streamStartPositionUs,
+        long streamOffsetUs,
+        long bufferTimestampAdjustmentUs,
+        long lastResetPositionUs) {
       // Ors because this method could be called multiple times on a stream offset change.
       pendingInputStreamOffsetChange |=
           inputStreamOffsetUs != streamOffsetUs
               || inputBufferTimestampAdjustmentUs != bufferTimestampAdjustmentUs;
+      inputStreamStartPositionUs = streamStartPositionUs;
       inputStreamOffsetUs = streamOffsetUs;
       inputBufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
+      this.lastResetPositionUs = lastResetPositionUs;
     }
 
     @Override
@@ -711,9 +718,55 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
     }
 
     @Override
-    public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) {
+    public boolean handleInputFrame(
+        long framePresentationTimeUs,
+        boolean isLastFrame,
+        long positionUs,
+        long elapsedRealtimeUs,
+        VideoFrameHandler videoFrameHandler)
+        throws VideoSinkException {
       checkState(isInitialized());
-      checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET);
+
+      // The sink takes in frames with monotonically increasing, non-offset frame
+      // timestamps. That is, with two ten-second long videos, the first frame of the second video
+      // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
+      // timestamp of the said frame would be 0s, but the streamOffset is incremented by 10s to
+      // include the duration of the first video. Thus this correction is needed to account for the
+      // different handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
+      //
+      // inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to
+      // the start of a composition) to the buffer timestamp (that corresponds to the player
+      // position).
+      long bufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs;
+      // The frame release action should be retrieved for all frames (even the ones that will be
+      // skipped), because the release control estimates the content frame rate from frame
+      // timestamps and we want to have this information known as early as possible, especially
+      // during seeking.
+      @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction;
+      try {
+        frameReleaseAction =
+            videoFrameReleaseControl.getFrameReleaseAction(
+                bufferPresentationTimeUs,
+                positionUs,
+                elapsedRealtimeUs,
+                inputStreamStartPositionUs,
+                isLastFrame,
+                frameReleaseInfo);
+      } catch (ExoPlaybackException e) {
+        throw new VideoSinkException(e, checkStateNotNull(inputFormat));
+      }
+      if (frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IGNORE) {
+        // The buffer is no longer valid and needs to be ignored.
+        return false;
+      }
+
+      if (bufferPresentationTimeUs < lastResetPositionUs && !isLastFrame) {
+        videoFrameHandler.skip();
+        return true;
+      }
+
+      // Drain the sink to make room for a new input frame.
+      render(positionUs, elapsedRealtimeUs);
 
       // An input stream is fully decoded, wait until all of its frames are released before queueing
       // input frame from the next input stream.
@@ -723,34 +776,27 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
           maybeRegisterInputStream();
           pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET;
         } else {
-          return C.TIME_UNSET;
+          return false;
         }
       }
-
       if (checkStateNotNull(videoFrameProcessor).getPendingInputFrameCount()
           >= videoFrameProcessorMaxPendingFrameCount) {
-        return C.TIME_UNSET;
+        return false;
       }
       if (!checkStateNotNull(videoFrameProcessor).registerInputFrame()) {
-        return C.TIME_UNSET;
+        return false;
       }
-      // The sink takes in frames with monotonically increasing, non-offset frame
+
-      // timestamps. That is, with two ten-second long videos, the first frame of the second video
-      // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
-      // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include
-      // the duration of the first video. Thus this correction is need to correct for the different
-      // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
-      //
-      // inputBufferTimestampAdjustmentUs adjusts the frame presentation time (which is relative to
-      // the start of a composition, to the buffer timestamp that is offset, and correspond to the
-      // player position).
-      long bufferPresentationTimeUs = framePresentationTimeUs - inputBufferTimestampAdjustmentUs;
       maybeSetStreamOffsetChange(bufferPresentationTimeUs);
       lastBufferPresentationTimeUs = bufferPresentationTimeUs;
       if (isLastFrame) {
         finalBufferPresentationTimeUs = bufferPresentationTimeUs;
       }
-      return framePresentationTimeUs * 1000;
+      // Use the frame presentation time as render time so that the SurfaceTexture is accompanied
+      // by this timestamp. Setting a realtime based release time is only relevant when rendering to
+      // a SurfaceView, but we render to a surface in this case.
+      videoFrameHandler.render(/* renderTimestampNs= */ framePresentationTimeUs * 1000);
+      return true;
     }
 
     @Override

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -737,8 +737,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
       // Flush the video sink first to ensure it stops reading textures that will be owned by
       // MediaCodec once the codec is flushed.
       videoSink.flush(/* resetPosition= */ true);
-      videoSink.setStreamOffsetAndAdjustmentUs(
+      videoSink.setStreamTimestampInfo(
-          getOutputStreamOffsetUs(), getBufferTimestampAdjustmentUs());
+          getOutputStreamStartPositionUs(),
+          getOutputStreamOffsetUs(),
+          getBufferTimestampAdjustmentUs(),
+          getLastResetPositionUs());
       videoSinkNeedsRegisterInputStream = true;
     }
     super.onPositionReset(positionUs, joining);
@@ -1404,6 +1407,34 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
     long outputStreamOffsetUs = getOutputStreamOffsetUs();
     long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
 
+    if (videoSink != null) {
+      long framePresentationTimeUs = bufferPresentationTimeUs + getBufferTimestampAdjustmentUs();
+      try {
+        return videoSink.handleInputFrame(
+            framePresentationTimeUs,
+            isLastBuffer,
+            positionUs,
+            elapsedRealtimeUs,
+            new VideoSink.VideoFrameHandler() {
+              @Override
+              public void render(long renderTimestampNs) {
+                renderOutputBuffer(codec, bufferIndex, presentationTimeUs, renderTimestampNs);
+              }
+
+              @Override
+              public void skip() {
+                skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
+              }
+            });
+      } catch (VideoSink.VideoSinkException e) {
+        throw createRendererException(
+            e, e.format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED);
+      }
+    }
+
+    // The frame release action should be retrieved for all frames (even the ones that will be
+    // skipped), because the release control estimates the content frame rate from frame timestamps
+    // and we want to have this information known as early as possible, especially during seeking.
     @VideoFrameReleaseControl.FrameReleaseAction
     int frameReleaseAction =
         videoFrameReleaseControl.getFrameReleaseAction(
@@ -1419,18 +1450,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
       return false;
     }
 
-    // Skip decode-only buffers, e.g. after seeking, immediately. This check must be performed after
+    // Skip decode-only buffers, e.g. after seeking, immediately.
-    // getting the release action from the video frame release control although not necessary.
-    // That's because the release control estimates the content frame rate from frame timestamps
-    // and we want to have this information known as early as possible, especially during seeking.
     if (isDecodeOnlyBuffer && !isLastBuffer) {
       skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
       return true;
     }
 
     // We are not rendering on a surface, the renderer will wait until a surface is set.
-    // Opportunistically render to VideoSink if it is enabled.
+    if (displaySurface == placeholderSurface) {
-    if (displaySurface == placeholderSurface && videoSink == null) {
       // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
       if (videoFrameReleaseInfo.getEarlyUs() < 30_000) {
         skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
@@ -1440,24 +1467,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
       return false;
     }
 
-    if (videoSink != null) {
-      try {
-        videoSink.render(positionUs, elapsedRealtimeUs);
-      } catch (VideoSink.VideoSinkException e) {
-        throw createRendererException(
-            e, e.format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED);
-      }
-
-      long releaseTimeNs =
-          videoSink.registerInputFrame(
-              bufferPresentationTimeUs + getBufferTimestampAdjustmentUs(), isLastBuffer);
-      if (releaseTimeNs == C.TIME_UNSET) {
-        return false;
-      }
-      renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
-      return true;
-    }
-
     switch (frameReleaseAction) {
       case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
         long releaseTimeNs = getClock().nanoTime();
@@ -1567,8 +1576,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
   protected void onProcessedStreamChange() {
     super.onProcessedStreamChange();
     if (videoSink != null) {
-      videoSink.setStreamOffsetAndAdjustmentUs(
+      videoSink.setStreamTimestampInfo(
-          getOutputStreamOffsetUs(), getBufferTimestampAdjustmentUs());
+          getOutputStreamStartPositionUs(),
+          getOutputStreamOffsetUs(),
+          getBufferTimestampAdjustmentUs(),
+          getLastResetPositionUs());
     } else {
       videoFrameReleaseControl.onProcessedStreamChange();
     }

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java

@@ -18,6 +18,7 @@ package androidx.media3.exoplayer.video;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
 import android.graphics.Bitmap;
+import android.os.SystemClock;
 import android.view.Surface;
 import androidx.annotation.FloatRange;
 import androidx.annotation.IntDef;
@@ -94,6 +95,21 @@ public interface VideoSink {
         };
   }
 
+  /** Handler for a video frame. */
+  interface VideoFrameHandler {
+
+    /**
+     * Renders the frame on the {@linkplain #getInputSurface() input surface}.
+     *
+     * @param renderTimestampNs The timestamp to associate with this frame when it is sent to the
+     *     surface.
+     */
+    void render(long renderTimestampNs);
+
+    /** Skips the frame. */
+    void skip();
+  }
+
   /**
    * Specifies how the input frames are made available to the video sink. One of {@link
    * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}.
@@ -193,14 +209,21 @@ public interface VideoSink {
   void setPendingVideoEffects(List<Effect> videoEffects);
 
   /**
-   * Sets the stream offset and buffer time adjustment.
+   * Sets information about the timestamps of the current input stream.
    *
+   * @param streamStartPositionUs The start position of the buffer presentation timestamps of the
+   *     current stream, in microseconds.
    * @param streamOffsetUs The offset that is added to the buffer presentation timestamps by the
    *     player, in microseconds.
    * @param bufferTimestampAdjustmentUs The timestamp adjustment to add to the buffer presentation
    *     timestamps to convert them to frame presentation timestamps, in microseconds.
+   * @param lastResetPositionUs The renderer last reset position, in microseconds.
    */
-  void setStreamOffsetAndAdjustmentUs(long streamOffsetUs, long bufferTimestampAdjustmentUs);
+  void setStreamTimestampInfo(
+      long streamStartPositionUs,
+      long streamOffsetUs,
+      long bufferTimestampAdjustmentUs,
+      long lastResetPositionUs);
 
   /** Sets the output surface info. */
   void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution);
@@ -236,19 +259,27 @@ public interface VideoSink {
   void registerInputStream(@InputType int inputType, Format format);
 
   /**
-   * Informs the video sink that a frame will be queued to its {@linkplain #getInputSurface() input
+   * Handles a video input frame.
-   * surface}.
    *
    * <p>Must be called after the corresponding stream is {@linkplain #registerInputStream(int,
    * Format) registered}.
    *
    * @param framePresentationTimeUs The frame's presentation time, in microseconds.
    * @param isLastFrame Whether this is the last frame of the video stream.
-   * @return A release timestamp, in nanoseconds, that should be associated when releasing this
+   * @param positionUs The current playback position, in microseconds.
-   *     frame, or {@link C#TIME_UNSET} if the sink was not able to register the frame and the
+   * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, taken
-   *     caller must try again later.
+   *     approximately at the time the playback position was {@code positionUs}.
+   * @param videoFrameHandler The {@link VideoFrameHandler} used to handle the input frame.
+   * @return Whether the frame was handled successfully. If {@code false}, the caller can try again
+   *     later.
    */
-  long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame);
+  boolean handleInputFrame(
+      long framePresentationTimeUs,
+      boolean isLastFrame,
+      long positionUs,
+      long elapsedRealtimeUs,
+      VideoFrameHandler videoFrameHandler)
+      throws VideoSinkException;
 
   /**
    * Provides an input {@link Bitmap} to the video sink.
@@ -259,8 +290,8 @@ public interface VideoSink {
    * @param inputBitmap The {@link Bitmap} queued to the video sink.
    * @param timestampIterator The times within the current stream that the bitmap should be shown
    *     at. The timestamps should be monotonically increasing.
-   * @return Whether the bitmap was queued successfully. A {@code false} value indicates the caller
+   * @return Whether the bitmap was queued successfully. If {@code false}, the caller can try again
-   *     must try again later.
+   *     later.
    */
   boolean queueBitmap(Bitmap inputBitmap, TimestampIterator timestampIterator);
 

libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java

@@ -302,6 +302,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     private @MonotonicNonNull EditedMediaItem editedMediaItem;
     @Nullable private ExoPlaybackException pendingExoPlaybackException;
     private boolean inputStreamPendingRegistration;
+    private long streamStartPositionUs;
     private long streamOffsetUs;
     private boolean mayRenderStartOfStream;
     private long offsetToCompositionTimeUs;
@@ -314,6 +315,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
           checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider);
       videoSink = compositingVideoSinkProvider.getSink();
       videoEffects = ImmutableList.of();
+      streamStartPositionUs = C.TIME_UNSET;
       streamOffsetUs = C.TIME_UNSET;
     }
 
@@ -399,6 +401,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         throws ExoPlaybackException {
       checkState(getTimeline().getWindowCount() == 1);
       super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
+      streamStartPositionUs = startPositionUs;
       streamOffsetUs = offsetUs;
       int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid);
       editedMediaItem =
@@ -425,10 +428,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     protected boolean processOutputBuffer(
         long positionUs, long elapsedRealtimeUs, Bitmap outputImage, long timeUs) {
       if (inputStreamPendingRegistration) {
+        checkState(streamStartPositionUs != C.TIME_UNSET);
         checkState(streamOffsetUs != C.TIME_UNSET);
         videoSink.setPendingVideoEffects(videoEffects);
-        videoSink.setStreamOffsetAndAdjustmentUs(
+        videoSink.setStreamTimestampInfo(
-            streamOffsetUs, /* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs);
+            streamStartPositionUs,
+            streamOffsetUs,
+            /* bufferTimestampAdjustmentUs= */ offsetToCompositionTimeUs,
+            getLastResetPositionUs());
         videoSink.registerInputStream(
             VideoSink.INPUT_TYPE_BITMAP,
             new Format.Builder()