Implement DefaultVideoSink.setStreamTimestampInfo

To do this, refactor stream start position handling so that
VideoFrameRenderControl is only passed a start position when it should
be applied, and therefore doesn't need to take a timestamp associated
with each start position anymore

PiperOrigin-RevId: 697544416

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DefaultVideoSink.java

@@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
 import android.graphics.Bitmap;
 import android.view.Surface;
 import androidx.annotation.Nullable;
+import androidx.media3.common.C;
 import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.util.Size;
@@ -49,6 +50,7 @@ import java.util.concurrent.Executor;
 
   @Nullable private Surface outputSurface;
   private Format inputFormat;
+  private long streamStartPositionUs;
 
   public DefaultVideoSink(
       VideoFrameReleaseControl videoFrameReleaseControl,
@@ -56,6 +58,7 @@ import java.util.concurrent.Executor;
     this.videoFrameReleaseControl = videoFrameReleaseControl;
     this.videoFrameRenderControl = videoFrameRenderControl;
     inputFormat = new Format.Builder().build();
+    streamStartPositionUs = C.TIME_UNSET;
   }
 
   @Override
@@ -149,7 +152,10 @@ import java.util.concurrent.Executor;
   @Override
   public void setStreamTimestampInfo(
       long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) {
-    throw new UnsupportedOperationException();
+    if (streamStartPositionUs != this.streamStartPositionUs) {
+      videoFrameRenderControl.onStreamStartPositionChanged(streamStartPositionUs);
+      this.streamStartPositionUs = streamStartPositionUs;
+    }
   }
 
   @Override

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java

@@ -46,6 +46,7 @@ import androidx.media3.common.VideoSize;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.HandlerWrapper;
 import androidx.media3.common.util.Size;
+import androidx.media3.common.util.TimedValueQueue;
 import androidx.media3.common.util.TimestampIterator;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
@@ -228,6 +229,13 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
 
   private final Context context;
   private final InputVideoSink inputVideoSink;
+
+  /**
+   * A queue of unprocessed input frame start positions. Each position is associated with the
+   * timestamp from which it should be applied.
+   */
+  private final TimedValueQueue<Long> streamStartPositionsUs;
+
   private final VideoFrameReleaseControl videoFrameReleaseControl;
   private final VideoFrameRenderControl videoFrameRenderControl;
   private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
@@ -240,6 +248,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
   private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener;
   private @MonotonicNonNull HandlerWrapper handler;
   private @MonotonicNonNull PreviewingVideoGraph videoGraph;
+  private long outputStreamStartPositionUs;
   @Nullable private Pair<Surface, Size> currentSurfaceAndSize;
   private int pendingFlushCount;
   private @State int state;
@@ -254,6 +263,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
   private PlaybackVideoGraphWrapper(Builder builder) {
     context = builder.context;
     inputVideoSink = new InputVideoSink(context);
+    streamStartPositionsUs = new TimedValueQueue<>();
     clock = builder.clock;
     videoFrameReleaseControl = builder.videoFrameReleaseControl;
     videoFrameReleaseControl.setClock(clock);
@@ -355,8 +365,16 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
     }
     // The frame presentation time is relative to the start of the Composition and without the
     // renderer offset
-    videoFrameRenderControl.onFrameAvailableForRendering(
-        framePresentationTimeUs - bufferTimestampAdjustmentUs);
+    long bufferPresentationTimeUs = framePresentationTimeUs - bufferTimestampAdjustmentUs;
+    Long newOutputStreamStartPositionUs =
+        streamStartPositionsUs.pollFloor(bufferPresentationTimeUs);
+    if (newOutputStreamStartPositionUs != null
+        && newOutputStreamStartPositionUs != outputStreamStartPositionUs) {
+      defaultVideoSink.setStreamTimestampInfo(
+          newOutputStreamStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET);
+      outputStreamStartPositionUs = newOutputStreamStartPositionUs;
+    }
+    videoFrameRenderControl.onFrameAvailableForRendering(bufferPresentationTimeUs);
   }
 
   @Override
@@ -454,6 +472,15 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
     }
     pendingFlushCount++;
     defaultVideoSink.flush(resetPosition);
+    while (streamStartPositionsUs.size() > 1) {
+      streamStartPositionsUs.pollFirst();
+    }
+    if (streamStartPositionsUs.size() == 1) {
+      long lastStartPositionUs = checkNotNull(streamStartPositionsUs.pollFirst());
+      // defaultVideoSink should use the latest startPositionUs if none is passed after flushing.
+      defaultVideoSink.setStreamTimestampInfo(
+          lastStartPositionUs, /* unused */ C.TIME_UNSET, /* unused */ C.TIME_UNSET);
+    }
     // Handle pending video graph callbacks to ensure video size changes reach the video render
     // control.
     checkStateNotNull(handler).post(() -> pendingFlushCount--);
@@ -472,12 +499,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
     this.bufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
   }
 
-  private void onStreamStartPositionChange(
-      long bufferPresentationTimeUs, long streamStartPositionUs) {
-    videoFrameRenderControl.onStreamStartPositionChanged(
-        bufferPresentationTimeUs, streamStartPositionUs);
-  }
-
   private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {
     if (inputColorInfo == null || !inputColorInfo.isDataSpaceValid()) {
       return ColorInfo.SDR_BT709_LIMITED;
@@ -499,7 +520,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
     private long inputStreamStartPositionUs;
     private long inputBufferTimestampAdjustmentUs;
     private long lastResetPositionUs;
-    private boolean pendingInputStartPositionChange;
 
     /** The buffer presentation time, in microseconds, of the final frame in the stream. */
     private long finalBufferPresentationTimeUs;
@@ -662,9 +682,13 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
     @Override
     public void setStreamTimestampInfo(
         long streamStartPositionUs, long bufferTimestampAdjustmentUs, long lastResetPositionUs) {
-      // Ors because this method could be called multiple times on a timestamp info change.
-      pendingInputStartPositionChange |= inputStreamStartPositionUs != streamStartPositionUs;
       inputStreamStartPositionUs = streamStartPositionUs;
+      // Input timestamps should always be positive because they are offset by ExoPlayer. Adding a
+      // position to the queue with timestamp 0 should therefore always apply it as long as it is
+      // the only position in the queue.
+      streamStartPositionsUs.add(
+          lastBufferPresentationTimeUs == C.TIME_UNSET ? 0 : lastBufferPresentationTimeUs + 1,
+          streamStartPositionUs);
       inputBufferTimestampAdjustmentUs = bufferTimestampAdjustmentUs;
       // The buffer timestamp adjustment is only allowed to change after a flush to make sure that
       // the buffer timestamps are increasing. We can update the buffer timestamp adjustment
@@ -767,7 +791,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
         return false;
       }
 
-      maybeSetStreamStartPosition(bufferPresentationTimeUs);
       lastBufferPresentationTimeUs = bufferPresentationTimeUs;
       if (isLastFrame) {
         finalBufferPresentationTimeUs = bufferPresentationTimeUs;
@@ -792,15 +815,10 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
         return false;
       }
 
-      // Create a copy of iterator because we need to take the next timestamp but we must not alter
-      // the state of the iterator.
-      TimestampIterator copyTimestampIterator = timestampIterator.copyOf();
-      long bufferPresentationTimeUs = copyTimestampIterator.next();
       // TimestampIterator generates frame time.
       long lastBufferPresentationTimeUs =
-          copyTimestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs;
+          timestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs;
       checkState(lastBufferPresentationTimeUs != C.TIME_UNSET);
-      maybeSetStreamStartPosition(bufferPresentationTimeUs);
       this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs;
       finalBufferPresentationTimeUs = lastBufferPresentationTimeUs;
       return true;
@@ -823,14 +841,6 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
 
     // Other methods
 
-    private void maybeSetStreamStartPosition(long bufferPresentationTimeUs) {
-      if (pendingInputStartPositionChange) {
-        PlaybackVideoGraphWrapper.this.onStreamStartPositionChange(
-            bufferPresentationTimeUs, inputStreamStartPositionUs);
-        pendingInputStartPositionChange = false;
-      }
-    }
-
     /**
      * Attempt to register any pending input stream to the video graph input and returns {@code
      * true} if a pending stream was registered and/or there is no pending input stream waiting for

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java

@@ -68,7 +68,13 @@ import androidx.media3.exoplayer.ExoPlaybackException;
    */
   private final TimedValueQueue<VideoSize> videoSizes;
 
+  /**
+   * A queue of unprocessed input frame start positions. Each position is associated with the
+   * timestamp from which it should be applied.
+   */
   private final TimedValueQueue<Long> streamStartPositionsUs;
+
+  /** A queue of unprocessed input frame timestamps. */
   private final LongArrayQueue presentationTimestampsUs;
 
   private long lastInputPresentationTimeUs;
@@ -180,6 +186,12 @@ import androidx.media3.exoplayer.ExoPlaybackException;
         new VideoSize(width, height));
   }
 
+  public void onStreamStartPositionChanged(long streamStartPositionUs) {
+    streamStartPositionsUs.add(
+        lastInputPresentationTimeUs == C.TIME_UNSET ? 0 : lastInputPresentationTimeUs + 1,
+        streamStartPositionUs);
+  }
+
   /**
    * Called when a frame is available for rendering.
    *
@@ -191,10 +203,6 @@ import androidx.media3.exoplayer.ExoPlaybackException;
     // TODO b/257464707 - Support extensively modified media.
   }
 
-  public void onStreamStartPositionChanged(long presentationTimeUs, long streamStartPositionUs) {
-    streamStartPositionsUs.add(presentationTimeUs, streamStartPositionUs);
-  }
-
   private void dropFrame() {
     presentationTimestampsUs.remove();
     frameRenderer.dropFrame();

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java

@@ -121,8 +121,7 @@ public class VideoFrameRenderControlTest {
     videoFrameReleaseControl.onStarted();
     videoFrameRenderControl.onVideoSizeChanged(
         /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
-    videoFrameRenderControl.onStreamStartPositionChanged(
-        /* presentationTimeUs= */ 0, /* streamStartPositionUs= */ 10_000);
+    videoFrameRenderControl.onStreamStartPositionChanged(/* streamStartPositionUs= */ 10_000);
     videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 0);
     videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
 
@@ -136,8 +135,7 @@ public class VideoFrameRenderControlTest {
 
     // 10 milliseconds pass
     clock.advanceTime(/* timeDiffMs= */ 10);
-    videoFrameRenderControl.onStreamStartPositionChanged(
-        /* presentationTimeUs= */ 10_000, /* streamStartPositionUs= */ 20_000);
+    videoFrameRenderControl.onStreamStartPositionChanged(/* streamStartPositionUs= */ 20_000);
     videoFrameRenderControl.onFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
     videoFrameRenderControl.render(/* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0);