Add VFP.queueInputBitmap api accepting timestamps

PiperOrigin-RevId: 555970312

RELEASENOTES.md

@@ -104,6 +104,8 @@
 *   Metadata:
 *   DRM:
 *   Effect:
+    *   Add `VideoFrameProcessor.queueInputBitmap(Bitmap, Iterator<Long>)`
+        queuing bitmap input by timestamp.
 *   Muxers:
 *   IMA extension:
 *   Session:

libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java

@@ -28,6 +28,7 @@ import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -150,7 +151,11 @@ public interface VideoFrameProcessor {
   long DROP_OUTPUT_FRAME = -2;
 
   /**
-   * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}.
+   * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor} to generate an input stream
+   * of frames.
+   *
+   * <p>Each call must be made after {@linkplain #registerInputStream registering a new input
+   * stream}.
    *
    * <p>Can be called on any thread.
    *
@@ -161,10 +166,26 @@ public interface VideoFrameProcessor {
    * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept
    *     {@linkplain #INPUT_TYPE_BITMAP bitmap input}.
    */
-  // TODO(b/262693274): Remove duration and frameRate parameters when EditedMediaItem can be
-  //  signalled down to the processors.
+  // TODO(b/262693274): Delete this method and usages in favor of the one below (Note it is not
+  //   deprecated because transformer still relies on this method for frame duplication).
   void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate);
 
+  /**
+   * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}.
+   *
+   * <p>Can be called many times after {@link #registerInputStream(int, List, FrameInfo) registering
+   * the input stream} to put multiple frames in the same input stream.
+   *
+   * <p>Can be called on any thread.
+   *
+   * @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}.
+   * @param inStreamOffsetsUs The times within the current stream that the bitmap should be shown
+   *     at. The timestamps should be monotonically increasing.
+   * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept
+   *     {@linkplain #INPUT_TYPE_BITMAP bitmap input}.
+   */
+  void queueInputBitmap(Bitmap inputBitmap, Iterator<Long> inStreamOffsetsUs);
+
   /**
    * Provides an input texture ID to the {@code VideoFrameProcessor}.
    *

libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java

@@ -19,10 +19,13 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
 import static com.google.common.truth.Truth.assertThat;
 
+import android.graphics.Bitmap;
+import android.util.Pair;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -112,8 +115,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
   @RequiresNonNull({"framesProduced", "testId"})
   public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs()
       throws Exception {
-    String testId =
-        "imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs";
     Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
     videoFrameProcessorTestRunner =
         getDefaultFrameProcessorTestRunnerBuilder(testId)
@@ -169,8 +170,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
   public void
       imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs()
           throws Exception {
-    String testId =
-        "imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs";
     Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
     videoFrameProcessorTestRunner =
         getDefaultFrameProcessorTestRunnerBuilder(testId)
@@ -192,6 +191,31 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
     assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder();
   }
 
+  @Test
+  @RequiresNonNull({"framesProduced", "testId"})
+  public void queueBitmapsWithTimestamps_outputsFramesAtTheCorrectPresentationTimesUs()
+      throws Exception {
+    Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
+    videoFrameProcessorTestRunner =
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add)
+            .build();
+    Bitmap bitmap1 = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+    Bitmap bitmap2 = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH);
+    Long offset1 = 0L;
+    Long offset2 = C.MICROS_PER_SECOND;
+    Long offset3 = 2 * C.MICROS_PER_SECOND;
+
+    videoFrameProcessorTestRunner.queueInputBitmaps(
+        bitmap1.getWidth(),
+        bitmap1.getHeight(),
+        Pair.create(bitmap1, ImmutableList.of(offset1).iterator()),
+        Pair.create(bitmap2, ImmutableList.of(offset2, offset3).iterator()));
+    videoFrameProcessorTestRunner.endFrameProcessing();
+
+    assertThat(actualPresentationTimesUs).containsExactly(offset1, offset2, offset3).inOrder();
+  }
+
   private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
       String testId) {
     return new VideoFrameProcessorTestRunner.Builder()

libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.effect;
 
+import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static java.lang.Math.round;
@@ -148,7 +149,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
 
     this.useHdr = useHdr;
+    // TODO(b/262693274): move frame duplication logic out of the texture manager. Note this will
+    //   involve removing the BitmapFrameSequenceInfo queue and using the FrameConsumptionManager
+    //   instead. It will also remove the framesToAdd variable
     int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
+    // framestoAdd > 0 otherwise the VFP will hang.
+    checkArgument(framesToAdd > 0);
     double frameDurationUs = C.MICROS_PER_SECOND / frameRate;
     pendingBitmaps.add(
         new BitmapFrameSequenceInfo(bitmap, frameInfo, frameDurationUs, framesToAdd));

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -58,6 +58,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -436,6 +437,26 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
     hasRefreshedNextInputFrameInfo = false;
   }
 
+  @Override
+  public void queueInputBitmap(Bitmap inputBitmap, Iterator<Long> inStreamOffsetsUs) {
+    FrameInfo frameInfo = checkNotNull(this.nextInputFrameInfo);
+    // TODO(b/262693274): move frame duplication logic out of the texture manager so
+    //   textureManager.queueInputBitmap() frame rate and duration parameters be removed.
+    while (inStreamOffsetsUs.hasNext()) {
+      long inStreamOffsetUs = inStreamOffsetsUs.next();
+      inputSwitcher
+          .activeTextureManager()
+          .queueInputBitmap(
+              inputBitmap,
+              /* durationUs= */ C.MICROS_PER_SECOND,
+              new FrameInfo.Builder(frameInfo)
+                  .setOffsetToAddUs(frameInfo.offsetToAddUs + inStreamOffsetUs)
+                  .build(),
+              /* frameRate= */ 1,
+              /* useHdr= */ false);
+    }
+  }
+
   @Override
   public void queueInputTexture(int textureId, long presentationTimeUs) {
     inputSwitcher.activeTextureManager().queueInputTexture(textureId, presentationTimeUs);

libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java

@@ -32,6 +32,7 @@ import android.graphics.PixelFormat;
 import android.media.Image;
 import android.media.ImageReader;
 import android.media.MediaFormat;
+import android.util.Pair;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
@@ -48,6 +49,7 @@ import androidx.media3.common.util.UnstableApi;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
@@ -357,6 +359,18 @@ public final class VideoFrameProcessorTestRunner {
     videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
   }
 
+  public void queueInputBitmaps(int width, int height, Pair<Bitmap, Iterator<Long>>... frames) {
+    videoFrameProcessor.registerInputStream(
+        INPUT_TYPE_BITMAP,
+        effects,
+        new FrameInfo.Builder(width, height)
+            .setPixelWidthHeightRatio(pixelWidthHeightRatio)
+            .build());
+    for (Pair<Bitmap, Iterator<Long>> frame : frames) {
+      videoFrameProcessor.queueInputBitmap(frame.first, frame.second);
+    }
+  }
+
   public void queueInputTexture(GlTextureInfo inputTexture, long pts) {
     videoFrameProcessor.registerInputStream(
         INPUT_TYPE_TEXTURE_ID,