From 48de9fa91643de8d7ead858cd494be92b2e25715 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 11 Aug 2023 15:56:44 +0000 Subject: [PATCH] Add VFP.queueInputBitmap api accepting timestamps PiperOrigin-RevId: 555970312 --- RELEASENOTES.md | 2 ++ .../media3/common/VideoFrameProcessor.java | 27 ++++++++++++++-- ...deoFrameProcessorImageFrameOutputTest.java | 32 ++++++++++++++++--- .../media3/effect/BitmapTextureManager.java | 6 ++++ .../effect/DefaultVideoFrameProcessor.java | 21 ++++++++++++ .../utils/VideoFrameProcessorTestRunner.java | 14 ++++++++ 6 files changed, 95 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6dda29044f..2117ceec04 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -104,6 +104,8 @@ * Metadata: * DRM: * Effect: + * Add `VideoFrameProcessor.queueInputBitmap(Bitmap, Iterator)` + queuing bitmap input by timestamp. * Muxers: * IMA extension: * Session: diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 599ecfe4ac..97b9c7f788 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -28,6 +28,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.Iterator; import java.util.List; import java.util.concurrent.Executor; @@ -150,7 +151,11 @@ public interface VideoFrameProcessor { long DROP_OUTPUT_FRAME = -2; /** - * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}. + * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor} to generate an input stream + * of frames. + * + *

Each call must be made after {@linkplain #registerInputStream registering a new input + * stream}. * *

Can be called on any thread. * @@ -161,10 +166,26 @@ public interface VideoFrameProcessor { * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept * {@linkplain #INPUT_TYPE_BITMAP bitmap input}. */ - // TODO(b/262693274): Remove duration and frameRate parameters when EditedMediaItem can be - // signalled down to the processors. + // TODO(b/262693274): Delete this method and usages in favor of the one below (Note it is not + // deprecated because transformer still relies on this method for frame duplication). void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate); + /** + * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}. + * + *

Can be called many times after {@link #registerInputStream(int, List, FrameInfo) registering + * the input stream} to put multiple frames in the same input stream. + * + *

Can be called on any thread. + * + * @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}. + * @param inStreamOffsetsUs The times within the current stream that the bitmap should be shown + * at. The timestamps should be monotonically increasing. + * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept + * {@linkplain #INPUT_TYPE_BITMAP bitmap input}. + */ + void queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs); + /** * Provides an input texture ID to the {@code VideoFrameProcessor}. * diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index acbb1c6d62..1f3bc1afba 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -19,10 +19,13 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static com.google.common.truth.Truth.assertThat; +import android.graphics.Bitmap; +import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; @@ -112,8 +115,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { @RequiresNonNull({"framesProduced", "testId"}) public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { - String testId = - "imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs"; Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) @@ -169,8 +170,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { public void imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs() throws Exception { - String testId = - "imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs"; Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) @@ -192,6 +191,31 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder(); } + @Test + @RequiresNonNull({"framesProduced", "testId"}) + public void queueBitmapsWithTimestamps_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) + .build(); + Bitmap bitmap1 = readBitmap(ORIGINAL_PNG_ASSET_PATH); + Bitmap bitmap2 = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH); + Long offset1 = 0L; + Long offset2 = C.MICROS_PER_SECOND; + Long offset3 = 2 * C.MICROS_PER_SECOND; + + videoFrameProcessorTestRunner.queueInputBitmaps( + bitmap1.getWidth(), + bitmap1.getHeight(), + Pair.create(bitmap1, ImmutableList.of(offset1).iterator()), + Pair.create(bitmap2, ImmutableList.of(offset2, offset3).iterator())); + videoFrameProcessorTestRunner.endFrameProcessing(); + + assertThat(actualPresentationTimesUs).containsExactly(offset1, offset2, offset3).inOrder(); + } + private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( String testId) { return new VideoFrameProcessorTestRunner.Builder() diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 3304844bfd..83dce85b97 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static java.lang.Math.round; @@ -148,7 +149,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } this.useHdr = useHdr; + // TODO(b/262693274): move frame duplication logic out of the texture manager. Note this will + // involve removing the BitmapFrameSequenceInfo queue and using the FrameConsumptionManager + // instead. It will also remove the framesToAdd variable int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); + // framestoAdd > 0 otherwise the VFP will hang. + checkArgument(framesToAdd > 0); double frameDurationUs = C.MICROS_PER_SECOND / frameRate; pendingBitmaps.add( new BitmapFrameSequenceInfo(bitmap, frameInfo, frameDurationUs, framesToAdd)); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index ae79d65e42..864bf68196 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -58,6 +58,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -436,6 +437,26 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { hasRefreshedNextInputFrameInfo = false; } + @Override + public void queueInputBitmap(Bitmap inputBitmap, Iterator inStreamOffsetsUs) { + FrameInfo frameInfo = checkNotNull(this.nextInputFrameInfo); + // TODO(b/262693274): move frame duplication logic out of the texture manager so + // textureManager.queueInputBitmap() frame rate and duration parameters be removed. + while (inStreamOffsetsUs.hasNext()) { + long inStreamOffsetUs = inStreamOffsetsUs.next(); + inputSwitcher + .activeTextureManager() + .queueInputBitmap( + inputBitmap, + /* durationUs= */ C.MICROS_PER_SECOND, + new FrameInfo.Builder(frameInfo) + .setOffsetToAddUs(frameInfo.offsetToAddUs + inStreamOffsetUs) + .build(), + /* frameRate= */ 1, + /* useHdr= */ false); + } + } + @Override public void queueInputTexture(int textureId, long presentationTimeUs) { inputSwitcher.activeTextureManager().queueInputTexture(textureId, presentationTimeUs); diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 9cac5ecc5b..5c5c354a0b 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -32,6 +32,7 @@ import android.graphics.PixelFormat; import android.media.Image; import android.media.ImageReader; import android.media.MediaFormat; +import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -48,6 +49,7 @@ import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.Iterator; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -357,6 +359,18 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); } + public void queueInputBitmaps(int width, int height, Pair>... frames) { + videoFrameProcessor.registerInputStream( + INPUT_TYPE_BITMAP, + effects, + new FrameInfo.Builder(width, height) + .setPixelWidthHeightRatio(pixelWidthHeightRatio) + .build()); + for (Pair> frame : frames) { + videoFrameProcessor.queueInputBitmap(frame.first, frame.second); + } + } + public void queueInputTexture(GlTextureInfo inputTexture, long pts) { videoFrameProcessor.registerInputStream( INPUT_TYPE_TEXTURE_ID,