From 95260e28a56e46c2bf5fb42af5fe1bd03e91dbfb Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Tue, 2 Jul 2024 05:38:56 -0700 Subject: [PATCH] Add input type for automatic frame registration Deprecate `setInputDefaultBufferSize` and `setRequireRegisteringAllInputFrames` as the new input stream type replaces these (as far as we know they are always used together). This is in preparation for supporting asset loaders signaling that they require these features, specifically for recording from a surface. PiperOrigin-RevId: 648686087 --- RELEASENOTES.md | 4 + .../media3/common/VideoFrameProcessor.java | 20 ++++- .../DefaultVideoFrameProcessorTest.java | 74 +++++++++++++++++++ .../effect/DefaultVideoFrameProcessor.java | 13 ++++ .../media3/effect/ExternalTextureManager.java | 13 +++- .../androidx/media3/effect/InputSwitcher.java | 56 +++++++------- .../media3/effect/TexIdTextureManager.java | 2 +- .../media3/effect/TextureManager.java | 11 ++- 8 files changed, 160 insertions(+), 33 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e11f91e0d1..502cf2f32c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -31,6 +31,10 @@ * Image: * DRM: * Effect: + * Deprecate `DefaultVideoFrameProcessor#setInputDefaultBufferSize` and + `DefaultVideoFrameProcessor.Builder#setRequireRegisteringAllInputFrames`. + Use the new frame processor input type + `INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION` instead. * Muxers: * IMA extension: * Session: diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index f891f04b3d..574946c36b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -48,12 +48,18 @@ import java.util.concurrent.Executor; public interface VideoFrameProcessor { /** * Specifies how the input frames are made available to the {@link VideoFrameProcessor}. One of - * {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP} or {@link #INPUT_TYPE_TEXTURE_ID}. + * {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP}, {@link #INPUT_TYPE_TEXTURE_ID} or + * {@link #INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION}. */ @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) - @IntDef({INPUT_TYPE_SURFACE, INPUT_TYPE_BITMAP, INPUT_TYPE_TEXTURE_ID}) + @IntDef({ + INPUT_TYPE_SURFACE, + INPUT_TYPE_BITMAP, + INPUT_TYPE_TEXTURE_ID, + INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, + }) @interface InputType {} /** @@ -73,6 +79,16 @@ public interface VideoFrameProcessor { */ int INPUT_TYPE_TEXTURE_ID = 3; + /** + * Input frames come from the {@linkplain #getInputSurface input surface} and don't need to be + * {@linkplain #registerInputFrame registered} (unlike with {@link #INPUT_TYPE_SURFACE}). + * + *

Every frame must use the {@linkplain #registerInputStream(int, List, FrameInfo) input + * stream's registered} frame info. Also sets the surface's {@linkplain + * android.graphics.SurfaceTexture#setDefaultBufferSize(int, int) default buffer size}. + */ + int INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION = 4; + /** A factory for {@link VideoFrameProcessor} instances. */ interface Factory { diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java index 65d1982baa..645eee84a0 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java @@ -16,12 +16,15 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmapUnpremultipliedAlpha; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; @@ -32,8 +35,10 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.ConstantRateTimestampIterator; +import androidx.media3.common.util.NullableType; import androidx.media3.common.util.SystemClock; import androidx.media3.common.util.Util; +import androidx.media3.test.utils.BitmapPixelTestUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; @@ -41,6 +46,7 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -288,6 +294,74 @@ public class DefaultVideoFrameProcessorTest { .isAtLeast(firstStreamLastFrameAvailableTimeMs.get()); } + @Test + public void registerInputStreamWithAutomaticFrameRegistration_succeeds() throws Exception { + CountDownLatch inputStreamRegisteredCountDownLatch = new CountDownLatch(1); + AtomicInteger outputFrameCount = new AtomicInteger(); + AtomicReference<@NullableType Exception> error = new AtomicReference<>(); + CountDownLatch endedCountDownLatch = new CountDownLatch(1); + defaultVideoFrameProcessor = + createDefaultVideoFrameProcessor( + new VideoFrameProcessor.Listener() { + @Override + public void onInputStreamRegistered( + @VideoFrameProcessor.InputType int inputType, + List effects, + FrameInfo frameInfo) { + inputStreamRegisteredCountDownLatch.countDown(); + } + + @Override + public void onOutputSizeChanged(int width, int height) {} + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + outputFrameCount.incrementAndGet(); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + error.set(exception); + } + + @Override + public void onEnded() { + endedCountDownLatch.countDown(); + } + }); + + Bitmap bitmap = BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); + defaultVideoFrameProcessor.registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, + /* effects= */ ImmutableList.of(), + new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, bitmap.getWidth(), bitmap.getHeight()) + .build()); + inputStreamRegisteredCountDownLatch.await(); + checkState(defaultVideoFrameProcessor.registerInputFrame()); + + int inputFrameCount = 2; + Surface surface = defaultVideoFrameProcessor.getInputSurface(); + for (int i = 0; i < inputFrameCount; i++) { + Canvas canvas = surface.lockCanvas(/* inOutDirty= */ null); + // Load the bitmap each time, as it's recycled after each use. + canvas.drawBitmap( + BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* left= */ 0f, + /* top= */ 0f, + /* paint= */ null); + // This causes a frame to become available on the input surface, which is processed by the + // video frame processor. + surface.unlockCanvasAndPost(canvas); + } + defaultVideoFrameProcessor.signalEndOfInput(); + + if (!endedCountDownLatch.await(TEST_TIMEOUT_MS, MILLISECONDS)) { + throw new IllegalStateException("Test timeout", error.get()); + } + assertThat(error.get()).isNull(); + assertThat(outputFrameCount.get()).isEqualTo(inputFrameCount); + } + private DefaultVideoFrameProcessor createDefaultVideoFrameProcessor( VideoFrameProcessor.Listener listener) throws Exception { return checkNotNull(factory) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index dd1e65e9d8..2957172d80 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -199,7 +199,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { *

Regardless of the value set, {@link #registerInputStream(int, List, FrameInfo)} must be * called for each input stream to specify the format for upcoming frames before calling * {@link #registerInputFrame()}. + * + * @param requireRegisteringAllInputFrames Whether registering every input frame is required. + * @deprecated For automatic frame registration ({@code + * setRequireRegisteringAllInputFrames(false)}), use {@link + * VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead. This call + * can be removed otherwise. */ + @Deprecated @CanIgnoreReturnValue public Builder setRequireRegisteringAllInputFrames(boolean requireRegisteringAllInputFrames) { this.requireRegisteringAllInputFrames = requireRegisteringAllInputFrames; @@ -514,7 +521,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * * @param width The default width for input buffers, in pixels. * @param height The default height for input buffers, in pixels. + * @deprecated Set the input type to {@link + * VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead, which sets + * the default buffer size automatically based on the registered frame info. */ + @Deprecated public void setInputDefaultBufferSize(int width, int height) { inputSwitcher.setInputDefaultBufferSize(width, height); } @@ -933,6 +944,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return "Bitmap"; case INPUT_TYPE_TEXTURE_ID: return "Texture ID"; + case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION: + return "Surface with automatic frame registration"; default: throw new IllegalArgumentException(String.valueOf(inputType)); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 860e3d4827..e9149a6096 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -87,7 +87,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final float[] textureTransformMatrix; private final Queue pendingFrames; private final ScheduledExecutorService scheduledExecutorService; - private final boolean repeatLastRegisteredFrame; private final boolean experimentalAdjustSurfaceTextureTransformationMatrix; // Must be accessed on the GL thread. @@ -98,6 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The frame that is sent downstream and is not done processing yet. @Nullable private FrameInfo currentFrame; @Nullable private FrameInfo lastRegisteredFrame; + private boolean repeatLastRegisteredFrame; @Nullable private Future forceSignalEndOfStreamFuture; private boolean shouldRejectIncomingFrames; @@ -234,6 +234,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; }); } + @Override + public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { + // Ignore inputFrameInfo when not automatically re-registering frames because it's also passed + // to registerInputFrame. + repeatLastRegisteredFrame = automaticReregistration; + if (repeatLastRegisteredFrame) { + lastRegisteredFrame = inputFrameInfo; + surfaceTexture.setDefaultBufferSize(inputFrameInfo.width, inputFrameInfo.height); + } + } + /** * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will * become available via the {@link SurfaceTexture} eventually. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 603bd43438..cd53301d99 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -19,6 +19,7 @@ package androidx.media3.effect; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; @@ -81,15 +82,16 @@ import org.checkerframework.checker.nullness.qual.Nullable; this.experimentalAdjustSurfaceTextureTransformationMatrix = experimentalAdjustSurfaceTextureTransformationMatrix; - // TODO(b/274109008): Investigate lazy instantiating the texture managers. - inputs.put( - INPUT_TYPE_SURFACE, + // TODO(b/274109008): Investigate lazily instantiating the texture managers. + Input surfaceInput = new Input( new ExternalTextureManager( glObjectsProvider, videoFrameProcessingTaskExecutor, repeatLastRegisteredFrame, - experimentalAdjustSurfaceTextureTransformationMatrix))); + experimentalAdjustSurfaceTextureTransformationMatrix)); + inputs.put(INPUT_TYPE_SURFACE, surfaceInput); + inputs.put(INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, surfaceInput); inputs.put( INPUT_TYPE_BITMAP, new Input( @@ -109,6 +111,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; DefaultShaderProgram samplingShaderProgram; switch (inputType) { case INPUT_TYPE_SURFACE: + case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION: samplingShaderProgram = DefaultShaderProgram.createWithExternalSampler( context, @@ -152,29 +155,28 @@ import org.checkerframework.checker.nullness.qual.Nullable; checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType); for (int i = 0; i < inputs.size(); i++) { - @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); - Input input = inputs.get(inputType); - if (inputType == newInputType) { - if (input.getInputColorInfo() == null - || !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) { - input.setSamplingGlShaderProgram( - createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType)); - input.setInputColorInfo(newInputFrameInfo.colorInfo); - } - input.setChainingListener( - new GatedChainingListenerWrapper( - glObjectsProvider, - checkNotNull(input.getSamplingGlShaderProgram()), - this.downstreamShaderProgram, - videoFrameProcessingTaskExecutor)); - input.setActive(true); - downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); - activeTextureManager = input.textureManager; - } else { - input.setActive(false); - } + inputs.get(inputs.keyAt(i)).setActive(false); } - checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo); + + // Activate the relevant input for the new input type. + Input input = inputs.get(newInputType); + if (input.getInputColorInfo() == null + || !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) { + input.setSamplingGlShaderProgram( + createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType)); + input.setInputColorInfo(newInputFrameInfo.colorInfo); + } + input.setChainingListener( + new GatedChainingListenerWrapper( + glObjectsProvider, + checkNotNull(input.getSamplingGlShaderProgram()), + this.downstreamShaderProgram, + videoFrameProcessingTaskExecutor)); + input.setActive(true); + downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); + activeTextureManager = input.textureManager; + boolean automaticRegistration = newInputType == INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION; + checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo, automaticRegistration); } /** Returns whether the {@code InputSwitcher} is connected to an active input. */ @@ -203,7 +205,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** * Returns the input {@link Surface}. * - * @return The input {@link Surface}, regardless if the current input is {@linkplain + * @return The input {@link Surface}, regardless of whether the current input is {@linkplain * #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. */ public Surface getInputSurface() { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java index 031433e8e2..e029660ad5 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java @@ -106,7 +106,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void setInputFrameInfo(FrameInfo inputFrameInfo) { + public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { this.inputFrameInfo = inputFrameInfo; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index ea884bfff0..2479c66c9d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -107,12 +107,19 @@ import androidx.media3.common.util.TimestampIterator; * Sets information about the input frames. * *

The new input information is applied from the next frame {@linkplain #registerInputFrame - * registered} or {@linkplain #queueInputTexture queued} onwards. + * registered} or {@linkplain #queueInputTexture queued} onwards. If the implementation requires + * frames to be registered, it may use the {@link FrameInfo} passed to {@link + * #registerInputFrame(FrameInfo)} instead of the one passed here. * *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output * frames' pixels have a ratio of 1. + * + * @param inputFrameInfo Information about the next input frame. + * @param automaticReregistration Whether the frames should be re-registered automatically, if + * using an input surface. Pass {@code false} if every frame will be registered before it is + * rendered to the surface. */ - public void setInputFrameInfo(FrameInfo inputFrameInfo) { + public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { // Do nothing. }