diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 502cf2f32c..e11f91e0d1 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -31,10 +31,6 @@ * Image: * DRM: * Effect: - * Deprecate `DefaultVideoFrameProcessor#setInputDefaultBufferSize` and - `DefaultVideoFrameProcessor.Builder#setRequireRegisteringAllInputFrames`. - Use the new frame processor input type - `INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION` instead. * Muxers: * IMA extension: * Session: diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 574946c36b..f891f04b3d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -48,18 +48,12 @@ import java.util.concurrent.Executor; public interface VideoFrameProcessor { /** * Specifies how the input frames are made available to the {@link VideoFrameProcessor}. One of - * {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP}, {@link #INPUT_TYPE_TEXTURE_ID} or - * {@link #INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION}. + * {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP} or {@link #INPUT_TYPE_TEXTURE_ID}. */ @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) - @IntDef({ - INPUT_TYPE_SURFACE, - INPUT_TYPE_BITMAP, - INPUT_TYPE_TEXTURE_ID, - INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, - }) + @IntDef({INPUT_TYPE_SURFACE, INPUT_TYPE_BITMAP, INPUT_TYPE_TEXTURE_ID}) @interface InputType {} /** @@ -79,16 +73,6 @@ public interface VideoFrameProcessor { */ int INPUT_TYPE_TEXTURE_ID = 3; - /** - * Input frames come from the {@linkplain #getInputSurface input surface} and don't need to be - * {@linkplain #registerInputFrame registered} (unlike with {@link #INPUT_TYPE_SURFACE}). - * - *

Every frame must use the {@linkplain #registerInputStream(int, List, FrameInfo) input - * stream's registered} frame info. Also sets the surface's {@linkplain - * android.graphics.SurfaceTexture#setDefaultBufferSize(int, int) default buffer size}. - */ - int INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION = 4; - /** A factory for {@link VideoFrameProcessor} instances. */ interface Factory { diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java index 645eee84a0..65d1982baa 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java @@ -16,15 +16,12 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmapUnpremultipliedAlpha; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; @@ -35,10 +32,8 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.ConstantRateTimestampIterator; -import androidx.media3.common.util.NullableType; import androidx.media3.common.util.SystemClock; import androidx.media3.common.util.Util; -import androidx.media3.test.utils.BitmapPixelTestUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; @@ -46,7 +41,6 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -294,74 +288,6 @@ public class DefaultVideoFrameProcessorTest { .isAtLeast(firstStreamLastFrameAvailableTimeMs.get()); } - @Test - public void registerInputStreamWithAutomaticFrameRegistration_succeeds() throws Exception { - CountDownLatch inputStreamRegisteredCountDownLatch = new CountDownLatch(1); - AtomicInteger outputFrameCount = new AtomicInteger(); - AtomicReference<@NullableType Exception> error = new AtomicReference<>(); - CountDownLatch endedCountDownLatch = new CountDownLatch(1); - defaultVideoFrameProcessor = - createDefaultVideoFrameProcessor( - new VideoFrameProcessor.Listener() { - @Override - public void onInputStreamRegistered( - @VideoFrameProcessor.InputType int inputType, - List effects, - FrameInfo frameInfo) { - inputStreamRegisteredCountDownLatch.countDown(); - } - - @Override - public void onOutputSizeChanged(int width, int height) {} - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - outputFrameCount.incrementAndGet(); - } - - @Override - public void onError(VideoFrameProcessingException exception) { - error.set(exception); - } - - @Override - public void onEnded() { - endedCountDownLatch.countDown(); - } - }); - - Bitmap bitmap = BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); - defaultVideoFrameProcessor.registerInputStream( - VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, - /* effects= */ ImmutableList.of(), - new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, bitmap.getWidth(), bitmap.getHeight()) - .build()); - inputStreamRegisteredCountDownLatch.await(); - checkState(defaultVideoFrameProcessor.registerInputFrame()); - - int inputFrameCount = 2; - Surface surface = defaultVideoFrameProcessor.getInputSurface(); - for (int i = 0; i < inputFrameCount; i++) { - Canvas canvas = surface.lockCanvas(/* inOutDirty= */ null); - // Load the bitmap each time, as it's recycled after each use. - canvas.drawBitmap( - BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* left= */ 0f, - /* top= */ 0f, - /* paint= */ null); - // This causes a frame to become available on the input surface, which is processed by the - // video frame processor. - surface.unlockCanvasAndPost(canvas); - } - defaultVideoFrameProcessor.signalEndOfInput(); - - if (!endedCountDownLatch.await(TEST_TIMEOUT_MS, MILLISECONDS)) { - throw new IllegalStateException("Test timeout", error.get()); - } - assertThat(error.get()).isNull(); - assertThat(outputFrameCount.get()).isEqualTo(inputFrameCount); - } - private DefaultVideoFrameProcessor createDefaultVideoFrameProcessor( VideoFrameProcessor.Listener listener) throws Exception { return checkNotNull(factory) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 2957172d80..dd1e65e9d8 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -199,14 +199,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { *

Regardless of the value set, {@link #registerInputStream(int, List, FrameInfo)} must be * called for each input stream to specify the format for upcoming frames before calling * {@link #registerInputFrame()}. - * - * @param requireRegisteringAllInputFrames Whether registering every input frame is required. - * @deprecated For automatic frame registration ({@code - * setRequireRegisteringAllInputFrames(false)}), use {@link - * VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead. This call - * can be removed otherwise. */ - @Deprecated @CanIgnoreReturnValue public Builder setRequireRegisteringAllInputFrames(boolean requireRegisteringAllInputFrames) { this.requireRegisteringAllInputFrames = requireRegisteringAllInputFrames; @@ -521,11 +514,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * * @param width The default width for input buffers, in pixels. * @param height The default height for input buffers, in pixels. - * @deprecated Set the input type to {@link - * VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead, which sets - * the default buffer size automatically based on the registered frame info. */ - @Deprecated public void setInputDefaultBufferSize(int width, int height) { inputSwitcher.setInputDefaultBufferSize(width, height); } @@ -944,8 +933,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return "Bitmap"; case INPUT_TYPE_TEXTURE_ID: return "Texture ID"; - case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION: - return "Surface with automatic frame registration"; default: throw new IllegalArgumentException(String.valueOf(inputType)); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index e9149a6096..860e3d4827 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -87,6 +87,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final float[] textureTransformMatrix; private final Queue pendingFrames; private final ScheduledExecutorService scheduledExecutorService; + private final boolean repeatLastRegisteredFrame; private final boolean experimentalAdjustSurfaceTextureTransformationMatrix; // Must be accessed on the GL thread. @@ -97,7 +98,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The frame that is sent downstream and is not done processing yet. @Nullable private FrameInfo currentFrame; @Nullable private FrameInfo lastRegisteredFrame; - private boolean repeatLastRegisteredFrame; @Nullable private Future forceSignalEndOfStreamFuture; private boolean shouldRejectIncomingFrames; @@ -234,17 +234,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; }); } - @Override - public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { - // Ignore inputFrameInfo when not automatically re-registering frames because it's also passed - // to registerInputFrame. - repeatLastRegisteredFrame = automaticReregistration; - if (repeatLastRegisteredFrame) { - lastRegisteredFrame = inputFrameInfo; - surfaceTexture.setDefaultBufferSize(inputFrameInfo.width, inputFrameInfo.height); - } - } - /** * Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will * become available via the {@link SurfaceTexture} eventually. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index cd53301d99..603bd43438 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -19,7 +19,6 @@ package androidx.media3.effect; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; @@ -82,16 +81,15 @@ import org.checkerframework.checker.nullness.qual.Nullable; this.experimentalAdjustSurfaceTextureTransformationMatrix = experimentalAdjustSurfaceTextureTransformationMatrix; - // TODO(b/274109008): Investigate lazily instantiating the texture managers. - Input surfaceInput = + // TODO(b/274109008): Investigate lazy instantiating the texture managers. + inputs.put( + INPUT_TYPE_SURFACE, new Input( new ExternalTextureManager( glObjectsProvider, videoFrameProcessingTaskExecutor, repeatLastRegisteredFrame, - experimentalAdjustSurfaceTextureTransformationMatrix)); - inputs.put(INPUT_TYPE_SURFACE, surfaceInput); - inputs.put(INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, surfaceInput); + experimentalAdjustSurfaceTextureTransformationMatrix))); inputs.put( INPUT_TYPE_BITMAP, new Input( @@ -111,7 +109,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; DefaultShaderProgram samplingShaderProgram; switch (inputType) { case INPUT_TYPE_SURFACE: - case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION: samplingShaderProgram = DefaultShaderProgram.createWithExternalSampler( context, @@ -155,28 +152,29 @@ import org.checkerframework.checker.nullness.qual.Nullable; checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType); for (int i = 0; i < inputs.size(); i++) { - inputs.get(inputs.keyAt(i)).setActive(false); + @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); + Input input = inputs.get(inputType); + if (inputType == newInputType) { + if (input.getInputColorInfo() == null + || !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) { + input.setSamplingGlShaderProgram( + createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType)); + input.setInputColorInfo(newInputFrameInfo.colorInfo); + } + input.setChainingListener( + new GatedChainingListenerWrapper( + glObjectsProvider, + checkNotNull(input.getSamplingGlShaderProgram()), + this.downstreamShaderProgram, + videoFrameProcessingTaskExecutor)); + input.setActive(true); + downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); + activeTextureManager = input.textureManager; + } else { + input.setActive(false); + } } - - // Activate the relevant input for the new input type. - Input input = inputs.get(newInputType); - if (input.getInputColorInfo() == null - || !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) { - input.setSamplingGlShaderProgram( - createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType)); - input.setInputColorInfo(newInputFrameInfo.colorInfo); - } - input.setChainingListener( - new GatedChainingListenerWrapper( - glObjectsProvider, - checkNotNull(input.getSamplingGlShaderProgram()), - this.downstreamShaderProgram, - videoFrameProcessingTaskExecutor)); - input.setActive(true); - downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); - activeTextureManager = input.textureManager; - boolean automaticRegistration = newInputType == INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION; - checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo, automaticRegistration); + checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo); } /** Returns whether the {@code InputSwitcher} is connected to an active input. */ @@ -205,7 +203,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** * Returns the input {@link Surface}. * - * @return The input {@link Surface}, regardless of whether the current input is {@linkplain + * @return The input {@link Surface}, regardless if the current input is {@linkplain * #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. */ public Surface getInputSurface() { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java index e029660ad5..031433e8e2 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java @@ -106,7 +106,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { + public void setInputFrameInfo(FrameInfo inputFrameInfo) { this.inputFrameInfo = inputFrameInfo; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 2479c66c9d..ea884bfff0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -107,19 +107,12 @@ import androidx.media3.common.util.TimestampIterator; * Sets information about the input frames. * *

The new input information is applied from the next frame {@linkplain #registerInputFrame - * registered} or {@linkplain #queueInputTexture queued} onwards. If the implementation requires - * frames to be registered, it may use the {@link FrameInfo} passed to {@link - * #registerInputFrame(FrameInfo)} instead of the one passed here. + * registered} or {@linkplain #queueInputTexture queued} onwards. * *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output * frames' pixels have a ratio of 1. - * - * @param inputFrameInfo Information about the next input frame. - * @param automaticReregistration Whether the frames should be re-registered automatically, if - * using an input surface. Pass {@code false} if every frame will be registered before it is - * rendered to the surface. */ - public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) { + public void setInputFrameInfo(FrameInfo inputFrameInfo) { // Do nothing. }