diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 3bf14f83ed..7158dc3e63 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -91,10 +91,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private @MonotonicNonNull GlObjectsProvider glObjectsProvider; private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener; private int textureOutputCapacity; + private boolean requireRegisteringAllInputFrames; /** Creates an instance. */ public Builder() { enableColorTransfers = true; + requireRegisteringAllInputFrames = true; } private Builder(Factory factory) { @@ -116,6 +118,33 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return this; } + /** + * Sets whether {@link VideoFrameProcessor#registerInputFrame() registering} every input frame + * is required. + * + *

The default value is {@code true}, meaning that all frames input to the {@link + * VideoFrameProcessor}'s input {@link #getInputSurface Surface} must be {@linkplain + * #registerInputFrame() registered} before they are rendered. In this mode the input format + * change between input streams is handled frame-exactly. If {@code false}, {@link + * #registerInputFrame} can be called only once for each {@linkplain #registerInputStream + * registered input stream} before rendering the first frame to the input {@link + * #getInputSurface() Surface}. The same registered {@link FrameInfo} is repeated for the + * subsequent frames. To ensure the format change between input streams is applied on the + * right frame, the caller needs to {@linkplain #registerInputStream(int, List, FrameInfo) + * register} the new input stream strictly after rendering all frames from the previous input + * stream. This mode should be used in streams where users don't have direct control over + * rendering frames, like in a camera feed. + * + *

Regardless of the value set, {@link #registerInputStream(int, List, FrameInfo)} must be + * called for each input stream to specify the format for upcoming frames before calling + * {@link #registerInputFrame()}. + */ + @CanIgnoreReturnValue + public Builder setRequireRegisteringAllInputFrames(boolean requireRegisteringAllInputFrames) { + this.requireRegisteringAllInputFrames = requireRegisteringAllInputFrames; + return this; + } + /** * Sets the {@link GlObjectsProvider}. * @@ -178,6 +207,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public DefaultVideoFrameProcessor.Factory build() { return new DefaultVideoFrameProcessor.Factory( enableColorTransfers, + /* repeatLastRegisteredFrame= */ !requireRegisteringAllInputFrames, glObjectsProvider == null ? new DefaultGlObjectsProvider() : glObjectsProvider, executorService, textureOutputListener, @@ -186,6 +216,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } private final boolean enableColorTransfers; + private final boolean repeatLastRegisteredFrame; private final GlObjectsProvider glObjectsProvider; @Nullable private final ExecutorService executorService; @Nullable private final GlTextureProducer.Listener textureOutputListener; @@ -193,11 +224,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private Factory( boolean enableColorTransfers, + boolean repeatLastRegisteredFrame, GlObjectsProvider glObjectsProvider, @Nullable ExecutorService executorService, @Nullable GlTextureProducer.Listener textureOutputListener, int textureOutputCapacity) { this.enableColorTransfers = enableColorTransfers; + this.repeatLastRegisteredFrame = repeatLastRegisteredFrame; this.glObjectsProvider = glObjectsProvider; this.executorService = executorService; this.textureOutputListener = textureOutputListener; @@ -264,7 +297,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { listener, glObjectsProvider, textureOutputListener, - textureOutputCapacity)); + textureOutputCapacity, + repeatLastRegisteredFrame)); try { return defaultVideoFrameProcessorFuture.get(); @@ -622,10 +656,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { boolean renderFramesAutomatically, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor videoFrameProcessorListenerExecutor, - VideoFrameProcessor.Listener listener, + Listener listener, GlObjectsProvider glObjectsProvider, @Nullable GlTextureProducer.Listener textureOutputListener, - int textureOutputCapacity) + int textureOutputCapacity, + boolean repeatLastRegisteredFrame) throws GlUtil.GlException, VideoFrameProcessingException { EGLDisplay eglDisplay = GlUtil.getDefaultEglDisplay(); int[] configAttributes = @@ -661,7 +696,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { videoFrameProcessingTaskExecutor, /* errorListenerExecutor= */ videoFrameProcessorListenerExecutor, /* samplingShaderProgramErrorListener= */ listener::onError, - enableColorTransfers); + enableColorTransfers, + repeatLastRegisteredFrame); FinalShaderProgramWrapper finalShaderProgramWrapper = new FinalShaderProgramWrapper( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index e57cb1c689..be8aa15255 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -68,6 +68,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Queue pendingFrames; private final ScheduledExecutorService forceEndOfStreamExecutorService; private final AtomicInteger externalShaderProgramInputCapacity; + private final boolean repeatLastRegisteredFrame; // Counts the frames that are registered before flush but are made available after flush. private int numberOfFramesToDropOnBecomingAvailable; @@ -76,6 +77,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The frame that is sent downstream and is not done processing yet. @Nullable private FrameInfo currentFrame; + @Nullable private FrameInfo lastRegisteredFrame; @Nullable private Future forceSignalEndOfStreamFuture; private boolean shouldRejectIncomingFrames; @@ -85,16 +87,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * * @param glObjectsProvider The {@link GlObjectsProvider} for using EGL and GLES. * @param videoFrameProcessingTaskExecutor The {@link VideoFrameProcessingTaskExecutor}. + * @param repeatLastRegisteredFrame If {@code true}, the last {@linkplain + * #registerInputFrame(FrameInfo) registered frame} is repeated for subsequent input textures + * made available on the {@linkplain #getInputSurface() input Surface}. This means the user + * can call {@link #registerInputFrame(FrameInfo)} only once. Else, every input frame needs to + * be {@linkplain #registerInputFrame(FrameInfo) registered} before they are made available on + * the {@linkplain #getInputSurface() input Surface}. * @throws VideoFrameProcessingException If a problem occurs while creating the external texture. */ // The onFrameAvailableListener will not be invoked until the constructor returns. @SuppressWarnings("nullness:method.invocation.invalid") public ExternalTextureManager( GlObjectsProvider glObjectsProvider, - VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) + VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, + boolean repeatLastRegisteredFrame) throws VideoFrameProcessingException { super(videoFrameProcessingTaskExecutor); this.glObjectsProvider = glObjectsProvider; + this.repeatLastRegisteredFrame = repeatLastRegisteredFrame; try { externalTexId = GlUtil.createExternalTexture(); } catch (GlUtil.GlException e) { @@ -190,7 +200,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ @Override public void registerInputFrame(FrameInfo frame) { - pendingFrames.add(frame); + lastRegisteredFrame = frame; + if (!repeatLastRegisteredFrame) { + pendingFrames.add(frame); + } videoFrameProcessingTaskExecutor.submit(() -> shouldRejectIncomingFrames = false); } @@ -198,6 +211,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Returns the number of {@linkplain #registerInputFrame(FrameInfo) registered} frames that have * not been sent to the downstream {@link ExternalShaderProgram} yet. * + *

This method always returns 0 if {@code ExternalTextureManager} is built with {@code + * repeatLastRegisteredFrame} equal to {@code true}. + * *

Can be called on any thread. */ @Override @@ -236,6 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; externalShaderProgramInputCapacity.set(0); currentFrame = null; pendingFrames.clear(); + lastRegisteredFrame = null; maybeExecuteAfterFlushTask(); } @@ -299,7 +316,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; surfaceTexture.updateTexImage(); availableFrameCount--; - FrameInfo currentFrame = pendingFrames.element(); + + FrameInfo currentFrame = + repeatLastRegisteredFrame ? checkNotNull(lastRegisteredFrame) : pendingFrames.element(); this.currentFrame = currentFrame; externalShaderProgramInputCapacity.decrementAndGet(); @@ -319,7 +338,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; currentFrame.width, currentFrame.height), presentationTimeUs); - checkStateNotNull(pendingFrames.remove()); + if (!repeatLastRegisteredFrame) { + checkStateNotNull(pendingFrames.remove()); + } DebugTraceUtil.logEvent(DebugTraceUtil.EVENT_VFP_QUEUE_FRAME, presentationTimeUs); // If the queued frame is the last frame, end of stream will be signaled onInputFrameProcessed. } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 3b4bd0de46..a865ac9412 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -63,7 +63,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor errorListenerExecutor, GlShaderProgram.ErrorListener samplingShaderProgramErrorListener, - boolean enableColorTransfers) + boolean enableColorTransfers, + boolean repeatLastRegisteredFrame) throws VideoFrameProcessingException { this.context = context; this.outputColorInfo = outputColorInfo; @@ -77,7 +78,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; // TODO(b/274109008): Investigate lazy instantiating the texture managers. inputs.put( INPUT_TYPE_SURFACE, - new Input(new ExternalTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor))); + new Input( + new ExternalTextureManager( + glObjectsProvider, videoFrameProcessingTaskExecutor, repeatLastRegisteredFrame))); inputs.put( INPUT_TYPE_BITMAP, new Input(new BitmapTextureManager(glObjectsProvider, videoFrameProcessingTaskExecutor)));