diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 10b0eab9a4..b0372a3b37 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

Public methods in this class can be called from any thread. */ @UnstableApi -/* package */ final class BitmapTextureManager implements InputHandler { +/* package */ final class BitmapTextureManager implements TextureManager { private final GlShaderProgram shaderProgram; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; // The queue holds all bitmaps with one or more frames pending to be sent downstream. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 32ad6073b5..5a6726112a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -251,11 +251,15 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final EGLDisplay eglDisplay; private final EGLContext eglContext; + private final InputSwitcher inputSwitcher; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; - private final InputHandler inputHandler; + // TODO(b/274109008) Use InputSwither to interact with texture manager. + // Owned and released by inputSwitcher. + private final TextureManager textureManager; private final boolean renderFramesAutomatically; private final FinalShaderProgramWrapper finalShaderProgramWrapper; - private final ImmutableList allShaderPrograms; + // Shader programs that apply Effects. + private final ImmutableList effectsShaderPrograms; // A queue of input streams that have not been fully processed identified by their input types. private final Queue<@InputType Integer> unprocessedInputStreams; @@ -268,41 +272,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private DefaultVideoFrameProcessor( EGLDisplay eglDisplay, EGLContext eglContext, + InputSwitcher inputSwitcher, @InputType int inputType, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, - ImmutableList shaderPrograms, - boolean renderFramesAutomatically) - throws VideoFrameProcessingException { - + ImmutableList effectsShaderPrograms, + boolean renderFramesAutomatically) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; + this.inputSwitcher = inputSwitcher; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; this.renderFramesAutomatically = renderFramesAutomatically; this.unprocessedInputStreams = new ConcurrentLinkedQueue<>(); - checkState(!shaderPrograms.isEmpty()); - checkState(getLast(shaderPrograms) instanceof FinalShaderProgramWrapper); + checkState(!effectsShaderPrograms.isEmpty()); + checkState(getLast(effectsShaderPrograms) instanceof FinalShaderProgramWrapper); - GlShaderProgram inputShaderProgram = shaderPrograms.get(0); - - switch (inputType) { - case VideoFrameProcessor.INPUT_TYPE_SURFACE: - checkState(inputShaderProgram instanceof ExternalShaderProgram); - inputHandler = - new ExternalTextureManager( - (ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor); - break; - case VideoFrameProcessor.INPUT_TYPE_BITMAP: - inputHandler = - new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor); - break; - case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through - default: - throw new VideoFrameProcessingException("Input type not supported yet"); - } - inputShaderProgram.setInputListener(inputHandler); - - finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms); + textureManager = inputSwitcher.switchToInput(inputType); + finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(effectsShaderPrograms); finalShaderProgramWrapper.setOnInputStreamProcessedListener( () -> { @InputType int currentInputType = unprocessedInputStreams.remove(); @@ -319,7 +305,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } return inputStreamEnded && unprocessedInputStreams.isEmpty(); }); - allShaderPrograms = shaderPrograms; + this.effectsShaderPrograms = effectsShaderPrograms; } /** Returns the task executor that runs video frame processing tasks. */ @@ -344,7 +330,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * @param height The default height for input buffers, in pixels. */ public void setInputDefaultBufferSize(int width, int height) { - inputHandler.setDefaultBufferSize(width, height); + textureManager.setDefaultBufferSize(width, height); } @Override @@ -352,7 +338,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { checkState( hasRefreshedNextInputFrameInfo, "setInputFrameInfo must be called before queueing another bitmap"); - inputHandler.queueInputBitmap( + textureManager.queueInputBitmap( inputBitmap, durationUs, checkNotNull(nextInputFrameInfo).offsetToAddUs, @@ -363,13 +349,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Override public Surface getInputSurface() { - return inputHandler.getInputSurface(); + return textureManager.getInputSurface(); } @Override public void registerInputStream(@InputType int inputType) { if (!unprocessedInputStreams.isEmpty()) { - inputHandler.signalEndOfCurrentInputStream(); + textureManager.signalEndOfCurrentInputStream(); // Wait until the current video is processed before continuing to the next input. if (checkNotNull(unprocessedInputStreams.peek()) == INPUT_TYPE_SURFACE) { latch = new CountDownLatch(1); @@ -396,13 +382,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { checkStateNotNull( nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); - inputHandler.registerInputFrame(nextInputFrameInfo); + textureManager.registerInputFrame(nextInputFrameInfo); hasRefreshedNextInputFrameInfo = false; } @Override public int getPendingInputFrameCount() { - return inputHandler.getPendingFrameCount(); + return textureManager.getPendingFrameCount(); } @Override @@ -423,8 +409,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public void signalEndOfInput() { checkState(!inputStreamEnded); inputStreamEnded = true; - inputHandler.signalEndOfCurrentInputStream(); - inputHandler.signalEndOfInput(); + textureManager.signalEndOfCurrentInputStream(); + inputSwitcher.signalEndOfInput(); } @Override @@ -432,10 +418,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { try { videoFrameProcessingTaskExecutor.flush(); CountDownLatch latch = new CountDownLatch(1); - inputHandler.setOnFlushCompleteListener(latch::countDown); + textureManager.setOnFlushCompleteListener(latch::countDown); videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush); latch.await(); - inputHandler.setOnFlushCompleteListener(null); + textureManager.setOnFlushCompleteListener(null); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -450,7 +436,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Thread.currentThread().interrupt(); throw new IllegalStateException(unexpected); } - inputHandler.release(); } /** @@ -526,35 +511,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { throw new VideoFrameProcessingException("BT.2020 PQ OpenGL output isn't supported."); } } + VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor = + new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener); + ColorInfo linearColorInfo = + outputColorInfo + .buildUpon() + .setColorTransfer(C.COLOR_TRANSFER_LINEAR) + .setHdrStaticInfo(null) + .build(); + InputSwitcher inputSwitcher = + new InputSwitcher( + context, + inputColorInfo, + /* outputColorInfo= */ linearColorInfo, + glObjectsProvider, + videoFrameProcessingTaskExecutor, + enableColorTransfers); - ImmutableList shaderPrograms = + ImmutableList effectsShaderPrograms = getGlShaderProgramsForGlEffects( context, effects, eglDisplay, eglContext, debugViewProvider, - inputColorInfo, + /* inputColorInfo= */ linearColorInfo, outputColorInfo, enableColorTransfers, - inputType, renderFramesAutomatically, executor, listener, glObjectsProvider, textureOutputListener); - setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider); - VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor = - new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener); + + // TODO(b/274109008): Register both image and video input. + inputSwitcher.registerInput(inputType); + inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0)); + + setGlObjectProviderOnShaderPrograms(effectsShaderPrograms, glObjectsProvider); chainShaderProgramsWithListeners( - shaderPrograms, videoFrameProcessingTaskExecutor, listener, executor); + effectsShaderPrograms, videoFrameProcessingTaskExecutor, listener, executor); return new DefaultVideoFrameProcessor( eglDisplay, eglContext, + inputSwitcher, inputType, videoFrameProcessingTaskExecutor, - shaderPrograms, + effectsShaderPrograms, renderFramesAutomatically); } @@ -566,8 +570,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { *

All {@link Effect} instances must be {@link GlEffect} instances. * * @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. The - * first is an {@link ExternalShaderProgram} and the last is a {@link - * FinalShaderProgramWrapper}. + * last is a {@link FinalShaderProgramWrapper}. */ private static ImmutableList getGlShaderProgramsForGlEffects( Context context, @@ -578,7 +581,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ColorInfo inputColorInfo, ColorInfo outputColorInfo, boolean enableColorTransfers, - @InputType int inputType, boolean renderFramesAutomatically, Executor executor, Listener listener, @@ -589,13 +591,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder rgbMatrixListBuilder = new ImmutableList.Builder<>(); - boolean sampleFromInputTexture = true; - ColorInfo linearColorInfo = - outputColorInfo - .buildUpon() - .setColorTransfer(C.COLOR_TRANSFER_LINEAR) - .setHdrStaticInfo(null) - .build(); for (int i = 0; i < effects.size(); i++) { Effect effect = effects.get(i); checkArgument( @@ -617,38 +612,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { matrixTransformationListBuilder.build(); ImmutableList rgbMatrices = rgbMatrixListBuilder.build(); boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo); - if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) { - DefaultShaderProgram defaultShaderProgram; - if (sampleFromInputTexture) { - if (inputType == INPUT_TYPE_SURFACE) { - defaultShaderProgram = - DefaultShaderProgram.createWithExternalSampler( - context, - matrixTransformations, - rgbMatrices, - inputColorInfo, - linearColorInfo, - enableColorTransfers); - } else { - defaultShaderProgram = - DefaultShaderProgram.createWithInternalSampler( - context, - matrixTransformations, - rgbMatrices, - inputColorInfo, - linearColorInfo, - enableColorTransfers, - inputType); - } - } else { - defaultShaderProgram = - DefaultShaderProgram.create( - context, matrixTransformations, rgbMatrices, isOutputTransferHdr); - } + if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) { + DefaultShaderProgram defaultShaderProgram = + DefaultShaderProgram.create( + context, matrixTransformations, rgbMatrices, isOutputTransferHdr); shaderProgramListBuilder.add(defaultShaderProgram); matrixTransformationListBuilder = new ImmutableList.Builder<>(); rgbMatrixListBuilder = new ImmutableList.Builder<>(); - sampleFromInputTexture = false; } shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr)); } @@ -661,11 +631,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { matrixTransformationListBuilder.build(), rgbMatrixListBuilder.build(), debugViewProvider, - /* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo, + inputColorInfo, outputColorInfo, enableColorTransfers, - sampleFromInputTexture, - inputType, renderFramesAutomatically, executor, listener, @@ -712,12 +680,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { */ private void releaseShaderProgramsAndDestroyGlContext() { try { - for (int i = 0; i < allShaderPrograms.size(); i++) { - try { - allShaderPrograms.get(i).release(); - } catch (Exception e) { - Log.e(TAG, "Error releasing shader program", e); + try { + inputSwitcher.release(); + for (int i = 0; i < effectsShaderPrograms.size(); i++) { + effectsShaderPrograms.get(i).release(); } + } catch (Exception e) { + Log.e(TAG, "Error releasing shader program", e); } } finally { try { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java index 2c13097d65..2958ddd058 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.graphics.Bitmap; @@ -35,7 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger; * Forwards externally produced frames that become available via a {@link SurfaceTexture} to an * {@link ExternalShaderProgram} for consumption. */ -/* package */ final class ExternalTextureManager implements InputHandler { +/* package */ final class ExternalTextureManager implements TextureManager { private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final ExternalShaderProgram externalShaderProgram; @@ -167,6 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ @Override public void registerInputFrame(FrameInfo frame) { + checkState(!inputStreamEnded); pendingFrames.add(frame); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index ef57aa623e..d3a07df56c 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; @@ -63,7 +62,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of * {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}. */ -/* package */ final class FinalShaderProgramWrapper implements ExternalShaderProgram { +/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram { /** Listener interface for the current input stream ending. */ interface OnInputStreamProcessedListener { @@ -82,15 +81,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final DebugViewProvider debugViewProvider; - private final boolean sampleFromInputTexture; - private final @VideoFrameProcessor.InputType int inputType; - private final ColorInfo inputColorInfo; private final ColorInfo outputColorInfo; private final boolean enableColorTransfers; private final boolean renderFramesAutomatically; private final Executor videoFrameProcessorListenerExecutor; private final VideoFrameProcessor.Listener videoFrameProcessorListener; - private final float[] textureTransformMatrix; private final Queue> availableFrames; @Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener; @@ -127,8 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ColorInfo inputColorInfo, ColorInfo outputColorInfo, boolean enableColorTransfers, - boolean sampleFromInputTexture, - @VideoFrameProcessor.InputType int inputType, boolean renderFramesAutomatically, Executor videoFrameProcessorListenerExecutor, VideoFrameProcessor.Listener videoFrameProcessorListener, @@ -140,9 +133,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.debugViewProvider = debugViewProvider; - this.sampleFromInputTexture = sampleFromInputTexture; - this.inputType = inputType; - this.inputColorInfo = inputColorInfo; this.outputColorInfo = outputColorInfo; this.enableColorTransfers = enableColorTransfers; this.renderFramesAutomatically = renderFramesAutomatically; @@ -151,7 +141,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.glObjectsProvider = glObjectsProvider; this.textureOutputListener = textureOutputListener; - textureTransformMatrix = GlUtil.create4x4IdentityMatrix(); inputListener = new InputListener() {}; availableFrames = new ConcurrentLinkedQueue<>(); } @@ -240,20 +229,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputListener.onReadyToAcceptInputFrame(); } - @Override - public void setTextureTransformMatrix(float[] textureTransformMatrix) { - System.arraycopy( - /* src= */ textureTransformMatrix, - /* srcPos= */ 0, - /* dest= */ this.textureTransformMatrix, - /* destPost= */ 0, - /* length= */ textureTransformMatrix.length); - - if (defaultShaderProgram != null) { - defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix); - } - } - @Override public synchronized void release() throws VideoFrameProcessingException { if (defaultShaderProgram != null) { @@ -479,38 +454,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; DefaultShaderProgram defaultShaderProgram; ImmutableList expandedMatrixTransformations = matrixTransformationListBuilder.build(); - if (sampleFromInputTexture) { - if (inputType == INPUT_TYPE_SURFACE) { - defaultShaderProgram = - DefaultShaderProgram.createWithExternalSampler( - context, - expandedMatrixTransformations, - rgbMatrices, - inputColorInfo, - outputColorInfo, - enableColorTransfers); - } else { - defaultShaderProgram = - DefaultShaderProgram.createWithInternalSampler( - context, - expandedMatrixTransformations, - rgbMatrices, - inputColorInfo, - outputColorInfo, - enableColorTransfers, - inputType); - } - } else { - defaultShaderProgram = - DefaultShaderProgram.createApplyingOetf( - context, - expandedMatrixTransformations, - rgbMatrices, - outputColorInfo, - enableColorTransfers); - } + defaultShaderProgram = + DefaultShaderProgram.createApplyingOetf( + context, + expandedMatrixTransformations, + rgbMatrices, + outputColorInfo, + enableColorTransfers); - defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix); Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); if (outputSurfaceInfo != null) { SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java new file mode 100644 index 0000000000..4d66b0726e --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -0,0 +1,288 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.util.SparseArray; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A switcher to switch between {@linkplain TextureManager texture managers} of different + * {@linkplain VideoFrameProcessor.InputType input types}. + */ +/* package */ final class InputSwitcher { + private final Context context; + private final ColorInfo inputColorInfo; + private final ColorInfo outputColorInfo; + private final GlObjectsProvider glObjectsProvider; + private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; + private final SparseArray inputs; + private final boolean enableColorTransfers; + + private @MonotonicNonNull GlShaderProgram downstreamShaderProgram; + private boolean inputEnded; + private int activeInputType; + + public InputSwitcher( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + GlObjectsProvider glObjectsProvider, + VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, + boolean enableColorTransfers) { + this.context = context; + this.inputColorInfo = inputColorInfo; + this.outputColorInfo = outputColorInfo; + this.glObjectsProvider = glObjectsProvider; + this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; + this.inputs = new SparseArray<>(); + this.enableColorTransfers = enableColorTransfers; + activeInputType = C.INDEX_UNSET; + } + + /** + * Registers for a new {@link VideoFrameProcessor.InputType input}. + * + *

Can be called multiple times on the same {@link VideoFrameProcessor.InputType inputType}, + * with the new inputs overwriting the old ones. For example, a new instance of {@link + * ExternalTextureManager} is created following each call to this method with {@link + * VideoFrameProcessor#INPUT_TYPE_SURFACE}. Effectively, the {@code inputSwitcher} keeps exactly + * one {@link TextureManager} per {@linkplain VideoFrameProcessor.InputType input type}. + * + *

Creates an {@link TextureManager} and an appropriate {@linkplain DefaultShaderProgram + * sampler} to sample from the input. + */ + public void registerInput(@VideoFrameProcessor.InputType int inputType) + throws VideoFrameProcessingException { + // TODO(b/274109008): Investigate lazy instantiating the texture managers. + DefaultShaderProgram samplingShaderProgram; + TextureManager textureManager; + // TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling. + switch (inputType) { + case VideoFrameProcessor.INPUT_TYPE_SURFACE: + samplingShaderProgram = + DefaultShaderProgram.createWithExternalSampler( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(), + inputColorInfo, + outputColorInfo, + enableColorTransfers); + samplingShaderProgram.setGlObjectsProvider(glObjectsProvider); + textureManager = + new ExternalTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor); + inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); + break; + case VideoFrameProcessor.INPUT_TYPE_BITMAP: + samplingShaderProgram = + DefaultShaderProgram.createWithInternalSampler( + context, + /* matrixTransformations= */ ImmutableList.of(), + /* rgbMatrices= */ ImmutableList.of(), + inputColorInfo, + outputColorInfo, + enableColorTransfers, + inputType); + samplingShaderProgram.setGlObjectsProvider(glObjectsProvider); + textureManager = + new BitmapTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor); + inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); + break; + case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through + default: + throw new VideoFrameProcessingException("Unsupported input type " + inputType); + } + } + + public void setDownstreamShaderProgram(GlShaderProgram downstreamShaderProgram) { + this.downstreamShaderProgram = downstreamShaderProgram; + + for (int i = 0; i < inputs.size(); i++) { + @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); + Input input = inputs.get(inputType); + input.setChainingListener( + new GatedChainingListenerWrapper( + input.samplingGlShaderProgram, + this.downstreamShaderProgram, + videoFrameProcessingTaskExecutor)); + } + } + + /** + * Switches to a new source of input. + * + *

Blocks until the current input stream is processed. + * + *

Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput + * registered}. + * + * @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to. + * @return The {@link TextureManager} associated with the {@code newInputType}. + */ + public TextureManager switchToInput(@VideoFrameProcessor.InputType int newInputType) { + checkStateNotNull(downstreamShaderProgram); + checkState(inputs.indexOfKey(newInputType) >= 0, "Input type not registered: " + newInputType); + + if (newInputType == activeInputType) { + return inputs.get(activeInputType).textureManager; + } + + @Nullable TextureManager activeTextureManager = null; + for (int i = 0; i < inputs.size(); i++) { + @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); + Input input = inputs.get(inputType); + if (inputType == newInputType) { + input.setActive(true); + downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); + activeTextureManager = input.textureManager; + } else { + input.setActive(false); + } + } + + activeInputType = newInputType; + return checkNotNull(activeTextureManager); + } + + /** Signals end of input to all {@linkplain #registerInput registered inputs}. */ + public void signalEndOfInput() { + checkState(!inputEnded); + inputEnded = true; + for (int i = 0; i < inputs.size(); i++) { + @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); + inputs.get(inputType).signalEndOfInput(); + } + } + + /** Releases the resources. */ + public void release() throws VideoFrameProcessingException { + for (int i = 0; i < inputs.size(); i++) { + inputs.get(inputs.keyAt(i)).release(); + } + } + + /** + * Wraps a {@link TextureManager} and an appropriate {@linkplain GlShaderProgram sampling shader + * program}. + * + *

The output is always an internal GL texture. + */ + private static final class Input { + public final TextureManager textureManager; + public final GlShaderProgram samplingGlShaderProgram; + + private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper; + + public Input(TextureManager textureManager, GlShaderProgram samplingGlShaderProgram) { + this.textureManager = textureManager; + this.samplingGlShaderProgram = samplingGlShaderProgram; + samplingGlShaderProgram.setInputListener(textureManager); + } + + public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) { + this.gatedChainingListenerWrapper = gatedChainingListenerWrapper; + samplingGlShaderProgram.setOutputListener(gatedChainingListenerWrapper); + } + + public void setActive(boolean active) { + checkStateNotNull(gatedChainingListenerWrapper); + gatedChainingListenerWrapper.setActive(active); + } + + public void signalEndOfInput() { + textureManager.signalEndOfInput(); + } + + public void release() throws VideoFrameProcessingException { + textureManager.release(); + samplingGlShaderProgram.release(); + } + } + + /** + * Wraps a {@link ChainingGlShaderProgramListener}, with the ability to turn off the event + * listening. + */ + private static final class GatedChainingListenerWrapper + implements GlShaderProgram.OutputListener, GlShaderProgram.InputListener { + + private final ChainingGlShaderProgramListener chainingGlShaderProgramListener; + private boolean isActive = false; + + public GatedChainingListenerWrapper( + GlShaderProgram producingGlShaderProgram, + GlShaderProgram consumingGlShaderProgram, + VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) { + this.chainingGlShaderProgramListener = + new ChainingGlShaderProgramListener( + producingGlShaderProgram, consumingGlShaderProgram, videoFrameProcessingTaskExecutor); + } + + @Override + public void onReadyToAcceptInputFrame() { + if (isActive) { + chainingGlShaderProgramListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void onInputFrameProcessed(GlTextureInfo inputTexture) { + if (isActive) { + chainingGlShaderProgramListener.onInputFrameProcessed(inputTexture); + } + } + + @Override + public synchronized void onFlush() { + if (isActive) { + chainingGlShaderProgramListener.onFlush(); + } + } + + @Override + public synchronized void onOutputFrameAvailable( + GlTextureInfo outputTexture, long presentationTimeUs) { + if (isActive) { + chainingGlShaderProgramListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); + } + } + + @Override + public synchronized void onCurrentOutputStreamEnded() { + if (isActive) { + chainingGlShaderProgramListener.onCurrentOutputStreamEnded(); + } + } + + public void setActive(boolean isActive) { + this.isActive = isActive; + } + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java similarity index 95% rename from libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java rename to libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 85ad7e01e7..c05cb9a167 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -25,7 +25,7 @@ import androidx.media3.common.FrameInfo; import androidx.media3.common.VideoFrameProcessor; /** A component that handles {@code DefaultVideoFrameProcessor}'s input. */ -/* package */ interface InputHandler extends GlShaderProgram.InputListener { +/* package */ interface TextureManager extends GlShaderProgram.InputListener { /** * See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. @@ -61,7 +61,7 @@ import androidx.media3.common.VideoFrameProcessor; throw new UnsupportedOperationException(); } - /** Informs the {@code InputHandler} that a frame will be queued. */ + /** Informs the {@code TextureManager} that a frame will be queued. */ default void registerInputFrame(FrameInfo frameInfo) { throw new UnsupportedOperationException(); }