diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 10bf77c127..97145db344 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -204,9 +204,15 @@ public interface VideoFrameProcessor { * *

Call {@link #setInputFrameInfo} before this method if the {@link FrameInfo} of the new input * stream differs from that of the current input stream. + * + * @param inputType The {@link InputType} of the new input stream. + * @param effects The list of {@link Effect effects} to apply to the new input stream. The list is + * ignored for the first input stream registered after {@linkplain Factory#create creating the + * VideoFrameProcessor}. The first input stream will use the effects passed in in {@link + * Factory#create}. */ - // TODO(b/274109008) Merge this and setInputFrameInfo. - void registerInputStream(@InputType int inputType); + // TODO(b/286032822) Merge this and setInputFrameInfo. + void registerInputStream(@InputType int inputType, List effects); /** * Sets information about the input frames. diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index 43c2cc6afc..504fdc51b3 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -347,7 +347,8 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { blankFrameProducer.configureGlObjects(); // A frame needs to be registered despite not queuing any external input to ensure // that the video frame processor knows about the stream offset. - checkNotNull(defaultVideoFrameProcessor).registerInputStream(INPUT_TYPE_SURFACE); + checkNotNull(defaultVideoFrameProcessor) + .registerInputStream(INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of()); defaultVideoFrameProcessor.setInputFrameInfo( new FrameInfo.Builder(WIDTH, HEIGHT).build()); blankFrameProducer.produceBlankFramesAndQueueEndOfStream(inputPresentationTimesUs); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 1bb02939f3..f67aaaa3d6 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -20,7 +20,7 @@ import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; -import static com.google.common.collect.Iterables.getLast; +import static com.google.common.collect.Iterables.getFirst; import android.content.Context; import android.graphics.Bitmap; @@ -52,6 +52,7 @@ import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.ArrayList; import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; @@ -285,6 +286,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private static final String THREAD_NAME = "Effect:GlThread"; private static final long RELEASE_WAIT_TIME_MS = 500; + private final Context context; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final InputSwitcher inputSwitcher; @@ -293,13 +295,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final Executor listenerExecutor; private final boolean renderFramesAutomatically; private final FinalShaderProgramWrapper finalShaderProgramWrapper; + // Shader programs that apply Effects. - private final ImmutableList effectsShaderPrograms; + private final List intermediateGlShaderPrograms; // A queue of input streams that have not been fully processed identified by their input types. @GuardedBy("lock") private final Queue<@InputType Integer> unprocessedInputStreams; + private final List activeEffects; private final Object lock; + private final ColorInfo outputColorInfo; + private final GlObjectsProvider glObjectsProvider; // CountDownLatch to wait for the current input stream to finish processing. private volatile @MonotonicNonNull CountDownLatch latch; @@ -308,14 +314,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private volatile boolean hasRefreshedNextInputFrameInfo; private DefaultVideoFrameProcessor( + Context context, EGLDisplay eglDisplay, EGLContext eglContext, InputSwitcher inputSwitcher, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, - VideoFrameProcessor.Listener listener, + Listener listener, Executor listenerExecutor, - ImmutableList effectsShaderPrograms, - boolean renderFramesAutomatically) { + ImmutableList intermediateGlShaderPrograms, + FinalShaderProgramWrapper finalShaderProgramWrapper, + boolean renderFramesAutomatically, + ColorInfo outputColorInfo, + GlObjectsProvider glObjectsProvider) { + this.context = context; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.inputSwitcher = inputSwitcher; @@ -324,12 +335,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { this.listenerExecutor = listenerExecutor; this.renderFramesAutomatically = renderFramesAutomatically; this.unprocessedInputStreams = new ConcurrentLinkedQueue<>(); + this.activeEffects = new ArrayList<>(); this.lock = new Object(); - - checkState(!effectsShaderPrograms.isEmpty()); - checkState(getLast(effectsShaderPrograms) instanceof FinalShaderProgramWrapper); - - finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(effectsShaderPrograms); + this.outputColorInfo = outputColorInfo; + this.glObjectsProvider = glObjectsProvider; + this.finalShaderProgramWrapper = finalShaderProgramWrapper; finalShaderProgramWrapper.setOnInputStreamProcessedListener( () -> { synchronized (lock) { @@ -340,7 +350,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return inputStreamEnded && unprocessedInputStreams.isEmpty(); } }); - this.effectsShaderPrograms = effectsShaderPrograms; + this.intermediateGlShaderPrograms = new ArrayList<>(intermediateGlShaderPrograms); } /** Returns the task executor that runs video frame processing tasks. */ @@ -400,11 +410,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } @Override - public void registerInputStream(@InputType int inputType) { + public void registerInputStream(@InputType int inputType, List effects) { synchronized (lock) { if (unprocessedInputStreams.isEmpty()) { inputSwitcher.switchToInput(inputType); unprocessedInputStreams.add(inputType); + activeEffects.clear(); + activeEffects.addAll(effects); return; } } @@ -418,10 +430,47 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Thread.currentThread().interrupt(); listenerExecutor.execute(() -> listener.onError(VideoFrameProcessingException.from(e))); } - inputSwitcher.switchToInput(inputType); + synchronized (lock) { unprocessedInputStreams.add(inputType); } + + if (!activeEffects.equals(effects)) { + // TODO(b/269424561) Investigate non blocking re-configuration. + // Shader program recreation must be on GL thread. Currently the calling thread is blocked + // until all shader programs are recreated, so that DefaultVideoFrameProcessor doesn't receive + // a new frame from the new input stream prematurely. + videoFrameProcessingTaskExecutor.submitAndBlock( + () -> { + try { + for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { + intermediateGlShaderPrograms.get(i).release(); + } + intermediateGlShaderPrograms.clear(); + intermediateGlShaderPrograms.addAll( + createGlShaderPrograms( + context, effects, outputColorInfo, finalShaderProgramWrapper)); + } catch (VideoFrameProcessingException e) { + listenerExecutor.execute(() -> listener.onError(e)); + return; + } + + inputSwitcher.setDownstreamShaderProgram( + getFirst( + intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper)); + setGlObjectProviderOnShaderPrograms(intermediateGlShaderPrograms, glObjectsProvider); + chainShaderProgramsWithListeners( + intermediateGlShaderPrograms, + finalShaderProgramWrapper, + videoFrameProcessingTaskExecutor, + listener, + listenerExecutor); + + activeEffects.clear(); + activeEffects.addAll(effects); + }); + } + inputSwitcher.switchToInput(inputType); } @Override @@ -506,7 +555,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public void release() { try { videoFrameProcessingTaskExecutor.release( - /* releaseTask= */ this::releaseShaderProgramsAndDestroyGlContext, RELEASE_WAIT_TIME_MS); + /* releaseTask= */ this::releaseGlObjects, RELEASE_WAIT_TIME_MS); } catch (InterruptedException unexpected) { Thread.currentThread().interrupt(); throw new IllegalStateException(unexpected); @@ -555,7 +604,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { boolean enableColorTransfers, boolean renderFramesAutomatically, ExecutorService singleThreadExecutorService, - Executor executor, + Executor videoFrameProcessorListenerExecutor, Listener listener, GlObjectsProvider glObjectsProvider, @Nullable TextureOutputListener textureOutputListener, @@ -602,10 +651,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { videoFrameProcessingTaskExecutor, enableColorTransfers); - ImmutableList effectsShaderPrograms = - getGlShaderProgramsForGlEffects( + FinalShaderProgramWrapper finalShaderProgramWrapper = + new FinalShaderProgramWrapper( context, - effects, eglDisplay, eglContext, debugViewProvider, @@ -613,12 +661,18 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { enableColorTransfers, renderFramesAutomatically, videoFrameProcessingTaskExecutor, - executor, + videoFrameProcessorListenerExecutor, listener, glObjectsProvider, textureOutputListener, textureOutputCapacity); + // TODO(b/269424561): Move effect creation to registerInputStream(). + // The GlShaderPrograms that should be inserted in between InputSwitcher and + // FinalShaderProgramWrapper. + ImmutableList intermediateGlShaderPrograms = + createGlShaderPrograms(context, effects, outputColorInfo, finalShaderProgramWrapper); + inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_SURFACE); if (!ColorInfo.isTransferHdr(inputColorInfo)) { // HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709. @@ -628,48 +682,53 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { // Image and textureId concatenation not supported. inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_TEXTURE_ID); } - inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0)); - setGlObjectProviderOnShaderPrograms(effectsShaderPrograms, glObjectsProvider); + inputSwitcher.setDownstreamShaderProgram( + getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper)); + + setGlObjectProviderOnShaderPrograms(intermediateGlShaderPrograms, glObjectsProvider); chainShaderProgramsWithListeners( - effectsShaderPrograms, videoFrameProcessingTaskExecutor, listener, executor); + intermediateGlShaderPrograms, + finalShaderProgramWrapper, + videoFrameProcessingTaskExecutor, + listener, + videoFrameProcessorListenerExecutor); return new DefaultVideoFrameProcessor( + context, eglDisplay, eglContext, inputSwitcher, videoFrameProcessingTaskExecutor, listener, - executor, - effectsShaderPrograms, - renderFramesAutomatically); + videoFrameProcessorListenerExecutor, + intermediateGlShaderPrograms, + finalShaderProgramWrapper, + renderFramesAutomatically, + outputColorInfo, + glObjectsProvider); } /** - * Combines consecutive {@link GlMatrixTransformation} and {@link RgbMatrix} instances into a - * single {@link DefaultShaderProgram} and converts all other {@link GlEffect} instances to - * separate {@link GlShaderProgram} instances. + * Combines consecutive {@link GlMatrixTransformation GlMatrixTransformations} and {@link + * RgbMatrix RgbMatrices} instances into a single {@link DefaultShaderProgram} and converts all + * other {@link GlEffect} instances to separate {@link GlShaderProgram} instances. * *

All {@link Effect} instances must be {@link GlEffect} instances. * - * @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. The - * last is a {@link FinalShaderProgramWrapper}. + * @param context The {@link Context}. + * @param effects The list of {@link GlEffect effects}. + * @param outputColorInfo The {@link ColorInfo} on {@code DefaultVideoFrameProcessor} output. + * @param finalShaderProgramWrapper The {@link FinalShaderProgramWrapper} to apply the {@link + * GlMatrixTransformation GlMatrixTransformations} and {@link RgbMatrix RgbMatrices} after all + * other {@link GlEffect GlEffects}. + * @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. */ - private static ImmutableList getGlShaderProgramsForGlEffects( + private static ImmutableList createGlShaderPrograms( Context context, List effects, - EGLDisplay eglDisplay, - EGLContext eglContext, - DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, - boolean enableColorTransfers, - boolean renderFramesAutomatically, - VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, - Executor executor, - Listener listener, - GlObjectsProvider glObjectsProvider, - @Nullable TextureOutputListener textureOutputListener, - int textureOutputCapacity) + FinalShaderProgramWrapper finalShaderProgramWrapper) throws VideoFrameProcessingException { ImmutableList.Builder shaderProgramListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder matrixTransformationListBuilder = @@ -707,29 +766,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr)); } - shaderProgramListBuilder.add( - new FinalShaderProgramWrapper( - context, - eglDisplay, - eglContext, - matrixTransformationListBuilder.build(), - rgbMatrixListBuilder.build(), - debugViewProvider, - outputColorInfo, - enableColorTransfers, - renderFramesAutomatically, - videoFrameProcessingTaskExecutor, - executor, - listener, - glObjectsProvider, - textureOutputListener, - textureOutputCapacity)); + finalShaderProgramWrapper.setMatrixTransformations( + matrixTransformationListBuilder.build(), rgbMatrixListBuilder.build()); return shaderProgramListBuilder.build(); } /** Sets the {@link GlObjectsProvider} on all of the {@linkplain GlShaderProgram}s provided. */ private static void setGlObjectProviderOnShaderPrograms( - ImmutableList shaderPrograms, GlObjectsProvider glObjectsProvider) { + List shaderPrograms, GlObjectsProvider glObjectsProvider) { for (int i = 0; i < shaderPrograms.size() - 1; i++) { GlShaderProgram shaderProgram = shaderPrograms.get(i); shaderProgram.setGlObjectsProvider(glObjectsProvider); @@ -741,13 +785,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * ChainingGlShaderProgramListener} instances. */ private static void chainShaderProgramsWithListeners( - ImmutableList shaderPrograms, + List shaderPrograms, + FinalShaderProgramWrapper finalShaderProgramWrapper, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Listener videoFrameProcessorListener, Executor videoFrameProcessorListenerExecutor) { - for (int i = 0; i < shaderPrograms.size() - 1; i++) { - GlShaderProgram producingGlShaderProgram = shaderPrograms.get(i); - GlShaderProgram consumingGlShaderProgram = shaderPrograms.get(i + 1); + ArrayList shaderProgramsToChain = new ArrayList<>(shaderPrograms); + shaderProgramsToChain.add(finalShaderProgramWrapper); + for (int i = 0; i < shaderProgramsToChain.size() - 1; i++) { + GlShaderProgram producingGlShaderProgram = shaderProgramsToChain.get(i); + GlShaderProgram consumingGlShaderProgram = shaderProgramsToChain.get(i + 1); ChainingGlShaderProgramListener chainingGlShaderProgramListener = new ChainingGlShaderProgramListener( producingGlShaderProgram, consumingGlShaderProgram, videoFrameProcessingTaskExecutor); @@ -763,12 +810,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * *

This method must be called on the {@linkplain #THREAD_NAME background thread}. */ - private void releaseShaderProgramsAndDestroyGlContext() { + private void releaseGlObjects() { try { try { inputSwitcher.release(); - for (int i = 0; i < effectsShaderPrograms.size(); i++) { - effectsShaderPrograms.get(i).release(); + for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { + intermediateGlShaderPrograms.get(i).release(); } } catch (Exception e) { Log.e(TAG, "Error releasing shader program", e); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index d9c48f3b57..e8615df432 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -45,6 +45,8 @@ import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; @@ -77,8 +79,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String TAG = "FinalShaderWrapper"; private final Context context; - private final ImmutableList matrixTransformations; - private final ImmutableList rgbMatrices; + private final List matrixTransformations; + private final List rgbMatrices; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final DebugViewProvider debugViewProvider; @@ -105,6 +107,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Nullable private SurfaceView debugSurfaceView; @Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener; private boolean frameProcessingStarted; + private boolean matrixTransformationsChanged; @GuardedBy("this") private boolean outputSurfaceInfoChanged; @@ -122,8 +125,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, EGLDisplay eglDisplay, EGLContext eglContext, - ImmutableList matrixTransformations, - ImmutableList rgbMatrices, DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, boolean enableColorTransfers, @@ -135,8 +136,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener, int textureOutputCapacity) { this.context = context; - this.matrixTransformations = matrixTransformations; - this.rgbMatrices = rgbMatrices; + this.matrixTransformations = new ArrayList<>(); + this.rgbMatrices = new ArrayList<>(); this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.debugViewProvider = debugViewProvider; @@ -227,6 +228,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } + /** + * Sets the list of {@link GlMatrixTransformation GlMatrixTransformations} and list of {@link + * RgbMatrix RgbMatrices} to apply to the next {@linkplain #queueInputFrame queued} frame. + * + *

The new transformations will be applied to the next {@linkplain #queueInputFrame queued} + * frame. + */ + public void setMatrixTransformations( + List matrixTransformations, List rgbMatrices) { + this.matrixTransformations.clear(); + this.matrixTransformations.addAll(matrixTransformations); + this.rgbMatrices.clear(); + this.rgbMatrices.addAll(rgbMatrices); + matrixTransformationsChanged = true; + } + public void releaseOutputFrame(long presentationTimeUs) { videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs)); } @@ -456,10 +473,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } this.debugSurfaceView = debugSurfaceView; - if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) { + if (defaultShaderProgram != null + && (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) { defaultShaderProgram.release(); defaultShaderProgram = null; outputSurfaceInfoChanged = false; + matrixTransformationsChanged = false; } if (defaultShaderProgram == null) { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index eadcf9cecd..01efca73ed 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -23,7 +23,6 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.util.SparseArray; -import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; @@ -47,7 +46,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private @MonotonicNonNull GlShaderProgram downstreamShaderProgram; private @MonotonicNonNull TextureManager activeTextureManager; private boolean inputEnded; - private int activeInputType; public InputSwitcher( Context context, @@ -61,7 +59,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; this.inputs = new SparseArray<>(); this.enableColorTransfers = enableColorTransfers; - activeInputType = C.INDEX_UNSET; } /** @@ -136,25 +133,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + /** Sets the {@link GlShaderProgram} that {@code InputSwitcher} outputs to. */ public void setDownstreamShaderProgram(GlShaderProgram downstreamShaderProgram) { this.downstreamShaderProgram = downstreamShaderProgram; - - for (int i = 0; i < inputs.size(); i++) { - @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); - Input input = inputs.get(inputType); - input.setChainingListener( - new GatedChainingListenerWrapper( - input.samplingGlShaderProgram, - this.downstreamShaderProgram, - videoFrameProcessingTaskExecutor)); - } } /** * Switches to a new source of input. * - *

Blocks until the current input stream is processed. - * *

Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput * registered}. * @@ -164,14 +150,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkStateNotNull(downstreamShaderProgram); checkState(inputs.indexOfKey(newInputType) >= 0, "Input type not registered: " + newInputType); - if (newInputType == activeInputType) { - activeTextureManager = inputs.get(activeInputType).textureManager; - } - for (int i = 0; i < inputs.size(); i++) { @VideoFrameProcessor.InputType int inputType = inputs.keyAt(i); Input input = inputs.get(inputType); if (inputType == newInputType) { + input.setChainingListener( + new GatedChainingListenerWrapper( + input.samplingGlShaderProgram, + this.downstreamShaderProgram, + videoFrameProcessingTaskExecutor)); input.setActive(true); downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper)); activeTextureManager = input.textureManager; @@ -179,7 +166,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; input.setActive(false); } } - activeInputType = newInputType; } /** @@ -240,7 +226,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } public void setActive(boolean active) { - checkStateNotNull(gatedChainingListenerWrapper); + if (gatedChainingListenerWrapper == null) { + return; + } gatedChainingListenerWrapper.setActive(active); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixUtils.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixUtils.java index 281aed461e..f91d4e1c31 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixUtils.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixUtils.java @@ -21,6 +21,7 @@ import android.opengl.Matrix; import androidx.media3.common.util.Size; import com.google.common.collect.ImmutableList; import java.util.Arrays; +import java.util.List; /** Utility functions for working with matrices, vertices, and polygons. */ /* package */ final class MatrixUtils { @@ -223,9 +224,7 @@ import java.util.Arrays; * GlMatrixTransformations} to an input frame with the given size. */ public static Size configureAndGetOutputSize( - int inputWidth, - int inputHeight, - ImmutableList matrixTransformations) { + int inputWidth, int inputHeight, List matrixTransformations) { checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive"); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoFrameProcessingTaskExecutor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoFrameProcessingTaskExecutor.java index be1bb6a64e..a5fdf33f6e 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoFrameProcessingTaskExecutor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoFrameProcessingTaskExecutor.java @@ -23,7 +23,9 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.UnstableApi; import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; @@ -52,7 +54,7 @@ import java.util.concurrent.RejectedExecutionException; private final Object lock; @GuardedBy("lock") - private final ArrayDeque highPriorityTasks; + private final Queue highPriorityTasks; @GuardedBy("lock") private boolean shouldCancelTasks; @@ -89,6 +91,28 @@ import java.util.concurrent.RejectedExecutionException; } } + /** + * Submits the given {@link VideoFrameProcessingTask} to execute, and returns after the task is + * executed. + */ + public void submitAndBlock(VideoFrameProcessingTask task) { + synchronized (lock) { + if (shouldCancelTasks) { + return; + } + } + + Future future = wrapTaskAndSubmitToExecutorService(task, /* isFlushOrReleaseTask= */ false); + try { + future.get(); + } catch (ExecutionException e) { + handleException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + handleException(e); + } + } + /** * Submits the given {@link VideoFrameProcessingTask} to be executed after the currently running * task and all previously submitted high-priority tasks have completed. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index f0ffd0d1eb..918624ae8e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2120,7 +2120,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { throw new IllegalStateException(); } }); - videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE); + + videoFrameProcessor.registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of()); this.initialStreamOffsetUs = initialStreamOffsetUs; } catch (Exception e) { throw renderer.createRendererException( diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index b4d0ffb632..410675eee5 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -312,7 +312,7 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessingEnded = true; } }); - videoFrameProcessor.registerInputStream(inputType); + videoFrameProcessor.registerInputStream(inputType, /* effects= */ ImmutableList.of()); } public void processFirstFrameAndEnd() throws Exception { @@ -327,7 +327,8 @@ public final class VideoFrameProcessorTestRunner { mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); - videoFrameProcessor.registerInputStream(INPUT_TYPE_SURFACE); + videoFrameProcessor.registerInputStream( + INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of()); videoFrameProcessor.registerInputFrame(); } @@ -347,7 +348,7 @@ public final class VideoFrameProcessorTestRunner { .setPixelWidthHeightRatio(pixelWidthHeightRatio) .setOffsetToAddUs(offsetToAddUs) .build()); - videoFrameProcessor.registerInputStream(INPUT_TYPE_BITMAP); + videoFrameProcessor.registerInputStream(INPUT_TYPE_BITMAP, /* effects= */ ImmutableList.of()); videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); } @@ -356,7 +357,8 @@ public final class VideoFrameProcessorTestRunner { new FrameInfo.Builder(inputTexture.getWidth(), inputTexture.getHeight()) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); - videoFrameProcessor.registerInputStream(INPUT_TYPE_TEXTURE_ID); + videoFrameProcessor.registerInputStream( + INPUT_TYPE_TEXTURE_ID, /* effects= */ ImmutableList.of()); videoFrameProcessor.setOnInputFrameProcessedListener( texId -> { try { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index f07cae11f9..8d57fee861 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -59,7 +59,6 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -74,6 +73,7 @@ import org.checkerframework.dataflow.qual.Pure; private final ColorInfo videoFrameProcessorInputColor; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; + @Nullable final Presentation presentation; private volatile boolean encoderExpectsTimestampZero; /** @@ -149,15 +149,12 @@ import org.checkerframework.dataflow.qual.Pure; videoFrameProcessorOutputColor = videoFrameProcessorInputColor; } - List effectsWithPresentation = new ArrayList<>(effects); - if (presentation != null) { - effectsWithPresentation.add(presentation); - } + this.presentation = presentation; try { videoFrameProcessor = videoFrameProcessorFactory.create( context, - effectsWithPresentation, + createEffectListWithPresentation(effects, presentation), debugViewProvider, videoFrameProcessorInputColor, videoFrameProcessorOutputColor, @@ -216,7 +213,8 @@ import org.checkerframework.dataflow.qual.Pure; if (trackFormat != null) { Size decodedSize = getDecodedSize(trackFormat); videoFrameProcessor.registerInputStream( - getInputType(checkNotNull(trackFormat.sampleMimeType))); + getInputType(checkNotNull(trackFormat.sampleMimeType)), + createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation)); videoFrameProcessor.setInputFrameInfo( new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight()) .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio) @@ -335,6 +333,16 @@ import org.checkerframework.dataflow.qual.Pure; return new Size(decodedWidth, decodedHeight); } + private static ImmutableList createEffectListWithPresentation( + List effects, @Nullable Presentation presentation) { + if (presentation == null) { + return ImmutableList.copyOf(effects); + } + ImmutableList.Builder effectsWithPresentationBuilder = new ImmutableList.Builder<>(); + effectsWithPresentationBuilder.addAll(effects).add(presentation); + return effectsWithPresentationBuilder.build(); + } + /** * Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}. *