diff --git a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java new file mode 100644 index 0000000000..eb9c8bb08b --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.common; + +import android.content.Context; +import androidx.media3.common.util.UnstableApi; +import java.util.List; +import java.util.concurrent.Executor; + +/** A {@link VideoGraph} specific to previewing. */ +@UnstableApi +public interface PreviewingVideoGraph extends VideoGraph { + + /** A factory for creating a {@link PreviewingVideoGraph}. */ + interface Factory { + /** + * Creates a new {@link PreviewingVideoGraph} instance. + * + * @param context A {@link Context}. + * @param inputColorInfo The {@link ColorInfo} for the input frames. + * @param outputColorInfo The {@link ColorInfo} for the output frames. + * @param debugViewProvider A {@link DebugViewProvider}. + * @param listener A {@link Listener}. + * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. + * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. + * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. + * @return A new instance. + * @throws VideoFrameProcessingException If a problem occurs while creating the {@link + * VideoFrameProcessor}. + */ + PreviewingVideoGraph create( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) + throws VideoFrameProcessingException; + } +} diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index adba883f42..496e16f7d4 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -34,6 +34,14 @@ public interface VideoGraph { */ void onOutputSizeChanged(int width, int height); + /** + * Called when an output frame with the given {@code presentationTimeUs} becomes available for + * rendering. + * + * @param presentationTimeUs The presentation time of the frame, in microseconds. + */ + void onOutputFrameAvailableForRendering(long presentationTimeUs); + /** * Called after the {@link VideoGraph} has rendered its final output frame. * diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java new file mode 100644 index 0000000000..f0568edfe7 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java @@ -0,0 +1,101 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import android.content.Context; +import androidx.annotation.Nullable; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.PreviewingVideoGraph; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.UnstableApi; +import java.util.List; +import java.util.concurrent.Executor; + +/** + * A {@link PreviewingVideoGraph Previewing} specific implementation of {@link + * SingleInputVideoGraph}. + */ +@UnstableApi +public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph + implements PreviewingVideoGraph { + + /** A factory for creating a {@link PreviewingSingleInputVideoGraph}. */ + public static final class Factory implements PreviewingVideoGraph.Factory { + + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + + @Override + public PreviewingVideoGraph create( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) { + @Nullable Presentation presentation = null; + for (int i = 0; i < compositionEffects.size(); i++) { + Effect effect = compositionEffects.get(i); + if (effect instanceof Presentation) { + presentation = (Presentation) effect; + } + } + return new PreviewingSingleInputVideoGraph( + context, + videoFrameProcessorFactory, + inputColorInfo, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + presentation, + initialTimestampOffsetUs); + } + } + + private PreviewingSingleInputVideoGraph( + Context context, + VideoFrameProcessor.Factory videoFrameProcessorFactory, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + Listener listener, + Executor listenerExecutor, + @Nullable Presentation presentation, + long initialTimestampOffsetUs) { + super( + context, + videoFrameProcessorFactory, + inputColorInfo, + outputColorInfo, + listener, + debugViewProvider, + listenerExecutor, + VideoCompositorSettings.DEFAULT, + // Previewing needs frame render timing. + /* renderFramesAutomatically= */ false, + presentation, + initialTimestampOffsetUs); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index b2133fb771..15fa3f1e11 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -134,6 +134,8 @@ public abstract class SingleInputVideoGraph implements VideoGraph { hasProducedFrameWithTimestampZero = true; } lastProcessedFramePresentationTimeUs = presentationTimeUs; + listenerExecutor.execute( + () -> listener.onOutputFrameAvailableForRendering(presentationTimeUs)); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index ee3d049f91..47015eff74 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -26,6 +26,7 @@ import android.os.Handler; import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; @@ -33,9 +34,11 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.FrameInfo; import androidx.media3.common.MimeTypes; +import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; import androidx.media3.common.VideoSize; import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; @@ -43,6 +46,7 @@ import androidx.media3.common.util.TimedValueQueue; import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import com.google.common.collect.ImmutableList; import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.util.ArrayList; @@ -57,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ final class CompositingVideoSinkProvider implements VideoSinkProvider { private final Context context; - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; private final VideoSink.RenderControl renderControl; @Nullable private VideoSinkImpl videoSinkImpl; @@ -70,8 +74,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoSink.RenderControl renderControl) { + this( + context, + new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory), + renderControl); + } + + @VisibleForTesting + /* package */ CompositingVideoSinkProvider( + Context context, + PreviewingVideoGraph.Factory previewingVideoGraphFactory, + VideoSink.RenderControl renderControl) { this.context = context; - this.videoFrameProcessorFactory = videoFrameProcessorFactory; + this.previewingVideoGraphFactory = previewingVideoGraphFactory; this.renderControl = renderControl; } @@ -82,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { videoSinkImpl = - new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat); + new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } @@ -147,10 +162,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener { + private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener { private final Context context; - private final RenderControl renderControl; + private final VideoSink.RenderControl renderControl; private final VideoFrameProcessor videoFrameProcessor; private final LongArrayQueue processedFramesBufferTimestampsUs; private final TimedValueQueue streamOffsets; @@ -196,7 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** Creates a new instance. */ public VideoSinkImpl( Context context, - VideoFrameProcessor.Factory videoFrameProcessorFactory, + PreviewingVideoGraph.Factory previewingVideoGraphFactory, RenderControl renderControl, Format sourceFormat) throws VideoFrameProcessingException { @@ -233,18 +248,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @SuppressWarnings("nullness:assignment") @Initialized VideoSinkImpl thisRef = this; - videoFrameProcessor = - videoFrameProcessorFactory.create( + PreviewingVideoGraph videoGraph = + previewingVideoGraphFactory.create( context, - DebugViewProvider.NONE, inputColorInfo, outputColorInfo, - /* renderFramesAutomatically= */ false, + DebugViewProvider.NONE, + /* listener= */ thisRef, /* listenerExecutor= */ handler::post, - thisRef); + /* compositionEffects= */ ImmutableList.of(), + /* initialTimestampOffsetUs= */ 0); + int videoGraphInputId = videoGraph.registerInput(); + videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); + if (currentSurfaceAndSize != null) { Size outputSurfaceSize = currentSurfaceAndSize.second; - videoFrameProcessor.setOutputSurfaceInfo( + videoGraph.setOutputSurfaceInfo( new SurfaceInfo( currentSurfaceAndSize.first, outputSurfaceSize.getWidth(), @@ -399,14 +418,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.playbackSpeed = speed; } - // VideoFrameProcessor.Listener impl - - @Override - public void onInputStreamRegistered( - @VideoFrameProcessor.InputType int inputType, List effects, FrameInfo frameInfo) { - // Do nothing. - } - @Override public void onOutputSizeChanged(int width, int height) { VideoSize newVideoSize = new VideoSize(width, height); @@ -454,12 +465,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void onEnded() { + public void onEnded(long finalFramePresentationTimeUs) { throw new IllegalStateException(); } - // Other methods - public void release() { videoFrameProcessor.release(); handler.removeCallbacksAndMessages(/* token= */ null); @@ -524,7 +533,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); } - /** Clears the set output surface info. */ public void clearOutputSurfaceInfo() { videoFrameProcessor.setOutputSurfaceInfo(null); currentSurfaceAndSize = null; @@ -616,4 +624,52 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } } + + /** + * Delays reflection for loading a {@linkplain PreviewingVideoGraph.Factory + * PreviewingSingleInputVideoGraph} instance. + */ + private static final class ReflectivePreviewingSingleInputVideoGraphFactory + implements PreviewingVideoGraph.Factory { + + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + + public ReflectivePreviewingSingleInputVideoGraphFactory( + VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + + @Override + public PreviewingVideoGraph create( + Context context, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + VideoGraph.Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) + throws VideoFrameProcessingException { + try { + Class previewingSingleInputVideoGraphFactoryClass = + Class.forName("androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory"); + PreviewingVideoGraph.Factory factory = + (PreviewingVideoGraph.Factory) + previewingSingleInputVideoGraphFactoryClass + .getConstructor(VideoFrameProcessor.Factory.class) + .newInstance(videoFrameProcessorFactory); + return factory.create( + context, + inputColorInfo, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + compositionEffects, + initialTimestampOffsetUs); + } catch (Exception e) { + throw VideoFrameProcessingException.from(e); + } + } + } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java index c502da81dc..79eeb6c17f 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -17,17 +17,21 @@ package androidx.media3.exoplayer.video; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.when; import android.content.Context; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.PreviewingVideoGraph; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; +import java.util.List; import java.util.concurrent.Executor; import org.junit.Test; import org.junit.runner.RunWith; @@ -128,29 +132,33 @@ public final class CompositingVideoSinkProviderTest { } private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() { - VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory(); VideoSink.RenderControl renderControl = new TestRenderControl(); return new CompositingVideoSinkProvider( - ApplicationProvider.getApplicationContext(), factory, renderControl); + ApplicationProvider.getApplicationContext(), + new TestPreviewingVideoGraphFactory(), + renderControl); } - private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory { + private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory { // Using a mock but we don't assert mock interactions. If needed to assert interactions, we // should a fake instead. + private final PreviewingVideoGraph previewingVideoGraph = + Mockito.mock(PreviewingVideoGraph.class); private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class); @Override - public VideoFrameProcessor create( + public PreviewingVideoGraph create( Context context, - DebugViewProvider debugViewProvider, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean renderFramesAutomatically, + DebugViewProvider debugViewProvider, + VideoGraph.Listener listener, Executor listenerExecutor, - VideoFrameProcessor.Listener listener) - throws VideoFrameProcessingException { + List compositionEffects, + long initialTimestampOffsetUs) { + when(previewingVideoGraph.getProcessor(anyInt())).thenReturn(videoFrameProcessor); when(videoFrameProcessor.registerInputFrame()).thenReturn(true); - return videoFrameProcessor; + return previewingVideoGraph; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 396efe2bc5..0e04ce596e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -513,6 +513,11 @@ import org.checkerframework.dataflow.qual.Pure; setOutputSurfaceInfo(surfaceInfo); } + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + // Do nothing. + } + @Override public void onEnded(long finalFramePresentationTimeUs) { VideoSampleExporter.this.finalFramePresentationTimeUs = finalFramePresentationTimeUs;