Add previewing specific video graph.

PiperOrigin-RevId: 569473178

libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.common;
+
+import android.content.Context;
+import androidx.media3.common.util.UnstableApi;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/** A {@link VideoGraph} specific to previewing. */
+@UnstableApi
+public interface PreviewingVideoGraph extends VideoGraph {
+
+  /** A factory for creating a {@link PreviewingVideoGraph}. */
+  interface Factory {
+    /**
+     * Creates a new {@link PreviewingVideoGraph} instance.
+     *
+     * @param context A {@link Context}.
+     * @param inputColorInfo The {@link ColorInfo} for the input frames.
+     * @param outputColorInfo The {@link ColorInfo} for the output frames.
+     * @param debugViewProvider A {@link DebugViewProvider}.
+     * @param listener A {@link Listener}.
+     * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked.
+     * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition.
+     * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds.
+     * @return A new instance.
+     * @throws VideoFrameProcessingException If a problem occurs while creating the {@link
+     *     VideoFrameProcessor}.
+     */
+    PreviewingVideoGraph create(
+        Context context,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
+        DebugViewProvider debugViewProvider,
+        Listener listener,
+        Executor listenerExecutor,
+        List<Effect> compositionEffects,
+        long initialTimestampOffsetUs)
+        throws VideoFrameProcessingException;
+  }
+}

libraries/common/src/main/java/androidx/media3/common/VideoGraph.java

@@ -34,6 +34,14 @@ public interface VideoGraph {
      */
     void onOutputSizeChanged(int width, int height);
 
+    /**
+     * Called when an output frame with the given {@code presentationTimeUs} becomes available for
+     * rendering.
+     *
+     * @param presentationTimeUs The presentation time of the frame, in microseconds.
+     */
+    void onOutputFrameAvailableForRendering(long presentationTimeUs);
+
     /**
      * Called after the {@link VideoGraph} has rendered its final output frame.
      *

libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.effect;
+
+import android.content.Context;
+import androidx.annotation.Nullable;
+import androidx.media3.common.ColorInfo;
+import androidx.media3.common.DebugViewProvider;
+import androidx.media3.common.Effect;
+import androidx.media3.common.PreviewingVideoGraph;
+import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.util.UnstableApi;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/**
+ * A {@link PreviewingVideoGraph Previewing} specific implementation of {@link
+ * SingleInputVideoGraph}.
+ */
+@UnstableApi
+public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph
+    implements PreviewingVideoGraph {
+
+  /** A factory for creating a {@link PreviewingSingleInputVideoGraph}. */
+  public static final class Factory implements PreviewingVideoGraph.Factory {
+
+    private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
+
+    public Factory(VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+      this.videoFrameProcessorFactory = videoFrameProcessorFactory;
+    }
+
+    @Override
+    public PreviewingVideoGraph create(
+        Context context,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
+        DebugViewProvider debugViewProvider,
+        Listener listener,
+        Executor listenerExecutor,
+        List<Effect> compositionEffects,
+        long initialTimestampOffsetUs) {
+      @Nullable Presentation presentation = null;
+      for (int i = 0; i < compositionEffects.size(); i++) {
+        Effect effect = compositionEffects.get(i);
+        if (effect instanceof Presentation) {
+          presentation = (Presentation) effect;
+        }
+      }
+      return new PreviewingSingleInputVideoGraph(
+          context,
+          videoFrameProcessorFactory,
+          inputColorInfo,
+          outputColorInfo,
+          debugViewProvider,
+          listener,
+          listenerExecutor,
+          presentation,
+          initialTimestampOffsetUs);
+    }
+  }
+
+  private PreviewingSingleInputVideoGraph(
+      Context context,
+      VideoFrameProcessor.Factory videoFrameProcessorFactory,
+      ColorInfo inputColorInfo,
+      ColorInfo outputColorInfo,
+      DebugViewProvider debugViewProvider,
+      Listener listener,
+      Executor listenerExecutor,
+      @Nullable Presentation presentation,
+      long initialTimestampOffsetUs) {
+    super(
+        context,
+        videoFrameProcessorFactory,
+        inputColorInfo,
+        outputColorInfo,
+        listener,
+        debugViewProvider,
+        listenerExecutor,
+        VideoCompositorSettings.DEFAULT,
+        // Previewing needs frame render timing.
+        /* renderFramesAutomatically= */ false,
+        presentation,
+        initialTimestampOffsetUs);
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java

@@ -134,6 +134,8 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
                   hasProducedFrameWithTimestampZero = true;
                 }
                 lastProcessedFramePresentationTimeUs = presentationTimeUs;
+                listenerExecutor.execute(
+                    () -> listener.onOutputFrameAvailableForRendering(presentationTimeUs));
               }
 
               @Override

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java

@@ -26,6 +26,7 @@ import android.os.Handler;
 import android.util.Pair;
 import android.view.Surface;
 import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.DebugViewProvider;
@@ -33,9 +34,11 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.FrameInfo;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.common.PreviewingVideoGraph;
 import androidx.media3.common.SurfaceInfo;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.VideoGraph;
 import androidx.media3.common.VideoSize;
 import androidx.media3.common.util.LongArrayQueue;
 import androidx.media3.common.util.Size;
@@ -43,6 +46,7 @@ import androidx.media3.common.util.TimedValueQueue;
 import androidx.media3.common.util.TimestampIterator;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
+import com.google.common.collect.ImmutableList;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
@@ -57,7 +61,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 /* package */ final class CompositingVideoSinkProvider implements VideoSinkProvider {
 
   private final Context context;
-  private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
+  private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
   private final VideoSink.RenderControl renderControl;
 
   @Nullable private VideoSinkImpl videoSinkImpl;
@@ -70,8 +74,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       Context context,
       VideoFrameProcessor.Factory videoFrameProcessorFactory,
       VideoSink.RenderControl renderControl) {
+    this(
+        context,
+        new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory),
+        renderControl);
+  }
+
+  @VisibleForTesting
+  /* package */ CompositingVideoSinkProvider(
+      Context context,
+      PreviewingVideoGraph.Factory previewingVideoGraphFactory,
+      VideoSink.RenderControl renderControl) {
     this.context = context;
-    this.videoFrameProcessorFactory = videoFrameProcessorFactory;
+    this.previewingVideoGraphFactory = previewingVideoGraphFactory;
     this.renderControl = renderControl;
   }
 
@@ -82,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     try {
       videoSinkImpl =
-          new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat);
+          new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat);
     } catch (VideoFrameProcessingException e) {
       throw new VideoSink.VideoSinkException(e, sourceFormat);
     }
@@ -147,10 +162,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
-  private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener {
+  private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener {
 
     private final Context context;
-    private final RenderControl renderControl;
+    private final VideoSink.RenderControl renderControl;
     private final VideoFrameProcessor videoFrameProcessor;
     private final LongArrayQueue processedFramesBufferTimestampsUs;
     private final TimedValueQueue<Long> streamOffsets;
@@ -196,7 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     /** Creates a new instance. */
     public VideoSinkImpl(
         Context context,
-        VideoFrameProcessor.Factory videoFrameProcessorFactory,
+        PreviewingVideoGraph.Factory previewingVideoGraphFactory,
         RenderControl renderControl,
         Format sourceFormat)
         throws VideoFrameProcessingException {
@@ -233,18 +248,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       @SuppressWarnings("nullness:assignment")
       @Initialized
       VideoSinkImpl thisRef = this;
-      videoFrameProcessor =
+      PreviewingVideoGraph videoGraph =
-          videoFrameProcessorFactory.create(
+          previewingVideoGraphFactory.create(
               context,
-              DebugViewProvider.NONE,
               inputColorInfo,
               outputColorInfo,
-              /* renderFramesAutomatically= */ false,
+              DebugViewProvider.NONE,
+              /* listener= */ thisRef,
               /* listenerExecutor= */ handler::post,
-              thisRef);
+              /* compositionEffects= */ ImmutableList.of(),
+              /* initialTimestampOffsetUs= */ 0);
+      int videoGraphInputId = videoGraph.registerInput();
+      videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId);
+
       if (currentSurfaceAndSize != null) {
         Size outputSurfaceSize = currentSurfaceAndSize.second;
-        videoFrameProcessor.setOutputSurfaceInfo(
+        videoGraph.setOutputSurfaceInfo(
             new SurfaceInfo(
                 currentSurfaceAndSize.first,
                 outputSurfaceSize.getWidth(),
@@ -399,14 +418,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       this.playbackSpeed = speed;
     }
 
-    // VideoFrameProcessor.Listener impl
-
-    @Override
-    public void onInputStreamRegistered(
-        @VideoFrameProcessor.InputType int inputType, List<Effect> effects, FrameInfo frameInfo) {
-      // Do nothing.
-    }
-
     @Override
     public void onOutputSizeChanged(int width, int height) {
       VideoSize newVideoSize = new VideoSize(width, height);
@@ -454,12 +465,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
 
     @Override
-    public void onEnded() {
+    public void onEnded(long finalFramePresentationTimeUs) {
       throw new IllegalStateException();
     }
 
-    // Other methods
-
     public void release() {
       videoFrameProcessor.release();
       handler.removeCallbacksAndMessages(/* token= */ null);
@@ -524,7 +533,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
     }
 
-    /** Clears the set output surface info. */
     public void clearOutputSurfaceInfo() {
       videoFrameProcessor.setOutputSurfaceInfo(null);
       currentSurfaceAndSize = null;
@@ -616,4 +624,52 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       }
     }
   }
+
+  /**
+   * Delays reflection for loading a {@linkplain PreviewingVideoGraph.Factory
+   * PreviewingSingleInputVideoGraph} instance.
+   */
+  private static final class ReflectivePreviewingSingleInputVideoGraphFactory
+      implements PreviewingVideoGraph.Factory {
+
+    private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
+
+    public ReflectivePreviewingSingleInputVideoGraphFactory(
+        VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+      this.videoFrameProcessorFactory = videoFrameProcessorFactory;
+    }
+
+    @Override
+    public PreviewingVideoGraph create(
+        Context context,
+        ColorInfo inputColorInfo,
+        ColorInfo outputColorInfo,
+        DebugViewProvider debugViewProvider,
+        VideoGraph.Listener listener,
+        Executor listenerExecutor,
+        List<Effect> compositionEffects,
+        long initialTimestampOffsetUs)
+        throws VideoFrameProcessingException {
+      try {
+        Class<?> previewingSingleInputVideoGraphFactoryClass =
+            Class.forName("androidx.media3.effect.PreviewingSingleInputVideoGraph$Factory");
+        PreviewingVideoGraph.Factory factory =
+            (PreviewingVideoGraph.Factory)
+                previewingSingleInputVideoGraphFactoryClass
+                    .getConstructor(VideoFrameProcessor.Factory.class)
+                    .newInstance(videoFrameProcessorFactory);
+        return factory.create(
+            context,
+            inputColorInfo,
+            outputColorInfo,
+            debugViewProvider,
+            listener,
+            listenerExecutor,
+            compositionEffects,
+            initialTimestampOffsetUs);
+      } catch (Exception e) {
+        throw VideoFrameProcessingException.from(e);
+      }
+    }
+  }
 }

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java

@@ -17,17 +17,21 @@ package androidx.media3.exoplayer.video;
 
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.when;
 
 import android.content.Context;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.DebugViewProvider;
+import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
-import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.PreviewingVideoGraph;
 import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.VideoGraph;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.common.collect.ImmutableList;
+import java.util.List;
 import java.util.concurrent.Executor;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -128,29 +132,33 @@ public final class CompositingVideoSinkProviderTest {
   }
 
   private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() {
-    VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory();
     VideoSink.RenderControl renderControl = new TestRenderControl();
     return new CompositingVideoSinkProvider(
-        ApplicationProvider.getApplicationContext(), factory, renderControl);
+        ApplicationProvider.getApplicationContext(),
+        new TestPreviewingVideoGraphFactory(),
+        renderControl);
   }
 
-  private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory {
+  private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory {
     // Using a mock but we don't assert mock interactions. If needed to assert interactions, we
     // should a fake instead.
+    private final PreviewingVideoGraph previewingVideoGraph =
+        Mockito.mock(PreviewingVideoGraph.class);
     private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class);
 
     @Override
-    public VideoFrameProcessor create(
+    public PreviewingVideoGraph create(
         Context context,
-        DebugViewProvider debugViewProvider,
         ColorInfo inputColorInfo,
         ColorInfo outputColorInfo,
-        boolean renderFramesAutomatically,
+        DebugViewProvider debugViewProvider,
+        VideoGraph.Listener listener,
         Executor listenerExecutor,
-        VideoFrameProcessor.Listener listener)
+        List<Effect> compositionEffects,
-        throws VideoFrameProcessingException {
+        long initialTimestampOffsetUs) {
+      when(previewingVideoGraph.getProcessor(anyInt())).thenReturn(videoFrameProcessor);
       when(videoFrameProcessor.registerInputFrame()).thenReturn(true);
-      return videoFrameProcessor;
+      return previewingVideoGraph;
     }
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -513,6 +513,11 @@ import org.checkerframework.dataflow.qual.Pure;
       setOutputSurfaceInfo(surfaceInfo);
     }
 
+    @Override
+    public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+      // Do nothing.
+    }
+
     @Override
     public void onEnded(long finalFramePresentationTimeUs) {
       VideoSampleExporter.this.finalFramePresentationTimeUs = finalFramePresentationTimeUs;