ExoPlayer: Add setVideoFrameProcessorFactory().

This allows apps to use a custom VideoFrameProcessor implementation for video
playback. This may be useful, for example, when outputting to a texture.

PiperOrigin-RevId: 534044831
(cherry picked from commit 438ae0ed6a)

RELEASENOTES.md

@@ -90,6 +90,9 @@
     *   Remove deprecated `ExoPlayer.retry()`, use `prepare()` instead.
     *   Remove deprecated zero-arg `DefaultTrackSelector` constructor, use
         `DefaultTrackSelector(Context)` instead.
+*   Core library:
+    *   Add `ExoPlayer.setVideoFrameProcessorFactory()` for using `Effect` with
+        a custom `VideoFrameProcessor.Factory` during video playback.
 
 ### 1.0.2 (2023-05-18)
 

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java

@@ -44,6 +44,7 @@ import androidx.media3.common.Player;
 import androidx.media3.common.PriorityTaskManager;
 import androidx.media3.common.Timeline;
 import androidx.media3.common.Tracks;
+import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoSize;
 import androidx.media3.common.text.CueGroup;
 import androidx.media3.common.util.Clock;
@@ -1562,6 +1563,19 @@ public interface ExoPlayer extends Player {
   @UnstableApi
   void setVideoEffects(List<Effect> videoEffects);
 
+  /**
+   * Sets a {@link VideoFrameProcessor.Factory} to create the {@link VideoFrameProcessor} that
+   * applies video effects set in {@link #setVideoEffects}.
+   *
+   * <p>See {@link #setVideoEffects} for limitations.
+   *
+   * @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} to use to apply the
+   *     video effects.
+   */
+  @RequiresApi(18)
+  @UnstableApi
+  void setVideoFrameProcessorFactory(VideoFrameProcessor.Factory videoFrameProcessorFactory);
+
   /**
    * Sets the {@link C.VideoScalingMode}.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java

@@ -32,6 +32,7 @@ import static androidx.media3.exoplayer.Renderer.MSG_SET_SCALING_MODE;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_SKIP_SILENCE_ENABLED;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_EFFECTS;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER;
+import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION;
 import static androidx.media3.exoplayer.Renderer.MSG_SET_VOLUME;
@@ -77,6 +78,7 @@ import androidx.media3.common.Timeline;
 import androidx.media3.common.TrackGroup;
 import androidx.media3.common.TrackSelectionParameters;
 import androidx.media3.common.Tracks;
+import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoSize;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.text.CueGroup;
@@ -1238,6 +1240,14 @@ import java.util.concurrent.TimeoutException;
     sendRendererMessage(TRACK_TYPE_VIDEO, MSG_SET_VIDEO_EFFECTS, videoEffects);
   }
 
+  @Override
+  public void setVideoFrameProcessorFactory(
+      VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+    verifyApplicationThread();
+    sendRendererMessage(
+        TRACK_TYPE_VIDEO, MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, videoFrameProcessorFactory);
+  }
+
   @Override
   public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) {
     verifyApplicationThread();

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java

@@ -27,6 +27,7 @@ import androidx.media3.common.C;
 import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.Player;
+import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
@@ -90,9 +91,9 @@ public interface Renderer extends PlayerMessage.Target {
    * #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link
    * #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link
    * #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link
-   * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS} or
+   * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS},
-   * {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May also be an app-defined value (see {@link
+   * {@link #MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY} or {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May
-   * #MSG_CUSTOM_BASE}).
+   * also be an app-defined value (see {@link #MSG_CUSTOM_BASE}).
    */
   @Documented
   @Retention(RetentionPolicy.SOURCE)
@@ -112,6 +113,7 @@ public interface Renderer extends PlayerMessage.Target {
         MSG_SET_AUDIO_SESSION_ID,
         MSG_SET_WAKEUP_LISTENER,
         MSG_SET_VIDEO_EFFECTS,
+        MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY,
         MSG_SET_VIDEO_OUTPUT_RESOLUTION
       })
   public @interface MessageType {}
@@ -219,12 +221,17 @@ public interface Renderer extends PlayerMessage.Target {
    * {@link List} containing {@linkplain Effect video effects}.
    */
   int MSG_SET_VIDEO_EFFECTS = 13;
+  /**
+   * The type of a message that can be passed to a video renderer. The message payload should be a
+   * {@link VideoFrameProcessor.Factory}.
+   */
+  int MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY = 14;
   /**
    * The type of a message that can be passed to a video renderer to set the desired output
    * resolution. The message payload should be a {@link Size} of the desired output width and
    * height. Use this method only when playing with video {@linkplain Effect effects}.
    */
-  int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 14;
+  int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 15;
   /**
    * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
    * renderers. These custom constants must be greater than or equal to this value.

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java

@@ -43,6 +43,7 @@ import androidx.media3.common.PriorityTaskManager;
 import androidx.media3.common.Timeline;
 import androidx.media3.common.TrackSelectionParameters;
 import androidx.media3.common.Tracks;
+import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoSize;
 import androidx.media3.common.text.CueGroup;
 import androidx.media3.common.util.Clock;
@@ -671,6 +672,13 @@ public class SimpleExoPlayer extends BasePlayer
     player.setVideoEffects(videoEffects);
   }
 
+  @Override
+  public void setVideoFrameProcessorFactory(
+      VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+    blockUntilConstructorFinished();
+    player.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
+  }
+
   @Override
   public void setSkipSilenceEnabled(boolean skipSilenceEnabled) {
     blockUntilConstructorFinished();

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -687,6 +687,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
         List<Effect> videoEffects = (List<Effect>) checkNotNull(message);
         videoFrameProcessorManager.setVideoEffects(videoEffects);
         break;
+      case MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY:
+        VideoFrameProcessor.Factory videoFrameProcessorFactory =
+            (VideoFrameProcessor.Factory) checkNotNull(message);
+        videoFrameProcessorManager.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
+        break;
       case MSG_SET_VIDEO_OUTPUT_RESOLUTION:
         Size outputResolution = (Size) checkNotNull(message);
         if (outputResolution.getWidth() != 0
@@ -1876,6 +1881,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats;
 
     private @MonotonicNonNull Handler handler;
+    private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
     @Nullable private VideoFrameProcessor videoFrameProcessor;
     @Nullable private CopyOnWriteArrayList<Effect> videoEffects;
     @Nullable private Format inputFormat;
@@ -1937,6 +1943,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       this.videoEffects.addAll(videoEffects);
     }
 
+    /** Sets the {@link VideoFrameProcessor.Factory}. */
+    public void setVideoFrameProcessorFactory(
+        VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+      this.videoFrameProcessorFactory = videoFrameProcessorFactory;
+    }
+
     /** Returns whether video frame processing is enabled. */
     public boolean isEnabled() {
       return videoFrameProcessor != null;
@@ -2009,66 +2021,69 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
               VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees));
         }
 
+        videoFrameProcessorFactory =
+            videoFrameProcessorFactory == null
+                ? VideoFrameProcessorAccessor.getFrameProcessorFactory()
+                : videoFrameProcessorFactory;
         videoFrameProcessor =
-            VideoFrameProcessorAccessor.getFrameProcessorFactory()
+            videoFrameProcessorFactory.create(
-                .create(
+                renderer.context,
-                    renderer.context,
+                checkNotNull(videoEffects),
-                    checkNotNull(videoEffects),
+                DebugViewProvider.NONE,
-                    DebugViewProvider.NONE,
+                inputAndOutputColorInfos.first,
-                    inputAndOutputColorInfos.first,
+                inputAndOutputColorInfos.second,
-                    inputAndOutputColorInfos.second,
+                /* renderFramesAutomatically= */ false,
-                    /* renderFramesAutomatically= */ false,
+                /* listenerExecutor= */ handler::post,
-                    /* listenerExecutor= */ handler::post,
+                new VideoFrameProcessor.Listener() {
-                    new VideoFrameProcessor.Listener() {
+                  @Override
-                      @Override
+                  public void onOutputSizeChanged(int width, int height) {
-                      public void onOutputSizeChanged(int width, int height) {
+                    @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
-                        @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
+                    checkStateNotNull(inputFormat);
-                        checkStateNotNull(inputFormat);
+                    // TODO(b/264889146): Handle Effect that changes output size based on pts.
-                        // TODO(b/264889146): Handle Effect that changes output size based on pts.
+                    processedFrameSize =
-                        processedFrameSize =
+                        new VideoSize(
-                            new VideoSize(
+                            width,
-                                width,
+                            height,
-                                height,
+                            // VideoFrameProcessor is configured to produce rotation free
-                                // VideoFrameProcessor is configured to produce rotation free
+                            // frames.
-                                // frames.
+                            /* unappliedRotationDegrees= */ 0,
-                                /* unappliedRotationDegrees= */ 0,
+                            // VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
-                                // VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
+                            /* pixelWidthHeightRatio= */ 1.f);
-                                /* pixelWidthHeightRatio= */ 1.f);
+                    pendingOutputSizeChange = true;
-                        pendingOutputSizeChange = true;
+                  }
-                      }
 
-                      @Override
+                  @Override
-                      public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+                  public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
-                        if (registeredLastFrame) {
+                    if (registeredLastFrame) {
-                          checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
+                      checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
-                        }
+                    }
-                        processedFramesTimestampsUs.add(presentationTimeUs);
+                    processedFramesTimestampsUs.add(presentationTimeUs);
-                        // TODO(b/257464707) Support extensively modified media.
+                    // TODO(b/257464707) Support extensively modified media.
-                        if (registeredLastFrame
+                    if (registeredLastFrame
-                            && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
+                        && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
-                          processedLastFrame = true;
+                      processedLastFrame = true;
-                        }
+                    }
-                        if (pendingOutputSizeChange) {
+                    if (pendingOutputSizeChange) {
-                          // Report the size change on releasing this frame.
+                      // Report the size change on releasing this frame.
-                          pendingOutputSizeChange = false;
+                      pendingOutputSizeChange = false;
-                          pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
+                      pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
-                        }
+                    }
-                      }
+                  }
 
-                      @Override
+                  @Override
-                      public void onError(VideoFrameProcessingException exception) {
+                  public void onError(VideoFrameProcessingException exception) {
-                        renderer.setPendingPlaybackException(
+                    renderer.setPendingPlaybackException(
-                            renderer.createRendererException(
+                        renderer.createRendererException(
-                                exception,
+                            exception,
-                                inputFormat,
+                            inputFormat,
-                                PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
+                            PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
-                      }
+                  }
 
-                      @Override
+                  @Override
-                      public void onEnded() {
+                  public void onEnded() {
-                        throw new IllegalStateException();
+                    throw new IllegalStateException();
-                      }
+                  }
-                    });
+                });
         videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE);
         this.initialStreamOffsetUs = initialStreamOffsetUs;
       } catch (Exception e) {

libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java

@@ -24,6 +24,7 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.Player;
 import androidx.media3.common.PriorityTaskManager;
+import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.exoplayer.DecoderCounters;
@@ -249,6 +250,12 @@ public class StubExoPlayer extends StubPlayer implements ExoPlayer {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public void setVideoFrameProcessorFactory(
+      VideoFrameProcessor.Factory videoFrameProcessorFactory) {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public void setVideoScalingMode(int videoScalingMode) {
     throw new UnsupportedOperationException();