From a09bdfe9953194910dcc87479568cd3b623ab65c Mon Sep 17 00:00:00 2001
From: claincly <claincly@google.com>
Date: Thu, 15 Dec 2022 03:03:46 +0000
Subject: [PATCH] Allow changing output surface in previewing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It covers the following cases:

| From/To     | `null` | `surface 0` | `surface 1` |
|-------------|--------|-------------|-------------|
| `null`      | 🆖      | 📺           | 📺           |
| `surface 0` | ❌      | 🔁           | 📺           |
| `surface 1` | ❌      | 📺           | 🔁           |

Where
- 🆖 means NOP
- ❌ means
  - Set `null` on FrameProcessor, effectively dropping all frames
- 📺 means
  - Notify the listener of video size
  - Set FrameProcessor output surface and size when MSG_SET_VIDEO_OUTPUT_SIZE is received
- 🔁 means
  - Notify the listener of video size

PiperOrigin-RevId: 495477620
---
 .../media3/exoplayer/ExoPlayerImpl.java       |  4 +--
 .../video/MediaCodecVideoRenderer.java        | 27 +++++++++++++++++--
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java
index 5e32b4d5de..e92e122c57 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java
@@ -2599,6 +2599,8 @@ import java.util.concurrent.TimeoutException;
       surfaceSize = new Size(width, height);
       listeners.sendEvent(
           EVENT_SURFACE_SIZE_CHANGED, listener -> listener.onSurfaceSizeChanged(width, height));
+      sendRendererMessage(
+          TRACK_TYPE_VIDEO, MSG_SET_VIDEO_OUTPUT_RESOLUTION, new Size(width, height));
     }
   }
 
@@ -2956,8 +2958,6 @@ import java.util.concurrent.TimeoutException;
     @Override
     public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
       maybeNotifySurfaceSizeChanged(width, height);
-      sendRendererMessage(
-          TRACK_TYPE_VIDEO, MSG_SET_VIDEO_OUTPUT_RESOLUTION, new Size(width, height));
     }
 
     @Override
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
index 4def7d237e..66b2ef3cc9 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@@ -699,7 +699,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
 
   private void setOutput(@Nullable Object output) throws ExoPlaybackException {
-    // TODO(b/238302341) Handle output surface change in previewing.
     // Handle unsupported (i.e., non-Surface) outputs by clearing the display surface.
     @Nullable Surface displaySurface = output instanceof Surface ? (Surface) output : null;
 
@@ -724,7 +723,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
 
       @State int state = getState();
       @Nullable MediaCodecAdapter codec = getCodec();
-      if (codec != null) {
+      // When FrameProcessorManager is enabled, set FrameProcessorManager's display surface when
+      // surface's resolution is set on receiving MSG_SET_VIDEO_OUTPUT_RESOLUTION.
+      if (codec != null && !frameProcessorManager.isEnabled()) {
         if (Util.SDK_INT >= 23 && displaySurface != null && !codecNeedsSetOutputSurfaceWorkaround) {
           setOutputSurfaceV23(codec, displaySurface);
         } else {
@@ -738,12 +739,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
         // We haven't rendered to the new display surface yet.
         clearRenderedFirstFrame();
         if (state == STATE_STARTED) {
+          // Set joining deadline to report MediaCodecVideoRenderer is ready.
           setJoiningDeadlineMs();
         }
       } else {
         // The display surface has been removed.
         clearReportedVideoSize();
         clearRenderedFirstFrame();
+        if (frameProcessorManager.isEnabled()) {
+          frameProcessorManager.clearOutputSurfaceInfo();
+        }
       }
     } else if (displaySurface != null && displaySurface != placeholderSurface) {
       // The display surface is set and unchanged. If we know the video size and/or have already
@@ -1811,6 +1816,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
      */
     private @MonotonicNonNull Pair<Long, Format> currentFrameFormat;
 
+    @Nullable private Pair<Surface, Size> currentSurfaceAndSize;
+
     private int frameProcessorMaxPendingFrameCount;
     private boolean canEnableFrameProcessing;
 
@@ -1961,12 +1968,28 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
      * @param outputResolution The {@link Size} of the output resolution.
      */
     public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
+      if (currentSurfaceAndSize != null
+          && currentSurfaceAndSize.first.equals(outputSurface)
+          && currentSurfaceAndSize.second.equals(outputResolution)) {
+        return;
+      }
       checkNotNull(frameProcessor)
           .setOutputSurfaceInfo(
               new SurfaceInfo(
                   outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
+      currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
     }
 
+    /**
+     * Clears the set output surface info.
+     *
+     * <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
+     * this method.
+     */
+    public void clearOutputSurfaceInfo() {
+      checkNotNull(frameProcessor).setOutputSurfaceInfo(null);
+      currentSurfaceAndSize = null;
+    }
     /**
      * Sets the input surface info.
      *