Support GL rendering with SimpleExoPlayer and PlayerView

PiperOrigin-RevId: 273760294

extensions/av1/README.md

@@ -79,23 +79,26 @@ a custom track selector the choice of `Renderer` is up to your implementation.
 You need to make sure you are passing a `Libgav1VideoRenderer` to the player and
 then you need to implement your own logic to use the renderer for a given track.
 
+## Rendering options ##
+
 There are two possibilities for rendering the output `Libgav1VideoRenderer`
 gets from the libgav1 decoder:
 
-* Native rendering with `ANativeWindow`
+* GL rendering using GL shader for color space conversion
-* OpenGL rendering.
+  * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
+    setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message
+    of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
+    `VideoDecoderOutputBufferRenderer` as its object.
 
-`SimpleExoPlayer` uses `ANativeWindow` rendering. To enable this mode send the
+* Native rendering using `ANativeWindow`
-renderer a message of type `C.MSG_SET_SURFACE` with a `Surface` as its object.
+  * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
-`Libgav1VideoRenderer` can also output to a `VideoDecoderSurfaceView` when
+    by default.
-not being used via `SimpleExoPlayer`, in which case color space conversion will
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of
-be performed using a GL shader. To enable this mode, send the renderer a message
+    type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
-of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
-its object.
 
 Note: Although the default option uses `ANativeWindow`, based on our testing the
-GL rendering mode has better performance, so should be preferred by apps that
+GL rendering mode has better performance, so should be preferred
-can use `VideoDecoderSurfaceView`.
 
 ## Links ##
 

extensions/vp9/README.md

@@ -107,11 +107,26 @@ a custom track selector the choice of `Renderer` is up to your implementation,
 so you need to make sure you are passing an `LibvpxVideoRenderer` to the
 player, then implement your own logic to use the renderer for a given track.
 
-`LibvpxVideoRenderer` can optionally output to a `VideoDecoderSurfaceView` when
+## Rendering options ##
-not being used via `SimpleExoPlayer`, in which case color space conversion will
+
-be performed using a GL shader. To enable this mode, send the renderer a message
+There are two possibilities for rendering the output `LibvpxVideoRenderer`
-of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
+gets from the libvpx decoder:
-its object, instead of sending `MSG_SET_SURFACE` with a `Surface`.
+
+* GL rendering using GL shader for color space conversion
+  * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
+    setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
+  * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
+    type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
+    `VideoDecoderOutputBufferRenderer` as its object.
+
+* Native rendering using `ANativeWindow`
+  * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
+    by default.
+  * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
+    type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
+
+Note: Although the default option uses `ANativeWindow`, based on our testing the
+GL rendering mode has better performance, so should be preferred.
 
 ## Links ##
 

extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java

@@ -124,7 +124,7 @@ public class VpxPlaybackTest {
       player
           .createMessage(videoRenderer)
           .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
-          .setPayload(new VideoDecoderSurfaceView(context))
+          .setPayload(new VideoDecoderSurfaceView(context).getOutputBufferRenderer())
           .send();
       player.prepare(mediaSource);
       player.setPlayWhenReady(true);

library/core/src/main/java/com/google/android/exoplayer2/Player.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.text.TextOutput;
 import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoListener;
 import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@@ -280,6 +281,13 @@ public interface Player {
      * @param textureView The texture view to clear.
      */
     void clearVideoTextureView(TextureView textureView);
+
+    /**
+     * Sets the output buffer renderer.
+     *
+     * @param outputBufferRenderer The output buffer renderer.
+     */
+    void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer);
   }
 
   /** The text component of a {@link Player}. */

library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java

@@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.Clock;
 import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.PriorityTaskManager;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@@ -584,8 +585,8 @@ public class SimpleExoPlayer extends BasePlayer
         Log.w(TAG, "Replacing existing SurfaceTextureListener.");
       }
       textureView.setSurfaceTextureListener(componentListener);
-      SurfaceTexture surfaceTexture = textureView.isAvailable() ? textureView.getSurfaceTexture()
+      SurfaceTexture surfaceTexture =
-          : null;
+          textureView.isAvailable() ? textureView.getSurfaceTexture() : null;
       if (surfaceTexture == null) {
         setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
         maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
@@ -604,6 +605,23 @@ public class SimpleExoPlayer extends BasePlayer
     }
   }
 
+  @Override
+  public void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer) {
+    verifyApplicationThread();
+    removeSurfaceCallbacks();
+    List<PlayerMessage> messages = new ArrayList<>();
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
+        messages.add(
+            player
+                .createMessage(renderer)
+                .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
+                .setPayload(outputBufferRenderer)
+                .send());
+      }
+    }
+  }
+
   @Override
   public void addAudioListener(AudioListener listener) {
     audioListeners.add(listener);
@@ -695,12 +713,12 @@ public class SimpleExoPlayer extends BasePlayer
 
   /**
    * Sets the stream type for audio playback, used by the underlying audio track.
-   * <p>
+   *
-   * Setting the stream type during playback may introduce a short gap in audio output as the audio
+   * <p>Setting the stream type during playback may introduce a short gap in audio output as the
-   * track is recreated. A new audio session id will also be generated.
+   * audio track is recreated. A new audio session id will also be generated.
-   * <p>
+   *
-   * Calling this method overwrites any attributes set previously by calling
+   * <p>Calling this method overwrites any attributes set previously by calling {@link
-   * {@link #setAudioAttributes(AudioAttributes)}.
+   * #setAudioAttributes(AudioAttributes)}.
    *
    * @deprecated Use {@link #setAudioAttributes(AudioAttributes)}.
    * @param streamType The stream type for audio playback.
@@ -1473,11 +1491,11 @@ public class SimpleExoPlayer extends BasePlayer
     }
 
     @Override
-    public void onVideoDecoderInitialized(String decoderName, long initializedTimestampMs,
+    public void onVideoDecoderInitialized(
-        long initializationDurationMs) {
+        String decoderName, long initializedTimestampMs, long initializationDurationMs) {
       for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
-        videoDebugListener.onVideoDecoderInitialized(decoderName, initializedTimestampMs,
+        videoDebugListener.onVideoDecoderInitialized(
-            initializationDurationMs);
+            decoderName, initializedTimestampMs, initializationDurationMs);
       }
     }
 
@@ -1497,8 +1515,8 @@ public class SimpleExoPlayer extends BasePlayer
     }
 
     @Override
-    public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees,
+    public void onVideoSizeChanged(
-        float pixelWidthHeightRatio) {
+        int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {
       for (com.google.android.exoplayer2.video.VideoListener videoListener : videoListeners) {
         // Prevent duplicate notification if a listener is both a VideoRendererEventListener and
         // a VideoListener, as they have the same method signature.
@@ -1508,8 +1526,8 @@ public class SimpleExoPlayer extends BasePlayer
         }
       }
       for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
-        videoDebugListener.onVideoSizeChanged(width, height, unappliedRotationDegrees,
+        videoDebugListener.onVideoSizeChanged(
-            pixelWidthHeightRatio);
+            width, height, unappliedRotationDegrees, pixelWidthHeightRatio);
       }
     }
 
@@ -1563,11 +1581,11 @@ public class SimpleExoPlayer extends BasePlayer
     }
 
     @Override
-    public void onAudioDecoderInitialized(String decoderName, long initializedTimestampMs,
+    public void onAudioDecoderInitialized(
-        long initializationDurationMs) {
+        String decoderName, long initializedTimestampMs, long initializationDurationMs) {
       for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
-        audioDebugListener.onAudioDecoderInitialized(decoderName, initializedTimestampMs,
+        audioDebugListener.onAudioDecoderInitialized(
-            initializationDurationMs);
+            decoderName, initializedTimestampMs, initializationDurationMs);
       }
     }
 
@@ -1580,8 +1598,8 @@ public class SimpleExoPlayer extends BasePlayer
     }
 
     @Override
-    public void onAudioSinkUnderrun(int bufferSize, long bufferSizeMs,
+    public void onAudioSinkUnderrun(
-        long elapsedSinceLastFeedMs) {
+        int bufferSize, long bufferSizeMs, long elapsedSinceLastFeedMs) {
       for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
         audioDebugListener.onAudioSinkUnderrun(bufferSize, bufferSizeMs, elapsedSinceLastFeedMs);
       }

library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java

@@ -28,7 +28,8 @@ import javax.microedition.khronos.opengles.GL10;
  * GLSurfaceView.Renderer implementation that can render YUV Frames returned by a video decoder
  * after decoding. It does the YUV to RGB color conversion in the Fragment Shader.
  */
-/* package */ class VideoDecoderRenderer implements GLSurfaceView.Renderer {
+/* package */ class VideoDecoderRenderer
+    implements GLSurfaceView.Renderer, VideoDecoderOutputBufferRenderer {
 
   private static final float[] kColorConversion601 = {
     1.164f, 1.164f, 1.164f,
@@ -82,6 +83,7 @@ import javax.microedition.khronos.opengles.GL10;
 
   private static final FloatBuffer TEXTURE_VERTICES =
       GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
+  private final GLSurfaceView surfaceView;
   private final int[] yuvTextures = new int[3];
   private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
 
@@ -98,7 +100,8 @@ import javax.microedition.khronos.opengles.GL10;
 
   private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
 
-  public VideoDecoderRenderer() {
+  public VideoDecoderRenderer(GLSurfaceView surfaceView) {
+    this.surfaceView = surfaceView;
     pendingOutputBufferReference = new AtomicReference<>();
     textureCoords = new FloatBuffer[3];
     texLocations = new int[3];
@@ -109,21 +112,6 @@ import javax.microedition.khronos.opengles.GL10;
     }
   }
 
-  /**
-   * Set a frame to be rendered. This should be followed by a call to
-   * VideoDecoderSurfaceView.requestRender() to actually render the frame.
-   *
-   * @param outputBuffer OutputBuffer containing the YUV Frame to be rendered
-   */
-  public void setFrame(VideoDecoderOutputBuffer outputBuffer) {
-    VideoDecoderOutputBuffer oldPendingOutputBuffer =
-        pendingOutputBufferReference.getAndSet(outputBuffer);
-    if (oldPendingOutputBuffer != null) {
-      // The old pending output buffer will never be used for rendering, so release it now.
-      oldPendingOutputBuffer.release();
-    }
-  }
-
   @Override
   public void onSurfaceCreated(GL10 unused, EGLConfig config) {
     program = GlUtil.compileProgram(VERTEX_SHADER, FRAGMENT_SHADER);
@@ -223,6 +211,17 @@ import javax.microedition.khronos.opengles.GL10;
     GlUtil.checkGlError();
   }
 
+  @Override
+  public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
+    VideoDecoderOutputBuffer oldPendingOutputBuffer =
+        pendingOutputBufferReference.getAndSet(outputBuffer);
+    if (oldPendingOutputBuffer != null) {
+      // The old pending output buffer will never be used for rendering, so release it now.
+      oldPendingOutputBuffer.release();
+    }
+    surfaceView.requestRender();
+  }
+
   private void setupTextures() {
     GLES20.glGenTextures(3, yuvTextures, 0);
     for (int i = 0; i < 3; i++) {

library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java

@@ -21,28 +21,40 @@ import android.util.AttributeSet;
 import androidx.annotation.Nullable;
 
 /** A GLSurfaceView extension that scales itself to the given aspect ratio. */
-public class VideoDecoderSurfaceView extends GLSurfaceView
+public class VideoDecoderSurfaceView extends GLSurfaceView {
-    implements VideoDecoderOutputBufferRenderer {
 
   private final VideoDecoderRenderer renderer;
 
+  /**
+   * Creates VideoDecoderSurfaceView.
+   *
+   * @param context A {@link Context}.
+   */
   public VideoDecoderSurfaceView(Context context) {
     this(context, /* attrs= */ null);
   }
 
+  /**
+   * Creates VideoDecoderSurfaceView.
+   *
+   * @param context A {@link Context}.
+   * @param attrs Custom attributes.
+   */
   public VideoDecoderSurfaceView(Context context, @Nullable AttributeSet attrs) {
     super(context, attrs);
-    renderer = new VideoDecoderRenderer();
+    renderer = new VideoDecoderRenderer(this);
     setPreserveEGLContextOnPause(true);
     setEGLContextClientVersion(2);
     setRenderer(renderer);
     setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
   }
 
-  @Override
+  /**
-  public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
+   * Returns the output buffer renderer used.
-    renderer.setFrame(outputBuffer);
+   *
-    requestRender();
+   * @return {@link VideoDecoderOutputBuffer}.
+   */
+  public VideoDecoderOutputBufferRenderer getOutputBufferRenderer() {
+    return renderer;
   }
-
 }

library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java

@@ -64,6 +64,7 @@ import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.ErrorMessageProvider;
 import com.google.android.exoplayer2.util.RepeatModeUtil;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderSurfaceView;
 import com.google.android.exoplayer2.video.VideoListener;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
@@ -276,6 +277,7 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
   private static final int SURFACE_TYPE_SURFACE_VIEW = 1;
   private static final int SURFACE_TYPE_TEXTURE_VIEW = 2;
   private static final int SURFACE_TYPE_MONO360_VIEW = 3;
+  private static final int SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW = 4;
   // LINT.ThenChange(../../../../../../res/values/attrs.xml)
 
   @Nullable private final AspectRatioFrameLayout contentFrame;
@@ -411,6 +413,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
           sphericalSurfaceView.setSingleTapListener(componentListener);
           surfaceView = sphericalSurfaceView;
           break;
+        case SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW:
+          surfaceView = new VideoDecoderSurfaceView(context);
+          break;
         default:
           surfaceView = new SurfaceView(context);
           break;
@@ -539,6 +544,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
           oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
         } else if (surfaceView instanceof SphericalSurfaceView) {
           ((SphericalSurfaceView) surfaceView).setVideoComponent(null);
+        } else if (surfaceView instanceof VideoDecoderSurfaceView) {
+          oldVideoComponent.setOutputBufferRenderer(null);
         } else if (surfaceView instanceof SurfaceView) {
           oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
         }
@@ -565,6 +572,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
           newVideoComponent.setVideoTextureView((TextureView) surfaceView);
         } else if (surfaceView instanceof SphericalSurfaceView) {
           ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
+        } else if (surfaceView instanceof VideoDecoderSurfaceView) {
+          newVideoComponent.setOutputBufferRenderer(
+              ((VideoDecoderSurfaceView) surfaceView).getOutputBufferRenderer());
         } else if (surfaceView instanceof SurfaceView) {
           newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
         }
@@ -736,8 +746,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
    * buffering spinner is not displayed by default.
    *
    * @param showBuffering The mode that defines when the buffering spinner is displayed. One of
-   *     {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and
+   *     {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and {@link
-   *     {@link #SHOW_BUFFERING_ALWAYS}.
+   *     #SHOW_BUFFERING_ALWAYS}.
    */
   public void setShowBuffering(@ShowBuffering int showBuffering) {
     if (this.showBuffering != showBuffering) {

library/ui/src/main/res/values/attrs.xml

@@ -30,6 +30,7 @@
     <enum name="surface_view" value="1"/>
     <enum name="texture_view" value="2"/>
     <enum name="spherical_view" value="3"/>
+    <enum name="video_decoder_surface_view" value="4"/>
   </attr>
 
   <!-- Must be kept in sync with RepeatModeUtil -->