Add OpenGL support to av1 extension: Libgav1VideoRenderer

Move reusable code from LibvpxVideoRenderer to SimpleDecoderVideoRenderer.
Pass outputBuffer to renderOutputBuffer method instead of keeping the reference in the renderer.

PiperOrigin-RevId: 272899549

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java

@@ -28,7 +28,6 @@ import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.decoder.SimpleDecoder;
 import com.google.android.exoplayer2.drm.DrmSessionManager;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
-import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer;
@@ -72,13 +71,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
   private final boolean disableLoopFilter;
   private final int threads;
 
-  private Surface surface;
-  private VideoDecoderOutputBufferRenderer outputBufferRenderer;
-  @C.VideoOutputMode private int outputMode;
-
   private VpxDecoder decoder;
-  private VideoDecoderOutputBuffer outputBuffer;
-
   private VideoFrameMetadataListener frameMetadataListener;
 
   /**
@@ -197,7 +190,6 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
     this.threads = threads;
     this.numInputBuffers = numInputBuffers;
     this.numOutputBuffers = numOutputBuffers;
-    outputMode = C.VIDEO_OUTPUT_MODE_NONE;
   }
 
   @Override
@@ -236,53 +228,37 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
             disableLoopFilter,
             enableRowMultiThreadMode,
             threads);
-    decoder.setOutputMode(outputMode);
     TraceUtil.endSection();
     return decoder;
   }
 
   @Override
-  @Nullable
+  protected void renderOutputBuffer(
-  protected VideoDecoderOutputBuffer dequeueOutputBuffer() throws VpxDecoderException {
+      VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
-    outputBuffer = decoder.dequeueOutputBuffer();
+      throws VideoDecoderException {
-    return outputBuffer;
-  }
-
-  @Override
-  protected void renderOutputBuffer(long presentationTimeUs, Format outputFormat)
-      throws VpxDecoderException {
     if (frameMetadataListener != null) {
       frameMetadataListener.onVideoFrameAboutToBeRendered(
           presentationTimeUs, System.nanoTime(), outputFormat);
     }
+    super.renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
+  }
 
-    int bufferMode = outputBuffer.mode;
+  @Override
-    boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;
+  protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
-    boolean renderYuv = bufferMode == C.VIDEO_OUTPUT_MODE_YUV && outputBufferRenderer != null;
+      throws VpxDecoderException {
-    if (!renderYuv && !renderSurface) {
+    if (decoder == null) {
-      dropOutputBuffer(outputBuffer);
+      throw new VpxDecoderException(
-    } else {
+          "Failed to render output buffer to surface: decoder is not initialized.");
-      maybeNotifyVideoSizeChanged(outputBuffer.width, outputBuffer.height);
-      if (renderYuv) {
-        outputBufferRenderer.setOutputBuffer(outputBuffer);
-        // The renderer will release the buffer.
-      } else { // renderSurface
-        decoder.renderToSurface(outputBuffer, surface);
-        outputBuffer.release();
-      }
-      onFrameRendered(surface);
     }
+    decoder.renderToSurface(outputBuffer, surface);
+    outputBuffer.release();
   }
 
   @Override
-  protected void clearOutputBuffer() {
+  protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) {
-    super.clearOutputBuffer();
+    if (decoder != null) {
-    outputBuffer = null;
+      decoder.setOutputMode(outputMode);
-  }
+    }
-
-  @Override
-  protected boolean hasOutputSurface() {
-    return outputMode != C.VIDEO_OUTPUT_MODE_NONE;
   }
 
   // PlayerMessage.Target implementation.
@@ -290,44 +266,13 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
   @Override
   public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
     if (messageType == C.MSG_SET_SURFACE) {
-      setOutput((Surface) message, null);
+      setOutputSurface((Surface) message);
     } else if (messageType == C.MSG_SET_OUTPUT_BUFFER_RENDERER) {
-      setOutput(null, (VideoDecoderOutputBufferRenderer) message);
+      setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
     } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
       frameMetadataListener = (VideoFrameMetadataListener) message;
     } else {
       super.handleMessage(messageType, message);
     }
   }
-
-  // Internal methods.
-
-  private void setOutput(
-      @Nullable Surface surface, @Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
-    // At most one output may be non-null. Both may be null if the output is being cleared.
-    Assertions.checkState(surface == null || outputBufferRenderer == null);
-    if (this.surface != surface || this.outputBufferRenderer != outputBufferRenderer) {
-      // The output has changed.
-      this.surface = surface;
-      this.outputBufferRenderer = outputBufferRenderer;
-      if (surface != null) {
-        outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
-      } else {
-        outputMode =
-            outputBufferRenderer != null ? C.VIDEO_OUTPUT_MODE_YUV : C.VIDEO_OUTPUT_MODE_NONE;
-      }
-      if (hasOutputSurface()) {
-        if (decoder != null) {
-          decoder.setOutputMode(outputMode);
-        }
-        onOutputSurfaceChanged();
-      } else {
-        // The output has been removed. We leave the outputMode of the underlying decoder unchanged
-        // in anticipation that a subsequent output will likely be of the same type.
-        onOutputSurfaceRemoved();
-      }
-    } else if (hasOutputSurface()) {
-      onOutputSurfaceReset(surface);
-    }
-  }
 }

library/core/src/main/java/com/google/android/exoplayer2/video/SimpleDecoderVideoRenderer.java

@@ -86,6 +86,10 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
       decoder;
   private VideoDecoderInputBuffer inputBuffer;
   private VideoDecoderOutputBuffer outputBuffer;
+  @Nullable private Surface surface;
+  @Nullable private VideoDecoderOutputBufferRenderer outputBufferRenderer;
+  @C.VideoOutputMode private int outputMode;
+
   @Nullable private DrmSession<ExoMediaCrypto> decoderDrmSession;
   @Nullable private DrmSession<ExoMediaCrypto> sourceDrmSession;
 
@@ -147,6 +151,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance();
     eventDispatcher = new EventDispatcher(eventHandler, eventListener);
     decoderReinitializationState = REINITIALIZATION_STATE_NONE;
+    outputMode = C.VIDEO_OUTPUT_MODE_NONE;
   }
 
   // BaseRenderer implementation.
@@ -210,7 +215,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     }
     if (inputFormat != null
         && (isSourceReady() || outputBuffer != null)
-        && (renderedFirstFrame || !hasOutputSurface())) {
+        && (renderedFirstFrame || !hasOutput())) {
       // Ready. If we were joining then we've now joined, so clear the joining deadline.
       joiningDeadlineMs = C.TIME_UNSET;
       return true;
@@ -227,6 +232,8 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     }
   }
 
+  // Protected methods.
+
   @Override
   protected void onEnabled(boolean joining) throws ExoPlaybackException {
     decoderCounters = new DecoderCounters();
@@ -316,7 +323,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
       inputBuffer = null;
       if (outputBuffer != null) {
         outputBuffer.release();
-        clearOutputBuffer();
+        outputBuffer = null;
       }
       decoder.flush();
       decoderReceivedBuffers = false;
@@ -327,7 +334,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
   @CallSuper
   protected void releaseDecoder() {
     inputBuffer = null;
-    clearOutputBuffer();
+    outputBuffer = null;
     decoderReinitializationState = REINITIALIZATION_STATE_NONE;
     decoderReceivedBuffers = false;
     buffersInCodecCount = 0;
@@ -339,16 +346,6 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     setDecoderDrmSession(null);
   }
 
-  private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
-    DrmSession.replaceSessionReferences(sourceDrmSession, session);
-    sourceDrmSession = session;
-  }
-
-  private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
-    DrmSession.replaceSessionReferences(decoderDrmSession, session);
-    decoderDrmSession = session;
-  }
-
   /**
    * Called when a new format is read from the upstream source.
    *
@@ -527,96 +524,126 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
       createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto)
           throws VideoDecoderException;
 
-  /**
-   * Dequeues output buffer.
-   *
-   * @return Dequeued video decoder output buffer, or null if an output buffer isn't available.
-   * @throws VideoDecoderException If an error occurs while dequeuing the output buffer.
-   */
-  @Nullable
-  protected abstract VideoDecoderOutputBuffer dequeueOutputBuffer() throws VideoDecoderException;
-
-  /** Clears output buffer. */
-  protected void clearOutputBuffer() {
-    outputBuffer = null;
-  }
-
   /**
    * Renders the specified output buffer.
    *
    * <p>The implementation of this method takes ownership of the output buffer and is responsible
    * for calling {@link VideoDecoderOutputBuffer#release()} either immediately or in the future.
    *
+   * @param outputBuffer {@link VideoDecoderOutputBuffer} to render.
    * @param presentationTimeUs Presentation time in microseconds.
-   * @param outputFormat Output format.
+   * @param outputFormat Output {@link Format}.
+   * @throws VideoDecoderException If an error occurs when rendering the output buffer.
    */
-  // TODO: The output buffer is not being passed to this method currently. Due to the need of
+  protected void renderOutputBuffer(
-  // decoder-specific output buffer type, the reference to the output buffer is being kept in the
+      VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
-  // subclass. Once the common output buffer is established, this method can be updated to receive
+      throws VideoDecoderException {
-  // the output buffer as an argument. See [Internal: b/139174707].
+    lastRenderTimeUs = C.msToUs(SystemClock.elapsedRealtime() * 1000);
-  protected abstract void renderOutputBuffer(long presentationTimeUs, Format outputFormat)
+    int bufferMode = outputBuffer.mode;
-      throws VideoDecoderException;
+    boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;
-
+    boolean renderYuv = bufferMode == C.VIDEO_OUTPUT_MODE_YUV && outputBufferRenderer != null;
-  /**
+    if (!renderYuv && !renderSurface) {
-   * Returns whether the renderer has output surface.
+      dropOutputBuffer(outputBuffer);
-   *
+    } else {
-   * @return Whether the renderer has output surface.
+      maybeNotifyVideoSizeChanged(outputBuffer.width, outputBuffer.height);
-   */
+      if (renderYuv) {
-  protected abstract boolean hasOutputSurface();
+        outputBufferRenderer.setOutputBuffer(outputBuffer);
-
+      } else {
-  /** Called when the output surface is changed. */
+        renderOutputBufferToSurface(outputBuffer, surface);
-  protected final void onOutputSurfaceChanged() {
+      }
-    // If we know the video size, report it again immediately.
+      consecutiveDroppedFrameCount = 0;
-    maybeRenotifyVideoSizeChanged();
+      decoderCounters.renderedOutputBufferCount++;
-    // We haven't rendered to the new output yet.
+      maybeNotifyRenderedFirstFrame();
-    clearRenderedFirstFrame();
-    if (getState() == STATE_STARTED) {
-      setJoiningDeadlineMs();
     }
   }
 
-  /** Called when the output surface is removed. */
+  /**
-  protected final void onOutputSurfaceRemoved() {
+   * Renders the specified output buffer to the passed surface.
-    clearReportedVideoSize();
+   *
-    clearRenderedFirstFrame();
+   * <p>The implementation of this method takes ownership of the output buffer and is responsible
-  }
+   * for calling {@link VideoDecoderOutputBuffer#release()} either immediately or in the future.
+   *
+   * @param outputBuffer {@link VideoDecoderOutputBuffer} to render.
+   * @param surface Output {@link Surface}.
+   * @throws VideoDecoderException If an error occurs when rendering the output buffer.
+   */
+  protected abstract void renderOutputBufferToSurface(
+      VideoDecoderOutputBuffer outputBuffer, Surface surface) throws VideoDecoderException;
 
   /**
-   * Called when the output surface is set again to the same non-null value.
+   * Sets output surface.
    *
-   * @param surface Output surface.
+   * @param surface Surface.
    */
-  protected final void onOutputSurfaceReset(Surface surface) {
+  protected final void setOutputSurface(@Nullable Surface surface) {
-    // The output is unchanged and non-null. If we know the video size and/or have already
+    if (this.surface != surface) {
-    // rendered to the output, report these again immediately.
+      // The output has changed.
-    maybeRenotifyVideoSizeChanged();
+      this.surface = surface;
-    maybeRenotifyRenderedFirstFrame(surface);
+      if (surface != null) {
-  }
+        outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
-
+        if (decoder != null) {
-  /**
+          setDecoderOutputMode(outputMode);
-   * Notifies event dispatcher if video size changed.
+        }
-   *
+        onOutputChanged();
-   * @param width New video width.
+      } else {
-   * @param height New video height.
+        // The output has been removed. We leave the outputMode of the underlying decoder unchanged
-   */
+        // in anticipation that a subsequent output will likely be of the same type.
-  protected final void maybeNotifyVideoSizeChanged(int width, int height) {
+        outputMode = C.VIDEO_OUTPUT_MODE_NONE;
-    if (reportedWidth != width || reportedHeight != height) {
+        onOutputRemoved();
-      reportedWidth = width;
+      }
-      reportedHeight = height;
+    } else if (surface != null) {
-      eventDispatcher.videoSizeChanged(
+      // The output is unchanged and non-null.
-          width, height, /* unappliedRotationDegrees= */ 0, /* pixelWidthHeightRatio= */ 1);
+      onOutputReset();
     }
   }
 
-  /** Called after rendering a frame. */
+  /**
-  protected final void onFrameRendered(Surface surface) {
+   * Sets output buffer renderer.
-    consecutiveDroppedFrameCount = 0;
+   *
-    decoderCounters.renderedOutputBufferCount++;
+   * @param outputBufferRenderer Output buffer renderer.
-    maybeNotifyRenderedFirstFrame(surface);
+   */
+  protected final void setOutputBufferRenderer(
+      @Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
+    if (this.outputBufferRenderer != outputBufferRenderer) {
+      // The output has changed.
+      this.outputBufferRenderer = outputBufferRenderer;
+      if (outputBufferRenderer != null) {
+        outputMode = C.VIDEO_OUTPUT_MODE_YUV;
+        if (decoder != null) {
+          setDecoderOutputMode(outputMode);
+        }
+        onOutputChanged();
+      } else {
+        // The output has been removed. We leave the outputMode of the underlying decoder unchanged
+        // in anticipation that a subsequent output will likely be of the same type.
+        outputMode = C.VIDEO_OUTPUT_MODE_NONE;
+        onOutputRemoved();
+      }
+    } else if (outputBufferRenderer != null) {
+      // The output is unchanged and non-null.
+      onOutputReset();
+    }
   }
 
+  /**
+   * Sets output mode of the decoder.
+   *
+   * @param outputMode Output mode.
+   */
+  protected abstract void setDecoderOutputMode(@C.VideoOutputMode int outputMode);
+
   // Internal methods.
 
+  private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession.replaceSessionReferences(sourceDrmSession, session);
+    sourceDrmSession = session;
+  }
+
+  private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession.replaceSessionReferences(decoderDrmSession, session);
+    decoderDrmSession = session;
+  }
+
   private void maybeInitDecoder() throws ExoPlaybackException {
     if (decoder != null) {
       return;
@@ -642,6 +669,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     try {
       long decoderInitializingTimestamp = SystemClock.elapsedRealtime();
       decoder = createDecoder(inputFormat, mediaCrypto);
+      setDecoderOutputMode(outputMode);
       long decoderInitializedTimestamp = SystemClock.elapsedRealtime();
       onDecoderInitialized(
           decoder.getName(),
@@ -731,7 +759,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
   private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
       throws ExoPlaybackException, VideoDecoderException {
     if (outputBuffer == null) {
-      outputBuffer = dequeueOutputBuffer();
+      outputBuffer = decoder.dequeueOutputBuffer();
       if (outputBuffer == null) {
         return false;
       }
@@ -746,7 +774,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
         maybeInitDecoder();
       } else {
         outputBuffer.release();
-        clearOutputBuffer();
+        outputBuffer = null;
         outputStreamEnded = true;
       }
       return false;
@@ -755,7 +783,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     boolean processedOutputBuffer = processOutputBuffer(positionUs, elapsedRealtimeUs);
     if (processedOutputBuffer) {
       onProcessedOutputBuffer(outputBuffer.timeUs);
-      clearOutputBuffer();
+      outputBuffer = null;
     }
     return processedOutputBuffer;
   }
@@ -777,7 +805,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     }
 
     long earlyUs = outputBuffer.timeUs - positionUs;
-    if (!hasOutputSurface()) {
+    if (!hasOutput()) {
       // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
       if (isBufferLate(earlyUs)) {
         skipOutputBuffer(outputBuffer);
@@ -797,8 +825,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     if (!renderedFirstFrame
         || (isStarted
             && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) {
-      lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
+      renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
-      renderOutputBuffer(presentationTimeUs, outputFormat);
       return true;
     }
 
@@ -815,14 +842,39 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     }
 
     if (earlyUs < 30000) {
-      lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
+      renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
-      renderOutputBuffer(presentationTimeUs, outputFormat);
       return true;
     }
 
     return false;
   }
 
+  private boolean hasOutput() {
+    return outputMode != C.VIDEO_OUTPUT_MODE_NONE;
+  }
+
+  private void onOutputChanged() {
+    // If we know the video size, report it again immediately.
+    maybeRenotifyVideoSizeChanged();
+    // We haven't rendered to the new output yet.
+    clearRenderedFirstFrame();
+    if (getState() == STATE_STARTED) {
+      setJoiningDeadlineMs();
+    }
+  }
+
+  private void onOutputRemoved() {
+    clearReportedVideoSize();
+    clearRenderedFirstFrame();
+  }
+
+  private void onOutputReset() {
+    // The output is unchanged and non-null. If we know the video size and/or have already
+    // rendered to the output, report these again immediately.
+    maybeRenotifyVideoSizeChanged();
+    maybeRenotifyRenderedFirstFrame();
+  }
+
   private boolean shouldWaitForKeys(boolean bufferEncrypted) throws ExoPlaybackException {
     if (decoderDrmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
       return false;
@@ -845,14 +897,14 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     renderedFirstFrame = false;
   }
 
-  private void maybeNotifyRenderedFirstFrame(Surface surface) {
+  private void maybeNotifyRenderedFirstFrame() {
     if (!renderedFirstFrame) {
       renderedFirstFrame = true;
       eventDispatcher.renderedFirstFrame(surface);
     }
   }
 
-  private void maybeRenotifyRenderedFirstFrame(Surface surface) {
+  private void maybeRenotifyRenderedFirstFrame() {
     if (renderedFirstFrame) {
       eventDispatcher.renderedFirstFrame(surface);
     }
@@ -863,6 +915,15 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
     reportedHeight = Format.NO_VALUE;
   }
 
+  private void maybeNotifyVideoSizeChanged(int width, int height) {
+    if (reportedWidth != width || reportedHeight != height) {
+      reportedWidth = width;
+      reportedHeight = height;
+      eventDispatcher.videoSizeChanged(
+          width, height, /* unappliedRotationDegrees= */ 0, /* pixelWidthHeightRatio= */ 1);
+    }
+  }
+
   private void maybeRenotifyVideoSizeChanged() {
     if (reportedWidth != Format.NO_VALUE || reportedHeight != Format.NO_VALUE) {
       eventDispatcher.videoSizeChanged(