Pass appropriate frame-rate to Surface.setFrameRate

PiperOrigin-RevId: 309746009

RELEASENOTES.md

@@ -83,6 +83,7 @@
         ([#7247](https://github.com/google/ExoPlayer/pull/7247)).
     *   Replace `CacheDataSinkFactory` and `CacheDataSourceFactory` with
         `CacheDataSink.Factory` and `CacheDataSource.Factory` respectively.
+*   Video: Pass frame rate hint to `Surface.setFrameRate` on Android R devices.
 *   Text:
     *   Parse `<ruby>` and `<rt>` tags in WebVTT subtitles (rendering is coming
         later).

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -374,7 +374,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   @Nullable private MediaCrypto mediaCrypto;
   private boolean mediaCryptoRequiresSecureDecoder;
   private long renderTimeLimitMs;
-  private float rendererOperatingRate;
+  private float operatingRate;
   @Nullable private MediaCodec codec;
   @Nullable private MediaCodecAdapter codecAdapter;
   @Nullable private Format codecFormat;
@@ -447,7 +447,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     formatQueue = new TimedValueQueue<>();
     decodeOnlyPresentationTimestamps = new ArrayList<>();
     outputBufferInfo = new MediaCodec.BufferInfo();
-    rendererOperatingRate = 1f;
+    operatingRate = 1f;
     renderTimeLimitMs = C.TIME_UNSET;
     mediaCodecOperationMode = OPERATION_MODE_SYNCHRONOUS;
     pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
@@ -710,8 +710,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   }
 
   @Override
-  public final void setOperatingRate(float operatingRate) throws ExoPlaybackException {
-    rendererOperatingRate = operatingRate;
+  public void setOperatingRate(float operatingRate) throws ExoPlaybackException {
+    this.operatingRate = operatingRate;
     if (codec != null
         && codecDrainAction != DRAIN_ACTION_REINITIALIZE
         && getState() != STATE_DISABLED) {
@@ -1031,7 +1031,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     float codecOperatingRate =
         Util.SDK_INT < 23
             ? CODEC_OPERATING_RATE_UNSET
-            : getCodecOperatingRateV23(rendererOperatingRate, inputFormat, getStreamFormats());
+            : getCodecOperatingRateV23(operatingRate, inputFormat, getStreamFormats());
     if (codecOperatingRate <= assumedMinimumCodecOperatingRate) {
       codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
     }
@@ -1561,6 +1561,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
                 && SystemClock.elapsedRealtime() < codecHotswapDeadlineMs));
   }
 
+  /** Returns the renderer operating rate, as set by {@link #setOperatingRate}. */
+  protected float getOperatingRate() {
+    return operatingRate;
+  }
+
   /**
    * Returns the {@link MediaFormat#KEY_OPERATING_RATE} value for a given renderer operating rate,
    * current {@link Format} and set of possible stream formats.
@@ -1589,7 +1594,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     }
 
     float newCodecOperatingRate =
-        getCodecOperatingRateV23(rendererOperatingRate, codecFormat, getStreamFormats());
+        getCodecOperatingRateV23(operatingRate, codecFormat, getStreamFormats());
     if (codecOperatingRate == newCodecOperatingRate) {
       // No change.
     } else if (newCodecOperatingRate == CODEC_OPERATING_RATE_UNSET) {

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher;
+import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.List;
@@ -98,6 +99,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   /** Magic frame render timestamp that indicates the EOS in tunneling mode. */
   private static final long TUNNELING_EOS_PRESENTATION_TIME_US = Long.MAX_VALUE;
 
+  // TODO: Remove reflection once we target API level 30.
+  @Nullable private static final Method surfaceSetFrameRateMethod;
+
+  static {
+    @Nullable Method setFrameRateMethod = null;
+    if (Util.SDK_INT >= 30) {
+      try {
+        setFrameRateMethod = Surface.class.getMethod("setFrameRate", float.class, int.class);
+      } catch (NoSuchMethodException e) {
+        // Do nothing.
+      }
+    }
+    surfaceSetFrameRateMethod = setFrameRateMethod;
+  }
+  // TODO: Remove these constants and use those defined by Surface once we target API level 30.
+  private static final int SURFACE_FRAME_RATE_COMPATIBILITY_DEFAULT = 0;
+  private static final int SURFACE_FRAME_RATE_COMPATIBILITY_FIXED_SOURCE = 1;
+
   private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround;
   private static boolean deviceNeedsSetOutputSurfaceWorkaround;
 
@@ -113,6 +132,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private boolean codecHandlesHdr10PlusOutOfBandMetadata;
 
   private Surface surface;
+  private float surfaceFrameRate;
   private Surface dummySurface;
   @VideoScalingMode private int scalingMode;
   private boolean renderedFirstFrameAfterReset;
@@ -135,6 +155,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private int currentHeight;
   private int currentUnappliedRotationDegrees;
   private float currentPixelWidthHeightRatio;
+  private float currentFrameRate;
   private int reportedWidth;
   private int reportedHeight;
   private int reportedUnappliedRotationDegrees;
@@ -409,6 +430,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
     totalVideoFrameProcessingOffsetUs = 0;
     videoFrameProcessingOffsetCount = 0;
+    updateSurfaceFrameRate(/* isNewSurface= */ false);
   }
 
   @Override
@@ -416,6 +438,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     joiningDeadlineMs = C.TIME_UNSET;
     maybeNotifyDroppedFrames();
     maybeNotifyVideoFrameProcessingOffset();
+    clearSurfaceFrameRate();
     super.onStopped();
   }
 
@@ -480,7 +503,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     }
     // We only need to update the codec if the surface has changed.
     if (this.surface != surface) {
+      clearSurfaceFrameRate();
       this.surface = surface;
+      updateSurfaceFrameRate(/* isNewSurface= */ true);
+
       @State int state = getState();
       MediaCodec codec = getCodec();
       if (codec != null) {
@@ -577,6 +603,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     buffersInCodecCount = 0;
   }
 
+  @Override
+  public void setOperatingRate(float operatingRate) throws ExoPlaybackException {
+    super.setOperatingRate(operatingRate);
+    updateSurfaceFrameRate(/* isNewSurface= */ false);
+  }
+
   @Override
   protected float getCodecOperatingRateV23(
       float operatingRate, Format format, Format[] streamFormats) {
@@ -682,6 +714,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       // On API level 20 and below the decoder does not apply the rotation.
       currentUnappliedRotationDegrees = outputFormat.rotationDegrees;
     }
+    currentFrameRate = outputFormat.frameRate;
+    updateSurfaceFrameRate(/* isNewSurface= */ false);
   }
 
   @Override
@@ -1049,6 +1083,52 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     maybeNotifyRenderedFirstFrame();
   }
 
+  /**
+   * Updates the frame-rate of the current {@link #surface} based on the renderer operating rate,
+   * frame-rate of the content, and whether the renderer is started.
+   *
+   * @param isNewSurface Whether the current {@link #surface} is new.
+   */
+  private void updateSurfaceFrameRate(boolean isNewSurface) {
+    if (Util.SDK_INT < 30 || surface == null || surface == dummySurface) {
+      return;
+    }
+    boolean shouldSetFrameRate = getState() == STATE_STARTED && currentFrameRate != Format.NO_VALUE;
+    float surfaceFrameRate = shouldSetFrameRate ? currentFrameRate * getOperatingRate() : 0;
+    // We always set the frame-rate if we have a new surface, since we have no way of knowing what
+    // it might have been set to previously.
+    if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) {
+      return;
+    }
+    this.surfaceFrameRate = surfaceFrameRate;
+    setSurfaceFrameRateV30(surface, surfaceFrameRate);
+  }
+
+  /** Clears the frame-rate of the current {@link #surface}. */
+  private void clearSurfaceFrameRate() {
+    if (Util.SDK_INT < 30 || surface == null || surface == dummySurface || surfaceFrameRate == 0) {
+      return;
+    }
+    surfaceFrameRate = 0;
+    setSurfaceFrameRateV30(surface, /* frameRate= */ 0);
+  }
+
+  @RequiresApi(30)
+  private void setSurfaceFrameRateV30(Surface surface, float frameRate) {
+    if (surfaceSetFrameRateMethod == null) {
+      Log.e(TAG, "Failed to call Surface.setFrameRate (method does not exist)");
+    }
+    int compatibility =
+        frameRate == 0
+            ? SURFACE_FRAME_RATE_COMPATIBILITY_DEFAULT
+            : SURFACE_FRAME_RATE_COMPATIBILITY_FIXED_SOURCE;
+    try {
+      surfaceSetFrameRateMethod.invoke(surface, frameRate, compatibility);
+    } catch (Exception e) {
+      Log.e(TAG, "Failed to call Surface.setFrameRate", e);
+    }
+  }
+
   private boolean shouldUseDummySurface(MediaCodecInfo codecInfo) {
     return Util.SDK_INT >= 23
         && !tunneling