From 862a6e4dd21b9376a9e6cfee46bbc301b960d72d Mon Sep 17 00:00:00 2001 From: olly Date: Mon, 4 May 2020 16:14:46 +0100 Subject: [PATCH] Pass appropriate frame-rate to Surface.setFrameRate PiperOrigin-RevId: 309746009 --- RELEASENOTES.md | 1 + .../mediacodec/MediaCodecRenderer.java | 17 ++-- .../video/MediaCodecVideoRenderer.java | 80 +++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8a1284c8e0..bd3441606a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -83,6 +83,7 @@ ([#7247](https://github.com/google/ExoPlayer/pull/7247)). * Replace `CacheDataSinkFactory` and `CacheDataSourceFactory` with `CacheDataSink.Factory` and `CacheDataSource.Factory` respectively. +* Video: Pass frame rate hint to `Surface.setFrameRate` on Android R devices. * Text: * Parse `` and `` tags in WebVTT subtitles (rendering is coming later). diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index eea8198989..b4b0d6ecd9 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -374,7 +374,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @Nullable private MediaCrypto mediaCrypto; private boolean mediaCryptoRequiresSecureDecoder; private long renderTimeLimitMs; - private float rendererOperatingRate; + private float operatingRate; @Nullable private MediaCodec codec; @Nullable private MediaCodecAdapter codecAdapter; @Nullable private Format codecFormat; @@ -447,7 +447,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { formatQueue = new TimedValueQueue<>(); decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); - rendererOperatingRate = 1f; + operatingRate = 1f; renderTimeLimitMs = C.TIME_UNSET; mediaCodecOperationMode = OPERATION_MODE_SYNCHRONOUS; pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; @@ -710,8 +710,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } @Override - public final void setOperatingRate(float operatingRate) throws ExoPlaybackException { - rendererOperatingRate = operatingRate; + public void setOperatingRate(float operatingRate) throws ExoPlaybackException { + this.operatingRate = operatingRate; if (codec != null && codecDrainAction != DRAIN_ACTION_REINITIALIZE && getState() != STATE_DISABLED) { @@ -1031,7 +1031,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { float codecOperatingRate = Util.SDK_INT < 23 ? CODEC_OPERATING_RATE_UNSET - : getCodecOperatingRateV23(rendererOperatingRate, inputFormat, getStreamFormats()); + : getCodecOperatingRateV23(operatingRate, inputFormat, getStreamFormats()); if (codecOperatingRate <= assumedMinimumCodecOperatingRate) { codecOperatingRate = CODEC_OPERATING_RATE_UNSET; } @@ -1561,6 +1561,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { && SystemClock.elapsedRealtime() < codecHotswapDeadlineMs)); } + /** Returns the renderer operating rate, as set by {@link #setOperatingRate}. */ + protected float getOperatingRate() { + return operatingRate; + } + /** * Returns the {@link MediaFormat#KEY_OPERATING_RATE} value for a given renderer operating rate, * current {@link Format} and set of possible stream formats. @@ -1589,7 +1594,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } float newCodecOperatingRate = - getCodecOperatingRateV23(rendererOperatingRate, codecFormat, getStreamFormats()); + getCodecOperatingRateV23(operatingRate, codecFormat, getStreamFormats()); if (codecOperatingRate == newCodecOperatingRate) { // No change. } else if (newCodecOperatingRate == CODEC_OPERATING_RATE_UNSET) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 91888cd906..847dac3a79 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -98,6 +99,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { /** Magic frame render timestamp that indicates the EOS in tunneling mode. */ private static final long TUNNELING_EOS_PRESENTATION_TIME_US = Long.MAX_VALUE; + // TODO: Remove reflection once we target API level 30. + @Nullable private static final Method surfaceSetFrameRateMethod; + + static { + @Nullable Method setFrameRateMethod = null; + if (Util.SDK_INT >= 30) { + try { + setFrameRateMethod = Surface.class.getMethod("setFrameRate", float.class, int.class); + } catch (NoSuchMethodException e) { + // Do nothing. + } + } + surfaceSetFrameRateMethod = setFrameRateMethod; + } + // TODO: Remove these constants and use those defined by Surface once we target API level 30. + private static final int SURFACE_FRAME_RATE_COMPATIBILITY_DEFAULT = 0; + private static final int SURFACE_FRAME_RATE_COMPATIBILITY_FIXED_SOURCE = 1; + private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; @@ -113,6 +132,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private boolean codecHandlesHdr10PlusOutOfBandMetadata; private Surface surface; + private float surfaceFrameRate; private Surface dummySurface; @VideoScalingMode private int scalingMode; private boolean renderedFirstFrameAfterReset; @@ -135,6 +155,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private int currentHeight; private int currentUnappliedRotationDegrees; private float currentPixelWidthHeightRatio; + private float currentFrameRate; private int reportedWidth; private int reportedHeight; private int reportedUnappliedRotationDegrees; @@ -409,6 +430,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000; totalVideoFrameProcessingOffsetUs = 0; videoFrameProcessingOffsetCount = 0; + updateSurfaceFrameRate(/* isNewSurface= */ false); } @Override @@ -416,6 +438,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { joiningDeadlineMs = C.TIME_UNSET; maybeNotifyDroppedFrames(); maybeNotifyVideoFrameProcessingOffset(); + clearSurfaceFrameRate(); super.onStopped(); } @@ -480,7 +503,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } // We only need to update the codec if the surface has changed. if (this.surface != surface) { + clearSurfaceFrameRate(); this.surface = surface; + updateSurfaceFrameRate(/* isNewSurface= */ true); + @State int state = getState(); MediaCodec codec = getCodec(); if (codec != null) { @@ -577,6 +603,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { buffersInCodecCount = 0; } + @Override + public void setOperatingRate(float operatingRate) throws ExoPlaybackException { + super.setOperatingRate(operatingRate); + updateSurfaceFrameRate(/* isNewSurface= */ false); + } + @Override protected float getCodecOperatingRateV23( float operatingRate, Format format, Format[] streamFormats) { @@ -682,6 +714,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // On API level 20 and below the decoder does not apply the rotation. currentUnappliedRotationDegrees = outputFormat.rotationDegrees; } + currentFrameRate = outputFormat.frameRate; + updateSurfaceFrameRate(/* isNewSurface= */ false); } @Override @@ -1049,6 +1083,52 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { maybeNotifyRenderedFirstFrame(); } + /** + * Updates the frame-rate of the current {@link #surface} based on the renderer operating rate, + * frame-rate of the content, and whether the renderer is started. + * + * @param isNewSurface Whether the current {@link #surface} is new. + */ + private void updateSurfaceFrameRate(boolean isNewSurface) { + if (Util.SDK_INT < 30 || surface == null || surface == dummySurface) { + return; + } + boolean shouldSetFrameRate = getState() == STATE_STARTED && currentFrameRate != Format.NO_VALUE; + float surfaceFrameRate = shouldSetFrameRate ? currentFrameRate * getOperatingRate() : 0; + // We always set the frame-rate if we have a new surface, since we have no way of knowing what + // it might have been set to previously. + if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) { + return; + } + this.surfaceFrameRate = surfaceFrameRate; + setSurfaceFrameRateV30(surface, surfaceFrameRate); + } + + /** Clears the frame-rate of the current {@link #surface}. */ + private void clearSurfaceFrameRate() { + if (Util.SDK_INT < 30 || surface == null || surface == dummySurface || surfaceFrameRate == 0) { + return; + } + surfaceFrameRate = 0; + setSurfaceFrameRateV30(surface, /* frameRate= */ 0); + } + + @RequiresApi(30) + private void setSurfaceFrameRateV30(Surface surface, float frameRate) { + if (surfaceSetFrameRateMethod == null) { + Log.e(TAG, "Failed to call Surface.setFrameRate (method does not exist)"); + } + int compatibility = + frameRate == 0 + ? SURFACE_FRAME_RATE_COMPATIBILITY_DEFAULT + : SURFACE_FRAME_RATE_COMPATIBILITY_FIXED_SOURCE; + try { + surfaceSetFrameRateMethod.invoke(surface, frameRate, compatibility); + } catch (Exception e) { + Log.e(TAG, "Failed to call Surface.setFrameRate", e); + } + } + private boolean shouldUseDummySurface(MediaCodecInfo codecInfo) { return Util.SDK_INT >= 23 && !tunneling