diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java b/library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java index 502729f018..a4315327f3 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java @@ -25,10 +25,8 @@ import java.util.Arrays; */ /* package */ final class FixedFrameRateEstimator { - /** - * The number of consecutive matching frame durations for the tracker to be considered in sync. - */ - @VisibleForTesting static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 10; + /** The number of consecutive matching frame durations required to detect a fixed frame rate. */ + public static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 15; /** * The maximum amount frame durations can differ for them to be considered matching, in * nanoseconds. @@ -44,13 +42,12 @@ import java.util.Arrays; private Matcher candidateMatcher; private boolean candidateMatcherActive; private boolean switchToCandidateMatcherWhenSynced; - private float formatFrameRate; private long lastFramePresentationTimeNs; + private int framesWithoutSyncCount; public FixedFrameRateEstimator() { currentMatcher = new Matcher(); candidateMatcher = new Matcher(); - formatFrameRate = Format.NO_VALUE; lastFramePresentationTimeNs = C.TIME_UNSET; } @@ -59,29 +56,8 @@ import java.util.Arrays; currentMatcher.reset(); candidateMatcher.reset(); candidateMatcherActive = false; - formatFrameRate = Format.NO_VALUE; lastFramePresentationTimeNs = C.TIME_UNSET; - } - - /** - * Called when the renderer's output format changes. - * - * @param formatFrameRate The format's frame rate, or {@link Format#NO_VALUE} if unknown. - */ - public void onFormatChanged(float formatFrameRate) { - // The format frame rate is only used to determine to what extent the estimator should be reset. - // Frame rate estimates are always calculated directly from frame presentation timestamps. - if (this.formatFrameRate != formatFrameRate) { - reset(); - } else { - // Keep the current matcher, but prefer to switch to a new matcher once synced even if the - // current one does not lose sync. This avoids an issue where the current matcher would - // continue to be used if a frame rate change has occurred that's too small to trigger sync - // loss (e.g., a change from 30fps to 29.97fps) and which is not represented in the format - // frame rates (e.g., because they're unset or only have integer precision). - switchToCandidateMatcherWhenSynced = true; - } - this.formatFrameRate = formatFrameRate; + framesWithoutSyncCount = 0; } /** @@ -113,6 +89,7 @@ import java.util.Arrays; switchToCandidateMatcherWhenSynced = false; } lastFramePresentationTimeNs = framePresentationTimeNs; + framesWithoutSyncCount = currentMatcher.isSynced() ? 0 : framesWithoutSyncCount + 1; } /** Returns whether the estimator has detected a fixed frame rate. */ @@ -120,6 +97,19 @@ import java.util.Arrays; return currentMatcher.isSynced(); } + /** Returns the number of frames since the estimator last detected a fixed frame rate. */ + public int getFramesWithoutSyncCount() { + return framesWithoutSyncCount; + } + + /** + * Returns the sum of all frame durations used to calculate the current fixed frame rate estimate, + * or {@link C#TIME_UNSET} if {@link #isSynced()} is {@code false}. + */ + public long getMatchingFrameDurationSumNs() { + return isSynced() ? currentMatcher.getMatchingFrameDurationSumNs() : C.TIME_UNSET; + } + /** * The currently detected fixed frame duration estimate in nanoseconds, or {@link C#TIME_UNSET} if * {@link #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link @@ -134,9 +124,9 @@ import java.util.Arrays; * #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link * #onNextFrame} is called with a new frame presentation timestamp. */ - public double getFrameRate() { + public float getFrameRate() { return isSynced() - ? (double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs() + ? (float) ((double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs()) : Format.NO_VALUE; } @@ -184,6 +174,10 @@ import java.util.Arrays; return recentFrameOutlierFlags[getRecentFrameOutlierIndex(frameCount - 1)]; } + public long getMatchingFrameDurationSumNs() { + return matchingFrameDurationSumNs; + } + public long getFrameDurationNs() { return matchingFrameCount == 0 ? 0 : (matchingFrameDurationSumNs / matchingFrameCount); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java index a78a288d2f..c646a2323f 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java @@ -51,6 +51,31 @@ public final class VideoFrameReleaseHelper { private static final String TAG = "VideoFrameReleaseHelper"; + /** + * The minimum sum of frame durations used to calculate the current fixed frame rate estimate, for + * the estimate to be treated as a high confidence estimate. + */ + private static final long MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS = 5_000_000_000L; + + /** + * The minimum change in media frame rate that will trigger a change in surface frame rate, given + * a high confidence estimate. + */ + private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE = 0.02f; + + /** + * The minimum change in media frame rate that will trigger a change in surface frame rate, given + * a low confidence estimate. + */ + private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE = 1f; + + /** + * The minimum number of frames without a frame rate estimate, for the surface frame rate to be + * cleared. + */ + private static final int MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE = + 2 * FixedFrameRateEstimator.CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; + /** The period between sampling display VSYNC timestamps, in milliseconds. */ private static final long VSYNC_SAMPLE_UPDATE_PERIOD_MS = 500; /** @@ -65,7 +90,7 @@ public final class VideoFrameReleaseHelper { */ private static final long VSYNC_OFFSET_PERCENTAGE = 80; - private final FixedFrameRateEstimator fixedFrameRateEstimator; + private final FixedFrameRateEstimator frameRateEstimator; @Nullable private final WindowManager windowManager; @Nullable private final VSyncSampler vsyncSampler; @Nullable private final DefaultDisplayListener displayListener; @@ -73,9 +98,18 @@ public final class VideoFrameReleaseHelper { private boolean started; @Nullable private Surface surface; - private float surfaceFrameRate; + /** The media frame rate specified in the {@link Format}. */ private float formatFrameRate; - private double playbackSpeed; + /** + * The media frame rate used to calculate the playback frame rate of the {@link Surface}. This may + * be different to {@link #formatFrameRate} if {@link #formatFrameRate} is unspecified or + * inaccurate. + */ + private float surfaceMediaFrameRate; + /** The playback frame rate set on the {@link Surface}. */ + private float surfacePlaybackFrameRate; + + private float playbackSpeed; private long vsyncDurationNs; private long vsyncOffsetNs; @@ -92,7 +126,7 @@ public final class VideoFrameReleaseHelper { * @param context A context from which information about the default display can be retrieved. */ public VideoFrameReleaseHelper(@Nullable Context context) { - fixedFrameRateEstimator = new FixedFrameRateEstimator(); + frameRateEstimator = new FixedFrameRateEstimator(); if (context != null) { context = context.getApplicationContext(); windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); @@ -116,7 +150,6 @@ public final class VideoFrameReleaseHelper { /** Called when the renderer is enabled. */ @TargetApi(17) // displayListener is null if Util.SDK_INT < 17. public void onEnabled() { - fixedFrameRateEstimator.reset(); if (windowManager != null) { checkNotNull(vsyncSampler).addObserver(); if (displayListener != null) { @@ -130,7 +163,7 @@ public final class VideoFrameReleaseHelper { public void onStarted() { started = true; resetAdjustment(); - updateSurfaceFrameRate(/* isNewSurface= */ false); + updateSurfacePlaybackFrameRate(/* isNewSurface= */ false); } /** @@ -148,7 +181,7 @@ public final class VideoFrameReleaseHelper { } clearSurfaceFrameRate(); this.surface = surface; - updateSurfaceFrameRate(/* isNewSurface= */ true); + updateSurfacePlaybackFrameRate(/* isNewSurface= */ true); } /** Called when the renderer's position is reset. */ @@ -162,10 +195,10 @@ public final class VideoFrameReleaseHelper { * * @param playbackSpeed The player's speed. */ - public void onPlaybackSpeed(double playbackSpeed) { + public void onPlaybackSpeed(float playbackSpeed) { this.playbackSpeed = playbackSpeed; resetAdjustment(); - updateSurfaceFrameRate(/* isNewSurface= */ false); + updateSurfacePlaybackFrameRate(/* isNewSurface= */ false); } /** @@ -175,8 +208,8 @@ public final class VideoFrameReleaseHelper { */ public void onFormatChanged(float formatFrameRate) { this.formatFrameRate = formatFrameRate; - fixedFrameRateEstimator.onFormatChanged(formatFrameRate); - updateSurfaceFrameRate(/* isNewSurface= */ false); + frameRateEstimator.reset(); + updateSurfaceMediaFrameRate(); } /** @@ -189,8 +222,9 @@ public final class VideoFrameReleaseHelper { lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex; lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs; } - fixedFrameRateEstimator.onNextFrame(framePresentationTimeUs * 1000); frameIndex++; + frameRateEstimator.onNextFrame(framePresentationTimeUs * 1000); + updateSurfaceMediaFrameRate(); } /** Called when the renderer is stopped. */ @@ -230,8 +264,8 @@ public final class VideoFrameReleaseHelper { // Until we know better, the adjustment will be a no-op. long adjustedReleaseTimeNs = releaseTimeNs; - if (lastAdjustedFrameIndex != C.INDEX_UNSET && fixedFrameRateEstimator.isSynced()) { - long frameDurationNs = fixedFrameRateEstimator.getFrameDurationNs(); + if (lastAdjustedFrameIndex != C.INDEX_UNSET && frameRateEstimator.isSynced()) { + long frameDurationNs = frameRateEstimator.getFrameDurationNs(); long candidateAdjustedReleaseTimeNs = lastAdjustedReleaseTimeNs + (long) ((frameDurationNs * (frameIndex - lastAdjustedFrameIndex)) / playbackSpeed); @@ -271,36 +305,78 @@ public final class VideoFrameReleaseHelper { // Surface frame rate adjustment. /** - * Updates the frame-rate of the current {@link #surface} based on the renderer operating rate, - * frame-rate of the content, and whether the renderer is started. - * - * @param isNewSurface Whether the current {@link #surface} is new. + * Updates the media frame rate that's used to calculate the playback frame rate of the current + * {@link #surface}. If the frame rate is updated then {@link #updateSurfacePlaybackFrameRate} is + * called to update the surface. */ - private void updateSurfaceFrameRate(boolean isNewSurface) { + private void updateSurfaceMediaFrameRate() { if (Util.SDK_INT < 30 || surface == null) { return; } - float surfaceFrameRate = 0; - // TODO: Hook up fixedFrameRateEstimator. - if (started && formatFrameRate != Format.NO_VALUE) { - surfaceFrameRate = (float) (formatFrameRate * playbackSpeed); + float candidateFrameRate = + frameRateEstimator.isSynced() ? frameRateEstimator.getFrameRate() : formatFrameRate; + if (candidateFrameRate == surfaceMediaFrameRate) { + return; + } + + // The candidate is different to the current surface media frame rate. Decide whether to update + // the surface media frame rate. + boolean shouldUpdate; + if (candidateFrameRate != Format.NO_VALUE && surfaceMediaFrameRate != Format.NO_VALUE) { + boolean candidateIsHighConfidence = + frameRateEstimator.isSynced() + && frameRateEstimator.getMatchingFrameDurationSumNs() + >= MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS; + float minimumChangeForUpdate = + candidateIsHighConfidence + ? MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE + : MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE; + shouldUpdate = Math.abs(candidateFrameRate - surfaceMediaFrameRate) >= minimumChangeForUpdate; + } else if (candidateFrameRate != Format.NO_VALUE) { + shouldUpdate = true; + } else { + shouldUpdate = + frameRateEstimator.getFramesWithoutSyncCount() + >= MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE; + } + + if (shouldUpdate) { + surfaceMediaFrameRate = candidateFrameRate; + updateSurfacePlaybackFrameRate(/* isNewSurface= */ false); + } + } + + /** + * Updates the playback frame rate of the current {@link #surface} based on the playback speed, + * frame rate of the content, and whether the renderer is started. + * + * @param isNewSurface Whether the current {@link #surface} is new. + */ + private void updateSurfacePlaybackFrameRate(boolean isNewSurface) { + if (Util.SDK_INT < 30 || surface == null) { + return; + } + + float surfacePlaybackFrameRate = 0; + if (started && surfaceMediaFrameRate != Format.NO_VALUE) { + surfacePlaybackFrameRate = surfaceMediaFrameRate * playbackSpeed; } // We always set the frame-rate if we have a new surface, since we have no way of knowing what // it might have been set to previously. - if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) { + if (!isNewSurface && this.surfacePlaybackFrameRate == surfacePlaybackFrameRate) { return; } - this.surfaceFrameRate = surfaceFrameRate; - setSurfaceFrameRateV30(surface, surfaceFrameRate); + this.surfacePlaybackFrameRate = surfacePlaybackFrameRate; + setSurfaceFrameRateV30(surface, surfacePlaybackFrameRate); } /** Clears the frame-rate of the current {@link #surface}. */ private void clearSurfaceFrameRate() { - if (Util.SDK_INT < 30 || surface == null || surfaceFrameRate == 0) { + if (Util.SDK_INT < 30 || surface == null || surfacePlaybackFrameRate == 0) { return; } - surfaceFrameRate = 0; + surfacePlaybackFrameRate = 0; setSurfaceFrameRateV30(surface, /* frameRate= */ 0); } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/video/FixedFrameRateEstimatorTest.java b/library/core/src/test/java/com/google/android/exoplayer2/video/FixedFrameRateEstimatorTest.java index cae24a8375..dbe2c6900e 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/video/FixedFrameRateEstimatorTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/video/FixedFrameRateEstimatorTest.java @@ -202,93 +202,6 @@ public final class FixedFrameRateEstimatorTest { } } - @Test - public void newFixedFrameRate_withFormatFrameRateChange_resyncs() { - long frameDurationNs = 33_333_333; - float frameRate = (float) C.NANOS_PER_SECOND / frameDurationNs; - FixedFrameRateEstimator estimator = new FixedFrameRateEstimator(); - estimator.onFormatChanged(frameRate); - - long framePresentationTimestampNs = 0; - estimator.onNextFrame(framePresentationTimestampNs); - for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) { - framePresentationTimestampNs += frameDurationNs; - estimator.onNextFrame(framePresentationTimestampNs); - } - - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs); - - // Frames durations are halved from this point. - long halfFrameDuration = frameDurationNs * 2; - float doubleFrameRate = (float) C.NANOS_PER_SECOND / halfFrameDuration; - estimator.onFormatChanged(doubleFrameRate); - - // Format frame rate change should cause immediate sync loss. - assertThat(estimator.isSynced()).isFalse(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(C.TIME_UNSET); - - // Frames with consistent durations, working toward establishing new sync. - for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) { - framePresentationTimestampNs += halfFrameDuration; - estimator.onNextFrame(framePresentationTimestampNs); - - assertThat(estimator.isSynced()).isFalse(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(C.TIME_UNSET); - } - - // This frame should establish sync. - framePresentationTimestampNs += halfFrameDuration; - estimator.onNextFrame(framePresentationTimestampNs); - - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(halfFrameDuration); - } - - @Test - public void smallFrameRateChange_withoutFormatFrameRateChange_keepsSyncAndAdjustsEstimate() { - long frameDurationNs = 33_333_333; // 30 fps - float roundedFrameRate = 30; - FixedFrameRateEstimator estimator = new FixedFrameRateEstimator(); - estimator.onFormatChanged(roundedFrameRate); - - long framePresentationTimestampNs = 0; - estimator.onNextFrame(framePresentationTimestampNs); - for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) { - framePresentationTimestampNs += frameDurationNs; - estimator.onNextFrame(framePresentationTimestampNs); - } - - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs); - - long newFrameDurationNs = 33_366_667; // 30 * (1000/1001) = 29.97 fps - estimator.onFormatChanged(roundedFrameRate); // Format frame rate is unchanged. - - // Previous estimate should remain valid for now because neither format specified a duration. - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs); - - // The estimate should start moving toward the new frame duration. If should not lose sync - // because the change in frame rate is very small. - for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC - 1; i++) { - framePresentationTimestampNs += newFrameDurationNs; - estimator.onNextFrame(framePresentationTimestampNs); - - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isGreaterThan(frameDurationNs); - assertThat(estimator.getFrameDurationNs()).isLessThan(newFrameDurationNs); - } - - framePresentationTimestampNs += newFrameDurationNs; - estimator.onNextFrame(framePresentationTimestampNs); - - // Frames with the previous frame duration should now be excluded from the estimate, so the - // estimate should become exact. - assertThat(estimator.isSynced()).isTrue(); - assertThat(estimator.getFrameDurationNs()).isEqualTo(newFrameDurationNs); - } - private static final long getNsWithMsPrecision(long presentationTimeNs) { return (presentationTimeNs / 1000000) * 1000000; }