sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-26 09:35:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update surface frame-rate based on fixed frame-rate estimate

Browse source 
PiperOrigin-RevId: 350442843
This commit is contained in:
olly 2021-01-06 23:47:00 +00:00 committed by Ian Baker
parent 25ed6b127d
commit 855db95fb3
3 changed files with 128 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java
View file

@ -25,10 +25,8 @@ import java.util.Arrays;
*/
/* package */ final class FixedFrameRateEstimator {
/**
* The number of consecutive matching frame durations for the tracker to be considered in sync.
*/
@VisibleForTesting static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 10;
/** The number of consecutive matching frame durations required to detect a fixed frame rate. */
public static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 15;
/**
* The maximum amount frame durations can differ for them to be considered matching, in
* nanoseconds.
@ -44,13 +42,12 @@ import java.util.Arrays;
private Matcher candidateMatcher;
private boolean candidateMatcherActive;
private boolean switchToCandidateMatcherWhenSynced;
private float formatFrameRate;
private long lastFramePresentationTimeNs;
private int framesWithoutSyncCount;
public FixedFrameRateEstimator() {
currentMatcher = new Matcher();
candidateMatcher = new Matcher();
formatFrameRate = Format.NO_VALUE;
lastFramePresentationTimeNs = C.TIME_UNSET;
}
@ -59,29 +56,8 @@ import java.util.Arrays;
currentMatcher.reset();
candidateMatcher.reset();
candidateMatcherActive = false;
formatFrameRate = Format.NO_VALUE;
lastFramePresentationTimeNs = C.TIME_UNSET;
}
/**
* Called when the renderer's output format changes.
*
* @param formatFrameRate The format's frame rate, or {@link Format#NO_VALUE} if unknown.
*/
public void onFormatChanged(float formatFrameRate) {
// The format frame rate is only used to determine to what extent the estimator should be reset.
// Frame rate estimates are always calculated directly from frame presentation timestamps.
if (this.formatFrameRate != formatFrameRate) {
reset();
} else {
// Keep the current matcher, but prefer to switch to a new matcher once synced even if the
// current one does not lose sync. This avoids an issue where the current matcher would
// continue to be used if a frame rate change has occurred that's too small to trigger sync
// loss (e.g., a change from 30fps to 29.97fps) and which is not represented in the format
// frame rates (e.g., because they're unset or only have integer precision).
switchToCandidateMatcherWhenSynced = true;
}
this.formatFrameRate = formatFrameRate;
framesWithoutSyncCount = 0;
}
/**
@ -113,6 +89,7 @@ import java.util.Arrays;
switchToCandidateMatcherWhenSynced = false;
}
lastFramePresentationTimeNs = framePresentationTimeNs;
framesWithoutSyncCount = currentMatcher.isSynced() ? 0 : framesWithoutSyncCount + 1;
}
/** Returns whether the estimator has detected a fixed frame rate. */
@ -120,6 +97,19 @@ import java.util.Arrays;
return currentMatcher.isSynced();
}
/** Returns the number of frames since the estimator last detected a fixed frame rate. */
public int getFramesWithoutSyncCount() {
return framesWithoutSyncCount;
}
/**
* Returns the sum of all frame durations used to calculate the current fixed frame rate estimate,
* or {@link C#TIME_UNSET} if {@link #isSynced()} is {@code false}.
*/
public long getMatchingFrameDurationSumNs() {
return isSynced() ? currentMatcher.getMatchingFrameDurationSumNs() : C.TIME_UNSET;
}
/**
* The currently detected fixed frame duration estimate in nanoseconds, or {@link C#TIME_UNSET} if
* {@link #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
@ -134,9 +124,9 @@ import java.util.Arrays;
* #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
* #onNextFrame} is called with a new frame presentation timestamp.
*/
public double getFrameRate() {
public float getFrameRate() {
return isSynced()
? (double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs()
? (float) ((double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs())
: Format.NO_VALUE;
}
@ -184,6 +174,10 @@ import java.util.Arrays;
return recentFrameOutlierFlags[getRecentFrameOutlierIndex(frameCount - 1)];
}
public long getMatchingFrameDurationSumNs() {
return matchingFrameDurationSumNs;
}
public long getFrameDurationNs() {
return matchingFrameCount == 0 ? 0 : (matchingFrameDurationSumNs / matchingFrameCount);
}

132
library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java
View file

@ -51,6 +51,31 @@ public final class VideoFrameReleaseHelper {
private static final String TAG = "VideoFrameReleaseHelper";
/**
* The minimum sum of frame durations used to calculate the current fixed frame rate estimate, for
* the estimate to be treated as a high confidence estimate.
*/
private static final long MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS = 5_000_000_000L;
/**
* The minimum change in media frame rate that will trigger a change in surface frame rate, given
* a high confidence estimate.
*/
private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE = 0.02f;
/**
* The minimum change in media frame rate that will trigger a change in surface frame rate, given
* a low confidence estimate.
*/
private static final float MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE = 1f;
/**
* The minimum number of frames without a frame rate estimate, for the surface frame rate to be
* cleared.
*/
private static final int MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE =
2 * FixedFrameRateEstimator.CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC;
/** The period between sampling display VSYNC timestamps, in milliseconds. */
private static final long VSYNC_SAMPLE_UPDATE_PERIOD_MS = 500;
/**
@ -65,7 +90,7 @@ public final class VideoFrameReleaseHelper {
*/
private static final long VSYNC_OFFSET_PERCENTAGE = 80;
private final FixedFrameRateEstimator fixedFrameRateEstimator;
private final FixedFrameRateEstimator frameRateEstimator;
@Nullable private final WindowManager windowManager;
@Nullable private final VSyncSampler vsyncSampler;
@Nullable private final DefaultDisplayListener displayListener;
@ -73,9 +98,18 @@ public final class VideoFrameReleaseHelper {
private boolean started;
@Nullable private Surface surface;
private float surfaceFrameRate;
/** The media frame rate specified in the {@link Format}. */
private float formatFrameRate;
private double playbackSpeed;
/**
* The media frame rate used to calculate the playback frame rate of the {@link Surface}. This may
* be different to {@link #formatFrameRate} if {@link #formatFrameRate} is unspecified or
* inaccurate.
*/
private float surfaceMediaFrameRate;
/** The playback frame rate set on the {@link Surface}. */
private float surfacePlaybackFrameRate;
private float playbackSpeed;
private long vsyncDurationNs;
private long vsyncOffsetNs;
@ -92,7 +126,7 @@ public final class VideoFrameReleaseHelper {
* @param context A context from which information about the default display can be retrieved.
*/
public VideoFrameReleaseHelper(@Nullable Context context) {
fixedFrameRateEstimator = new FixedFrameRateEstimator();
frameRateEstimator = new FixedFrameRateEstimator();
if (context != null) {
context = context.getApplicationContext();
windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
@ -116,7 +150,6 @@ public final class VideoFrameReleaseHelper {
/** Called when the renderer is enabled. */
@TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
public void onEnabled() {
fixedFrameRateEstimator.reset();
if (windowManager != null) {
checkNotNull(vsyncSampler).addObserver();
if (displayListener != null) {
@ -130,7 +163,7 @@ public final class VideoFrameReleaseHelper {
public void onStarted() {
started = true;
resetAdjustment();
updateSurfaceFrameRate(/* isNewSurface= */ false);
updateSurfacePlaybackFrameRate(/* isNewSurface= */ false);
}
/**
@ -148,7 +181,7 @@ public final class VideoFrameReleaseHelper {
}
clearSurfaceFrameRate();
this.surface = surface;
updateSurfaceFrameRate(/* isNewSurface= */ true);
updateSurfacePlaybackFrameRate(/* isNewSurface= */ true);
}
/** Called when the renderer's position is reset. */
@ -162,10 +195,10 @@ public final class VideoFrameReleaseHelper {
*
* @param playbackSpeed The player's speed.
*/
public void onPlaybackSpeed(double playbackSpeed) {
public void onPlaybackSpeed(float playbackSpeed) {
this.playbackSpeed = playbackSpeed;
resetAdjustment();
updateSurfaceFrameRate(/* isNewSurface= */ false);
updateSurfacePlaybackFrameRate(/* isNewSurface= */ false);
}
/**
@ -175,8 +208,8 @@ public final class VideoFrameReleaseHelper {
*/
public void onFormatChanged(float formatFrameRate) {
this.formatFrameRate = formatFrameRate;
fixedFrameRateEstimator.onFormatChanged(formatFrameRate);
updateSurfaceFrameRate(/* isNewSurface= */ false);
frameRateEstimator.reset();
updateSurfaceMediaFrameRate();
}
/**
@ -189,8 +222,9 @@ public final class VideoFrameReleaseHelper {
lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex;
lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs;
}
fixedFrameRateEstimator.onNextFrame(framePresentationTimeUs * 1000);
frameIndex++;
frameRateEstimator.onNextFrame(framePresentationTimeUs * 1000);
updateSurfaceMediaFrameRate();
}
/** Called when the renderer is stopped. */
@ -230,8 +264,8 @@ public final class VideoFrameReleaseHelper {
// Until we know better, the adjustment will be a no-op.
long adjustedReleaseTimeNs = releaseTimeNs;
if (lastAdjustedFrameIndex != C.INDEX_UNSET && fixedFrameRateEstimator.isSynced()) {
long frameDurationNs = fixedFrameRateEstimator.getFrameDurationNs();
if (lastAdjustedFrameIndex != C.INDEX_UNSET && frameRateEstimator.isSynced()) {
long frameDurationNs = frameRateEstimator.getFrameDurationNs();
long candidateAdjustedReleaseTimeNs =
lastAdjustedReleaseTimeNs
+ (long) ((frameDurationNs * (frameIndex - lastAdjustedFrameIndex)) / playbackSpeed);
@ -271,36 +305,78 @@ public final class VideoFrameReleaseHelper {
// Surface frame rate adjustment.
/**
* Updates the frame-rate of the current {@link #surface} based on the renderer operating rate,
* frame-rate of the content, and whether the renderer is started.
*
* @param isNewSurface Whether the current {@link #surface} is new.
* Updates the media frame rate that's used to calculate the playback frame rate of the current
* {@link #surface}. If the frame rate is updated then {@link #updateSurfacePlaybackFrameRate} is
* called to update the surface.
*/
private void updateSurfaceFrameRate(boolean isNewSurface) {
private void updateSurfaceMediaFrameRate() {
if (Util.SDK_INT < 30 || surface == null) {
return;
}
float surfaceFrameRate = 0;
// TODO: Hook up fixedFrameRateEstimator.
if (started && formatFrameRate != Format.NO_VALUE) {
surfaceFrameRate = (float) (formatFrameRate * playbackSpeed);
float candidateFrameRate =
frameRateEstimator.isSynced() ? frameRateEstimator.getFrameRate() : formatFrameRate;
if (candidateFrameRate == surfaceMediaFrameRate) {
return;
}
// The candidate is different to the current surface media frame rate. Decide whether to update
// the surface media frame rate.
boolean shouldUpdate;
if (candidateFrameRate != Format.NO_VALUE && surfaceMediaFrameRate != Format.NO_VALUE) {
boolean candidateIsHighConfidence =
frameRateEstimator.isSynced()
&& frameRateEstimator.getMatchingFrameDurationSumNs()
>= MINIMUM_MATCHING_FRAME_DURATION_FOR_HIGH_CONFIDENCE_NS;
float minimumChangeForUpdate =
candidateIsHighConfidence
? MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_HIGH_CONFIDENCE
: MINIMUM_MEDIA_FRAME_RATE_CHANGE_FOR_UPDATE_LOW_CONFIDENCE;
shouldUpdate = Math.abs(candidateFrameRate - surfaceMediaFrameRate) >= minimumChangeForUpdate;
} else if (candidateFrameRate != Format.NO_VALUE) {
shouldUpdate = true;
} else {
shouldUpdate =
frameRateEstimator.getFramesWithoutSyncCount()
>= MINIMUM_FRAMES_WITHOUT_SYNC_TO_CLEAR_SURFACE_FRAME_RATE;
}
if (shouldUpdate) {
surfaceMediaFrameRate = candidateFrameRate;
updateSurfacePlaybackFrameRate(/* isNewSurface= */ false);
}
}
/**
* Updates the playback frame rate of the current {@link #surface} based on the playback speed,
* frame rate of the content, and whether the renderer is started.
*
* @param isNewSurface Whether the current {@link #surface} is new.
*/
private void updateSurfacePlaybackFrameRate(boolean isNewSurface) {
if (Util.SDK_INT < 30 || surface == null) {
return;
}
float surfacePlaybackFrameRate = 0;
if (started && surfaceMediaFrameRate != Format.NO_VALUE) {
surfacePlaybackFrameRate = surfaceMediaFrameRate * playbackSpeed;
}
// We always set the frame-rate if we have a new surface, since we have no way of knowing what
// it might have been set to previously.
if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) {
if (!isNewSurface && this.surfacePlaybackFrameRate == surfacePlaybackFrameRate) {
return;
}
this.surfaceFrameRate = surfaceFrameRate;
setSurfaceFrameRateV30(surface, surfaceFrameRate);
this.surfacePlaybackFrameRate = surfacePlaybackFrameRate;
setSurfaceFrameRateV30(surface, surfacePlaybackFrameRate);
}
/** Clears the frame-rate of the current {@link #surface}. */
private void clearSurfaceFrameRate() {
if (Util.SDK_INT < 30 || surface == null || surfaceFrameRate == 0) {
if (Util.SDK_INT < 30 || surface == null || surfacePlaybackFrameRate == 0) {
return;
}
surfaceFrameRate = 0;
surfacePlaybackFrameRate = 0;
setSurfaceFrameRateV30(surface, /* frameRate= */ 0);
}

87
library/core/src/test/java/com/google/android/exoplayer2/video/FixedFrameRateEstimatorTest.java
View file

@ -202,93 +202,6 @@ public final class FixedFrameRateEstimatorTest {
}
}
@Test
public void newFixedFrameRate_withFormatFrameRateChange_resyncs() {
long frameDurationNs = 33_333_333;
float frameRate = (float) C.NANOS_PER_SECOND / frameDurationNs;
FixedFrameRateEstimator estimator = new FixedFrameRateEstimator();
estimator.onFormatChanged(frameRate);
long framePresentationTimestampNs = 0;
estimator.onNextFrame(framePresentationTimestampNs);
for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) {
framePresentationTimestampNs += frameDurationNs;
estimator.onNextFrame(framePresentationTimestampNs);
}
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs);
// Frames durations are halved from this point.
long halfFrameDuration = frameDurationNs * 2;
float doubleFrameRate = (float) C.NANOS_PER_SECOND / halfFrameDuration;
estimator.onFormatChanged(doubleFrameRate);
// Format frame rate change should cause immediate sync loss.
assertThat(estimator.isSynced()).isFalse();
assertThat(estimator.getFrameDurationNs()).isEqualTo(C.TIME_UNSET);
// Frames with consistent durations, working toward establishing new sync.
for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) {
framePresentationTimestampNs += halfFrameDuration;
estimator.onNextFrame(framePresentationTimestampNs);
assertThat(estimator.isSynced()).isFalse();
assertThat(estimator.getFrameDurationNs()).isEqualTo(C.TIME_UNSET);
}
// This frame should establish sync.
framePresentationTimestampNs += halfFrameDuration;
estimator.onNextFrame(framePresentationTimestampNs);
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isEqualTo(halfFrameDuration);
}
@Test
public void smallFrameRateChange_withoutFormatFrameRateChange_keepsSyncAndAdjustsEstimate() {
long frameDurationNs = 33_333_333; // 30 fps
float roundedFrameRate = 30;
FixedFrameRateEstimator estimator = new FixedFrameRateEstimator();
estimator.onFormatChanged(roundedFrameRate);
long framePresentationTimestampNs = 0;
estimator.onNextFrame(framePresentationTimestampNs);
for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC; i++) {
framePresentationTimestampNs += frameDurationNs;
estimator.onNextFrame(framePresentationTimestampNs);
}
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs);
long newFrameDurationNs = 33_366_667; // 30 * (1000/1001) = 29.97 fps
estimator.onFormatChanged(roundedFrameRate); // Format frame rate is unchanged.
// Previous estimate should remain valid for now because neither format specified a duration.
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isEqualTo(frameDurationNs);
// The estimate should start moving toward the new frame duration. If should not lose sync
// because the change in frame rate is very small.
for (int i = 0; i < CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC - 1; i++) {
framePresentationTimestampNs += newFrameDurationNs;
estimator.onNextFrame(framePresentationTimestampNs);
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isGreaterThan(frameDurationNs);
assertThat(estimator.getFrameDurationNs()).isLessThan(newFrameDurationNs);
}
framePresentationTimestampNs += newFrameDurationNs;
estimator.onNextFrame(framePresentationTimestampNs);
// Frames with the previous frame duration should now be excluded from the estimate, so the
// estimate should become exact.
assertThat(estimator.isSynced()).isTrue();
assertThat(estimator.getFrameDurationNs()).isEqualTo(newFrameDurationNs);
}
private static final long getNsWithMsPrecision(long presentationTimeNs) {
return (presentationTimeNs / 1000000) * 1000000;
}