sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-03 10:55:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Simplify first frame logic

Browse source 
We currently use 3 different booleans to track the state of the first
frame rendering, which implies that there are 8 distinct possible
overall states. However, this is actually a staged process and there
are only 3 different overall states in the current code. This means
it's clearer and easier to reason about if the variables are combined
to a single state value. Overall, this should be a complete no-op.

State mapping:
 - rFFAReset=false, rFFAEnable=false, mayRenderFFAEINS=false
   => FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED
 - rFFAReset=false and/or rFFAEnable=false, mayRenderFFAEINS=any
   => FIRST_FRAME_NOT_RENDERED
 - rFFAReset=true, rFFAEnable=true, mayRenderFFAEINS=any
   => FIRST_FRAME_RENDERED

PiperOrigin-RevId: 552857802
This commit is contained in:
tonihei 2023-08-01 18:05:50 +00:00 committed by Tianyi Feng
parent 11648e6c8e
commit 79e05ad049
3 changed files with 109 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
libraries/common/src/main/java/androidx/media3/common/C.java
View file

@ -1506,6 +1506,36 @@ public final class C {
*/
@UnstableApi public static final int FORMAT_UNSUPPORTED_TYPE = 0b000;
/**
* State of the first frame in a video renderer.
*
* <p>One of {@link #FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED}, {@link
* #FIRST_FRAME_NOT_RENDERED} or {@link #FIRST_FRAME_RENDERED}. The stages are ordered and
* comparable, i.e., a value implies that all stages with higher values are not reached yet.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@UnstableApi
@IntDef({
FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED,
FIRST_FRAME_NOT_RENDERED,
FIRST_FRAME_RENDERED
})
public @interface FirstFrameState {}
/**
* The first frame was not rendered yet, and is only allowed to be rendered if the renderer is
* started.
*/
@UnstableApi public static final int FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED = 0;
/** The first frame was not rendered after the last reset, output surface or stream change. */
@UnstableApi public static final int FIRST_FRAME_NOT_RENDERED = 1;
/** The first frame was rendered. */
@UnstableApi public static final int FIRST_FRAME_RENDERED = 2;
/**
* @deprecated Use {@link Util#usToMs(long)}.
*/

66
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java
View file

@ -21,6 +21,7 @@ import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_RE
import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO;
import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.os.Handler;
@ -136,9 +137,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
private @ReinitializationState int decoderReinitializationState;
private boolean decoderReceivedBuffers;
private boolean renderedFirstFrameAfterReset;
private boolean mayRenderFirstFrameAfterEnableIfNotStarted;
private boolean renderedFirstFrameAfterEnable;
private @C.FirstFrameState int firstFrameState;
private long initialPositionUs;
private long joiningDeadlineMs;
private boolean waitingForFirstSampleInFormat;
@ -181,6 +180,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
}
// BaseRenderer implementation.
@ -238,7 +238,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
public boolean isReady() {
if (inputFormat != null
&& (isSourceReady() || outputBuffer != null)
&& (renderedFirstFrameAfterReset || !hasOutput())) {
&& (firstFrameState == C.FIRST_FRAME_RENDERED || !hasOutput())) {
// Ready. If we were joining then we've now joined, so clear the joining deadline.
joiningDeadlineMs = C.TIME_UNSET;
return true;
@ -276,20 +276,24 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
throws ExoPlaybackException {
decoderCounters = new DecoderCounters();
eventDispatcher.enabled(decoderCounters);
mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream;
renderedFirstFrameAfterEnable = false;
firstFrameState =
mayRenderStartOfStream
? C.FIRST_FRAME_NOT_RENDERED
: C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
}
@Override
public void enableMayRenderStartOfStream() {
mayRenderFirstFrameAfterEnableIfNotStarted = true;
if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) {
firstFrameState = C.FIRST_FRAME_NOT_RENDERED;
}
}
@Override
protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
inputStreamEnded = false;
outputStreamEnded = false;
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
initialPositionUs = C.TIME_UNSET;
consecutiveDroppedFrameCount = 0;
if (decoder != null) {
@ -320,7 +324,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
protected void onDisabled() {
inputFormat = null;
clearReportedVideoSize();
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED);
try {
setSourceDrmSession(null);
releaseDecoder();
@ -854,20 +858,12 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
outputFormat = format;
}
long elapsedRealtimeNowUs = msToUs(SystemClock.elapsedRealtime());
long elapsedSinceLastRenderUs = elapsedRealtimeNowUs - lastRenderTimeUs;
boolean isStarted = getState() == STATE_STARTED;
boolean shouldRenderFirstFrame =
!renderedFirstFrameAfterEnable
? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted)
: !renderedFirstFrameAfterReset;
// TODO: We shouldn't force render while we are joining an ongoing playback.
if (shouldRenderFirstFrame
|| (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs))) {
if (shouldForceRender(earlyUs)) {
renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
return true;
}
boolean isStarted = getState() == STATE_STARTED;
if (!isStarted || positionUs == initialPositionUs) {
return false;
}
@ -889,6 +885,23 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
return false;
}
/** Returns whether a buffer or a processed frame should be force rendered. */
private boolean shouldForceRender(long earlyUs) {
// TODO: We shouldn't force render while we are joining an ongoing playback.
boolean isStarted = getState() == STATE_STARTED;
switch (firstFrameState) {
case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED:
return isStarted;
case C.FIRST_FRAME_NOT_RENDERED:
return true;
case C.FIRST_FRAME_RENDERED:
long elapsedSinceLastRenderUs = msToUs(SystemClock.elapsedRealtime()) - lastRenderTimeUs;
return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs);
default:
throw new IllegalStateException();
}
}
private boolean hasOutput() {
return outputMode != C.VIDEO_OUTPUT_MODE_NONE;
}
@ -897,7 +910,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
// We haven't rendered to the new output yet.
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
if (getState() == STATE_STARTED) {
setJoiningDeadlineMs();
}
@ -905,7 +918,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
private void onOutputRemoved() {
clearReportedVideoSize();
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
}
private void onOutputReset() {
@ -922,20 +935,19 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
: C.TIME_UNSET;
}
private void clearRenderedFirstFrame() {
renderedFirstFrameAfterReset = false;
private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) {
this.firstFrameState = min(this.firstFrameState, firstFrameState);
}
private void maybeNotifyRenderedFirstFrame() {
renderedFirstFrameAfterEnable = true;
if (!renderedFirstFrameAfterReset) {
renderedFirstFrameAfterReset = true;
if (firstFrameState != C.FIRST_FRAME_RENDERED) {
firstFrameState = C.FIRST_FRAME_RENDERED;
eventDispatcher.renderedFirstFrame(output);
}
}
private void maybeRenotifyRenderedFirstFrame() {
if (renderedFirstFrameAfterReset) {
if (firstFrameState == C.FIRST_FRAME_RENDERED) {
eventDispatcher.renderedFirstFrame(output);
}
}

67
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View file

@ -166,9 +166,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Nullable private PlaceholderSurface placeholderSurface;
private boolean haveReportedFirstFrameRenderedForCurrentSurface;
private @C.VideoScalingMode int scalingMode;
private boolean renderedFirstFrameAfterReset;
private boolean mayRenderFirstFrameAfterEnableIfNotStarted;
private boolean renderedFirstFrameAfterEnable;
private @C.FirstFrameState int firstFrameState;
private long initialPositionUs;
private long joiningDeadlineMs;
private long droppedFrameAccumulationStartTimeMs;
@ -412,6 +410,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
decodedVideoSize = VideoSize.UNKNOWN;
tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET;
firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
clearReportedVideoSize();
}
@ -601,13 +600,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
releaseCodec();
}
eventDispatcher.enabled(decoderCounters);
mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream;
renderedFirstFrameAfterEnable = false;
firstFrameState =
mayRenderStartOfStream
? C.FIRST_FRAME_NOT_RENDERED
: C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED;
}
@Override
public void enableMayRenderStartOfStream() {
mayRenderFirstFrameAfterEnableIfNotStarted = true;
if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) {
firstFrameState = C.FIRST_FRAME_NOT_RENDERED;
}
}
@Override
@ -616,7 +619,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
if (videoFrameProcessorManager.isEnabled()) {
videoFrameProcessorManager.flush();
}
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
frameReleaseHelper.onPositionReset();
lastBufferPresentationTimeUs = C.TIME_UNSET;
initialPositionUs = C.TIME_UNSET;
@ -641,7 +644,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
public boolean isReady() {
if (super.isReady()
&& (!videoFrameProcessorManager.isEnabled() || videoFrameProcessorManager.isReady())
&& (renderedFirstFrameAfterReset
&& (firstFrameState == C.FIRST_FRAME_RENDERED
|| (placeholderSurface != null && displaySurface == placeholderSurface)
|| getCodec() == null
|| tunneling)) {
@ -685,7 +688,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
protected void onDisabled() {
clearReportedVideoSize();
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED);
haveReportedFirstFrameRenderedForCurrentSurface = false;
tunnelingOnFrameRenderedListener = null;
try {
@ -801,7 +804,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
// We haven't rendered to the new display surface yet.
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
if (state == STATE_STARTED) {
// Set joining deadline to report MediaCodecVideoRenderer is ready.
setJoiningDeadlineMs();
@ -814,7 +817,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
} else {
// The display surface has been removed.
clearReportedVideoSize();
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
if (videoFrameProcessorManager.isEnabled()) {
videoFrameProcessorManager.clearOutputSurfaceInfo();
}
@ -1334,17 +1337,28 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
/** Returns whether a buffer or a processed frame should be force rendered. */
private boolean shouldForceRender(long positionUs, long earlyUs) {
if (joiningDeadlineMs != C.TIME_UNSET) {
// No force rendering during joining.
return false;
}
if (positionUs < getOutputStreamOffsetUs()) {
// No force rendering if we haven't reached the stream start position.
// TODO: b/160461756 - This is a bug because it compares against the offset and not the start
// position and also should only be applied when transitioning streams, not after every reset.
return false;
}
boolean isStarted = getState() == STATE_STARTED;
boolean shouldRenderFirstFrame =
!renderedFirstFrameAfterEnable
? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted)
: !renderedFirstFrameAfterReset;
long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs;
// Don't force output until we joined and the position reached the current stream.
return joiningDeadlineMs == C.TIME_UNSET
&& positionUs >= getOutputStreamOffsetUs()
&& (shouldRenderFirstFrame
|| (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs)));
switch (firstFrameState) {
case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED:
return isStarted;
case C.FIRST_FRAME_NOT_RENDERED:
return true;
case C.FIRST_FRAME_RENDERED:
long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs;
return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs);
default:
throw new IllegalStateException();
}
}
/**
@ -1417,7 +1431,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
protected void onProcessedStreamChange() {
super.onProcessedStreamChange();
clearRenderedFirstFrame();
lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED);
}
/**
@ -1701,8 +1715,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
: C.TIME_UNSET;
}
private void clearRenderedFirstFrame() {
renderedFirstFrameAfterReset = false;
private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) {
this.firstFrameState = min(this.firstFrameState, firstFrameState);
// The first frame notification is triggered by renderOutputBuffer or renderOutputBufferV21 for
// non-tunneled playback, onQueueInputBuffer for tunneled playback prior to API level 23, and
// OnFrameRenderedListenerV23.onFrameRenderedListener for tunneled playback on API level 23 and
@ -1717,9 +1731,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
/* package */ void maybeNotifyRenderedFirstFrame() {
renderedFirstFrameAfterEnable = true;
if (!renderedFirstFrameAfterReset) {
renderedFirstFrameAfterReset = true;
if (firstFrameState != C.FIRST_FRAME_RENDERED) {
firstFrameState = C.FIRST_FRAME_RENDERED;
eventDispatcher.renderedFirstFrame(displaySurface);
haveReportedFirstFrameRenderedForCurrentSurface = true;
}