diff --git a/libraries/common/src/main/java/androidx/media3/common/C.java b/libraries/common/src/main/java/androidx/media3/common/C.java index f4c3b54d4e..061bc40f2b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/C.java +++ b/libraries/common/src/main/java/androidx/media3/common/C.java @@ -1506,6 +1506,36 @@ public final class C { */ @UnstableApi public static final int FORMAT_UNSUPPORTED_TYPE = 0b000; + /** + * State of the first frame in a video renderer. + * + *

One of {@link #FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED}, {@link + * #FIRST_FRAME_NOT_RENDERED} or {@link #FIRST_FRAME_RENDERED}. The stages are ordered and + * comparable, i.e., a value implies that all stages with higher values are not reached yet. + */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @UnstableApi + @IntDef({ + FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED, + FIRST_FRAME_NOT_RENDERED, + FIRST_FRAME_RENDERED + }) + public @interface FirstFrameState {} + + /** + * The first frame was not rendered yet, and is only allowed to be rendered if the renderer is + * started. + */ + @UnstableApi public static final int FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED = 0; + + /** The first frame was not rendered after the last reset, output surface or stream change. */ + @UnstableApi public static final int FIRST_FRAME_NOT_RENDERED = 1; + + /** The first frame was rendered. */ + @UnstableApi public static final int FIRST_FRAME_RENDERED = 2; + /** * @deprecated Use {@link Util#usToMs(long)}. */ diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java index 330d35a1ec..a6997b8788 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java @@ -21,6 +21,7 @@ import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_RE import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; import static java.lang.Math.max; +import static java.lang.Math.min; import static java.lang.annotation.ElementType.TYPE_USE; import android.os.Handler; @@ -136,9 +137,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { private @ReinitializationState int decoderReinitializationState; private boolean decoderReceivedBuffers; - private boolean renderedFirstFrameAfterReset; - private boolean mayRenderFirstFrameAfterEnableIfNotStarted; - private boolean renderedFirstFrameAfterEnable; + private @C.FirstFrameState int firstFrameState; private long initialPositionUs; private long joiningDeadlineMs; private boolean waitingForFirstSampleInFormat; @@ -181,6 +180,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { eventDispatcher = new EventDispatcher(eventHandler, eventListener); decoderReinitializationState = REINITIALIZATION_STATE_NONE; outputMode = C.VIDEO_OUTPUT_MODE_NONE; + firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; } // BaseRenderer implementation. @@ -238,7 +238,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { public boolean isReady() { if (inputFormat != null && (isSourceReady() || outputBuffer != null) - && (renderedFirstFrameAfterReset || !hasOutput())) { + && (firstFrameState == C.FIRST_FRAME_RENDERED || !hasOutput())) { // Ready. If we were joining then we've now joined, so clear the joining deadline. joiningDeadlineMs = C.TIME_UNSET; return true; @@ -276,20 +276,24 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { throws ExoPlaybackException { decoderCounters = new DecoderCounters(); eventDispatcher.enabled(decoderCounters); - mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream; - renderedFirstFrameAfterEnable = false; + firstFrameState = + mayRenderStartOfStream + ? C.FIRST_FRAME_NOT_RENDERED + : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; } @Override public void enableMayRenderStartOfStream() { - mayRenderFirstFrameAfterEnableIfNotStarted = true; + if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { + firstFrameState = C.FIRST_FRAME_NOT_RENDERED; + } } @Override protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { inputStreamEnded = false; outputStreamEnded = false; - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); initialPositionUs = C.TIME_UNSET; consecutiveDroppedFrameCount = 0; if (decoder != null) { @@ -320,7 +324,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { protected void onDisabled() { inputFormat = null; clearReportedVideoSize(); - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); try { setSourceDrmSession(null); releaseDecoder(); @@ -854,20 +858,12 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { outputFormat = format; } - long elapsedRealtimeNowUs = msToUs(SystemClock.elapsedRealtime()); - long elapsedSinceLastRenderUs = elapsedRealtimeNowUs - lastRenderTimeUs; - boolean isStarted = getState() == STATE_STARTED; - boolean shouldRenderFirstFrame = - !renderedFirstFrameAfterEnable - ? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted) - : !renderedFirstFrameAfterReset; - // TODO: We shouldn't force render while we are joining an ongoing playback. - if (shouldRenderFirstFrame - || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs))) { + if (shouldForceRender(earlyUs)) { renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat); return true; } + boolean isStarted = getState() == STATE_STARTED; if (!isStarted || positionUs == initialPositionUs) { return false; } @@ -889,6 +885,23 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { return false; } + /** Returns whether a buffer or a processed frame should be force rendered. */ + private boolean shouldForceRender(long earlyUs) { + // TODO: We shouldn't force render while we are joining an ongoing playback. + boolean isStarted = getState() == STATE_STARTED; + switch (firstFrameState) { + case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: + return isStarted; + case C.FIRST_FRAME_NOT_RENDERED: + return true; + case C.FIRST_FRAME_RENDERED: + long elapsedSinceLastRenderUs = msToUs(SystemClock.elapsedRealtime()) - lastRenderTimeUs; + return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs); + default: + throw new IllegalStateException(); + } + } + private boolean hasOutput() { return outputMode != C.VIDEO_OUTPUT_MODE_NONE; } @@ -897,7 +910,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { // If we know the video size, report it again immediately. maybeRenotifyVideoSizeChanged(); // We haven't rendered to the new output yet. - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (getState() == STATE_STARTED) { setJoiningDeadlineMs(); } @@ -905,7 +918,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { private void onOutputRemoved() { clearReportedVideoSize(); - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); } private void onOutputReset() { @@ -922,20 +935,19 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { : C.TIME_UNSET; } - private void clearRenderedFirstFrame() { - renderedFirstFrameAfterReset = false; + private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { + this.firstFrameState = min(this.firstFrameState, firstFrameState); } private void maybeNotifyRenderedFirstFrame() { - renderedFirstFrameAfterEnable = true; - if (!renderedFirstFrameAfterReset) { - renderedFirstFrameAfterReset = true; + if (firstFrameState != C.FIRST_FRAME_RENDERED) { + firstFrameState = C.FIRST_FRAME_RENDERED; eventDispatcher.renderedFirstFrame(output); } } private void maybeRenotifyRenderedFirstFrame() { - if (renderedFirstFrameAfterReset) { + if (firstFrameState == C.FIRST_FRAME_RENDERED) { eventDispatcher.renderedFirstFrame(output); } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index fd7d255ae6..09df4fca08 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -166,9 +166,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Nullable private PlaceholderSurface placeholderSurface; private boolean haveReportedFirstFrameRenderedForCurrentSurface; private @C.VideoScalingMode int scalingMode; - private boolean renderedFirstFrameAfterReset; - private boolean mayRenderFirstFrameAfterEnableIfNotStarted; - private boolean renderedFirstFrameAfterEnable; + private @C.FirstFrameState int firstFrameState; private long initialPositionUs; private long joiningDeadlineMs; private long droppedFrameAccumulationStartTimeMs; @@ -412,6 +410,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; decodedVideoSize = VideoSize.UNKNOWN; tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET; + firstFrameState = C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; clearReportedVideoSize(); } @@ -601,13 +600,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { releaseCodec(); } eventDispatcher.enabled(decoderCounters); - mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream; - renderedFirstFrameAfterEnable = false; + firstFrameState = + mayRenderStartOfStream + ? C.FIRST_FRAME_NOT_RENDERED + : C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED; } @Override public void enableMayRenderStartOfStream() { - mayRenderFirstFrameAfterEnableIfNotStarted = true; + if (firstFrameState == C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED) { + firstFrameState = C.FIRST_FRAME_NOT_RENDERED; + } } @Override @@ -616,7 +619,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (videoFrameProcessorManager.isEnabled()) { videoFrameProcessorManager.flush(); } - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); frameReleaseHelper.onPositionReset(); lastBufferPresentationTimeUs = C.TIME_UNSET; initialPositionUs = C.TIME_UNSET; @@ -641,7 +644,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { public boolean isReady() { if (super.isReady() && (!videoFrameProcessorManager.isEnabled() || videoFrameProcessorManager.isReady()) - && (renderedFirstFrameAfterReset + && (firstFrameState == C.FIRST_FRAME_RENDERED || (placeholderSurface != null && displaySurface == placeholderSurface) || getCodec() == null || tunneling)) { @@ -685,7 +688,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected void onDisabled() { clearReportedVideoSize(); - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED); haveReportedFirstFrameRenderedForCurrentSurface = false; tunnelingOnFrameRenderedListener = null; try { @@ -801,7 +804,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // If we know the video size, report it again immediately. maybeRenotifyVideoSizeChanged(); // We haven't rendered to the new display surface yet. - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (state == STATE_STARTED) { // Set joining deadline to report MediaCodecVideoRenderer is ready. setJoiningDeadlineMs(); @@ -814,7 +817,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } else { // The display surface has been removed. clearReportedVideoSize(); - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); if (videoFrameProcessorManager.isEnabled()) { videoFrameProcessorManager.clearOutputSurfaceInfo(); } @@ -1334,17 +1337,28 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { /** Returns whether a buffer or a processed frame should be force rendered. */ private boolean shouldForceRender(long positionUs, long earlyUs) { + if (joiningDeadlineMs != C.TIME_UNSET) { + // No force rendering during joining. + return false; + } + if (positionUs < getOutputStreamOffsetUs()) { + // No force rendering if we haven't reached the stream start position. + // TODO: b/160461756 - This is a bug because it compares against the offset and not the start + // position and also should only be applied when transitioning streams, not after every reset. + return false; + } boolean isStarted = getState() == STATE_STARTED; - boolean shouldRenderFirstFrame = - !renderedFirstFrameAfterEnable - ? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted) - : !renderedFirstFrameAfterReset; - long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs; - // Don't force output until we joined and the position reached the current stream. - return joiningDeadlineMs == C.TIME_UNSET - && positionUs >= getOutputStreamOffsetUs() - && (shouldRenderFirstFrame - || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs))); + switch (firstFrameState) { + case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: + return isStarted; + case C.FIRST_FRAME_NOT_RENDERED: + return true; + case C.FIRST_FRAME_RENDERED: + long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs; + return isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs); + default: + throw new IllegalStateException(); + } } /** @@ -1417,7 +1431,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected void onProcessedStreamChange() { super.onProcessedStreamChange(); - clearRenderedFirstFrame(); + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); } /** @@ -1701,8 +1715,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { : C.TIME_UNSET; } - private void clearRenderedFirstFrame() { - renderedFirstFrameAfterReset = false; + private void lowerFirstFrameState(@C.FirstFrameState int firstFrameState) { + this.firstFrameState = min(this.firstFrameState, firstFrameState); // The first frame notification is triggered by renderOutputBuffer or renderOutputBufferV21 for // non-tunneled playback, onQueueInputBuffer for tunneled playback prior to API level 23, and // OnFrameRenderedListenerV23.onFrameRenderedListener for tunneled playback on API level 23 and @@ -1717,9 +1731,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } /* package */ void maybeNotifyRenderedFirstFrame() { - renderedFirstFrameAfterEnable = true; - if (!renderedFirstFrameAfterReset) { - renderedFirstFrameAfterReset = true; + if (firstFrameState != C.FIRST_FRAME_RENDERED) { + firstFrameState = C.FIRST_FRAME_RENDERED; eventDispatcher.renderedFirstFrame(displaySurface); haveReportedFirstFrameRenderedForCurrentSurface = true; }