From 913dcb3e1dafc1f9340e5592df099ee460cb5c26 Mon Sep 17 00:00:00 2001 From: tonihei Date: Tue, 4 Jun 2019 14:20:51 +0100 Subject: [PATCH] Display last frame when seeking to end of stream. We currently don't display the last frame because the seek time is behind the last frame's timestamps and it's thus marked as decodeOnly. This case can be detected by checking whether all data sent to the codec is marked as decodeOnly at the time we read the end of stream signal. If so, we can re-enable the last frame. This should work for almost all cases because the end-of-stream signal is read in the same feedInputBuffer loop as the last frame and we therefore haven't released the last frame buffer yet. Issue:#2568 PiperOrigin-RevId: 251425870 --- RELEASENOTES.md | 5 +++ .../audio/MediaCodecAudioRenderer.java | 5 ++- .../mediacodec/MediaCodecRenderer.java | 42 ++++++++++++++----- .../source/ProgressiveMediaPeriod.java | 2 +- .../video/MediaCodecVideoRenderer.java | 25 ++++++----- .../testutil/DebugRenderersFactory.java | 8 ++-- 6 files changed, 61 insertions(+), 26 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 246a968256..b20a278713 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,10 @@ # Release notes # +### 2.10.3 ### + +* Display last frame when seeking to end of stream + ([#2568](https://github.com/google/ExoPlayer/issues/2568)). + ### 2.10.2 ### * Add `ResolvingDataSource` for just-in-time resolution of `DataSpec`s diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index d43bd6cbf8..ace7ebbcc6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -695,7 +695,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, Format format) throws ExoPlaybackException { if (codecNeedsEosBufferTimestampWorkaround @@ -711,7 +712,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media return true; } - if (shouldSkip) { + if (isDecodeOnlyBuffer) { codec.releaseOutputBuffer(bufferIndex, false); decoderCounters.skippedOutputBufferCount++; audioSink.handleDiscontinuity(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 5f7f5d60b7..c3072a1590 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -328,14 +328,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private int inputIndex; private int outputIndex; private ByteBuffer outputBuffer; - private boolean shouldSkipOutputBuffer; + private boolean isDecodeOnlyOutputBuffer; + private boolean isLastOutputBuffer; private boolean codecReconfigured; @ReconfigurationState private int codecReconfigurationState; @DrainState private int codecDrainState; @DrainAction private int codecDrainAction; private boolean codecReceivedBuffers; private boolean codecReceivedEos; - + private long lastBufferInStreamPresentationTimeUs; + private long largestQueuedPresentationTimeUs; private boolean inputStreamEnded; private boolean outputStreamEnded; private boolean waitingForKeys; @@ -598,6 +600,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { waitingForKeys = false; codecHotswapDeadlineMs = C.TIME_UNSET; decodeOnlyPresentationTimestamps.clear(); + largestQueuedPresentationTimeUs = C.TIME_UNSET; + lastBufferInStreamPresentationTimeUs = C.TIME_UNSET; try { if (codec != null) { decoderCounters.decoderReleaseCount++; @@ -704,10 +708,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer { waitingForFirstSyncSample = true; codecNeedsAdaptationWorkaroundBuffer = false; shouldSkipAdaptationWorkaroundOutputBuffer = false; - shouldSkipOutputBuffer = false; + isDecodeOnlyOutputBuffer = false; + isLastOutputBuffer = false; waitingForKeys = false; decodeOnlyPresentationTimestamps.clear(); + largestQueuedPresentationTimeUs = C.TIME_UNSET; + lastBufferInStreamPresentationTimeUs = C.TIME_UNSET; codecDrainState = DRAIN_STATE_NONE; codecDrainAction = DRAIN_ACTION_NONE; // Reconfiguration data sent shortly before the flush may not have been processed by the @@ -881,7 +888,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { codecDrainAction = DRAIN_ACTION_NONE; codecNeedsAdaptationWorkaroundBuffer = false; shouldSkipAdaptationWorkaroundOutputBuffer = false; - shouldSkipOutputBuffer = false; + isDecodeOnlyOutputBuffer = false; + isLastOutputBuffer = false; waitingForFirstSyncSample = true; decoderCounters.decoderInitCount++; @@ -1016,6 +1024,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { result = readSource(formatHolder, buffer, false); } + if (hasReadStreamToEnd()) { + // Notify output queue of the last buffer's timestamp. + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; + } + if (result == C.RESULT_NOTHING_READ) { return false; } @@ -1088,6 +1101,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { formatQueue.add(presentationTimeUs, inputFormat); waitingForFirstSampleInFormat = false; } + largestQueuedPresentationTimeUs = + Math.max(largestQueuedPresentationTimeUs, presentationTimeUs); buffer.flip(); onQueueInputBuffer(buffer); @@ -1458,7 +1473,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer { outputBuffer.position(outputBufferInfo.offset); outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size); } - shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs); + isDecodeOnlyOutputBuffer = isDecodeOnlyBuffer(outputBufferInfo.presentationTimeUs); + isLastOutputBuffer = + lastBufferInStreamPresentationTimeUs == outputBufferInfo.presentationTimeUs; updateOutputFormatForTime(outputBufferInfo.presentationTimeUs); } @@ -1474,7 +1491,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { outputIndex, outputBufferInfo.flags, outputBufferInfo.presentationTimeUs, - shouldSkipOutputBuffer, + isDecodeOnlyOutputBuffer, + isLastOutputBuffer, outputFormat); } catch (IllegalStateException e) { processEndOfStream(); @@ -1494,7 +1512,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { outputIndex, outputBufferInfo.flags, outputBufferInfo.presentationTimeUs, - shouldSkipOutputBuffer, + isDecodeOnlyOutputBuffer, + isLastOutputBuffer, outputFormat); } @@ -1561,7 +1580,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer { * @param bufferIndex The index of the output buffer. * @param bufferFlags The flags attached to the output buffer. * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds. - * @param shouldSkip Whether the buffer should be skipped (i.e. not rendered). + * @param isDecodeOnlyBuffer Whether the buffer was marked with {@link C#BUFFER_FLAG_DECODE_ONLY} + * by the source. + * @param isLastBuffer Whether the buffer is the last sample of the current stream. * @param format The format associated with the buffer. * @return Whether the output buffer was fully processed (e.g. rendered or skipped). * @throws ExoPlaybackException If an error occurs processing the output buffer. @@ -1574,7 +1595,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, Format format) throws ExoPlaybackException; @@ -1654,7 +1676,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { codecDrainAction = DRAIN_ACTION_NONE; } - private boolean shouldSkipOutputBuffer(long presentationTimeUs) { + private boolean isDecodeOnlyBuffer(long presentationTimeUs) { // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would // box presentationTimeUs, creating a Long object that would need to be garbage collected. int size = decodeOnlyPresentationTimestamps.size(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java b/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java index d9f0008a7f..4dafa0ba76 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java @@ -733,7 +733,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; if (prepared) { SeekMap seekMap = getPreparedState().seekMap; Assertions.checkState(isPendingReset()); - if (durationUs != C.TIME_UNSET && pendingResetPositionUs >= durationUs) { + if (durationUs != C.TIME_UNSET && pendingResetPositionUs > durationUs) { loadingFinished = true; pendingResetPositionUs = C.TIME_UNSET; return; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index e75a3866b6..8d5b890c7f 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -679,7 +679,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, Format format) throws ExoPlaybackException { if (initialPositionUs == C.TIME_UNSET) { @@ -688,7 +689,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; - if (shouldSkip) { + if (isDecodeOnlyBuffer && !isLastBuffer) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } @@ -736,10 +737,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs); earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; - if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs) + if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer) && maybeDropBuffersToKeyframe(codec, bufferIndex, presentationTimeUs, positionUs)) { return false; - } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) { + } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastBuffer)) { dropOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } @@ -807,8 +808,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { /** * Returns the offset that should be subtracted from {@code bufferPresentationTimeUs} in {@link - * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, Format)} to - * get the playback position with respect to the media. + * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, boolean, + * Format)} to get the playback position with respect to the media. */ protected long getOutputStreamOffsetUs() { return outputStreamOffsetUs; @@ -860,9 +861,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { * indicates that the buffer is late. * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, * measured at the start of the current iteration of the rendering loop. + * @param isLastBuffer Whether the buffer is the last buffer in the current stream. */ - protected boolean shouldDropOutputBuffer(long earlyUs, long elapsedRealtimeUs) { - return isBufferLate(earlyUs); + protected boolean shouldDropOutputBuffer( + long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + return isBufferLate(earlyUs) && !isLastBuffer; } /** @@ -873,9 +876,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { * negative value indicates that the buffer is late. * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, * measured at the start of the current iteration of the rendering loop. + * @param isLastBuffer Whether the buffer is the last buffer in the current stream. */ - protected boolean shouldDropBuffersToKeyframe(long earlyUs, long elapsedRealtimeUs) { - return isBufferVeryLate(earlyUs); + protected boolean shouldDropBuffersToKeyframe( + long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + return isBufferVeryLate(earlyUs) && !isLastBuffer; } /** diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java index 9feaf6863a..8b11a89d8d 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java @@ -163,14 +163,15 @@ public class DebugRenderersFactory extends DefaultRenderersFactory { int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, Format format) throws ExoPlaybackException { if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) { // After the codec has been initialized, don't render the first frame until we've caught up // to the playback position. Else test runs on devices that do not support dummy surface // will drop frames between rendering the first one and catching up [Internal: b/66494991]. - shouldSkip = true; + isDecodeOnlyBuffer = true; } return super.processOutputBuffer( positionUs, @@ -180,7 +181,8 @@ public class DebugRenderersFactory extends DefaultRenderersFactory { bufferIndex, bufferFlags, bufferPresentationTimeUs, - shouldSkip, + isDecodeOnlyBuffer, + isLastBuffer, format); }