From b3d462df398578836139f14cd708230fdf8a9b04 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Wed, 4 Oct 2017 00:59:14 -0700 Subject: [PATCH] Catch up video rendering by dropping to keyframes If the current output buffer is very late and the playback position is in a later group of pictures, drop all buffers to the keyframe preceding the playback position. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=170975259 --- .../exoplayer2/decoder/DecoderCounters.java | 9 ++ .../video/MediaCodecVideoRenderer.java | 107 ++++++++++++++++-- .../exoplayer2/ui/DebugTextViewHelper.java | 3 +- .../testutil/DebugRenderersFactory.java | 2 +- 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java index 7a532110d3..e1dff12e52 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java @@ -65,6 +65,14 @@ public final class DecoderCounters { * Skipped output buffers are ignored for the purposes of calculating this value. */ public int maxConsecutiveDroppedOutputBufferCount; + /** + * The number of times all buffers to a keyframe were dropped. + *

+ * Each time buffers to a keyframe are dropped, this counter is increased by one, and the dropped + * output buffer counters are increased by one (for the current output buffer) plus the number of + * buffers dropped from the source to advance to the keyframe. + */ + public int droppedToKeyframeCount; /** * Should be called to ensure counter values are made visible across threads. The playback thread @@ -91,6 +99,7 @@ public final class DecoderCounters { droppedOutputBufferCount += other.droppedOutputBufferCount; maxConsecutiveDroppedOutputBufferCount = Math.max(maxConsecutiveDroppedOutputBufferCount, other.maxConsecutiveDroppedOutputBufferCount); + droppedToKeyframeCount += other.droppedToKeyframeCount; } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 4c1f4c0eb2..84073f9338 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -25,6 +25,7 @@ import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Handler; import android.os.SystemClock; +import android.support.annotation.CallSuper; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.util.Log; @@ -85,10 +86,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @C.VideoScalingMode private int scalingMode; private boolean renderedFirstFrame; + private boolean forceRenderFrame; private long joiningDeadlineMs; private long droppedFrameAccumulationStartTimeMs; private int droppedFrames; private int consecutiveDroppedFrameCount; + private int buffersInCodecCount; private int pendingRotationDegrees; private float pendingPixelWidthHeightRatio; @@ -414,11 +417,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } + @CallSuper @Override protected void releaseCodec() { try { super.releaseCodec(); } finally { + buffersInCodecCount = 0; + forceRenderFrame = false; if (dummySurface != null) { if (surface == dummySurface) { surface = null; @@ -429,6 +435,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } + @CallSuper + @Override + protected void flushCodec() throws ExoPlaybackException { + super.flushCodec(); + buffersInCodecCount = 0; + forceRenderFrame = false; + } + @Override protected void onCodecInitialized(String name, long initializedTimestampMs, long initializationDurationMs) { @@ -444,8 +458,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { pendingRotationDegrees = getRotationDegrees(newFormat); } + /** + * Called immediately before an input buffer is queued into the codec. + * + * @param buffer The buffer to be queued. + */ + @CallSuper @Override protected void onQueueInputBuffer(DecoderInputBuffer buffer) { + buffersInCodecCount++; if (Util.SDK_INT < 23 && tunneling) { maybeNotifyRenderedFirstFrame(); } @@ -492,7 +513,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip) { + boolean shouldSkip) throws ExoPlaybackException { while (pendingOutputStreamOffsetCount != 0 && bufferPresentationTimeUs >= pendingOutputStreamOffsetsUs[0]) { outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0]; @@ -517,7 +538,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } - if (!renderedFirstFrame) { + if (!renderedFirstFrame || forceRenderFrame) { + forceRenderFrame = false; if (Util.SDK_INT >= 21) { renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime()); } else { @@ -544,7 +566,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { bufferPresentationTimeUs, unadjustedFrameReleaseTimeNs); earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; - if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) { + if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs) + && maybeDropBuffersToKeyframe(codec, bufferIndex, presentationTimeUs, positionUs)) { + forceRenderFrame = true; + return true; + } else if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) { dropOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } @@ -577,6 +603,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } + /** + * Called when an output buffer is successfully processed. + * + * @param presentationTimeUs The timestamp associated with the output buffer. + */ + @CallSuper + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + buffersInCodecCount--; + } + /** * Returns whether the buffer being processed should be dropped. * @@ -589,6 +626,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return isBufferLate(earlyUs); } + /** + * Returns whether to drop all buffers from the buffer being processed to the keyframe at or after + * the current playback position, if possible. + * + * @param earlyUs The time until the current buffer should be presented in microseconds. A + * negative value indicates that the buffer is late. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * measured at the start of the current iteration of the rendering loop. + */ + protected boolean shouldDropBuffersToKeyframe(long earlyUs, long elapsedRealtimeUs) { + return isBufferVeryLate(earlyUs); + } + /** * Skips the output buffer with the specified index. * @@ -614,12 +664,48 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { TraceUtil.beginSection("dropVideoBuffer"); codec.releaseOutputBuffer(index, false); TraceUtil.endSection(); - decoderCounters.droppedOutputBufferCount++; - droppedFrames++; - consecutiveDroppedFrameCount++; + updateDroppedBufferCounters(1); + } + + /** + * Drops frames from the current output buffer to the next keyframe at or before the playback + * position. If no such keyframe exists, as the playback position is inside the same group of + * pictures as the buffer being processed, returns {@code false}. Returns {@code true} otherwise. + * + * @param codec The codec that owns the output buffer. + * @param index The index of the output buffer to drop. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + * @param positionUs The current playback position, in microseconds. + * @return Whether any buffers were dropped. + * @throws ExoPlaybackException If an error occurs flushing the codec. + */ + protected boolean maybeDropBuffersToKeyframe(MediaCodec codec, int index, long presentationTimeUs, + long positionUs) throws ExoPlaybackException { + int droppedSourceBufferCount = skipSource(positionUs); + if (droppedSourceBufferCount == 0) { + return false; + } + decoderCounters.droppedToKeyframeCount++; + // We dropped some buffers to catch up, so update the decoder counters and flush the codec, + // which releases all pending buffers buffers including the current output buffer. + updateDroppedBufferCounters(buffersInCodecCount + droppedSourceBufferCount); + flushCodec(); + return true; + } + + /** + * Updates decoder counters to reflect that {@code droppedBufferCount} additional buffers were + * dropped. + * + * @param droppedBufferCount The number of additional dropped buffers. + */ + protected void updateDroppedBufferCounters(int droppedBufferCount) { + decoderCounters.droppedOutputBufferCount += droppedBufferCount; + droppedFrames += droppedBufferCount; + consecutiveDroppedFrameCount += droppedBufferCount; decoderCounters.maxConsecutiveDroppedOutputBufferCount = Math.max(consecutiveDroppedFrameCount, decoderCounters.maxConsecutiveDroppedOutputBufferCount); - if (droppedFrames == maxDroppedFramesToNotify) { + if (droppedFrames >= maxDroppedFramesToNotify) { maybeNotifyDroppedFrames(); } } @@ -740,10 +826,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } private static boolean isBufferLate(long earlyUs) { - // Class a buffer as late if it should have been presented more than 30ms ago. + // Class a buffer as late if it should have been presented more than 30 ms ago. return earlyUs < -30000; } + private static boolean isBufferVeryLate(long earlyUs) { + // Class a buffer as very late if it should have been presented more than 500 ms ago. + return earlyUs < -500000; + } + @TargetApi(23) private static void setOutputSurfaceV23(MediaCodec codec, Surface surface) { codec.setOutputSurface(surface); diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java index f443c2a06f..9d9272f10e 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java @@ -155,7 +155,8 @@ public final class DebugTextViewHelper extends Player.DefaultEventListener imple + " sb:" + counters.skippedOutputBufferCount + " rb:" + counters.renderedOutputBufferCount + " db:" + counters.droppedOutputBufferCount - + " mcdb:" + counters.maxConsecutiveDroppedOutputBufferCount; + + " mcdb:" + counters.maxConsecutiveDroppedOutputBufferCount + + " dk:" + counters.droppedToKeyframeCount; } private static String getPixelAspectRatioString(float pixelAspectRatio) { diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java index b63afd3984..392a4907d4 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java @@ -124,7 +124,7 @@ public class DebugRenderersFactory extends DefaultRenderersFactory { @Override protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip) { + boolean shouldSkip) throws ExoPlaybackException { if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) { // After the codec has been initialized, don't render the first frame until we've caught up // to the playback position. Else test runs on devices that do not support dummy surface