From 9c58e57127dcdc03410f34db1278865fd95c7e44 Mon Sep 17 00:00:00 2001 From: christosts Date: Wed, 12 Feb 2020 16:53:24 +0000 Subject: [PATCH] Video frame processing offset in DecoderCounters Add fields in DecoderCounters for computing the average video frame processing offset. The MediaCodecVideoRenderer reports the video frame processing offset and the demo app presents it on the debug information. PiperOrigin-RevId: 294677878 --- RELEASENOTES.md | 3 + .../exoplayer2/decoder/DecoderCounters.java | 44 +++++++++ .../video/MediaCodecVideoRenderer.java | 5 + .../decoder/DecoderCountersTest.java | 96 +++++++++++++++++++ .../exoplayer2/ui/DebugTextViewHelper.java | 12 +++ .../testutil/DecoderCountersUtil.java | 16 ++++ 6 files changed, 176 insertions(+) create mode 100644 library/core/src/test/java/com/google/android/exoplayer2/decoder/DecoderCountersTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9bff079475..383be1025c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -3,6 +3,9 @@ ### dev-v2 (not yet released) ### * Core library: + * Add fields `videoFrameProcessingOffsetUsSum` and + `videoFrameProcessingOffsetUsCount` in `DecoderCounters` to compute + the average video frame processing offset. * Add playlist API ([#6161](https://github.com/google/ExoPlayer/issues/6161)). * Add `play` and `pause` methods to `Player`. * Add `Player.getCurrentLiveOffset` to conveniently return the live offset. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java index 8409bab558..3993739967 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.decoder; +import com.google.android.exoplayer2.util.Util; + /** * Maintains decoder event counts, for debugging purposes only. *

@@ -73,6 +75,23 @@ public final class DecoderCounters { * dropped from the source to advance to the keyframe. */ public int droppedToKeyframeCount; + /** + * The sum of video frame processing offset samples in microseconds. + * + *

Video frame processing offset measures how early a video frame was processed by a video + * renderer compared to the player's current position. + * + *

Note: Use {@link #addVideoFrameProcessingOffsetSample(long)} to update this field instead of + * updating it directly. + */ + public long videoFrameProcessingOffsetUsSum; + /** + * The number of video frame processing offset samples added. + * + *

Note: Use {@link #addVideoFrameProcessingOffsetSample(long)} to update this field instead of + * updating it directly. + */ + public int videoFrameProcessingOffsetUsCount; /** * Should be called to ensure counter values are made visible across threads. The playback thread @@ -100,6 +119,31 @@ public final class DecoderCounters { maxConsecutiveDroppedBufferCount = Math.max(maxConsecutiveDroppedBufferCount, other.maxConsecutiveDroppedBufferCount); droppedToKeyframeCount += other.droppedToKeyframeCount; + + addVideoFrameProcessingOffsetSamples( + other.videoFrameProcessingOffsetUsSum, other.videoFrameProcessingOffsetUsCount); } + /** + * Adds a video frame processing offset sample to {@link #videoFrameProcessingOffsetUsSum} and + * increases {@link #videoFrameProcessingOffsetUsCount} by one. + * + *

This method checks if adding {@code sampleUs} to {@link #videoFrameProcessingOffsetUsSum} + * will cause an overflow, in which case this method is a no-op. + * + * @param sampleUs The sample in microseconds. + */ + public void addVideoFrameProcessingOffsetSample(long sampleUs) { + addVideoFrameProcessingOffsetSamples(sampleUs, /* count= */ 1); + } + + private void addVideoFrameProcessingOffsetSamples(long sampleUs, int count) { + long overflowFlag = videoFrameProcessingOffsetUsSum > 0 ? Long.MIN_VALUE : Long.MAX_VALUE; + long newSampleSum = + Util.addWithOverflowDefault(videoFrameProcessingOffsetUsSum, sampleUs, overflowFlag); + if (newSampleSum != overflowFlag) { + videoFrameProcessingOffsetUsCount += count; + videoFrameProcessingOffsetUsSum = newSampleSum; + } + } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 4bab19a355..d2ec6c267c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -812,6 +812,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. if (isBufferLate(earlyUs)) { skipOutputBuffer(codec, bufferIndex, presentationTimeUs); + decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs); return true; } return false; @@ -834,6 +835,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } else { renderOutputBuffer(codec, bufferIndex, presentationTimeUs); } + decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs); return true; } @@ -866,6 +868,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } else { dropOutputBuffer(codec, bufferIndex, presentationTimeUs); } + decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs); return true; } @@ -875,6 +878,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { notifyFrameMetadataListener( presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat); renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); + decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs); return true; } } else { @@ -894,6 +898,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { notifyFrameMetadataListener( presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat); renderOutputBuffer(codec, bufferIndex, presentationTimeUs); + decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs); return true; } } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/decoder/DecoderCountersTest.java b/library/core/src/test/java/com/google/android/exoplayer2/decoder/DecoderCountersTest.java new file mode 100644 index 0000000000..65b0ac5a55 --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/decoder/DecoderCountersTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.decoder; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link DecoderCounters}. */ +@RunWith(AndroidJUnit4.class) +public class DecoderCountersTest { + private DecoderCounters decoderCounters; + + @Before + public void setUp() { + decoderCounters = new DecoderCounters(); + } + + @Test + public void maybeAddVideoFrameProcessingOffsetSample_addsSamples() { + long sampleSum = 0; + for (int i = 0; i < 100; i++) { + long sample = (i + 10) * 10L; + sampleSum += sample; + decoderCounters.addVideoFrameProcessingOffsetSample(sample); + } + + assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(sampleSum); + assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(100); + } + + @Test + public void addVideoFrameProcessingOffsetSample_sumReachesMaxLong_addsValues() { + long highSampleValue = Long.MAX_VALUE - 10; + long additionalSample = Long.MAX_VALUE - highSampleValue; + + decoderCounters.addVideoFrameProcessingOffsetSample(highSampleValue); + decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample); + + assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(Long.MAX_VALUE); + assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(2); + } + + @Test + public void addVideoFrameProcessingOffsetSample_sumOverflows_isNoOp() { + long highSampleValue = Long.MAX_VALUE - 10; + long additionalSample = Long.MAX_VALUE - highSampleValue + 10; + + decoderCounters.addVideoFrameProcessingOffsetSample(highSampleValue); + decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample); + + assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(highSampleValue); + assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(1); + } + + @Test + public void addVideoFrameProcessingOffsetSample_sumReachesMinLong_addsValues() { + long lowSampleValue = Long.MIN_VALUE + 10; + long additionalSample = Long.MIN_VALUE - lowSampleValue; + + decoderCounters.addVideoFrameProcessingOffsetSample(lowSampleValue); + decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample); + + assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(Long.MIN_VALUE); + assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(2); + } + + @Test + public void addVideoFrameProcessingOffsetSample_sumUnderflows_isNoOp() { + long lowSampleValue = Long.MIN_VALUE + 10; + long additionalSample = Long.MIN_VALUE - lowSampleValue - 10; + + decoderCounters.addVideoFrameProcessingOffsetSample(lowSampleValue); + decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample); + + assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(lowSampleValue); + assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(1); + } +} diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java index cd5713624b..fe3301130c 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java @@ -157,6 +157,8 @@ public class DebugTextViewHelper implements Player.EventListener, Runnable { + format.height + getPixelAspectRatioString(format.pixelWidthHeightRatio) + getDecoderCountersBufferCountString(decoderCounters) + + " vfpo: " + + getVideoFrameProcessingOffsetAverageString(decoderCounters) + ")"; } @@ -197,4 +199,14 @@ public class DebugTextViewHelper implements Player.EventListener, Runnable { : (" par:" + String.format(Locale.US, "%.02f", pixelAspectRatio)); } + private static String getVideoFrameProcessingOffsetAverageString(DecoderCounters counters) { + counters.ensureUpdated(); + int sampleCount = counters.videoFrameProcessingOffsetUsCount; + if (sampleCount == 0) { + return "N/A"; + } else { + long averageUs = (long) ((double) counters.videoFrameProcessingOffsetUsSum / sampleCount); + return String.valueOf(averageUs); + } + } } diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java index 307443718c..0c9a1a79b8 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java @@ -98,4 +98,20 @@ public final class DecoderCountersUtil { .isAtMost(limit); } + public static void assertVideoFrameProcessingOffsetSampleCount( + String name, DecoderCounters counters, int minCount, int maxCount) { + int actual = counters.videoFrameProcessingOffsetUsCount; + assertWithMessage( + "Codec(" + + name + + ") videoFrameProcessingOffsetSampleCount " + + actual + + ". Expected in range [" + + minCount + + ", " + + maxCount + + "].") + .that(minCount <= actual && actual <= maxCount) + .isTrue(); + } }