Video frame processing offset in DecoderCounters

Add fields in DecoderCounters for computing the average video frame
processing offset.

The MediaCodecVideoRenderer reports the video frame processing offset
and the demo app presents it on the debug information.

PiperOrigin-RevId: 294677878

RELEASENOTES.md

@@ -3,6 +3,9 @@
 ### dev-v2 (not yet released) ###
 
 * Core library:
+  * Add fields `videoFrameProcessingOffsetUsSum` and
+    `videoFrameProcessingOffsetUsCount` in `DecoderCounters` to compute
+    the average video frame processing offset.
   * Add playlist API ([#6161](https://github.com/google/ExoPlayer/issues/6161)).
   * Add `play` and `pause` methods to `Player`.
   * Add `Player.getCurrentLiveOffset` to conveniently return the live offset.

library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java

@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.decoder;
 
+import com.google.android.exoplayer2.util.Util;
+
 /**
  * Maintains decoder event counts, for debugging purposes only.
  * <p>
@@ -73,6 +75,23 @@ public final class DecoderCounters {
    * dropped from the source to advance to the keyframe.
    */
   public int droppedToKeyframeCount;
+  /**
+   * The sum of video frame processing offset samples in microseconds.
+   *
+   * <p>Video frame processing offset measures how early a video frame was processed by a video
+   * renderer compared to the player's current position.
+   *
+   * <p>Note: Use {@link #addVideoFrameProcessingOffsetSample(long)} to update this field instead of
+   * updating it directly.
+   */
+  public long videoFrameProcessingOffsetUsSum;
+  /**
+   * The number of video frame processing offset samples added.
+   *
+   * <p>Note: Use {@link #addVideoFrameProcessingOffsetSample(long)} to update this field instead of
+   * updating it directly.
+   */
+  public int videoFrameProcessingOffsetUsCount;
 
   /**
    * Should be called to ensure counter values are made visible across threads. The playback thread
@@ -100,6 +119,31 @@ public final class DecoderCounters {
     maxConsecutiveDroppedBufferCount = Math.max(maxConsecutiveDroppedBufferCount,
         other.maxConsecutiveDroppedBufferCount);
     droppedToKeyframeCount += other.droppedToKeyframeCount;
+
+    addVideoFrameProcessingOffsetSamples(
+        other.videoFrameProcessingOffsetUsSum, other.videoFrameProcessingOffsetUsCount);
   }
 
+  /**
+   * Adds a video frame processing offset sample to {@link #videoFrameProcessingOffsetUsSum} and
+   * increases {@link #videoFrameProcessingOffsetUsCount} by one.
+   *
+   * <p>This method checks if adding {@code sampleUs} to {@link #videoFrameProcessingOffsetUsSum}
+   * will cause an overflow, in which case this method is a no-op.
+   *
+   * @param sampleUs The sample in microseconds.
+   */
+  public void addVideoFrameProcessingOffsetSample(long sampleUs) {
+    addVideoFrameProcessingOffsetSamples(sampleUs, /* count= */ 1);
+  }
+
+  private void addVideoFrameProcessingOffsetSamples(long sampleUs, int count) {
+    long overflowFlag = videoFrameProcessingOffsetUsSum > 0 ? Long.MIN_VALUE : Long.MAX_VALUE;
+    long newSampleSum =
+        Util.addWithOverflowDefault(videoFrameProcessingOffsetUsSum, sampleUs, overflowFlag);
+    if (newSampleSum != overflowFlag) {
+      videoFrameProcessingOffsetUsCount += count;
+      videoFrameProcessingOffsetUsSum = newSampleSum;
+    }
+  }
 }

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -812,6 +812,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       // Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
       if (isBufferLate(earlyUs)) {
         skipOutputBuffer(codec, bufferIndex, presentationTimeUs);
+        decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs);
         return true;
       }
       return false;
@@ -834,6 +835,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       } else {
         renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
       }
+      decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs);
       return true;
     }
 
@@ -866,6 +868,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       } else {
         dropOutputBuffer(codec, bufferIndex, presentationTimeUs);
       }
+      decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs);
       return true;
     }
 
@@ -875,6 +878,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
         notifyFrameMetadataListener(
             presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat);
         renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs);
+        decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs);
         return true;
       }
     } else {
@@ -894,6 +898,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
         notifyFrameMetadataListener(
             presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat);
         renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
+        decoderCounters.addVideoFrameProcessingOffsetSample(earlyUs);
         return true;
       }
     }

library/core/src/test/java/com/google/android/exoplayer2/decoder/DecoderCountersTest.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.decoder;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link DecoderCounters}. */
+@RunWith(AndroidJUnit4.class)
+public class DecoderCountersTest {
+  private DecoderCounters decoderCounters;
+
+  @Before
+  public void setUp() {
+    decoderCounters = new DecoderCounters();
+  }
+
+  @Test
+  public void maybeAddVideoFrameProcessingOffsetSample_addsSamples() {
+    long sampleSum = 0;
+    for (int i = 0; i < 100; i++) {
+      long sample = (i + 10) * 10L;
+      sampleSum += sample;
+      decoderCounters.addVideoFrameProcessingOffsetSample(sample);
+    }
+
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(sampleSum);
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(100);
+  }
+
+  @Test
+  public void addVideoFrameProcessingOffsetSample_sumReachesMaxLong_addsValues() {
+    long highSampleValue = Long.MAX_VALUE - 10;
+    long additionalSample = Long.MAX_VALUE - highSampleValue;
+
+    decoderCounters.addVideoFrameProcessingOffsetSample(highSampleValue);
+    decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample);
+
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(Long.MAX_VALUE);
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(2);
+  }
+
+  @Test
+  public void addVideoFrameProcessingOffsetSample_sumOverflows_isNoOp() {
+    long highSampleValue = Long.MAX_VALUE - 10;
+    long additionalSample = Long.MAX_VALUE - highSampleValue + 10;
+
+    decoderCounters.addVideoFrameProcessingOffsetSample(highSampleValue);
+    decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample);
+
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(highSampleValue);
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(1);
+  }
+
+  @Test
+  public void addVideoFrameProcessingOffsetSample_sumReachesMinLong_addsValues() {
+    long lowSampleValue = Long.MIN_VALUE + 10;
+    long additionalSample = Long.MIN_VALUE - lowSampleValue;
+
+    decoderCounters.addVideoFrameProcessingOffsetSample(lowSampleValue);
+    decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample);
+
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(Long.MIN_VALUE);
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(2);
+  }
+
+  @Test
+  public void addVideoFrameProcessingOffsetSample_sumUnderflows_isNoOp() {
+    long lowSampleValue = Long.MIN_VALUE + 10;
+    long additionalSample = Long.MIN_VALUE - lowSampleValue - 10;
+
+    decoderCounters.addVideoFrameProcessingOffsetSample(lowSampleValue);
+    decoderCounters.addVideoFrameProcessingOffsetSample(additionalSample);
+
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsSum).isEqualTo(lowSampleValue);
+    assertThat(decoderCounters.videoFrameProcessingOffsetUsCount).isEqualTo(1);
+  }
+}

library/ui/src/main/java/com/google/android/exoplayer2/ui/DebugTextViewHelper.java

@@ -157,6 +157,8 @@ public class DebugTextViewHelper implements Player.EventListener, Runnable {
         + format.height
         + getPixelAspectRatioString(format.pixelWidthHeightRatio)
         + getDecoderCountersBufferCountString(decoderCounters)
+        + " vfpo: "
+        + getVideoFrameProcessingOffsetAverageString(decoderCounters)
         + ")";
   }
 
@@ -197,4 +199,14 @@ public class DebugTextViewHelper implements Player.EventListener, Runnable {
         : (" par:" + String.format(Locale.US, "%.02f", pixelAspectRatio));
   }
 
+  private static String getVideoFrameProcessingOffsetAverageString(DecoderCounters counters) {
+    counters.ensureUpdated();
+    int sampleCount = counters.videoFrameProcessingOffsetUsCount;
+    if (sampleCount == 0) {
+      return "N/A";
+    } else {
+      long averageUs = (long) ((double) counters.videoFrameProcessingOffsetUsSum / sampleCount);
+      return String.valueOf(averageUs);
+    }
+  }
 }

testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java

@@ -98,4 +98,20 @@ public final class DecoderCountersUtil {
         .isAtMost(limit);
   }
 
+  public static void assertVideoFrameProcessingOffsetSampleCount(
+      String name, DecoderCounters counters, int minCount, int maxCount) {
+    int actual = counters.videoFrameProcessingOffsetUsCount;
+    assertWithMessage(
+            "Codec("
+                + name
+                + ") videoFrameProcessingOffsetSampleCount "
+                + actual
+                + ". Expected in range ["
+                + minCount
+                + ", "
+                + maxCount
+                + "].")
+        .that(minCount <= actual && actual <= maxCount)
+        .isTrue();
+  }
 }