Improve Format propagation within the MediaCodecRenderer.

For example, fix handling of pixel aspect ratio changes in
playlists where video resolution does not change.

Issue:#6646
PiperOrigin-RevId: 281276023

RELEASENOTES.md

@@ -15,6 +15,13 @@
   [embedded in Matroska streams](https://matroska.org/technical/specs/subtitles/index.html).
 * Use `ExoMediaDrm.Provider` in `OfflineLicenseHelper` to avoid `ExoMediaDrm`
   leaks ([#4721](https://github.com/google/ExoPlayer/issues/4721)).
+* Improve `Format` propagation within the `MediaCodecRenderer` and subclasses.
+  For example, fix handling of pixel aspect ratio changes in playlists where
+  video resolution does not change.
+  ([#6646](https://github.com/google/ExoPlayer/issues/6646)).
+* Rename `MediaCodecRenderer.onOutputFormatChanged` to
+  `MediaCodecRenderer.onOutputMediaFormatChanged`, further
+  clarifying the distinction between `Format` and `MediaFormat`.
 * Fix byte order of HDR10+ static metadata to match CTA-861.3.
 * Reconfigure audio sink when PCM encoding changes
   ([#6601](https://github.com/google/ExoPlayer/issues/6601)).

library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java

@@ -475,10 +475,10 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   protected @KeepCodecResult int canKeepCodec(
       MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) {
     // TODO: We currently rely on recreating the codec when encoder delay or padding is non-zero.
-    // Re-creating the codec is necessary to guarantee that onOutputFormatChanged is called, which
+    // Re-creating the codec is necessary to guarantee that onOutputMediaFormatChanged is called,
-    // is where encoder delay and padding are propagated to the sink. We should find a better way to
+    // which is where encoder delay and padding are propagated to the sink. We should find a better
-    // propagate these values, and then allow the codec to be re-used in cases where this would
+    // way to propagate these values, and then allow the codec to be re-used in cases where this
-    // otherwise be possible.
+    // would otherwise be possible.
     if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize
         || oldFormat.encoderDelay != 0
         || oldFormat.encoderPadding != 0
@@ -558,7 +558,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   }
 
   @Override
-  protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
+  protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
       throws ExoPlaybackException {
     @C.Encoding int encoding;
     MediaFormat mediaFormat;

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -565,7 +565,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     return true;
   }
 
-  /**
+  /*
    * Returns whether the codec needs the renderer to propagate the end-of-stream signal directly,
    * rather than by using an end-of-stream buffer queued to the codec.
    */
@@ -574,17 +574,17 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   }
 
   /**
-   * Polls the pending output format queue for a given buffer timestamp. If a format is present, it
+   * Polls the pending output format queue for a given buffer timestamp. If a format is present,
-   * is removed and returned. Otherwise returns {@code null}. Subclasses should only call this
+   * {@link #onOutputFormatChanged(Format)} is called. Subclasses should only call this method if
-   * method if they are taking over responsibility for output format propagation (e.g., when using
+   * they are taking over responsibility for output format propagation (e.g., when using video
-   * video tunneling).
+   * tunneling).
    */
-  protected final @Nullable Format updateOutputFormatForTime(long presentationTimeUs) {
+  protected final void updateOutputFormatForTime(long presentationTimeUs) {
-    Format format = formatQueue.pollFloor(presentationTimeUs);
+    @Nullable Format format = formatQueue.pollFloor(presentationTimeUs);
     if (format != null) {
       outputFormat = format;
+      onOutputFormatChanged(outputFormat);
     }
-    return format;
   }
 
   protected final MediaCodec getCodec() {
@@ -1305,11 +1305,22 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * @param outputMediaFormat The new output {@link MediaFormat}.
    * @throws ExoPlaybackException Thrown if an error occurs handling the new output media format.
    */
-  protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
+  protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
       throws ExoPlaybackException {
     // Do nothing.
   }
 
+  /**
+   * Called when the output {@link Format} changes from the format queue.
+   *
+   * <p>The default implementation is a no-op.
+   *
+   * @param outputFormat The new output {@link Format}.
+   */
+  protected void onOutputFormatChanged(Format outputFormat) {
+    // Do nothing.
+  }
+
   /**
    * Handles supplemental data associated with an input buffer.
    *
@@ -1504,7 +1515,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
 
       if (outputIndex < 0) {
         if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
-          processOutputFormat();
+          processOutputMediaFormat();
           return true;
         } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
           processOutputBuffersChanged();
@@ -1596,7 +1607,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   }
 
   /** Processes a new output {@link MediaFormat}. */
-  private void processOutputFormat() throws ExoPlaybackException {
+  private void processOutputMediaFormat() throws ExoPlaybackException {
     MediaFormat mediaFormat = codec.getOutputFormat();
     if (codecAdaptationWorkaroundMode != ADAPTATION_WORKAROUND_MODE_NEVER
         && mediaFormat.getInteger(MediaFormat.KEY_WIDTH) == ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT
@@ -1609,7 +1620,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     if (codecNeedsMonoChannelCountWorkaround) {
       mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
     }
-    onOutputFormatChanged(codec, mediaFormat);
+    onOutputMediaFormatChanged(codec, mediaFormat);
   }
 
   /**

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -143,9 +143,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private int buffersInCodecCount;
   private long lastRenderTimeUs;
 
-  private int pendingRotationDegrees;
-  private float pendingPixelWidthHeightRatio;
   @Nullable private MediaFormat currentMediaFormat;
+  private int mediaFormatWidth;
+  private int mediaFormatHeight;
   private int currentWidth;
   private int currentHeight;
   private int currentUnappliedRotationDegrees;
@@ -353,8 +353,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     joiningDeadlineMs = C.TIME_UNSET;
     currentWidth = Format.NO_VALUE;
     currentHeight = Format.NO_VALUE;
+    mediaFormatWidth = Format.NO_VALUE;
+    mediaFormatHeight = Format.NO_VALUE;
     currentPixelWidthHeightRatio = Format.NO_VALUE;
-    pendingPixelWidthHeightRatio = Format.NO_VALUE;
     scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
     clearReportedVideoSize();
   }
@@ -749,10 +750,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   @Override
   protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybackException {
     super.onInputFormatChanged(formatHolder);
-    Format newFormat = formatHolder.format;
+    eventDispatcher.inputFormatChanged(formatHolder.format);
-    eventDispatcher.inputFormatChanged(newFormat);
-    pendingPixelWidthHeightRatio = newFormat.pixelWidthHeightRatio;
-    pendingRotationDegrees = newFormat.rotationDegrees;
   }
 
   /**
@@ -773,26 +771,56 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
 
   @Override
-  protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat) {
+  protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat) {
     currentMediaFormat = outputMediaFormat;
+
     boolean hasCrop =
         outputMediaFormat.containsKey(KEY_CROP_RIGHT)
             && outputMediaFormat.containsKey(KEY_CROP_LEFT)
             && outputMediaFormat.containsKey(KEY_CROP_BOTTOM)
             && outputMediaFormat.containsKey(KEY_CROP_TOP);
-    int width =
+    mediaFormatWidth =
         hasCrop
             ? outputMediaFormat.getInteger(KEY_CROP_RIGHT)
                 - outputMediaFormat.getInteger(KEY_CROP_LEFT)
                 + 1
             : outputMediaFormat.getInteger(MediaFormat.KEY_WIDTH);
-    int height =
+    mediaFormatHeight =
         hasCrop
             ? outputMediaFormat.getInteger(KEY_CROP_BOTTOM)
                 - outputMediaFormat.getInteger(KEY_CROP_TOP)
                 + 1
             : outputMediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
-    processOutputFormat(codec, width, height);
+
+    // Must be applied each time the output MediaFormat changes.
+    codec.setVideoScalingMode(scalingMode);
+  }
+
+  @Override
+  protected void onOutputFormatChanged(Format outputFormat) {
+    if (tunneling) {
+      currentWidth = outputFormat.width;
+      currentHeight = outputFormat.height;
+    } else {
+      currentWidth = mediaFormatWidth;
+      currentHeight = mediaFormatHeight;
+    }
+
+    currentPixelWidthHeightRatio = outputFormat.pixelWidthHeightRatio;
+    if (Util.SDK_INT >= 21) {
+      // On API level 21 and above the decoder applies the rotation when rendering to the surface.
+      // Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
+      // to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
+      if (outputFormat.rotationDegrees == 90 || outputFormat.rotationDegrees == 270) {
+        int rotatedHeight = currentWidth;
+        currentWidth = currentHeight;
+        currentHeight = rotatedHeight;
+        currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
+      }
+    } else {
+      // On API level 20 and below the decoder does not apply the rotation.
+      currentUnappliedRotationDegrees = outputFormat.rotationDegrees;
+    }
   }
 
   @Override
@@ -945,28 +973,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     return false;
   }
 
-  private void processOutputFormat(MediaCodec codec, int width, int height) {
-    currentWidth = width;
-    currentHeight = height;
-    currentPixelWidthHeightRatio = pendingPixelWidthHeightRatio;
-    if (Util.SDK_INT >= 21) {
-      // On API level 21 and above the decoder applies the rotation when rendering to the surface.
-      // Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
-      // to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
-      if (pendingRotationDegrees == 90 || pendingRotationDegrees == 270) {
-        int rotatedHeight = currentWidth;
-        currentWidth = currentHeight;
-        currentHeight = rotatedHeight;
-        currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
-      }
-    } else {
-      // On API level 20 and below the decoder does not apply the rotation.
-      currentUnappliedRotationDegrees = pendingRotationDegrees;
-    }
-    // Must be applied each time the output MediaFormat changes.
-    codec.setVideoScalingMode(scalingMode);
-  }
-
   private void notifyFrameMetadataListener(
       long presentationTimeUs, long releaseTimeNs, Format format, MediaFormat mediaFormat) {
     if (frameMetadataListener != null) {
@@ -986,10 +992,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
 
   /** Called when a buffer was processed in tunneling mode. */
   protected void onProcessedTunneledBuffer(long presentationTimeUs) {
-    @Nullable Format format = updateOutputFormatForTime(presentationTimeUs);
+    updateOutputFormatForTime(presentationTimeUs);
-    if (format != null) {
-      processOutputFormat(getCodec(), format.width, format.height);
-    }
     maybeNotifyVideoSizeChanged();
     maybeNotifyRenderedFirstFrame();
     onProcessedOutputBuffer(presentationTimeUs);