From 1c66010b4a6edd99fb3b392b5e7efb30aa2c8ff3 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Fri, 11 Oct 2019 15:37:57 +0100 Subject: [PATCH] Add MediaFormat on video frame metadata listener This is useful for apps that want to access HDR metadata that MediaCodec puts in its output format. PiperOrigin-RevId: 274169985 --- RELEASENOTES.md | 1 + .../exoplayer2/ext/vp9/LibvpxVideoRenderer.java | 2 +- .../exoplayer2/video/MediaCodecVideoRenderer.java | 15 ++++++++++----- .../video/VideoFrameMetadataListener.java | 11 ++++++++++- .../exoplayer2/ui/spherical/SceneRenderer.java | 6 +++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index b2f5931e02..d1a5f13eb3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -113,6 +113,7 @@ * Add support for ID3-in-EMSG in HLS streams ([spec](https://aomediacodec.github.io/av1-id3/)). * Make show and hide player controls accessible for TalkBack in `PlayerView`. +* Pass the codec output `MediaFormat` to `VideoFrameMetadataListener`. ### 2.10.5 (2019-09-20) ### diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java index ec4f3f7903..31195a3070 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java @@ -238,7 +238,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer { throws VideoDecoderException { if (frameMetadataListener != null) { frameMetadataListener.onVideoFrameAboutToBeRendered( - presentationTimeUs, System.nanoTime(), outputFormat); + presentationTimeUs, System.nanoTime(), outputFormat, /* mediaFormat= */ null); } super.renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index c66ce82614..c8f13cb6de 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -142,6 +142,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private int pendingRotationDegrees; private float pendingPixelWidthHeightRatio; + @Nullable private MediaFormat currentMediaFormat; private int currentWidth; private int currentHeight; private int currentUnappliedRotationDegrees; @@ -502,6 +503,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { lastInputTimeUs = C.TIME_UNSET; outputStreamOffsetUs = C.TIME_UNSET; pendingOutputStreamOffsetCount = 0; + currentMediaFormat = null; clearReportedVideoSize(); clearRenderedFirstFrame(); frameReleaseTimeHelper.disable(); @@ -720,6 +722,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputFormat) { + currentMediaFormat = outputFormat; boolean hasCrop = outputFormat.containsKey(KEY_CROP_RIGHT) && outputFormat.containsKey(KEY_CROP_LEFT) && outputFormat.containsKey(KEY_CROP_BOTTOM) && outputFormat.containsKey(KEY_CROP_TOP); @@ -810,7 +813,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs))); if (forceRenderOutputBuffer) { long releaseTimeNs = System.nanoTime(); - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format, currentMediaFormat); if (Util.SDK_INT >= 21) { renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs); } else { @@ -854,7 +857,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. if (earlyUs < 50000) { - notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); + notifyFrameMetadataListener( + presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat); renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); return true; } @@ -872,7 +876,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } } - notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); + notifyFrameMetadataListener( + presentationTimeUs, adjustedReleaseTimeNs, format, currentMediaFormat); renderOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } @@ -905,10 +910,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } private void notifyFrameMetadataListener( - long presentationTimeUs, long releaseTimeNs, Format format) { + long presentationTimeUs, long releaseTimeNs, Format format, MediaFormat mediaFormat) { if (frameMetadataListener != null) { frameMetadataListener.onVideoFrameAboutToBeRendered( - presentationTimeUs, releaseTimeNs, format); + presentationTimeUs, releaseTimeNs, format, mediaFormat); } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java index b467d0f421..746903a101 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.video; +import android.media.MediaFormat; +import androidx.annotation.Nullable; import com.google.android.exoplayer2.Format; /** A listener for metadata corresponding to video frame being rendered. */ @@ -26,6 +28,13 @@ public interface VideoFrameMetadataListener { * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds. * If the platform API version of the device is less than 21, then this is the best effort. * @param format The format associated with the frame. + * @param mediaFormat The framework media format associated with the frame, or {@code null} if not + * known or not applicable (e.g., because the frame was not output by a {@link + * android.media.MediaCodec MediaCodec}). */ - void onVideoFrameAboutToBeRendered(long presentationTimeUs, long releaseTimeNs, Format format); + void onVideoFrameAboutToBeRendered( + long presentationTimeUs, + long releaseTimeNs, + Format format, + @Nullable MediaFormat mediaFormat); } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java index b70fd277a9..4343800500 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java @@ -18,6 +18,7 @@ package com.google.android.exoplayer2.ui.spherical; import static com.google.android.exoplayer2.util.GlUtil.checkGlError; import android.graphics.SurfaceTexture; +import android.media.MediaFormat; import android.opengl.GLES20; import android.opengl.Matrix; import androidx.annotation.Nullable; @@ -142,7 +143,10 @@ public final class SceneRenderer implements VideoFrameMetadataListener, CameraMo @Override public void onVideoFrameAboutToBeRendered( - long presentationTimeUs, long releaseTimeNs, Format format) { + long presentationTimeUs, + long releaseTimeNs, + Format format, + @Nullable MediaFormat mediaFormat) { sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs); setProjection(format.projectionData, format.stereoMode, releaseTimeNs); }