From f69a2f5131bce4b9d7385da86173d10d0b14cb95 Mon Sep 17 00:00:00 2001 From: claincly Date: Tue, 31 Jan 2023 17:16:29 +0000 Subject: [PATCH] Render PQ to DebugSurface in transformer. For HLG input in transformer, FinalWrapper is configured to only output HLG to encoder. But since DebugPreview is configured to take PQ for HDR content, the color will not look correct. This CL allows overriding the MatrixTP output transfer function, so that FinalWrapper can output - HLG to encoder - PQ to debug preview PiperOrigin-RevId: 506022840 --- .../FinalMatrixTextureProcessorWrapper.java | 17 +++++++------ .../media3/effect/MatrixTextureProcessor.java | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java index b7ccb57fc4..22621f8118 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTextureProcessorWrapper.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -422,7 +423,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; debugSurfaceViewWrapper.maybeRenderToSurfaceView( () -> { GlUtil.clearOutputFrame(); + @C.ColorTransfer + int configuredColorTransfer = matrixTextureProcessor.getOutputColorTransfer(); + matrixTextureProcessor.setOutputColorTransfer( + checkNotNull(debugSurfaceViewWrapper).outputColorTransfer); matrixTextureProcessor.drawFrame(inputTexture.texId, presentationTimeUs); + matrixTextureProcessor.setOutputColorTransfer(configuredColorTransfer); }); } catch (FrameProcessingException | GlUtil.GlException e) { Log.d(TAG, "Error rendering to debug preview", e); @@ -434,9 +440,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * and makes rendering a no-op if not. */ private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { + public final @C.ColorTransfer int outputColorTransfer; private final EGLDisplay eglDisplay; private final EGLContext eglContext; - private final boolean useHdr; @GuardedBy("this") @Nullable @@ -453,7 +459,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLDisplay eglDisplay, EGLContext eglContext, boolean useHdr, SurfaceView surfaceView) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; - this.useHdr = useHdr; + // Screen output supports only BT.2020 PQ (ST2084) for HDR. + this.outputColorTransfer = useHdr ? C.COLOR_TRANSFER_ST2084 : C.COLOR_TRANSFER_SDR; surfaceView.getHolder().addCallback(this); surface = surfaceView.getHolder().getSurface(); width = surfaceView.getWidth(); @@ -473,13 +480,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (eglSurface == null) { - // Screen output supports only BT.2020 PQ (ST2084). eglSurface = GlUtil.createEglSurface( - eglDisplay, - surface, - useHdr ? C.COLOR_TRANSFER_ST2084 : C.COLOR_TRANSFER_SDR, - /* isEncoderInputSurface= */ false); + eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false); } EGLSurface eglSurface = this.eglSurface; GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java index bde02e0306..3dad757001 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java @@ -127,6 +127,7 @@ import java.util.List; private ImmutableList visiblePolygon; private final GlProgram glProgram; + private @C.ColorTransfer int outputColorTransfer; /** * Creates a new instance. @@ -158,6 +159,7 @@ import java.util.List; glProgram, ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(rgbMatrices), + /* outputColorTransfer= */ C.COLOR_TRANSFER_LINEAR, useHdr); } @@ -244,6 +246,7 @@ import java.util.List; glProgram, ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(rgbMatrices), + outputColorInfo.colorTransfer, isInputTransferHdr); } @@ -298,6 +301,7 @@ import java.util.List; glProgram, ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(rgbMatrices), + outputColorInfo.colorTransfer, outputIsHdr); } @@ -309,6 +313,7 @@ import java.util.List; * apply to each frame in order. Can be empty to apply no vertex transformations. * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be * empty to apply no color transformations. + * @param outputColorTransfer The output {@link C.ColorTransfer}. * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code * EXT_YUV_target} OpenGL extension. */ @@ -316,9 +321,11 @@ import java.util.List; GlProgram glProgram, ImmutableList matrixTransformations, ImmutableList rgbMatrices, + int outputColorTransfer, boolean useHdr) { super(useHdr); this.glProgram = glProgram; + this.outputColorTransfer = outputColorTransfer; this.matrixTransformations = matrixTransformations; this.rgbMatrices = rgbMatrices; this.useHdr = useHdr; @@ -393,6 +400,23 @@ import java.util.List; } } + /** + * Sets the output {@link C.ColorTransfer}. + * + *

This method must not be called on {@code MatrixTextureProcessor} instances that output + * {@linkplain C#COLOR_TRANSFER_LINEAR linear colors}. + */ + public void setOutputColorTransfer(@C.ColorTransfer int colorTransfer) { + checkState(outputColorTransfer != C.COLOR_TRANSFER_LINEAR); + outputColorTransfer = colorTransfer; + glProgram.setIntUniform("uOutputColorTransfer", colorTransfer); + } + + /** Returns the output {@link C.ColorTransfer}. */ + public @C.ColorTransfer int getOutputColorTransfer() { + return outputColorTransfer; + } + /** * Updates {@link #compositeTransformationMatrixArray} and {@link #visiblePolygon} based on the * given frame timestamp.