From 7089253beff5019fb768326e5a8aa41694a948fa Mon Sep 17 00:00:00 2001 From: tofunmi Date: Mon, 29 Apr 2024 03:24:19 -0700 Subject: [PATCH] Effect:migrate setEnableColorTransfers to setSdrWorkingColorSpace Part of a two stage change to remove the conversion to linear colors in the SDR effects pipeline by default. Changes the boolean to an intdef, introducing a third option that gets all sdr input into the same colorspace. This is a planned API breaking change, but this change should not change the behavior of the pipeline. PiperOrigin-RevId: 629013747 --- .../DefaultVideoFrameProcessorPixelTest.java | 5 +- ...hader_transformation_sdr_external_es2.glsl | 83 ++++++++++------ ...hader_transformation_sdr_internal_es2.glsl | 79 ++++++++++----- .../media3/effect/DefaultShaderProgram.java | 31 +++--- .../effect/DefaultVideoFrameProcessor.java | 98 +++++++++++++------ .../effect/FinalShaderProgramWrapper.java | 22 +++-- .../androidx/media3/effect/InputSwitcher.java | 11 ++- .../media3/transformer/mh/HdrEditingTest.java | 5 +- .../ToneMapHdrToSdrUsingOpenGlPixelTest.java | 5 +- 9 files changed, 222 insertions(+), 117 deletions(-) diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java index b3e010d60e..78106c0e77 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java @@ -17,6 +17,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; @@ -145,7 +146,7 @@ public final class DefaultVideoFrameProcessorPixelTest { getDefaultFrameProcessorTestRunnerBuilder(testId) .setVideoFrameProcessorFactory( new DefaultVideoFrameProcessor.Factory.Builder() - .setEnableColorTransfers(false) + .setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL) .build()) .build(); Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); @@ -209,7 +210,7 @@ public final class DefaultVideoFrameProcessorPixelTest { getDefaultFrameProcessorTestRunnerBuilder(testId) .setVideoFrameProcessorFactory( new DefaultVideoFrameProcessor.Factory.Builder() - .setEnableColorTransfers(false) + .setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL) .build()) .setEffects(NO_OP_EFFECT) .build(); diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl index 30065f5580..e5837d38e3 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl @@ -16,8 +16,9 @@ // ES 2 fragment shader that: // 1. Samples from an external texture with uTexSampler copying from this // texture to the current output. -// 2. Transforms the electrical colors to optical colors using the SMPTE 170M -// EOTF. +// 2. Transforms the electrical colors to "working" colors which is the input +// colorspace with the colors transferred to either linear or SMPTE 170M as +// requested by uSdrWorkingColorSpace. // 3. Applies a 4x4 RGB color matrix to change the pixel colors. // 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR // for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to @@ -31,12 +32,16 @@ varying vec2 vTexSamplingCoord; // C.java#ColorTransfer value. // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed. uniform int uOutputColorTransfer; -uniform int uEnableColorTransfer; +uniform int uSdrWorkingColorSpace; const float inverseGamma = 0.4500; const float gamma = 1.0 / inverseGamma; const int GL_FALSE = 0; const int GL_TRUE = 1; +// LINT.IfChange(working_color_space) +const int WORKING_COLOR_SPACE_DEFAULT = 0; +const int WORKING_COLOR_SPACE_ORIGINAL = 1; +const int WORKING_COLOR_SPACE_LINEAR = 2; // Transforms a single channel from electrical to optical SDR using the SMPTE // 170M OETF. @@ -71,39 +76,57 @@ vec3 smpte170mOetf(vec3 opticalColor) { smpte170mOetfSingleChannel(opticalColor.b)); } -// Applies the appropriate OETF to convert linear optical signals to nonlinear -// electrical signals. Input and output are both normalized to [0, 1]. -highp vec3 applyOetf(highp vec3 linearColor) { - // LINT.IfChange(color_transfer) - const int COLOR_TRANSFER_LINEAR = 1; - const int COLOR_TRANSFER_SDR_VIDEO = 3; - if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR || - uEnableColorTransfer == GL_FALSE) { - return linearColor; - } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { - return smpte170mOetf(linearColor); - } else { - // Output red as an obviously visible error. - return vec3(1.0, 0.0, 0.0); - } -} - -vec3 applyEotf(vec3 electricalColor) { - if (uEnableColorTransfer == GL_TRUE) { - return smpte170mEotf(electricalColor); - } else if (uEnableColorTransfer == GL_FALSE) { - return electricalColor; +// Optionally applies the appropriate EOTF to convert nonlinear electrical +// signals to linear optical signals. Input and output are both normalized to +// [0, 1]. +vec3 convertToWorkingColors(vec3 inputColor) { + if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT || + uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) { + return inputColor; + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { + return smpte170mEotf(inputColor); } else { // Output blue as an obviously visible error. return vec3(0.0, 0.0, 1.0); } } +// Optionally applies the appropriate OETF to convert linear optical signals to +// nonlinear electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 convertToOutputColors(highp vec3 workingColors) { + // LINT.IfChange(color_transfer) + const int COLOR_TRANSFER_LINEAR = 1; + const int COLOR_TRANSFER_SDR_VIDEO = 3; + if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) { + if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) { + return smpte170mEotf(workingColors); + } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return workingColors; + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) { + return workingColors; + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { + if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) { + return workingColors; + } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return smpte170mOetf(workingColors); + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } +} + void main() { vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); - vec3 linearInputColor = applyEotf(inputColor.rgb); - - vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1); - - gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); + vec3 workingColors = convertToWorkingColors(inputColor.rgb); + vec4 transformedColors = uRgbMatrix * vec4(workingColors, 1); + gl_FragColor = + vec4(convertToOutputColors(transformedColors.rgb), inputColor.a); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl index 59426e4822..b44c410aff 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl @@ -17,8 +17,9 @@ // 1. Samples from an input texture created from an internal texture (e.g. a // texture created from a bitmap), with uTexSampler copying from this texture // to the current output. -// 2. Transforms the electrical colors to optical colors using the SMPTE 170M -// EOTF or the sRGB EOTF, as requested by uInputColorTransfer. +// 2. Transforms the electrical colors to "working" colors which is the input +// colorspace with the colors transferred to either linear or SMPTE 170M as +// requested by uSdrWorkingColorSpace. // 3. Applies a 4x4 RGB color matrix to change the pixel colors. // 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR // for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to @@ -34,7 +35,7 @@ uniform int uInputColorTransfer; // C.java#ColorTransfer value. // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed. uniform int uOutputColorTransfer; -uniform int uEnableColorTransfer; +uniform int uSdrWorkingColorSpace; const float inverseGamma = 0.4500; const float gamma = 1.0 / inverseGamma; @@ -44,6 +45,10 @@ const int GL_TRUE = 1; const int COLOR_TRANSFER_LINEAR = 1; const int COLOR_TRANSFER_SRGB = 2; const int COLOR_TRANSFER_SDR_VIDEO = 3; +// LINT.IfChange(working_color_space) +const int WORKING_COLOR_SPACE_DEFAULT = 0; +const int WORKING_COLOR_SPACE_ORIGINAL = 1; +const int WORKING_COLOR_SPACE_LINEAR = 2; // Transforms a single channel from electrical to optical SDR using the sRGB // EOTF. @@ -56,7 +61,7 @@ float srgbEotfSingleChannel(float electricalChannel) { } // Transforms electrical to optical SDR using the sRGB EOTF. -vec3 srgbEotf(const vec3 electricalColor) { +vec3 srgbEotf(vec3 electricalColor) { return vec3(srgbEotfSingleChannel(electricalColor.r), srgbEotfSingleChannel(electricalColor.g), srgbEotfSingleChannel(electricalColor.b)); @@ -94,34 +99,60 @@ vec3 smpte170mOetf(vec3 opticalColor) { smpte170mOetfSingleChannel(opticalColor.g), smpte170mOetfSingleChannel(opticalColor.b)); } -// Applies the appropriate EOTF to convert nonlinear electrical signals to -// linear optical signals. Input and output are both normalized to [0, 1]. -vec3 applyEotf(vec3 electricalColor) { - if (uEnableColorTransfer == GL_TRUE) { + +// Optionally applies the appropriate EOTF to convert nonlinear electrical +// signals to linear optical signals. Input and output are both normalized to +// [0, 1]. +vec3 convertToWorkingColors(vec3 inputColor) { + if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) { if (uInputColorTransfer == COLOR_TRANSFER_SRGB) { - return srgbEotf(electricalColor); + return smpte170mOetf(srgbEotf(inputColor)); } else if (uInputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { - return smpte170mEotf(electricalColor); + return inputColor; + } else { + // Output blue as an obviously visible error. + return vec3(0.0, 0.0, 1.0); + } + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) { + return inputColor; + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { + if (uInputColorTransfer == COLOR_TRANSFER_SRGB) { + return srgbEotf(inputColor); + } else if (uInputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return smpte170mEotf(inputColor); } else { // Output blue as an obviously visible error. return vec3(0.0, 0.0, 1.0); } - } else if (uEnableColorTransfer == GL_FALSE) { - return electricalColor; } else { // Output blue as an obviously visible error. return vec3(0.0, 0.0, 1.0); } } -// Applies the appropriate OETF to convert linear optical signals to nonlinear -// electrical signals. Input and output are both normalized to [0, 1]. -highp vec3 applyOetf(highp vec3 linearColor) { - if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR || - uEnableColorTransfer == GL_FALSE) { - return linearColor; - } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { - return smpte170mOetf(linearColor); +// Optionally applies the appropriate OETF to convert linear optical signals to +// nonlinear electrical signals. Input and output are both normalized to [0, 1]. +highp vec3 convertToOutputColors(highp vec3 workingColors) { + if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) { + if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) { + return smpte170mEotf(workingColors); + } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return workingColors; + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) { + return workingColors; + } else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { + if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) { + return workingColors; + } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return smpte170mOetf(workingColors); + } else { + // Output red as an obviously visible error. + return vec3(1.0, 0.0, 0.0); + } } else { // Output red as an obviously visible error. return vec3(1.0, 0.0, 0.0); @@ -143,8 +174,8 @@ vec2 getAdjustedTexSamplingCoord(vec2 originalTexSamplingCoord) { void main() { vec4 inputColor = texture2D(uTexSampler, getAdjustedTexSamplingCoord(vTexSamplingCoord)); - vec3 linearInputColor = applyEotf(inputColor.rgb); - vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1); - - gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); + vec3 workingColors = convertToWorkingColors(inputColor.rgb); + vec4 transformedColors = uRgbMatrix * vec4(workingColors, 1); + gl_FragColor = + vec4(convertToOutputColors(transformedColors.rgb), inputColor.a); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 2156f5ae10..7a182c60d6 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -20,6 +20,7 @@ import static android.opengl.GLES20.GL_TRUE; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR; import android.content.Context; import android.graphics.Bitmap; @@ -38,6 +39,7 @@ import androidx.media3.common.util.GlUtil.GlException; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.Arrays; @@ -182,6 +184,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; createGlProgram( context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH); + // TODO: b/263306471 - when default working color space changes to WORKING_COLOR_SPACE_DEFAULT, + // make sure no color transfers are applied in shader. + // No transfer functions needed, because input and output are both optical colors. return new DefaultShaderProgram( glProgram, @@ -206,8 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}. * If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain * ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. - * @param enableColorTransfers Whether to transfer colors to an intermediate color space when - * applying effects. If the input or output is HDR, this must be {@code true}. + * @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in. * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an * OpenGL operation fails or is unsupported. */ @@ -215,7 +219,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean enableColorTransfers, + @WorkingColorSpace int sdrWorkingColorSpace, @InputType int inputType) throws VideoFrameProcessingException { checkState( @@ -242,7 +246,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; "uApplyHdrToSdrToneMapping", outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020 ? GL_TRUE : GL_FALSE); } - return createWithSampler(glProgram, inputColorInfo, outputColorInfo, enableColorTransfers); + return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace); } /** @@ -262,8 +266,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}. * If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain * ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not. - * @param enableColorTransfers Whether to transfer colors to an intermediate color space when - * applying effects. If the input or output is HDR, this must be {@code true}. + * @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in. * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an * OpenGL operation fails or is unsupported. */ @@ -271,7 +274,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Context context, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean enableColorTransfers) + @WorkingColorSpace int sdrWorkingColorSpace) throws VideoFrameProcessingException { boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); String vertexShaderFilePath = @@ -300,7 +303,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020 ? GL_TRUE : GL_FALSE); } - return createWithSampler(glProgram, inputColorInfo, outputColorInfo, enableColorTransfers); + return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace); } /** @@ -319,6 +322,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be * empty to apply no color transformations. * @param outputColorInfo The electrical (non-linear) {@link ColorInfo} describing output colors. + * @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in. * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an * OpenGL operation fails or is unsupported. */ @@ -327,15 +331,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; List matrixTransformations, List rgbMatrices, ColorInfo outputColorInfo, - boolean enableColorTransfers) + @WorkingColorSpace int sdrWorkingColorSpace) throws VideoFrameProcessingException { boolean outputIsHdr = ColorInfo.isTransferHdr(outputColorInfo); + boolean shouldApplyOetf = sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR; String vertexShaderFilePath = outputIsHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; String fragmentShaderFilePath = outputIsHdr ? FRAGMENT_SHADER_OETF_ES3_PATH - : enableColorTransfers + : shouldApplyOetf ? FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH : FRAGMENT_SHADER_TRANSFORMATION_PATH; GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); @@ -346,7 +351,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputColorTransfer == C.COLOR_TRANSFER_HLG || outputColorTransfer == C.COLOR_TRANSFER_ST2084); glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer); - } else if (enableColorTransfers) { + } else if (shouldApplyOetf) { checkArgument( outputColorTransfer == C.COLOR_TRANSFER_SDR || outputColorTransfer == C.COLOR_TRANSFER_GAMMA_2_2); @@ -365,7 +370,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlProgram glProgram, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean enableColorTransfers) { + @WorkingColorSpace int sdrWorkingColorSpace) { boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); boolean isExpandingColorGamut = (inputColorInfo.colorSpace == C.COLOR_SPACE_BT709 @@ -384,7 +389,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } else if (isExpandingColorGamut) { glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer); } else { - glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE); + glProgram.setIntUniform("uSdrWorkingColorSpace", sdrWorkingColorSpace); checkArgument( outputColorTransfer == C.COLOR_TRANSFER_SDR || outputColorTransfer == C.COLOR_TRANSFER_LINEAR); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 8fadcdb1f6..60304f4ca4 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -25,6 +25,7 @@ import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_REGISTER_NEW_INPUT import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_SIGNAL_ENDED; import static androidx.media3.effect.DebugTraceUtil.logEvent; import static com.google.common.collect.Iterables.getFirst; +import static java.lang.annotation.ElementType.TYPE_USE; import android.content.Context; import android.graphics.Bitmap; @@ -35,6 +36,7 @@ import android.opengl.GLES20; import android.opengl.GLES30; import android.view.Surface; import androidx.annotation.GuardedBy; +import androidx.annotation.IntDef; import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; @@ -57,6 +59,10 @@ import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -80,13 +86,52 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { void release(long presentationTimeUs); } + // LINT.IfChange(working_color_space) + /** + * Specifies the color space that frames passed to intermediate {@link GlShaderProgram}s will be + * represented in. + */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({WORKING_COLOR_SPACE_DEFAULT, WORKING_COLOR_SPACE_ORIGINAL, WORKING_COLOR_SPACE_LINEAR}) + public @interface WorkingColorSpace {} + + /** + * Use BT709 color primaries with the standard SDR transfer function (SMPTE 170m) as the working + * color space. + * + *

Any SDR content in a different color space will be transferred to this one. + */ + public static final int WORKING_COLOR_SPACE_DEFAULT = 0; + + /** + * Use the original color space of the input as the working color space when the input is SDR. + * + *

Tonemapped HDR content will be represented with BT709 color primaries and the standard SDR + * transfer function (SMPTE 170m). + * + *

No color transfers will be applied when the input is SDR. + */ + public static final int WORKING_COLOR_SPACE_ORIGINAL = 1; + + /** + * The working color space will have the same primaries as the input and a linear transfer + * function. + * + *

This option is not recommended for SDR content since it may lead to color banding since + * 8-bit colors are used in SDR processing. It may also cause effects that modify a frame's output + * colors (for example {@linkplain OverlayEffect overlays}) to have incorrect output colors. + */ + public static final int WORKING_COLOR_SPACE_LINEAR = 2; + /** A factory for {@link DefaultVideoFrameProcessor} instances. */ public static final class Factory implements VideoFrameProcessor.Factory { private static final String THREAD_NAME = "Effect:DefaultVideoFrameProcessor:GlThread"; /** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */ public static final class Builder { - private boolean enableColorTransfers; + private @WorkingColorSpace int sdrWorkingColorSpace; @Nullable private ExecutorService executorService; private @MonotonicNonNull GlObjectsProvider glObjectsProvider; private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener; @@ -95,12 +140,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { /** Creates an instance. */ public Builder() { - enableColorTransfers = true; + sdrWorkingColorSpace = WORKING_COLOR_SPACE_LINEAR; requireRegisteringAllInputFrames = true; } private Builder(Factory factory) { - enableColorTransfers = factory.enableColorTransfers; + sdrWorkingColorSpace = factory.sdrWorkingColorSpace; executorService = factory.executorService; glObjectsProvider = factory.glObjectsProvider; textureOutputListener = factory.textureOutputListener; @@ -108,22 +153,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { requireRegisteringAllInputFrames = !factory.repeatLastRegisteredFrame; } + // TODO: b/263306471 - Change default to WORKING_COLOR_SPACE_DEFAULT. /** - * Sets whether to transfer colors to an intermediate color space when applying effects. + * Sets the {@link WorkingColorSpace} in which frames passed to intermediate effects will be + * represented. * - *

The default value is {@code true}. + *

The default value is {@link #WORKING_COLOR_SPACE_LINEAR}. * - *

If the output is HDR, this is ignored as the working color space must have a linear - * transfer function. - * - *

If all input and output content will be SDR, it's recommended to set this value to - * {@code false}. This is because 8-bit colors in SDR may result in color banding. - * - *

This doesn't currently work with overlay effects (ex. {@link TextureOverlay}). + *

This setter doesn't affect the working color space for HDR output, since the working + * color space must have a linear transfer function for HDR output. */ @CanIgnoreReturnValue - public Builder setEnableColorTransfers(boolean enableColorTransfers) { - this.enableColorTransfers = enableColorTransfers; + public Builder setSdrWorkingColorSpace(@WorkingColorSpace int sdrWorkingColorSpace) { + this.sdrWorkingColorSpace = sdrWorkingColorSpace; return this; } @@ -215,7 +257,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { /** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */ public DefaultVideoFrameProcessor.Factory build() { return new DefaultVideoFrameProcessor.Factory( - enableColorTransfers, + sdrWorkingColorSpace, /* repeatLastRegisteredFrame= */ !requireRegisteringAllInputFrames, glObjectsProvider == null ? new DefaultGlObjectsProvider() : glObjectsProvider, executorService, @@ -224,7 +266,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } } - private final boolean enableColorTransfers; + private final @WorkingColorSpace int sdrWorkingColorSpace; private final boolean repeatLastRegisteredFrame; private final GlObjectsProvider glObjectsProvider; @Nullable private final ExecutorService executorService; @@ -232,13 +274,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final int textureOutputCapacity; private Factory( - boolean enableColorTransfers, + @WorkingColorSpace int sdrWorkingColorSpace, boolean repeatLastRegisteredFrame, GlObjectsProvider glObjectsProvider, @Nullable ExecutorService executorService, @Nullable GlTextureProducer.Listener textureOutputListener, int textureOutputCapacity) { - this.enableColorTransfers = enableColorTransfers; + this.sdrWorkingColorSpace = sdrWorkingColorSpace; this.repeatLastRegisteredFrame = repeatLastRegisteredFrame; this.glObjectsProvider = glObjectsProvider; this.executorService = executorService; @@ -298,7 +340,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { context, debugViewProvider, outputColorInfo, - enableColorTransfers, + sdrWorkingColorSpace, renderFramesAutomatically, videoFrameProcessingTaskExecutor, listenerExecutor, @@ -488,10 +530,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * {@link C#COLOR_TRANSFER_GAMMA_2_2}, for consistency with other tone-mapping and color behavior * in the Android ecosystem (for example, MediaFormat's COLOR_TRANSFER_SDR_VIDEO is defined as * SMPTE 170M, but most OEMs process it as Gamma 2.2). - * - *

If either {@link FrameInfo#colorInfo} or {@code outputColorInfo} {@linkplain - * ColorInfo#isTransferHdr} are HDR}, color transfers must {@linkplain - * Factory.Builder#setEnableColorTransfers be enabled}. */ @Override public void registerInputStream( @@ -666,7 +704,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Context context, DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, - boolean enableColorTransfers, + @WorkingColorSpace int sdrWorkingColorSpace, boolean renderFramesAutomatically, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor videoFrameProcessorListenerExecutor, @@ -693,7 +731,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ColorInfo intermediateColorInfo = ColorInfo.isTransferHdr(outputColorInfo) ? linearColorInfo - : enableColorTransfers ? linearColorInfo : outputColorInfo; + : sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR + ? linearColorInfo + : outputColorInfo; InputSwitcher inputSwitcher = new InputSwitcher( context, @@ -702,7 +742,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { videoFrameProcessingTaskExecutor, /* errorListenerExecutor= */ videoFrameProcessorListenerExecutor, /* samplingShaderProgramErrorListener= */ listener::onError, - enableColorTransfers, + sdrWorkingColorSpace, repeatLastRegisteredFrame); FinalShaderProgramWrapper finalShaderProgramWrapper = @@ -712,13 +752,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { eglContext, debugViewProvider, outputColorInfo, - enableColorTransfers, - renderFramesAutomatically, videoFrameProcessingTaskExecutor, videoFrameProcessorListenerExecutor, listener, textureOutputListener, - textureOutputCapacity); + textureOutputCapacity, + sdrWorkingColorSpace, + renderFramesAutomatically); return new DefaultVideoFrameProcessor( context, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 025d525645..30463ae82a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -17,6 +17,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR; import android.content.Context; import android.opengl.EGL14; @@ -44,6 +45,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; +import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; @@ -81,8 +83,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final EGLContext eglContext; private final DebugViewProvider debugViewProvider; private final ColorInfo outputColorInfo; - private final boolean enableColorTransfers; - private final boolean renderFramesAutomatically; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final Executor videoFrameProcessorListenerExecutor; private final VideoFrameProcessor.Listener videoFrameProcessorListener; @@ -91,6 +91,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool. private final LongArrayQueue syncObjects; @Nullable private final GlTextureProducer.Listener textureOutputListener; + private final @WorkingColorSpace int sdrWorkingColorSpace; + private final boolean renderFramesAutomatically; private int inputWidth; private int inputHeight; @@ -122,13 +124,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLContext eglContext, DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, - boolean enableColorTransfers, - boolean renderFramesAutomatically, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor videoFrameProcessorListenerExecutor, VideoFrameProcessor.Listener videoFrameProcessorListener, - @Nullable GlTextureProducer.Listener textureOutputListener, - int textureOutputCapacity) { + @Nullable Listener textureOutputListener, + int textureOutputCapacity, + @WorkingColorSpace int sdrWorkingColorSpace, + boolean renderFramesAutomatically) { this.context = context; this.matrixTransformations = new ArrayList<>(); this.rgbMatrices = new ArrayList<>(); @@ -136,12 +138,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.eglContext = eglContext; this.debugViewProvider = debugViewProvider; this.outputColorInfo = outputColorInfo; - this.enableColorTransfers = enableColorTransfers; - this.renderFramesAutomatically = renderFramesAutomatically; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor; this.videoFrameProcessorListener = videoFrameProcessorListener; this.textureOutputListener = textureOutputListener; + this.sdrWorkingColorSpace = sdrWorkingColorSpace; + this.renderFramesAutomatically = renderFramesAutomatically; inputListener = new InputListener() {}; availableFrames = new ConcurrentLinkedQueue<>(); @@ -525,7 +527,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; expandedMatrixTransformations, rgbMatrices, outputColorInfo, - enableColorTransfers); + sdrWorkingColorSpace); Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); if (outputSurfaceInfo != null) { @@ -545,7 +547,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .maybeRenderToSurfaceView( () -> { GlUtil.clearFocusedBuffers(); - if (enableColorTransfers) { + if (sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { @C.ColorTransfer int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer(); defaultShaderProgram.setOutputColorTransfer( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 7931cb1094..a412794a1a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -35,6 +35,7 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace; import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -51,7 +52,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; private final GlShaderProgram.ErrorListener samplingShaderProgramErrorListener; private final Executor errorListenerExecutor; private final SparseArray inputs; - private final boolean enableColorTransfers; + private final @WorkingColorSpace int sdrWorkingColorSpace; private @MonotonicNonNull GlShaderProgram downstreamShaderProgram; private @MonotonicNonNull TextureManager activeTextureManager; @@ -63,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor errorListenerExecutor, GlShaderProgram.ErrorListener samplingShaderProgramErrorListener, - boolean enableColorTransfers, + @WorkingColorSpace int sdrWorkingColorSpace, boolean repeatLastRegisteredFrame) throws VideoFrameProcessingException { this.context = context; @@ -73,7 +74,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; this.errorListenerExecutor = errorListenerExecutor; this.samplingShaderProgramErrorListener = samplingShaderProgramErrorListener; this.inputs = new SparseArray<>(); - this.enableColorTransfers = enableColorTransfers; + this.sdrWorkingColorSpace = sdrWorkingColorSpace; // TODO(b/274109008): Investigate lazy instantiating the texture managers. inputs.put( @@ -98,13 +99,13 @@ import org.checkerframework.checker.nullness.qual.Nullable; case INPUT_TYPE_SURFACE: samplingShaderProgram = DefaultShaderProgram.createWithExternalSampler( - context, inputColorInfo, outputColorInfo, enableColorTransfers); + context, inputColorInfo, outputColorInfo, sdrWorkingColorSpace); break; case INPUT_TYPE_BITMAP: case INPUT_TYPE_TEXTURE_ID: samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( - context, inputColorInfo, outputColorInfo, enableColorTransfers, inputType); + context, inputColorInfo, outputColorInfo, sdrWorkingColorSpace, inputType); break; default: throw new VideoFrameProcessingException("Unsupported input type " + inputType); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java index ad4c41ceab..308732c9ce 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer.mh; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL; import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat; import static androidx.media3.transformer.AndroidTestUtil.FORCE_TRANSCODE_VIDEO_EFFECTS; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10; @@ -210,7 +211,7 @@ public final class HdrEditingTest { } @Test - public void exportAndTranscodeHdr_withDisabledColorTransfers_whenHdrEditingIsSupported() + public void exportAndTranscodeHdr_ignoringSdrWorkingColorSpace_whenHdrEditingIsSupported() throws Exception { Context context = ApplicationProvider.getApplicationContext(); Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; @@ -222,7 +223,7 @@ public final class HdrEditingTest { new Transformer.Builder(context) .setVideoFrameProcessorFactory( new DefaultVideoFrameProcessor.Factory.Builder() - .setEnableColorTransfers(false) + .setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL) .build()) .build(); EditedMediaItem editedMediaItem = diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java index 796735e6fd..dfb1ef9c1e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer.mh; import static androidx.media3.common.MimeTypes.VIDEO_H265; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL; import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; @@ -280,14 +281,14 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { } @Test - public void toneMap_withDisabledColorTransfers_matchesGoldenFile() throws Exception { + public void toneMap_withWorkingColorSpaceSetToOriginal_matchesGoldenFile() throws Exception { assumeDeviceSupportsOpenGlToneMapping(testId, HLG_ASSET_FORMAT); videoFrameProcessorTestRunner = new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory( new DefaultVideoFrameProcessor.Factory.Builder() - .setEnableColorTransfers(false) + .setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL) .build()) .setVideoAssetPath(HLG_ASSET_STRING) .setOutputColorInfo(TONE_MAP_SDR_COLOR)