From 178a3238970139b20905840cc8bb420dabd82dce Mon Sep 17 00:00:00 2001 From: tofunmi Date: Wed, 26 Apr 2023 16:19:19 +0100 Subject: [PATCH] Update sdr internal fsh to accept input color transfer PiperOrigin-RevId: 527271212 --- ...hader_transformation_sdr_internal_es2.glsl | 72 ++++++++++++++----- .../media3/effect/DefaultShaderProgram.java | 8 ++- .../effect/DefaultVideoFrameProcessor.java | 3 +- .../effect/FinalShaderProgramWrapper.java | 3 +- 4 files changed, 64 insertions(+), 22 deletions(-) diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl index da26704bd7..4fb21bfec0 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl @@ -19,7 +19,7 @@ // texture created from a bitmap), with uTexSampler copying from this texture // to the current output. // 2. Transforms the electrical colors to optical colors using the SMPTE 170M -// EOTF. +// EOTF or the sRGB EOTF, as requested by uInputColorTransfer. // 3. Applies a 4x4 RGB color matrix to change the pixel colors. // 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR // for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to @@ -31,6 +31,7 @@ uniform mat4 uRgbMatrix; varying vec2 vTexSamplingCoord; // C.java#ColorTransfer value. // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed. +uniform int uInputColorTransfer; uniform int uOutputColorTransfer; uniform int uEnableColorTransfer; @@ -38,6 +39,10 @@ const float inverseGamma = 0.4500; const float gamma = 1.0 / inverseGamma; const int GL_FALSE = 0; const int GL_TRUE = 1; +// LINT.IfChange(color_transfer) +const int COLOR_TRANSFER_LINEAR = 1; +const int COLOR_TRANSFER_SRGB = 2; +const int COLOR_TRANSFER_SDR_VIDEO = 3; // Transforms a single channel from electrical to optical SDR using the sRGB // EOTF. @@ -58,6 +63,24 @@ vec3 srgbEotf(const vec3 electricalColor) { ); } +// Transforms a single channel from electrical to optical SDR using the SMPTE +// 170M OETF. +float smpte170mEotfSingleChannel(float electricalChannel) { + // Specification: + // https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en + return electricalChannel < 0.0812 + ? electricalChannel / 4.500 + : pow((electricalChannel + 0.099) / 1.099, gamma); +} + +// Transforms electrical to optical SDR using the SMPTE 170M EOTF. +vec3 smpte170mEotf(vec3 electricalColor) { + return vec3( + smpte170mEotfSingleChannel(electricalColor.r), + smpte170mEotfSingleChannel(electricalColor.g), + smpte170mEotfSingleChannel(electricalColor.b)); +} + // Transforms a single channel from optical to electrical SDR. float smpte170mOetfSingleChannel(float opticalChannel) { // Specification: @@ -74,13 +97,29 @@ vec3 smpte170mOetf(vec3 opticalColor) { smpte170mOetfSingleChannel(opticalColor.g), smpte170mOetfSingleChannel(opticalColor.b)); } +// Applies the appropriate EOTF to convert nonlinear electrical signals to linear +// optical signals. Input and output are both normalized to [0, 1]. +vec3 applyEotf(vec3 electricalColor){ + if (uEnableColorTransfer == GL_TRUE){ + if (uInputColorTransfer == COLOR_TRANSFER_SRGB){ + return srgbEotf(electricalColor) ; + } else if (uInputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) { + return smpte170mEotf(electricalColor); + } else { + // Output blue as an obviously visible error. + return vec3(0.0, 0.0, 1.0); + } + } else if (uEnableColorTransfer == GL_FALSE) { + return electricalColor; + } else { + // Output blue as an obviously visible error. + return vec3(0.0, 0.0, 1.0); + } +} // Applies the appropriate OETF to convert linear optical signals to nonlinear // electrical signals. Input and output are both normalized to [0, 1]. highp vec3 applyOetf(highp vec3 linearColor) { - // LINT.IfChange(color_transfer) - const int COLOR_TRANSFER_LINEAR = 1; - const int COLOR_TRANSFER_SDR_VIDEO = 3; if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR || uEnableColorTransfer == GL_FALSE) { return linearColor; @@ -92,27 +131,22 @@ highp vec3 applyOetf(highp vec3 linearColor) { } } -vec3 applyEotf(vec3 electricalColor){ - if (uEnableColorTransfer == GL_TRUE){ - return srgbEotf(electricalColor) ; - } else if (uEnableColorTransfer == GL_FALSE) { - return electricalColor; +vec2 getAdjustedTexSamplingCoord(vec2 originalTexSamplingCoord){ + if (uInputColorTransfer == COLOR_TRANSFER_SRGB){ + // Whereas the Android system uses the top-left corner as (0,0) of the + // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the + // texture gets flipped. We flip the texture vertically to ensure the + // orientation of the output is correct. + return vec2(originalTexSamplingCoord.x, 1.0 - originalTexSamplingCoord.y); } else { - // Output blue as an obviously visible error. - return vec3(0.0, 0.0, 1.0); + return originalTexSamplingCoord; } } void main() { - vec2 vTexSamplingCoordFlipped = - vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y); - // Whereas the Android system uses the top-left corner as (0,0) of the - // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the - // texture gets flipped. We flip the texture vertically to ensure the - // orientation of the output is correct. - vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoordFlipped); + vec4 inputColor = texture2D( + uTexSampler, getAdjustedTexSamplingCoord(vTexSamplingCoord)); vec3 linearInputColor = applyEotf(inputColor.rgb); - vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1); gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 42defabd3a..0e1a94deea 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -15,6 +15,7 @@ */ package androidx.media3.effect; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; @@ -25,6 +26,7 @@ import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor.InputType; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; @@ -199,16 +201,20 @@ import java.util.List; List rgbMatrices, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean enableColorTransfers) + boolean enableColorTransfers, + @InputType int inputType) throws VideoFrameProcessingException { checkState( !ColorInfo.isTransferHdr(inputColorInfo), "DefaultShaderProgram doesn't support HDR internal sampler input yet."); + checkState( + inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB || inputType == INPUT_TYPE_BITMAP); GlProgram glProgram = createGlProgram( context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH); + glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer); return createWithSampler( glProgram, matrixTransformations, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index adda1683eb..33bc03137d 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -595,7 +595,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { rgbMatrices, inputColorInfo, linearColorInfo, - enableColorTransfers); + enableColorTransfers, + inputType); } } else { defaultShaderProgram = diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 5cdda92e6c..d06a3ac044 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -477,7 +477,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; rgbMatrices, inputColorInfo, outputColorInfo, - enableColorTransfers); + enableColorTransfers, + inputType); } } else { defaultShaderProgram =