sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-16 13:05:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Effect: Add support for disabling color transfers for SDR

Browse source 
PiperOrigin-RevId: 521805477
This commit is contained in:
tofunmi 2023-04-04 18:42:23 +01:00 committed by Marc Baechinger
parent c3f25651bd
commit 08a9ccc743
5 changed files with 145 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
View file

@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOutputColorTransfer;
uniform int uEnableColorTransfer;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
const int GL_FALSE = 0;
const int GL_TRUE = 1;
// Transforms a single channel from electrical to optical SDR using the SMPTE
// 170M OETF.
@ -77,7 +80,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
|| uEnableColorTransfer == GL_FALSE) {
return linearColor;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(linearColor);
@ -87,9 +91,20 @@ highp vec3 applyOetf(highp vec3 linearColor) {
}
}
vec3 applyEotf(vec3 electricalColor){
if (uEnableColorTransfer == GL_TRUE){
return smpte170mEotf(electricalColor);
} else if (uEnableColorTransfer == GL_FALSE) {
return electricalColor;
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
}
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
vec3 linearInputColor = smpte170mEotf(inputColor.rgb);
vec3 linearInputColor = applyEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);

19
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl
View file

@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOutputColorTransfer;
uniform int uEnableColorTransfer;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
const int GL_FALSE = 0;
const int GL_TRUE = 1;
// Transforms a single channel from electrical to optical SDR using the sRGB
// EOTF.
@ -78,7 +81,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
|| uEnableColorTransfer == GL_FALSE) {
return linearColor;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(linearColor);
@ -88,6 +92,17 @@ highp vec3 applyOetf(highp vec3 linearColor) {
}
}
vec3 applyEotf(vec3 electricalColor){
if (uEnableColorTransfer == GL_TRUE){
return srgbEotf(electricalColor) ;
} else if (uEnableColorTransfer == GL_FALSE) {
return electricalColor;
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
}
void main() {
vec2 vTexSamplingCoordFlipped =
vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y);
@ -96,7 +111,7 @@ void main() {
// texture gets flipped. We flip the texture vertically to ensure the
// orientation of the output is correct.
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoordFlipped);
vec3 linearInputColor = srgbEotf(inputColor.rgb);
vec3 linearInputColor = applyEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);

28
libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java
View file

@ -92,6 +92,9 @@ import java.util.List;
1.6853f, -0.6530f, 0.0000f,
};
private static final int GL_FALSE = 0;
private static final int GL_TRUE = 1;
/** The {@link MatrixTransformation MatrixTransformations} to apply. */
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
/** The {@link RgbMatrix RgbMatrices} to apply. */
@ -185,6 +188,8 @@ import java.util.List;
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @param enableColorTransfers Whether to transfer colors to an intermediate color space when
* applying effects. If the input or output is HDR, this must be {@code true}.
* @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
* OpenGL operation fails or is unsupported.
*/
@ -193,7 +198,8 @@ import java.util.List;
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
ColorInfo outputColorInfo,
boolean enableColorTransfers)
throws VideoFrameProcessingException {
checkState(
!ColorInfo.isTransferHdr(inputColorInfo),
@ -204,7 +210,12 @@ import java.util.List;
VERTEX_SHADER_TRANSFORMATION_PATH,
FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH);
return createWithSampler(
glProgram, matrixTransformations, rgbMatrices, inputColorInfo, outputColorInfo);
glProgram,
matrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo,
enableColorTransfers);
}
/**
@ -229,6 +240,8 @@ import java.util.List;
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @param enableColorTransfers Whether to transfer colors to an intermediate color space when
* applying effects. If the input or output is HDR, this must be {@code true}.
* @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
* OpenGL operation fails or is unsupported.
*/
@ -237,7 +250,8 @@ import java.util.List;
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
ColorInfo outputColorInfo,
boolean enableColorTransfers)
throws VideoFrameProcessingException {
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
String vertexShaderFilePath =
@ -253,7 +267,8 @@ import java.util.List;
matrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo);
outputColorInfo,
enableColorTransfers);
}
/**
@ -316,12 +331,15 @@ import java.util.List;
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
ColorInfo outputColorInfo,
boolean enableColorTransfers)
throws VideoFrameProcessingException {
glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE);
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
@C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
if (isInputTransferHdr) {
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
checkArgument(enableColorTransfers);
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {

74
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View file

@ -47,6 +47,7 @@ import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
@ -64,9 +65,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
public static final class Factory implements VideoFrameProcessor.Factory {
/** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */
public static final class Builder {
private boolean enableColorTransfers;
/** Creates an instance. */
public Builder() {
enableColorTransfers = true;
}
/**
* Sets whether to transfer colors to an intermediate color space when applying effects.
*
* <p>If the input or output is HDR, this must be {@code true}.
*/
@CanIgnoreReturnValue
public DefaultVideoFrameProcessor.Factory.Builder setEnableColorTransfers(
boolean enableColorTransfers) {
this.enableColorTransfers = enableColorTransfers;
return this;
}
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
public DefaultVideoFrameProcessor.Factory build() {
return new DefaultVideoFrameProcessor.Factory(enableColorTransfers);
}
}
/** Whether to transfer colors to an intermediate color space when applying effects. */
public final boolean enableColorTransfers;
private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
private boolean outputToTexture;
private Factory(boolean enableColorTransfers) {
this.enableColorTransfers = enableColorTransfers;
}
// TODO(276913828): Remove and change all calls to a builder.
/**
* @deprecated Use {@link DefaultVideoFrameProcessor.Factory.Builder} instead.
*/
@Deprecated
public Factory() {
this(/* enableColorTransfers= */ true);
}
// TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
/**
* {@inheritDoc}
*
@ -79,10 +125,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return this;
}
// TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
/**
* Sets whether to output to a texture for testing.
*
* <p>Must be called before {@link #create}.
* <p>Must be called before {@link VideoFrameProcessor.Factory#create}.
*
* <p>The default value is {@code false}.
*/
@ -114,6 +161,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* are HDR}, textures will use {@link GLES30#GL_RGBA16F} and {@link GLES30#GL_HALF_FLOAT}.
* Otherwise, textures will use {@link GLES20#GL_RGBA} and {@link GLES20#GL_UNSIGNED_BYTE}.
*
* <p>If {@code inputColorInfo} or {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr}
* are HDR}, color transfers must be enabled.
*
* <p>If {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr is HDR}, the context will
* be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_1010102}. Otherwise, the context
* will be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_8888}.
@ -139,6 +189,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
checkArgument(outputColorInfo.isValid());
checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
if (ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) {
checkArgument(enableColorTransfers);
}
if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
|| ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
@ -163,6 +216,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
debugViewProvider,
inputColorInfo,
outputColorInfo,
enableColorTransfers,
isInputTextureExternal,
releaseFramesAutomatically,
singleThreadExecutorService,
@ -407,6 +461,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean isInputTextureExternal,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
@ -452,6 +507,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
debugViewProvider,
inputColorInfo,
outputColorInfo,
enableColorTransfers,
isInputTextureExternal,
releaseFramesAutomatically,
executor,
@ -492,6 +548,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean isInputTextureExternal,
boolean releaseFramesAutomatically,
Executor executor,
@ -537,11 +594,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
if (isInputTextureExternal) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
context,
matrixTransformations,
rgbMatrices,
inputColorInfo,
linearColorInfo,
enableColorTransfers);
} else {
defaultShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
context,
matrixTransformations,
rgbMatrices,
inputColorInfo,
linearColorInfo,
enableColorTransfers);
}
} else {
defaultShaderProgram =
@ -566,6 +633,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
debugViewProvider,
/* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
outputColorInfo,
enableColorTransfers,
sampleFromInputTexture,
isInputTextureExternal,
releaseFramesAutomatically,

25
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View file

@ -77,6 +77,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final boolean isInputTextureExternal;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
private final boolean releaseFramesAutomatically;
private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
@ -115,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean sampleFromInputTexture,
boolean isInputTextureExternal,
boolean releaseFramesAutomatically,
@ -132,6 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.isInputTextureExternal = isInputTextureExternal;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
this.releaseFramesAutomatically = releaseFramesAutomatically;
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
this.videoFrameProcessorListener = videoFrameProcessorListener;
@ -491,7 +494,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
expandedMatrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo);
outputColorInfo,
enableColorTransfers);
} else {
defaultShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
@ -499,7 +503,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
expandedMatrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo);
outputColorInfo,
enableColorTransfers);
}
} else {
defaultShaderProgram =
@ -522,12 +527,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.maybeRenderToSurfaceView(
() -> {
GlUtil.clearOutputFrame();
@C.ColorTransfer
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(
debugSurfaceViewWrapper.outputColorTransfer);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
if (enableColorTransfers) {
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
} else {
@C.ColorTransfer
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(
debugSurfaceViewWrapper.outputColorTransfer);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
}
},
glObjectsProvider);
} catch (VideoFrameProcessingException | GlUtil.GlException e) {