Effect: Add support for disabling color transfers for SDR

PiperOrigin-RevId: 521805477

libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl

@@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
 // C.java#ColorTransfer value.
 // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
 uniform int uOutputColorTransfer;
+uniform int uEnableColorTransfer;
 
 const float inverseGamma = 0.4500;
 const float gamma = 1.0 / inverseGamma;
+const int GL_FALSE = 0;
+const int GL_TRUE = 1;
 
 // Transforms a single channel from electrical to optical SDR using the SMPTE 
 // 170M OETF.
@@ -77,7 +80,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
   // LINT.IfChange(color_transfer)
   const int COLOR_TRANSFER_LINEAR = 1;
   const int COLOR_TRANSFER_SDR_VIDEO = 3;
-  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
+  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
+    || uEnableColorTransfer == GL_FALSE) {
     return linearColor;
   } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
     return smpte170mOetf(linearColor);
@@ -87,9 +91,20 @@ highp vec3 applyOetf(highp vec3 linearColor) {
   }
 }
 
+vec3 applyEotf(vec3 electricalColor){
+  if (uEnableColorTransfer == GL_TRUE){
+    return smpte170mEotf(electricalColor);
+  } else if (uEnableColorTransfer == GL_FALSE) {
+    return electricalColor;
+  } else {
+    // Output blue as an obviously visible error.
+    return vec3(0.0, 0.0, 1.0);
+  }
+}
+
 void main() {
   vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
-  vec3 linearInputColor = smpte170mEotf(inputColor.rgb);
+  vec3 linearInputColor = applyEotf(inputColor.rgb);
 
   vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
 

libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl

@@ -32,9 +32,12 @@ varying vec2 vTexSamplingCoord;
 // C.java#ColorTransfer value.
 // Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
 uniform int uOutputColorTransfer;
+uniform int uEnableColorTransfer;
 
 const float inverseGamma = 0.4500;
 const float gamma = 1.0 / inverseGamma;
+const int GL_FALSE = 0;
+const int GL_TRUE = 1;
 
 // Transforms a single channel from electrical to optical SDR using the sRGB
 // EOTF.
@@ -78,7 +81,8 @@ highp vec3 applyOetf(highp vec3 linearColor) {
   // LINT.IfChange(color_transfer)
   const int COLOR_TRANSFER_LINEAR = 1;
   const int COLOR_TRANSFER_SDR_VIDEO = 3;
-  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
+  if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR
+    || uEnableColorTransfer == GL_FALSE) {
     return linearColor;
   } else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
     return smpte170mOetf(linearColor);
@@ -88,6 +92,17 @@ highp vec3 applyOetf(highp vec3 linearColor) {
   }
 }
 
+vec3 applyEotf(vec3 electricalColor){
+  if (uEnableColorTransfer == GL_TRUE){
+    return srgbEotf(electricalColor) ;
+  } else if (uEnableColorTransfer == GL_FALSE) {
+    return electricalColor;
+  } else {
+    // Output blue as an obviously visible error.
+    return vec3(0.0, 0.0, 1.0);
+  }
+}
+
 void main() {
   vec2 vTexSamplingCoordFlipped =
       vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y);
@@ -96,7 +111,7 @@ void main() {
   // texture gets flipped. We flip the texture vertically to ensure the
   // orientation of the output is correct.
   vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoordFlipped);
-  vec3 linearInputColor = srgbEotf(inputColor.rgb);
+  vec3 linearInputColor = applyEotf(inputColor.rgb);
 
   vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
 

libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java

@@ -92,6 +92,9 @@ import java.util.List;
     1.6853f, -0.6530f, 0.0000f,
   };
 
+  private static final int GL_FALSE = 0;
+  private static final int GL_TRUE = 1;
+
   /** The {@link MatrixTransformation MatrixTransformations} to apply. */
   private final ImmutableList<GlMatrixTransformation> matrixTransformations;
   /** The {@link RgbMatrix RgbMatrices} to apply. */
@@ -185,6 +188,8 @@ import java.util.List;
    * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
    *     If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
    *     ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
+   * @param enableColorTransfers Whether to transfer colors to an intermediate color space when
+   *     applying effects. If the input or output is HDR, this must be {@code true}.
    * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
    *     OpenGL operation fails or is unsupported.
    */
@@ -193,7 +198,8 @@ import java.util.List;
       List<GlMatrixTransformation> matrixTransformations,
       List<RgbMatrix> rgbMatrices,
       ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
       throws VideoFrameProcessingException {
     checkState(
         !ColorInfo.isTransferHdr(inputColorInfo),
@@ -204,7 +210,12 @@ import java.util.List;
             VERTEX_SHADER_TRANSFORMATION_PATH,
             FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH);
     return createWithSampler(
-        glProgram, matrixTransformations, rgbMatrices, inputColorInfo, outputColorInfo);
+        glProgram,
+        matrixTransformations,
+        rgbMatrices,
+        inputColorInfo,
+        outputColorInfo,
+        enableColorTransfers);
   }
 
   /**
@@ -229,6 +240,8 @@ import java.util.List;
    * @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
    *     If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
    *     ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
+   * @param enableColorTransfers Whether to transfer colors to an intermediate color space when
+   *     applying effects. If the input or output is HDR, this must be {@code true}.
    * @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
    *     OpenGL operation fails or is unsupported.
    */
@@ -237,7 +250,8 @@ import java.util.List;
       List<GlMatrixTransformation> matrixTransformations,
       List<RgbMatrix> rgbMatrices,
       ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
       throws VideoFrameProcessingException {
     boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
     String vertexShaderFilePath =
@@ -253,7 +267,8 @@ import java.util.List;
         matrixTransformations,
         rgbMatrices,
         inputColorInfo,
-        outputColorInfo);
+        outputColorInfo,
+        enableColorTransfers);
   }
 
   /**
@@ -316,12 +331,15 @@ import java.util.List;
       List<GlMatrixTransformation> matrixTransformations,
       List<RgbMatrix> rgbMatrices,
       ColorInfo inputColorInfo,
-      ColorInfo outputColorInfo)
+      ColorInfo outputColorInfo,
+      boolean enableColorTransfers)
       throws VideoFrameProcessingException {
+    glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE);
     boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
     @C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
     if (isInputTransferHdr) {
       checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
+      checkArgument(enableColorTransfers);
 
       // In HDR editing mode the decoder output is sampled in YUV.
       if (!GlUtil.isYuvTargetExtensionSupported()) {

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -47,6 +47,7 @@ import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -64,9 +65,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
 
   /** A factory for {@link DefaultVideoFrameProcessor} instances. */
   public static final class Factory implements VideoFrameProcessor.Factory {
+
+    /** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */
+    public static final class Builder {
+      private boolean enableColorTransfers;
+
+      /** Creates an instance. */
+      public Builder() {
+        enableColorTransfers = true;
+      }
+
+      /**
+       * Sets whether to transfer colors to an intermediate color space when applying effects.
+       *
+       * <p>If the input or output is HDR, this must be {@code true}.
+       */
+      @CanIgnoreReturnValue
+      public DefaultVideoFrameProcessor.Factory.Builder setEnableColorTransfers(
+          boolean enableColorTransfers) {
+        this.enableColorTransfers = enableColorTransfers;
+        return this;
+      }
+
+      /** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
+      public DefaultVideoFrameProcessor.Factory build() {
+        return new DefaultVideoFrameProcessor.Factory(enableColorTransfers);
+      }
+    }
+
+    /** Whether to transfer colors to an intermediate color space when applying effects. */
+    public final boolean enableColorTransfers;
+
     private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
     private boolean outputToTexture;
 
+    private Factory(boolean enableColorTransfers) {
+      this.enableColorTransfers = enableColorTransfers;
+    }
+
+    // TODO(276913828): Remove and change all calls to a builder.
+    /**
+     * @deprecated Use {@link DefaultVideoFrameProcessor.Factory.Builder} instead.
+     */
+    @Deprecated
+    public Factory() {
+      this(/* enableColorTransfers= */ true);
+    }
+
+    // TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
     /**
      * {@inheritDoc}
      *
@@ -79,10 +125,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       return this;
     }
 
+    // TODO(276913828): Move this setter to the DefaultVideoFrameProcessor.Factory.Builder.
     /**
      * Sets whether to output to a texture for testing.
      *
-     * <p>Must be called before {@link #create}.
+     * <p>Must be called before {@link VideoFrameProcessor.Factory#create}.
      *
      * <p>The default value is {@code false}.
      */
@@ -114,6 +161,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
      * are HDR}, textures will use {@link GLES30#GL_RGBA16F} and {@link GLES30#GL_HALF_FLOAT}.
      * Otherwise, textures will use {@link GLES20#GL_RGBA} and {@link GLES20#GL_UNSIGNED_BYTE}.
      *
+     * <p>If {@code inputColorInfo} or {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr}
+     * are HDR}, color transfers must be enabled.
+     *
      * <p>If {@code outputColorInfo} {@linkplain ColorInfo#isTransferHdr is HDR}, the context will
      * be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_1010102}. Otherwise, the context
      * will be configured with {@link GlUtil#EGL_CONFIG_ATTRIBUTES_RGBA_8888}.
@@ -139,6 +189,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
       checkArgument(outputColorInfo.isValid());
       checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
+      if (ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo)) {
+        checkArgument(enableColorTransfers);
+      }
 
       if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
           || ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
@@ -163,6 +216,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
                       debugViewProvider,
                       inputColorInfo,
                       outputColorInfo,
+                      enableColorTransfers,
                       isInputTextureExternal,
                       releaseFramesAutomatically,
                       singleThreadExecutorService,
@@ -407,6 +461,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       DebugViewProvider debugViewProvider,
       ColorInfo inputColorInfo,
       ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
       boolean isInputTextureExternal,
       boolean releaseFramesAutomatically,
       ExecutorService singleThreadExecutorService,
@@ -452,6 +507,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
             debugViewProvider,
             inputColorInfo,
             outputColorInfo,
+            enableColorTransfers,
             isInputTextureExternal,
             releaseFramesAutomatically,
             executor,
@@ -492,6 +548,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
       DebugViewProvider debugViewProvider,
       ColorInfo inputColorInfo,
       ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
       boolean isInputTextureExternal,
       boolean releaseFramesAutomatically,
       Executor executor,
@@ -537,11 +594,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
           if (isInputTextureExternal) {
             defaultShaderProgram =
                 DefaultShaderProgram.createWithExternalSampler(
-                    context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
+                    context,
+                    matrixTransformations,
+                    rgbMatrices,
+                    inputColorInfo,
+                    linearColorInfo,
+                    enableColorTransfers);
           } else {
             defaultShaderProgram =
                 DefaultShaderProgram.createWithInternalSampler(
-                    context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
+                    context,
+                    matrixTransformations,
+                    rgbMatrices,
+                    inputColorInfo,
+                    linearColorInfo,
+                    enableColorTransfers);
           }
         } else {
           defaultShaderProgram =
@@ -566,6 +633,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
             debugViewProvider,
             /* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
             outputColorInfo,
+            enableColorTransfers,
             sampleFromInputTexture,
             isInputTextureExternal,
             releaseFramesAutomatically,

libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java

@@ -77,6 +77,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final boolean isInputTextureExternal;
   private final ColorInfo inputColorInfo;
   private final ColorInfo outputColorInfo;
+  private final boolean enableColorTransfers;
   private final boolean releaseFramesAutomatically;
   private final Executor videoFrameProcessorListenerExecutor;
   private final VideoFrameProcessor.Listener videoFrameProcessorListener;
@@ -115,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       DebugViewProvider debugViewProvider,
       ColorInfo inputColorInfo,
       ColorInfo outputColorInfo,
+      boolean enableColorTransfers,
       boolean sampleFromInputTexture,
       boolean isInputTextureExternal,
       boolean releaseFramesAutomatically,
@@ -132,6 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.isInputTextureExternal = isInputTextureExternal;
     this.inputColorInfo = inputColorInfo;
     this.outputColorInfo = outputColorInfo;
+    this.enableColorTransfers = enableColorTransfers;
     this.releaseFramesAutomatically = releaseFramesAutomatically;
     this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
     this.videoFrameProcessorListener = videoFrameProcessorListener;
@@ -491,7 +494,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 expandedMatrixTransformations,
                 rgbMatrices,
                 inputColorInfo,
-                outputColorInfo);
+                outputColorInfo,
+                enableColorTransfers);
       } else {
         defaultShaderProgram =
             DefaultShaderProgram.createWithInternalSampler(
@@ -499,7 +503,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 expandedMatrixTransformations,
                 rgbMatrices,
                 inputColorInfo,
-                outputColorInfo);
+                outputColorInfo,
+                enableColorTransfers);
       }
     } else {
       defaultShaderProgram =
@@ -522,12 +527,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
           .maybeRenderToSurfaceView(
               () -> {
                 GlUtil.clearOutputFrame();
-                @C.ColorTransfer
+                if (enableColorTransfers) {
-                int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
+                  defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
-                defaultShaderProgram.setOutputColorTransfer(
+                } else {
-                    debugSurfaceViewWrapper.outputColorTransfer);
+                  @C.ColorTransfer
-                defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
+                  int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
-                defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
+                  defaultShaderProgram.setOutputColorTransfer(
+                      debugSurfaceViewWrapper.outputColorTransfer);
+                  defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
+                  defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
+                }
               },
               glObjectsProvider);
     } catch (VideoFrameProcessingException | GlUtil.GlException e) {