From 84c81b8575f7a366bf81e6c216641ad32c708887 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Tue, 20 Dec 2022 18:52:03 +0000 Subject: [PATCH] HDR: Implement Transformer HDR to SDR GL tone-mapping API Note that we simply use GlEffectsFrameProcessor in-app / GL tone-mapping, so PQ->SDR tone-mapping isn't yet implemented. Tested manually using the demo on Pixel 7, to confirm that device and in-app tone mapping behave similarly. PiperOrigin-RevId: 496700231 --- .../transformer/ConfigurationActivity.java | 7 +- .../media3/transformer/mh/HdrEditingTest.java | 4 +- ...> ToneMapHdrToSdrUsingMediaCodecTest.java} | 15 +++-- .../transformer/TransformationRequest.java | 47 ++++++++++---- .../VideoTranscodingSamplePipeline.java | 64 ++++++++++++++----- 5 files changed, 96 insertions(+), 41 deletions(-) rename libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/{HdrToSdrToneMapTest.java => ToneMapHdrToSdrUsingMediaCodecTest.java} (96%) diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 9bd5d4be39..5710bba348 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -176,7 +176,12 @@ public final class ConfigurationActivity extends AppCompatActivity { HDR_MODE_DESCRIPTIONS = new ImmutableMap.Builder() .put("Keep HDR", TransformationRequest.HDR_MODE_KEEP_HDR) - .put("Tone-map HDR to SDR", TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR) + .put( + "MediaCodec tone-map HDR to SDR", + TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC) + .put( + "OpenGL tone-map HDR to SDR", + TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) .put( "Force Interpret HDR as SDR", TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java index 2a3449a03d..fb04691e66 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java @@ -183,7 +183,7 @@ public class HdrEditingTest { .isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR); isToneMappingFallbackApplied.set( fallbackTransformationRequest.hdrMode - == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR); + == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC); } }) .build(); @@ -236,7 +236,7 @@ public class HdrEditingTest { .isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR); isToneMappingFallbackApplied.set( fallbackTransformationRequest.hdrMode - == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR); + == TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC); } }) .build(); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrToSdrToneMapTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingMediaCodecTest.java similarity index 96% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrToSdrToneMapTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingMediaCodecTest.java index 4b171ed129..683d14d2c8 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrToSdrToneMapTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingMediaCodecTest.java @@ -38,11 +38,12 @@ import org.junit.runner.RunWith; /** * {@link Transformer} instrumentation test for applying an {@linkplain - * TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR HDR to SDR tone mapping edit}. + * TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC HDR to SDR tone mapping + * edit}. */ @RunWith(AndroidJUnit4.class) -public class HdrToSdrToneMapTest { - public static final String TAG = "HdrToSdrToneMapTest"; +public class ToneMapHdrToSdrUsingMediaCodecTest { + public static final String TAG = "ToneMapHdrToSdrUsingMediaCodecTest"; @Test public void transform_toneMapNoRequestedTranscode_hdr10File_toneMapsOrThrows() throws Exception { @@ -53,7 +54,7 @@ public class HdrToSdrToneMapTest { new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR) + .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC) .build()) .addListener( new Transformer.Listener() { @@ -98,7 +99,7 @@ public class HdrToSdrToneMapTest { new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR) + .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC) .build()) .addListener( new Transformer.Listener() { @@ -143,7 +144,7 @@ public class HdrToSdrToneMapTest { new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR) + .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC) .setRotationDegrees(180) .build()) .addListener( @@ -189,7 +190,7 @@ public class HdrToSdrToneMapTest { new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR) + .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC) .setRotationDegrees(180) .build()) .addListener( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index bc193f6970..a1c82595c2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -40,7 +40,8 @@ public final class TransformationRequest { /** * The strategy to use to transcode or edit High Dynamic Range (HDR) input video. * - *

One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR}, or {@link + *

One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, + * {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL}, or {@link * #HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. * *

Standard Dynamic Range (SDR) input video is unaffected by these settings. @@ -50,8 +51,9 @@ public final class TransformationRequest { @Target(TYPE_USE) @IntDef({ HDR_MODE_KEEP_HDR, - HDR_MODE_TONE_MAP_HDR_TO_SDR, - HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR + HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC, + HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL, + HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR, }) public @interface HdrMode {} /** @@ -60,26 +62,43 @@ public final class TransformationRequest { *

Supported on API 31+, by some device and HDR format combinations. * *

If not supported, {@link Transformer} may fall back to {@link - * #HDR_MODE_TONE_MAP_HDR_TO_SDR}. + * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}. */ public static final int HDR_MODE_KEEP_HDR = 0; /** - * Tone map HDR input to SDR before processing, to generate SDR output. + * Tone map HDR input to SDR before processing, to generate SDR output, using the {@link + * android.media.MediaCodec} decoder tone-mapper. * *

Supported on API 31+, by some device and HDR format combinations. Tone-mapping is only * guaranteed to be supported from Android T onwards. * - *

If not supported, {@link Transformer} may throw a {@link TransformationException}. + *

If not supported, {@link Transformer} throws a {@link TransformationException}. */ - public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR = 1; + public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC = 1; /** - * Interpret HDR input as SDR, resulting in washed out video. + * Tone map HDR input to SDR before processing, to generate SDR output, using an OpenGL + * tone-mapper. + * + *

Supported on API 29+, for HLG input. + * + *

This may exhibit mild differences from {@link + * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, depending on the device's tone-mapping + * implementation, but should have much wider support and have more consistent results across + * devices. + * + *

If not supported, {@link Transformer} throws a {@link TransformationException}. + */ + // TODO(b/239735341): Implement PQ tone-mapping to remove the HLG reference. + public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL = 2; + /** + * Interpret HDR input as SDR, likely with a washed out look. * *

Supported on API 29+. * *

This is much more widely supported than {@link #HDR_MODE_KEEP_HDR} and {@link - * #HDR_MODE_TONE_MAP_HDR_TO_SDR}. However, as HDR transfer functions and metadata will be - * ignored, contents will be displayed incorrectly, likely with a washed out look. + * #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}. However, as HDR transfer functions and + * metadata will be ignored, contents will be displayed incorrectly, likely with a washed out + * look. * *

Use of this flag may result in {@code * TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED} or {@code @@ -87,7 +106,7 @@ public final class TransformationRequest { * *

This field is experimental, and will be renamed or removed in a future release. */ - public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 2; + public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 3; /** A builder for {@link TransformationRequest} instances. */ public static final class Builder { @@ -285,14 +304,14 @@ public final class TransformationRequest { /** * @deprecated This method is now a no-op if {@code false}, and sets {@code - * setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR)} if {@code true}. Use {@link #setHdrMode} with - * {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR} instead. + * setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)} if {@code true}. Use {@link + * #setHdrMode} with {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} instead. */ @Deprecated @CanIgnoreReturnValue public Builder setEnableRequestSdrToneMapping(boolean enableRequestSdrToneMapping) { if (enableRequestSdrToneMapping) { - return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR); + return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC); } return this; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index cbfa876055..4823dc2419 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -19,6 +19,10 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.SDK_INT; +import static androidx.media3.transformer.TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; +import static androidx.media3.transformer.TransformationRequest.HDR_MODE_KEEP_HDR; +import static androidx.media3.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; +import static androidx.media3.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import android.content.Context; import android.media.MediaCodec; @@ -88,9 +92,10 @@ import org.checkerframework.dataflow.qual.Pure; DebugViewProvider debugViewProvider) throws TransformationException { super(inputFormat, streamStartPositionUs, muxerWrapper); + + boolean isGlToneMapping = false; if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) { - if (transformationRequest.hdrMode - == TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) { + if (transformationRequest.hdrMode == HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) { if (SDK_INT < 29) { throw TransformationException.createForCodec( new IllegalArgumentException("Interpreting HDR video as SDR is not supported."), @@ -101,6 +106,18 @@ import org.checkerframework.dataflow.qual.Pure; TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED); } inputFormat = inputFormat.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); + } else if (transformationRequest.hdrMode == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) { + if (SDK_INT < 29) { + throw TransformationException.createForCodec( + new IllegalArgumentException( + "OpenGL-based HDR to SDR tone mapping is not supported."), + /* isVideo= */ true, + /* isDecoder= */ true, + inputFormat, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED); + } + isGlToneMapping = true; } else if (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround()) { throw TransformationException.createForCodec( new IllegalArgumentException("HDR editing and tone mapping is not supported."), @@ -149,19 +166,30 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, fallbackListener); - // HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing. - // This implies that the OpenGL EXT_YUV_target extension is supported and hence the - // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone - // mapping is applied, which ensures the decoder outputs SDR output for an HDR input. - ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor(); + ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor(); + // If not tone mapping using OpenGL, the decoder will output the encoderInputColor, + // possibly by tone mapping. + ColorInfo frameProcessorInputColor = + isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor; + // For consistency with the Android platform, OpenGL tone mapping outputs colors with + // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as + // C.COLOR_TRANSFER_SDR to the encoder. + ColorInfo frameProcessorOutputColor = + isGlToneMapping + ? new ColorInfo( + C.COLOR_SPACE_BT709, + C.COLOR_RANGE_LIMITED, + C.COLOR_TRANSFER_GAMMA_2_2, + /* hdrStaticInfo= */ null) + : encoderInputColor; try { frameProcessor = frameProcessorFactory.create( context, effectsListBuilder.build(), debugViewProvider, - /* inputColorInfo= */ encoderSupportedInputColor, - /* outputColorInfo= */ encoderSupportedInputColor, + frameProcessorInputColor, + frameProcessorOutputColor, /* releaseFramesAutomatically= */ true, MoreExecutors.directExecutor(), new FrameProcessor.Listener() { @@ -209,12 +237,12 @@ import org.checkerframework.dataflow.qual.Pure; new FrameInfo( decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs)); - boolean isToneMappingRequired = + boolean isDecoderToneMappingRequired = ColorInfo.isTransferHdr(inputFormat.colorInfo) - && !ColorInfo.isTransferHdr(encoderWrapper.getSupportedInputColor()); + && !ColorInfo.isTransferHdr(frameProcessorInputColor); decoder = decoderFactory.createForVideoDecoding( - inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired); + inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired); maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } @@ -432,7 +460,7 @@ import org.checkerframework.dataflow.qual.Pure; /** Returns the {@link ColorInfo} expected from the input surface. */ public ColorInfo getSupportedInputColor() { boolean isHdrEditingEnabled = - transformationRequest.hdrMode == TransformationRequest.HDR_MODE_KEEP_HDR + transformationRequest.hdrMode == HDR_MODE_KEEP_HDR && !supportedEncoderNamesForHdrEditing.isEmpty(); boolean isInputToneMapped = !isHdrEditingEnabled && ColorInfo.isTransferHdr(inputFormat.colorInfo); @@ -504,10 +532,12 @@ import org.checkerframework.dataflow.qual.Pure; boolean isInputToneMapped = ColorInfo.isTransferHdr(inputFormat.colorInfo) && !ColorInfo.isTransferHdr(requestedEncoderFormat.colorInfo); + // HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to + // HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC. @TransformationRequest.HdrMode - int hdrMode = - isInputToneMapped - ? TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR + int supportedFallbackHdrMode = + isInputToneMapped && transformationRequest.hdrMode == HDR_MODE_KEEP_HDR + ? HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC : transformationRequest.hdrMode; fallbackListener.onTransformationRequestFinalized( @@ -516,7 +546,7 @@ import org.checkerframework.dataflow.qual.Pure; /* hasOutputFormatRotation= */ flipOrientation, requestedEncoderFormat, encoderSupportedFormat, - hdrMode)); + supportedFallbackHdrMode)); encoderSurfaceInfo = new SurfaceInfo(