From 35528fd1bfe980a5279d80e9abe8ad44f939be4e Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 28 Mar 2022 16:29:56 +0100 Subject: [PATCH] Add support for requesting color transfer to SDR From Android T onwards `MediaCodec` supports requesting tone-mapping down to SDR. Add an option to request this behavior and document that it isn't supported before T. Also add an option in the demo app to try it out. Tested manually on a prerelease build. PiperOrigin-RevId: 437765325 --- .../ConfigurationActivity.java | 11 +++++++ .../transformerdemo/TransformerActivity.java | 2 ++ .../res/layout/configuration_activity.xml | 10 +++++++ .../src/main/res/values/strings.xml | 3 +- .../android/exoplayer2/transformer/Codec.java | 4 ++- .../transformer/DefaultDecoderFactory.java | 7 ++++- .../transformer/TransformationRequest.java | 30 ++++++++++++++++++- .../transformer/TransformerVideoRenderer.java | 3 ++ .../VideoTranscodingSamplePipeline.java | 5 +++- 9 files changed, 70 insertions(+), 5 deletions(-) diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java index c1202f49cb..986553fde9 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java @@ -54,6 +54,7 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; public static final String ENABLE_FALLBACK = "enable_fallback"; + public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; private static final String[] INPUT_URIS = { "https://html5demos.com/assets/dizzy.mp4", @@ -84,6 +85,7 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; private @MonotonicNonNull CheckBox enableFallbackCheckBox; + private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; private int inputUriPosition; @@ -150,6 +152,7 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox); + enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); } @@ -179,6 +182,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "scaleSpinner", "rotateSpinner", "enableFallbackCheckBox", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void startTransformation(View view) { @@ -211,6 +215,8 @@ public final class ConfigurationActivity extends AppCompatActivity { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked()); + bundle.putBoolean( + ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); transformerIntent.putExtras(bundle); @@ -243,6 +249,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void onRemoveAudio(View view) { @@ -261,6 +268,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void onRemoveVideo(View view) { @@ -278,6 +286,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { @@ -286,6 +295,7 @@ public final class ConfigurationActivity extends AppCompatActivity { resolutionHeightSpinner.setEnabled(isVideoEnabled); scaleSpinner.setEnabled(isVideoEnabled); rotateSpinner.setEnabled(isVideoEnabled); + enableRequestSdrToneMappingCheckBox.setEnabled(isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); @@ -293,6 +303,7 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.resolution_height_text_view).setEnabled(isVideoEnabled); findViewById(R.id.scale).setEnabled(isVideoEnabled); findViewById(R.id.rotate).setEnabled(isVideoEnabled); + findViewById(R.id.request_sdr_tone_mapping).setEnabled(isVideoEnabled); findViewById(R.id.hdr_editing).setEnabled(isVideoEnabled); } } diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index ab7ea58def..f2c5bf2b5d 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -225,6 +225,8 @@ public final class TransformerActivity extends AppCompatActivity { bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); requestBuilder.setRotationDegrees(rotateDegrees); + requestBuilder.setEnableRequestSdrToneMapping( + bundle.getBoolean(ConfigurationActivity.ENABLE_REQUEST_SDR_TONE_MAPPING)); requestBuilder.experimental_setEnableHdrEditing( bundle.getBoolean(ConfigurationActivity.ENABLE_HDR_EDITING)); transformerBuilder diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 1ff3cafc6b..c973bf4137 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -169,6 +169,16 @@ android:layout_gravity="right" android:checked="true"/> + + + + diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 8e8f97ecf9..8ab9fe25f2 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -27,8 +27,9 @@ Scale video Rotate video (degrees) Enable fallback - Transform + Request SDR tone-mapping [Experimental] HDR editing + Transform Debug preview: No debug preview available. Transformation started diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java index c2167a2495..72c86653ce 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java @@ -56,10 +56,12 @@ public interface Codec { * @param format The {@link Format} (of the input data) used to determine the underlying decoder * and its configuration values. * @param outputSurface The {@link Surface} to which the decoder output is rendered. + * @param enableRequestSdrToneMapping Whether to request tone-mapping to SDR. * @return A {@link Codec} for video decoding. * @throws TransformationException If no suitable {@link Codec} can be created. */ - Codec createForVideoDecoding(Format format, Surface outputSurface) + Codec createForVideoDecoding( + Format format, Surface outputSurface, boolean enableRequestSdrToneMapping) throws TransformationException; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultDecoderFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultDecoderFactory.java index 3a302937dc..661d553431 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultDecoderFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultDecoderFactory.java @@ -49,7 +49,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public Codec createForVideoDecoding(Format format, Surface outputSurface) + public Codec createForVideoDecoding( + Format format, Surface outputSurface, boolean enableRequestSdrToneMapping) throws TransformationException { MediaFormat mediaFormat = MediaFormat.createVideoFormat( @@ -63,6 +64,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // cycle. This key ensures no frame dropping when the decoder's output surface is full. mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); } + if (SDK_INT >= 31 && enableRequestSdrToneMapping) { + mediaFormat.setInteger( + MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + } @Nullable String mediaCodecName = EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index 35b9907dd8..faf8d3436b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -38,6 +38,7 @@ public final class TransformationRequest { private int outputHeight; @Nullable private String audioMimeType; @Nullable private String videoMimeType; + private boolean enableRequestSdrToneMapping; private boolean enableHdrEditing; /** @@ -60,6 +61,7 @@ public final class TransformationRequest { this.outputHeight = transformationRequest.outputHeight; this.audioMimeType = transformationRequest.audioMimeType; this.videoMimeType = transformationRequest.videoMimeType; + this.enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping; this.enableHdrEditing = transformationRequest.enableHdrEditing; } @@ -192,13 +194,30 @@ public final class TransformationRequest { return this; } + /** + * Sets whether to request tone-mapping to standard dynamic range (SDR). If enabled and + * supported, high dynamic range (HDR) input will be tone-mapped into an SDR opto-electrical + * transfer function before processing. + * + *

The setting has no effect if the input is already in SDR, or if tone-mapping is not + * supported. Currently tone-mapping is only guaranteed to be supported from Android T onwards. + * + * @param enableRequestSdrToneMapping Whether to request tone-mapping down to SDR. + * @return This builder. + */ + public Builder setEnableRequestSdrToneMapping(boolean enableRequestSdrToneMapping) { + this.enableRequestSdrToneMapping = enableRequestSdrToneMapping; + return this; + } + /** * Sets whether to attempt to process any input video stream as a high dynamic range (HDR) * signal. * *

This method is experimental, and will be renamed or removed in a future release. The HDR * editing feature is under development and is intended for developing/testing HDR processing - * and encoding support. + * and encoding support. HDR editing can't be enabled at the same time as {@link + * #setEnableRequestSdrToneMapping(boolean) SDR tone-mapping}. * * @param enableHdrEditing Whether to attempt to process any input video stream as a high * dynamic range (HDR) signal. @@ -219,6 +238,7 @@ public final class TransformationRequest { outputHeight, audioMimeType, videoMimeType, + enableRequestSdrToneMapping, enableHdrEditing); } } @@ -269,6 +289,9 @@ public final class TransformationRequest { * @see Builder#setVideoMimeType(String) */ @Nullable public final String videoMimeType; + /** Whether to request tone-mapping to standard dynamic range (SDR). */ + public final boolean enableRequestSdrToneMapping; + /** * Whether to attempt to process any input video stream as a high dynamic range (HDR) signal. * @@ -284,7 +307,9 @@ public final class TransformationRequest { int outputHeight, @Nullable String audioMimeType, @Nullable String videoMimeType, + boolean enableRequestSdrToneMapping, boolean enableHdrEditing) { + checkArgument(!enableHdrEditing || !enableRequestSdrToneMapping); this.flattenForSlowMotion = flattenForSlowMotion; this.scaleX = scaleX; this.scaleY = scaleY; @@ -292,6 +317,7 @@ public final class TransformationRequest { this.outputHeight = outputHeight; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; + this.enableRequestSdrToneMapping = enableRequestSdrToneMapping; this.enableHdrEditing = enableHdrEditing; } @@ -311,6 +337,7 @@ public final class TransformationRequest { && outputHeight == that.outputHeight && Util.areEqual(audioMimeType, that.audioMimeType) && Util.areEqual(videoMimeType, that.videoMimeType) + && enableRequestSdrToneMapping == that.enableRequestSdrToneMapping && enableHdrEditing == that.enableHdrEditing; } @@ -323,6 +350,7 @@ public final class TransformationRequest { result = 31 * result + outputHeight; result = 31 * result + (audioMimeType != null ? audioMimeType.hashCode() : 0); result = 31 * result + (videoMimeType != null ? videoMimeType.hashCode() : 0); + result = 31 * result + (enableRequestSdrToneMapping ? 1 : 0); result = 31 * result + (enableHdrEditing ? 1 : 0); return result; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index c0c8adcbfa..70b87f78d5 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -107,6 +107,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; if (encoderFactory.videoNeedsEncoding()) { return false; } + if (transformationRequest.enableRequestSdrToneMapping) { + return false; + } if (transformationRequest.enableHdrEditing) { return false; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 0d61832395..72d3893ed2 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -124,7 +124,10 @@ import org.checkerframework.dataflow.qual.Pure; encoderSupportedFormat.width, encoderSupportedFormat.height)); decoder = - decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface()); + decoderFactory.createForVideoDecoding( + inputFormat, + frameProcessorChain.getInputSurface(), + transformationRequest.enableRequestSdrToneMapping); maxPendingFrameCount = getMaxPendingFrameCount(); }