sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-08 11:45:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

HDR: Implement Transformer HDR to SDR GL tone-mapping API

Browse source 
Note that we simply use GlEffectsFrameProcessor in-app / GL tone-mapping, so PQ->SDR tone-mapping isn't yet implemented.

Tested manually using the demo on Pixel 7, to confirm that device and in-app tone
mapping behave similarly.

PiperOrigin-RevId: 496700231
This commit is contained in:
huangdarwin 2022-12-20 18:52:03 +00:00 committed by Tianyi Feng
parent 9465fe2f98
commit 84c81b8575
5 changed files with 96 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
View file

@ -176,7 +176,12 @@ public final class ConfigurationActivity extends AppCompatActivity {
HDR_MODE_DESCRIPTIONS =
new ImmutableMap.Builder<String, @TransformationRequest.HdrMode Integer>()
.put("Keep HDR", TransformationRequest.HDR_MODE_KEEP_HDR)
.put("Tone-map HDR to SDR", TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
.put(
"MediaCodec tone-map HDR to SDR",
TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
.put(
"OpenGL tone-map HDR to SDR",
TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.put(
"Force Interpret HDR as SDR",
TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR)

4
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java
View file

@ -183,7 +183,7 @@ public class HdrEditingTest {
.isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR);
isToneMappingFallbackApplied.set(
fallbackTransformationRequest.hdrMode
== TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR);
== TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
}
})
.build();
@ -236,7 +236,7 @@ public class HdrEditingTest {
.isEqualTo(TransformationRequest.HDR_MODE_KEEP_HDR);
isToneMappingFallbackApplied.set(
fallbackTransformationRequest.hdrMode
== TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR);
== TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
}
})
.build();

15
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrToSdrToneMapTest.java → libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingMediaCodecTest.java
View file

@ -38,11 +38,12 @@ import org.junit.runner.RunWith;
/**
* {@link Transformer} instrumentation test for applying an {@linkplain
* TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR HDR to SDR tone mapping edit}.
* TransformationRequest#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC HDR to SDR tone mapping
* edit}.
*/
@RunWith(AndroidJUnit4.class)
public class HdrToSdrToneMapTest {
public static final String TAG = "HdrToSdrToneMapTest";
public class ToneMapHdrToSdrUsingMediaCodecTest {
public static final String TAG = "ToneMapHdrToSdrUsingMediaCodecTest";
@Test
public void transform_toneMapNoRequestedTranscode_hdr10File_toneMapsOrThrows() throws Exception {
@ -53,7 +54,7 @@ public class HdrToSdrToneMapTest {
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
.build())
.addListener(
new Transformer.Listener() {
@ -98,7 +99,7 @@ public class HdrToSdrToneMapTest {
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
.build())
.addListener(
new Transformer.Listener() {
@ -143,7 +144,7 @@ public class HdrToSdrToneMapTest {
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
.setRotationDegrees(180)
.build())
.addListener(
@ -189,7 +190,7 @@ public class HdrToSdrToneMapTest {
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR)
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)
.setRotationDegrees(180)
.build())
.addListener(

47
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java
View file

@ -40,7 +40,8 @@ public final class TransformationRequest {
/**
* The strategy to use to transcode or edit High Dynamic Range (HDR) input video.
*
* <p>One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR}, or {@link
* <p>One of {@link #HDR_MODE_KEEP_HDR}, {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC},
* {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL}, or {@link
* #HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}.
*
* <p>Standard Dynamic Range (SDR) input video is unaffected by these settings.
@ -50,8 +51,9 @@ public final class TransformationRequest {
@Target(TYPE_USE)
@IntDef({
HDR_MODE_KEEP_HDR,
HDR_MODE_TONE_MAP_HDR_TO_SDR,
HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR
HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC,
HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL,
HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR,
})
public @interface HdrMode {}
/**
@ -60,26 +62,43 @@ public final class TransformationRequest {
* <p>Supported on API 31+, by some device and HDR format combinations.
*
* <p>If not supported, {@link Transformer} may fall back to {@link
* #HDR_MODE_TONE_MAP_HDR_TO_SDR}.
* #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}.
*/
public static final int HDR_MODE_KEEP_HDR = 0;
/**
* Tone map HDR input to SDR before processing, to generate SDR output.
* Tone map HDR input to SDR before processing, to generate SDR output, using the {@link
* android.media.MediaCodec} decoder tone-mapper.
*
* <p>Supported on API 31+, by some device and HDR format combinations. Tone-mapping is only
* guaranteed to be supported from Android T onwards.
*
* <p>If not supported, {@link Transformer} may throw a {@link TransformationException}.
* <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
*/
public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR = 1;
public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC = 1;
/**
* Interpret HDR input as SDR, resulting in washed out video.
* Tone map HDR input to SDR before processing, to generate SDR output, using an OpenGL
* tone-mapper.
*
* <p>Supported on API 29+, for HLG input.
*
* <p>This may exhibit mild differences from {@link
* #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}, depending on the device's tone-mapping
* implementation, but should have much wider support and have more consistent results across
* devices.
*
* <p>If not supported, {@link Transformer} throws a {@link TransformationException}.
*/
// TODO(b/239735341): Implement PQ tone-mapping to remove the HLG reference.
public static final int HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL = 2;
/**
* Interpret HDR input as SDR, likely with a washed out look.
*
* <p>Supported on API 29+.
*
* <p>This is much more widely supported than {@link #HDR_MODE_KEEP_HDR} and {@link
* #HDR_MODE_TONE_MAP_HDR_TO_SDR}. However, as HDR transfer functions and metadata will be
* ignored, contents will be displayed incorrectly, likely with a washed out look.
* #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC}. However, as HDR transfer functions and
* metadata will be ignored, contents will be displayed incorrectly, likely with a washed out
* look.
*
* <p>Use of this flag may result in {@code
* TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED} or {@code
@ -87,7 +106,7 @@ public final class TransformationRequest {
*
* <p>This field is experimental, and will be renamed or removed in a future release.
*/
public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 2;
public static final int HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR = 3;
/** A builder for {@link TransformationRequest} instances. */
public static final class Builder {
@ -285,14 +304,14 @@ public final class TransformationRequest {
/**
* @deprecated This method is now a no-op if {@code false}, and sets {@code
* setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR)} if {@code true}. Use {@link #setHdrMode} with
* {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR} instead.
* setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC)} if {@code true}. Use {@link
* #setHdrMode} with {@link #HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} instead.
*/
@Deprecated
@CanIgnoreReturnValue
public Builder setEnableRequestSdrToneMapping(boolean enableRequestSdrToneMapping) {
if (enableRequestSdrToneMapping) {
return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR);
return setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC);
}
return this;
}

64
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View file

@ -19,6 +19,10 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Util.SDK_INT;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_KEEP_HDR;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC;
import static androidx.media3.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL;
import android.content.Context;
import android.media.MediaCodec;
@ -88,9 +92,10 @@ import org.checkerframework.dataflow.qual.Pure;
DebugViewProvider debugViewProvider)
throws TransformationException {
super(inputFormat, streamStartPositionUs, muxerWrapper);
boolean isGlToneMapping = false;
if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) {
if (transformationRequest.hdrMode
== TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) {
if (transformationRequest.hdrMode == HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) {
if (SDK_INT < 29) {
throw TransformationException.createForCodec(
new IllegalArgumentException("Interpreting HDR video as SDR is not supported."),
@ -101,6 +106,18 @@ import org.checkerframework.dataflow.qual.Pure;
TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED);
}
inputFormat = inputFormat.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
} else if (transformationRequest.hdrMode == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) {
if (SDK_INT < 29) {
throw TransformationException.createForCodec(
new IllegalArgumentException(
"OpenGL-based HDR to SDR tone mapping is not supported."),
/* isVideo= */ true,
/* isDecoder= */ true,
inputFormat,
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_HDR_DECODING_UNSUPPORTED);
}
isGlToneMapping = true;
} else if (SDK_INT < 31 || deviceNeedsNoToneMappingWorkaround()) {
throw TransformationException.createForCodec(
new IllegalArgumentException("HDR editing and tone mapping is not supported."),
@ -149,19 +166,30 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest,
fallbackListener);
// HDR colors are only used if the MediaCodec encoder supports FEATURE_HdrEditing.
// This implies that the OpenGL EXT_YUV_target extension is supported and hence the
// default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone
// mapping is applied, which ensures the decoder outputs SDR output for an HDR input.
ColorInfo encoderSupportedInputColor = encoderWrapper.getSupportedInputColor();
ColorInfo encoderInputColor = encoderWrapper.getSupportedInputColor();
// If not tone mapping using OpenGL, the decoder will output the encoderInputColor,
// possibly by tone mapping.
ColorInfo frameProcessorInputColor =
isGlToneMapping ? checkNotNull(inputFormat.colorInfo) : encoderInputColor;
// For consistency with the Android platform, OpenGL tone mapping outputs colors with
// C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as
// C.COLOR_TRANSFER_SDR to the encoder.
ColorInfo frameProcessorOutputColor =
isGlToneMapping
? new ColorInfo(
C.COLOR_SPACE_BT709,
C.COLOR_RANGE_LIMITED,
C.COLOR_TRANSFER_GAMMA_2_2,
/* hdrStaticInfo= */ null)
: encoderInputColor;
try {
frameProcessor =
frameProcessorFactory.create(
context,
effectsListBuilder.build(),
debugViewProvider,
/* inputColorInfo= */ encoderSupportedInputColor,
/* outputColorInfo= */ encoderSupportedInputColor,
frameProcessorInputColor,
frameProcessorOutputColor,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new FrameProcessor.Listener() {
@ -209,12 +237,12 @@ import org.checkerframework.dataflow.qual.Pure;
new FrameInfo(
decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
boolean isToneMappingRequired =
boolean isDecoderToneMappingRequired =
ColorInfo.isTransferHdr(inputFormat.colorInfo)
&& !ColorInfo.isTransferHdr(encoderWrapper.getSupportedInputColor());
&& !ColorInfo.isTransferHdr(frameProcessorInputColor);
decoder =
decoderFactory.createForVideoDecoding(
inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
inputFormat, frameProcessor.getInputSurface(), isDecoderToneMappingRequired);
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
}
@ -432,7 +460,7 @@ import org.checkerframework.dataflow.qual.Pure;
/** Returns the {@link ColorInfo} expected from the input surface. */
public ColorInfo getSupportedInputColor() {
boolean isHdrEditingEnabled =
transformationRequest.hdrMode == TransformationRequest.HDR_MODE_KEEP_HDR
transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
&& !supportedEncoderNamesForHdrEditing.isEmpty();
boolean isInputToneMapped =
!isHdrEditingEnabled && ColorInfo.isTransferHdr(inputFormat.colorInfo);
@ -504,10 +532,12 @@ import org.checkerframework.dataflow.qual.Pure;
boolean isInputToneMapped =
ColorInfo.isTransferHdr(inputFormat.colorInfo)
&& !ColorInfo.isTransferHdr(requestedEncoderFormat.colorInfo);
// HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to
// HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC.
@TransformationRequest.HdrMode
int hdrMode =
isInputToneMapped
? TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR
int supportedFallbackHdrMode =
isInputToneMapped && transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
? HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC
: transformationRequest.hdrMode;
fallbackListener.onTransformationRequestFinalized(
@ -516,7 +546,7 @@ import org.checkerframework.dataflow.qual.Pure;
/* hasOutputFormatRotation= */ flipOrientation,
requestedEncoderFormat,
encoderSupportedFormat,
hdrMode));
supportedFallbackHdrMode));
encoderSurfaceInfo =
new SurfaceInfo(