sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-25 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Don't set codec color info for default SDR

Browse source 
Some media can read color info values from the bitstream
and may partially set some of the SDR default values in
Format.ColorInfo. Setting these default values for SDR can
confuse some codecs and may also prevent adaptive ABR
switches if not all ColorInfo values are set in exactly the
same way.

We can avoid any influence of HDR color info handling by
disabling setting the color info MediaFormat keys for SDR
video and also avoid codec reset at format changes if both
formats are SDR with slightly different ColorInfo settings.

To identify "SDR" ColorInfo instances, we need to do some
fuzzy matching as many of the default values are assumed to
match the SDR profile even if not set.

Issue: androidx/media#1158
PiperOrigin-RevId: 617473937
This commit is contained in:
tonihei 2024-03-20 04:55:06 -07:00 committed by Copybara-Service
parent 3272ad50f3
commit 3a7d31a599
6 changed files with 135 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
RELEASENOTES.md
View file

@ -50,6 +50,9 @@
* Add workaround that ensures the first frame is always rendered while
tunneling even if the device does not do this automatically as required
by the API ([#1169](https://github.com/androidx/media/issues/1169)).
* Fix issue where HDR color info handling causes codec mishavior and
prevents adaptive format switches for SDR video tracks
([#1158](https://github.com/androidx/media/issues/1158)).
* Text:
* WebVTT: Prevent directly consecutive cues from creating spurious
additional `CuesWithTiming` instances from `WebvttParser.parse`

42
libraries/common/src/main/java/androidx/media3/common/ColorInfo.java
View file

@ -21,6 +21,7 @@ import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.util.Arrays;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.dataflow.qual.Pure;
/**
@ -172,6 +173,34 @@ public final class ColorInfo implements Bundleable {
.setColorTransfer(C.COLOR_TRANSFER_SRGB)
.build();
/**
* Returns whether the given color info is equivalent to values for a standard dynamic range video
* that could generally be assumed if no further information is given.
*
* <p>The color info is deemed to be equivalent to SDR video if it either has unset values or
* values matching a 8-bit (chroma+luma), BT.709 or BT.601 color space, SDR transfer and Limited
* range color info.
*
* @param colorInfo The color info to evaluate.
* @return Whether the given color info is equivalent to the assumed default SDR color info.
*/
@EnsuresNonNullIf(result = false, expression = "#1")
public static boolean isEquivalentToAssumedSdrDefault(@Nullable ColorInfo colorInfo) {
if (colorInfo == null) {
return true;
}
return (colorInfo.colorSpace == Format.NO_VALUE
|| colorInfo.colorSpace == C.COLOR_SPACE_BT709
|| colorInfo.colorSpace == C.COLOR_SPACE_BT601)
&& (colorInfo.colorRange == Format.NO_VALUE
|| colorInfo.colorRange == C.COLOR_RANGE_LIMITED)
&& (colorInfo.colorTransfer == Format.NO_VALUE
|| colorInfo.colorTransfer == C.COLOR_TRANSFER_SDR)
&& colorInfo.hdrStaticInfo == null
&& (colorInfo.chromaBitdepth == Format.NO_VALUE || colorInfo.chromaBitdepth == 8)
&& (colorInfo.lumaBitdepth == Format.NO_VALUE || colorInfo.lumaBitdepth == 8);
}
/**
* Returns the {@link C.ColorSpace} corresponding to the given ISO color primary code, as per
* table A.7.21.1 in Rec. ITU-T T.832 (03/2009), or {@link Format#NO_VALUE} if no mapping can be
@ -232,22 +261,25 @@ public final class ColorInfo implements Bundleable {
|| colorInfo.colorTransfer == C.COLOR_TRANSFER_ST2084);
}
/** The {@link C.ColorSpace}. */
/** The {@link C.ColorSpace}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorSpace int colorSpace;
/** The {@link C.ColorRange}. */
/** The {@link C.ColorRange}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorRange int colorRange;
/** The {@link C.ColorTransfer}. */
/** The {@link C.ColorTransfer}, or {@link Format#NO_VALUE} if not set. */
public final @C.ColorTransfer int colorTransfer;
/** HdrStaticInfo as defined in CTA-861.3, or null if none specified. */
@Nullable public final byte[] hdrStaticInfo;
/** The bit depth of the luma samples of the video. */
/** The bit depth of the luma samples of the video, or {@link Format#NO_VALUE} if not set. */
public final int lumaBitdepth;
/** The bit depth of the chroma samples of the video. It may differ from the luma bit depth. */
/**
* The bit depth of the chroma samples of the video, or {@link Format#NO_VALUE} if not set. It may
* differ from the luma bit depth.
*/
public final int chromaBitdepth;
// Lazily initialized hashcode.

2
libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java
View file

@ -252,7 +252,7 @@ public final class MediaFormatUtil {
*/
@SuppressWarnings("InlinedApi")
public static void maybeSetColorInfo(MediaFormat format, @Nullable ColorInfo colorInfo) {
if (colorInfo != null) {
if (!ColorInfo.isEquivalentToAssumedSdrDefault(colorInfo)) {
maybeSetInteger(format, MediaFormat.KEY_COLOR_TRANSFER, colorInfo.colorTransfer);
maybeSetInteger(format, MediaFormat.KEY_COLOR_STANDARD, colorInfo.colorSpace);
maybeSetInteger(format, MediaFormat.KEY_COLOR_RANGE, colorInfo.colorRange);

14
libraries/common/src/test/java/androidx/media3/common/util/MediaFormatUtilTest.java
View file

@ -233,9 +233,23 @@ public class MediaFormatUtilTest {
@Test
public void createMediaFormatFromFormat_withCustomPcmEncoding_setsCustomPcmEncodingEntry() {
Format format = new Format.Builder().setPcmEncoding(C.ENCODING_PCM_16BIT_BIG_ENDIAN).build();
MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);
assertThat(mediaFormat.getInteger(MediaFormatUtil.KEY_PCM_ENCODING_EXTENDED))
.isEqualTo(C.ENCODING_PCM_16BIT_BIG_ENDIAN);
assertThat(mediaFormat.containsKey(MediaFormat.KEY_PCM_ENCODING)).isFalse();
}
@Test
public void createMediaFormatFromFormat_withSdrColorInfo_omitsMediaFormatColorInfoKeys() {
Format format = new Format.Builder().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);
assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_TRANSFER)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_RANGE)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_COLOR_STANDARD)).isFalse();
assertThat(mediaFormat.containsKey(MediaFormat.KEY_HDR_STATIC_INFO)).isFalse();
}
}

6
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
View file

@ -41,6 +41,7 @@ import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Assertions;
@ -422,7 +423,10 @@ public final class MediaCodecInfo {
&& (oldFormat.width != newFormat.width || oldFormat.height != newFormat.height)) {
discardReasons |= DISCARD_REASON_VIDEO_RESOLUTION_CHANGED;
}
if (!Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) {
if ((!ColorInfo.isEquivalentToAssumedSdrDefault(oldFormat.colorInfo)
|| !ColorInfo.isEquivalentToAssumedSdrDefault(newFormat.colorInfo))
&& !Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo)) {
// Don't perform detailed checks if both ColorInfos fall within the default SDR assumption.
discardReasons |= DISCARD_REASON_VIDEO_COLOR_INFO_CHANGED;
}
if (needsAdaptationReconfigureWorkaround(name)

96
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java
View file

@ -74,7 +74,7 @@ public final class MediaCodecInfoTest {
.build();
@Test
public void canKeepCodec_withDifferentMimeType_returnsNo() {
public void canReuseCodec_withDifferentMimeType_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);
Format hdAv1Format = FORMAT_H264_HD.buildUpon().setSampleMimeType(VIDEO_AV1).build();
@ -89,7 +89,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_withRotation_returnsNo() {
public void canReuseCodec_withRotation_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);
Format hdRotatedFormat = FORMAT_H264_HD.buildUpon().setRotationDegrees(90).build();
@ -104,7 +104,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() {
public void canReuseCodec_withResolutionChange_adaptiveCodec_returnsYesWithReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ true);
assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K))
@ -118,7 +118,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() {
public void canReuseCodec_withResolutionChange_nonAdaptiveCodec_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
assertThat(codecInfo.canReuseCodec(FORMAT_H264_HD, FORMAT_H264_4K))
@ -132,7 +132,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() {
public void canReuseCodec_noResolutionChange_nonAdaptiveCodec_returnsYesWithReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdVariantFormat =
@ -148,11 +148,11 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_colorInfoOmittedFromNewFormat_returnsNo() {
public void canReuseCodec_hdrToSdr_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(codecInfo.canReuseCodec(hdrVariantFormat, FORMAT_H264_4K))
.isEqualTo(
new DecoderReuseEvaluation(
@ -164,11 +164,11 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_colorInfoOmittedFromOldFormat_returnsNo() {
public void canReuseCodec_sdrToHdr_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, hdrVariantFormat))
.isEqualTo(
new DecoderReuseEvaluation(
@ -180,13 +180,13 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_colorInfoChange_returnsNo() {
public void canReuseCodec_hdrColorInfoChange_returnsNo() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat1 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
Format hdrVariantFormat2 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build();
assertThat(codecInfo.canReuseCodec(hdrVariantFormat1, hdrVariantFormat2))
.isEqualTo(
new DecoderReuseEvaluation(
@ -198,7 +198,61 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_audioWithDifferentChannelCounts_returnsNo() {
public void canReuseCodec_nullColorInfoToSdr_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format variantWithColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
assertThat(codecInfo.canReuseCodec(FORMAT_H264_4K, variantWithColorInfo))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
FORMAT_H264_4K,
variantWithColorInfo,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}
@Test
public void canReuseCodec_sdrToNullColorInfo_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format variantWithColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
assertThat(codecInfo.canReuseCodec(variantWithColorInfo, FORMAT_H264_4K))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
variantWithColorInfo,
FORMAT_H264_4K,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}
@Test
public void canReuseCodec_sdrToSdrWithPartialInformation_returnsYesWithoutReconfiguration() {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format variantWithFullColorInfo =
FORMAT_H264_4K.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build();
Format variantWithPartialColorInfo =
FORMAT_H264_4K
.buildUpon()
.setColorInfo(
ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(Format.NO_VALUE).build())
.build();
assertThat(codecInfo.canReuseCodec(variantWithFullColorInfo, variantWithPartialColorInfo))
.isEqualTo(
new DecoderReuseEvaluation(
codecInfo.name,
variantWithFullColorInfo,
variantWithPartialColorInfo,
DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION,
/* discardReasons= */ 0));
}
@Test
public void canReuseCodec_audioWithDifferentChannelCounts_returnsNo() {
MediaCodecInfo codecInfo = buildAacCodecInfo();
assertThat(codecInfo.canReuseCodec(FORMAT_AAC_STEREO, FORMAT_AAC_SURROUND))
@ -212,7 +266,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_audioWithSameChannelCounts_returnsYesWithFlush() {
public void canReuseCodec_audioWithSameChannelCounts_returnsYesWithFlush() {
MediaCodecInfo codecInfo = buildAacCodecInfo();
Format stereoVariantFormat = FORMAT_AAC_STEREO.buildUpon().setAverageBitrate(100).build();
@ -227,7 +281,7 @@ public final class MediaCodecInfoTest {
}
@Test
public void canKeepCodec_audioWithDifferentInitializationData_returnsNo() {
public void canReuseCodec_audioWithDifferentInitializationData_returnsNo() {
MediaCodecInfo codecInfo = buildAacCodecInfo();
Format stereoVariantFormat =
@ -310,7 +364,7 @@ public final class MediaCodecInfoTest {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ true))
@ -323,7 +377,7 @@ public final class MediaCodecInfoTest {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat, FORMAT_H264_4K, /* isNewFormatComplete= */ false))
@ -336,7 +390,7 @@ public final class MediaCodecInfoTest {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
FORMAT_H264_4K, hdrVariantFormat, /* isNewFormatComplete= */ true))
@ -349,9 +403,9 @@ public final class MediaCodecInfoTest {
MediaCodecInfo codecInfo = buildH264CodecInfo(/* adaptive= */ false);
Format hdrVariantFormat1 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT601)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT601)).build();
Format hdrVariantFormat2 =
FORMAT_H264_4K.buildUpon().setColorInfo(buildColorInfo(C.COLOR_SPACE_BT709)).build();
FORMAT_H264_4K.buildUpon().setColorInfo(buildHdrColorInfo(C.COLOR_SPACE_BT709)).build();
assertThat(
codecInfo.isSeamlessAdaptationSupported(
hdrVariantFormat1, hdrVariantFormat2, /* isNewFormatComplete= */ true))
@ -429,7 +483,7 @@ public final class MediaCodecInfoTest {
/* secure= */ false);
}
private static ColorInfo buildColorInfo(@C.ColorSpace int colorSpace) {
private static ColorInfo buildHdrColorInfo(@C.ColorSpace int colorSpace) {
return new ColorInfo.Builder()
.setColorSpace(colorSpace)
.setColorRange(C.COLOR_RANGE_FULL)