diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1cd9050639..a55d2fc1d3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -26,6 +26,8 @@ live edge ((#9784)[https://github.com/google/ExoPlayer/issues/9784]). * Fix Maven dependency resolution ((#8353)[https://github.com/google/ExoPlayer/issues/8353]). + * Fix decoder fallback logic for Dolby Atmos (E-AC3-JOC) and Dolby Vision + to use a compatible base decoder (E-AC3 or H264/H265) if needed. * Android 12 compatibility: * Upgrade the Cast extension to depend on `com.google.android.gms:play-services-cast-framework:20.1.0`. Earlier diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 4a2b5314d2..42a5945f00 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -56,9 +56,8 @@ import com.google.android.exoplayer2.util.MediaClock; import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -381,27 +380,29 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media throws DecoderQueryException { @Nullable String mimeType = format.sampleMimeType; if (mimeType == null) { - return Collections.emptyList(); + return ImmutableList.of(); } if (audioSink.supportsFormat(format)) { // The format is supported directly, so a codec is only needed for decryption. @Nullable MediaCodecInfo codecInfo = MediaCodecUtil.getDecryptOnlyDecoderInfo(); if (codecInfo != null) { - return Collections.singletonList(codecInfo); + return ImmutableList.of(codecInfo); } } List decoderInfos = mediaCodecSelector.getDecoderInfos( mimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false); - if (MimeTypes.AUDIO_E_AC3_JOC.equals(mimeType)) { - // E-AC3 decoders can decode JOC streams, but in 2-D rather than 3-D. - List decoderInfosWithEac3 = new ArrayList<>(decoderInfos); - decoderInfosWithEac3.addAll( - mediaCodecSelector.getDecoderInfos( - MimeTypes.AUDIO_E_AC3, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false)); - decoderInfos = decoderInfosWithEac3; + @Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format); + if (alternativeMimeType == null) { + return ImmutableList.copyOf(decoderInfos); } - return Collections.unmodifiableList(decoderInfos); + List alternativeDecoderInfos = + mediaCodecSelector.getDecoderInfos( + alternativeMimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false); + return ImmutableList.builder() + .addAll(decoderInfos) + .addAll(alternativeDecoderInfos) + .build(); } @Override diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java index 6d4c11603a..5b1cb66095 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java @@ -241,7 +241,11 @@ public final class MediaCodecInfo { * @throws MediaCodecUtil.DecoderQueryException Thrown if an error occurs while querying decoders. */ public boolean isFormatSupported(Format format) throws MediaCodecUtil.DecoderQueryException { - if (!isCodecSupported(format)) { + if (!isSampleMimeTypeSupported(format)) { + return false; + } + + if (!isCodecProfileAndLevelSupported(format)) { return false; } @@ -268,25 +272,15 @@ public final class MediaCodecInfo { } } - /** - * Whether the decoder supports the codec of the given {@code format}. If there is insufficient - * information to decide, returns true. - * - * @param format The input media format. - * @return True if the codec of the given {@code format} is supported by the decoder. - */ - public boolean isCodecSupported(Format format) { - if (format.codecs == null || mimeType == null) { + private boolean isSampleMimeTypeSupported(Format format) { + return mimeType.equals(format.sampleMimeType) + || mimeType.equals(MediaCodecUtil.getAlternativeCodecMimeType(format)); + } + + private boolean isCodecProfileAndLevelSupported(Format format) { + if (format.codecs == null) { return true; } - String codecMimeType = MimeTypes.getMediaMimeType(format.codecs); - if (codecMimeType == null) { - return true; - } - if (!mimeType.equals(codecMimeType)) { - logNoSupport("codec.mime " + format.codecs + ", " + codecMimeType); - return false; - } Pair codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format); if (codecProfileAndLevel == null) { // If we don't know any better, we assume that the profile and level are supported. @@ -294,6 +288,19 @@ public final class MediaCodecInfo { } int profile = codecProfileAndLevel.first; int level = codecProfileAndLevel.second; + if (MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)) { + // If this codec is H264 or H265, we only support the Dolby Vision base layer and need to map + // the Dolby Vision profile to the corresponding base layer profile. Also assume all levels of + // this base layer profile are supported. + if (MimeTypes.VIDEO_H264.equals(mimeType)) { + profile = CodecProfileLevel.AVCProfileHigh; + level = 0; + } else if (MimeTypes.VIDEO_H265.equals(mimeType)) { + profile = CodecProfileLevel.HEVCProfileMain10; + level = 0; + } + } + if (!isVideo && profile != CodecProfileLevel.AACObjectXHE) { // Some devices/builds underreport audio capabilities, so assume support except for xHE-AAC // which may not be widely supported. See https://github.com/google/ExoPlayer/issues/5145. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java index c60836fc2b..4f1443c7d1 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java @@ -35,6 +35,7 @@ import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.ColorInfo; import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -181,9 +182,9 @@ public final class MediaCodecUtil { } } applyWorkarounds(mimeType, decoderInfos); - List unmodifiableDecoderInfos = Collections.unmodifiableList(decoderInfos); - decoderInfosCache.put(key, unmodifiableDecoderInfos); - return unmodifiableDecoderInfos; + ImmutableList immutableDecoderInfos = ImmutableList.copyOf(decoderInfos); + decoderInfosCache.put(key, immutableDecoderInfos); + return immutableDecoderInfos; } /** @@ -266,6 +267,41 @@ public final class MediaCodecUtil { } } + /** + * Returns an alternative codec MIME type (besides the default {@link Format#sampleMimeType}) that + * can be used to decode samples of the provided {@link Format}. + * + * @param format The media format. + * @return An alternative MIME type of a codec that be used decode samples of the provided {@code + * Format} (besides the default {@link Format#sampleMimeType}), or null if no such alternative + * exists. + */ + @Nullable + public static String getAlternativeCodecMimeType(Format format) { + if (MimeTypes.AUDIO_E_AC3_JOC.equals(format.sampleMimeType)) { + // E-AC3 decoders can decode JOC streams, but in 2-D rather than 3-D. + return MimeTypes.AUDIO_E_AC3; + } + if (MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)) { + // H.264/AVC or H.265/HEVC decoders can decode the base layer of some DV profiles. This can't + // be done for profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile + // CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward + // compatible and the second one is deprecated and is not always backward compatible. + @Nullable + Pair codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format); + if (codecProfileAndLevel != null) { + int profile = codecProfileAndLevel.first; + if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr + || profile == CodecProfileLevel.DolbyVisionProfileDvheSt) { + return MimeTypes.VIDEO_H265; + } else if (profile == CodecProfileLevel.DolbyVisionProfileDvavSe) { + return MimeTypes.VIDEO_H264; + } + } + } + return null; + } + // Internal methods. /** diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index c25d915b26..7f53747ea3 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -67,7 +67,6 @@ import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.List; /** @@ -462,41 +461,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { throws DecoderQueryException { @Nullable String mimeType = format.sampleMimeType; if (mimeType == null) { - return Collections.emptyList(); + return ImmutableList.of(); } List decoderInfos = mediaCodecSelector.getDecoderInfos( mimeType, requiresSecureDecoder, requiresTunnelingDecoder); - if (MimeTypes.VIDEO_DOLBY_VISION.equals(mimeType)) { - // Fall back to H.264/AVC or H.265/HEVC for the relevant DV profiles. This can't be done for - // profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile - // CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward - // compatible and the second one is deprecated and is not always backward compatible. - @Nullable - Pair codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format); - if (codecProfileAndLevel != null) { - List fallbackDecoderInfos; - int profile = codecProfileAndLevel.first; - if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr - || profile == CodecProfileLevel.DolbyVisionProfileDvheSt) { - fallbackDecoderInfos = - mediaCodecSelector.getDecoderInfos( - MimeTypes.VIDEO_H265, requiresSecureDecoder, requiresTunnelingDecoder); - } else if (profile == CodecProfileLevel.DolbyVisionProfileDvavSe) { - fallbackDecoderInfos = - mediaCodecSelector.getDecoderInfos( - MimeTypes.VIDEO_H264, requiresSecureDecoder, requiresTunnelingDecoder); - } else { - fallbackDecoderInfos = ImmutableList.of(); - } - decoderInfos = - ImmutableList.builder() - .addAll(decoderInfos) - .addAll(fallbackDecoderInfos) - .build(); - } + @Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format); + if (alternativeMimeType == null) { + return ImmutableList.copyOf(decoderInfos); } - return Collections.unmodifiableList(decoderInfos); + List alternativeDecoderInfos = + mediaCodecSelector.getDecoderInfos( + alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder); + return ImmutableList.builder() + .addAll(decoderInfos) + .addAll(alternativeDecoderInfos) + .build(); } @Override diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java index 54e5511035..8571040181 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java @@ -18,6 +18,7 @@ package com.google.android.exoplayer2.audio; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.format; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -38,6 +39,8 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.ExoPlaybackException; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.PlaybackException; +import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.RendererCapabilities.Capabilities; import com.google.android.exoplayer2.RendererConfiguration; import com.google.android.exoplayer2.analytics.PlayerId; import com.google.android.exoplayer2.drm.DrmSessionEventListener; @@ -84,8 +87,14 @@ public class MediaCodecAudioRendererTest { // audioSink isEnded can always be true because the MediaCodecAudioRenderer isEnded = // super.isEnded && audioSink.isEnded. when(audioSink.isEnded()).thenReturn(true); - when(audioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(audioSink.supportsFormat(any())) + .thenAnswer( + invocation -> { + Format format = invocation.getArgument(/* index= */ 0, Format.class); + return MimeTypes.AUDIO_RAW.equals(format.sampleMimeType) + && format.pcmEncoding == C.ENCODING_PCM_16BIT; + }); mediaCodecSelector = (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> @@ -315,6 +324,43 @@ public class MediaCodecAudioRendererTest { verify(audioRendererEventListener).onAudioSinkError(error); } + @Test + public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception { + Format mediaFormat = + new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_E_AC3_JOC) + .setCodecs(MimeTypes.CODEC_E_AC3_JOC) + .build(); + MediaCodecSelector mediaCodecSelector = + (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + !mimeType.equals(MimeTypes.AUDIO_E_AC3) + ? ImmutableList.of() + : ImmutableList.of( + MediaCodecInfo.newInstance( + /* name= */ "eac3-codec", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)); + MediaCodecAudioRenderer renderer = + new MediaCodecAudioRenderer( + ApplicationProvider.getApplicationContext(), + mediaCodecSelector, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(Looper.getMainLooper()), + audioRendererEventListener, + audioSink); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + @Capabilities int capabilities = renderer.supportsFormat(mediaFormat); + + assertThat(RendererCapabilities.getFormatSupport(capabilities)).isEqualTo(C.FORMAT_HANDLED); + } + private static Format getAudioSinkFormat(Format inputFormat) { return new Format.Builder() .setSampleMimeType(MimeTypes.AUDIO_RAW) diff --git a/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java index 58b9424bb6..08816add89 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java @@ -27,6 +27,8 @@ import static org.mockito.Mockito.verify; import static org.robolectric.Shadows.shadowOf; import android.graphics.SurfaceTexture; +import android.media.MediaCodecInfo.CodecCapabilities; +import android.media.MediaCodecInfo.CodecProfileLevel; import android.media.MediaFormat; import android.os.Handler; import android.os.Looper; @@ -39,7 +41,9 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Renderer; import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.RendererCapabilities.Capabilities; import com.google.android.exoplayer2.RendererConfiguration; +import com.google.android.exoplayer2.analytics.PlayerId; import com.google.android.exoplayer2.drm.DrmSessionEventListener; import com.google.android.exoplayer2.drm.DrmSessionManager; import com.google.android.exoplayer2.mediacodec.MediaCodecInfo; @@ -506,4 +510,103 @@ public class MediaCodecVideoRendererTest { verify(eventListener, times(2)) .onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong()); } + + @Test + public void supportsFormat_withDolbyVisionMedia_returnsTrueWhenFallbackToH265orH264Allowed() + throws Exception { + // Create Dolby media formats that could fall back to H265 or H264. + Format formatDvheDtrFallbackToH265 = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.04.01") + .build(); + Format formatDvheStFallbackToH265 = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.08.01") + .build(); + Format formatDvavSeFallbackToH264 = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvav.09.01") + .build(); + Format formatNoFallbackPossible = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvav.01.01") + .build(); + // Only provide H264 and H265 decoders with codec profiles needed for fallback. + MediaCodecSelector mediaCodecSelector = + (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> { + switch (mimeType) { + case MimeTypes.VIDEO_H264: + CodecCapabilities capabilitiesH264 = new CodecCapabilities(); + capabilitiesH264.profileLevels = + new CodecProfileLevel[] {new CodecProfileLevel(), new CodecProfileLevel()}; + capabilitiesH264.profileLevels[0].profile = CodecProfileLevel.AVCProfileBaseline; + capabilitiesH264.profileLevels[0].level = CodecProfileLevel.AVCLevel42; + capabilitiesH264.profileLevels[1].profile = CodecProfileLevel.AVCProfileHigh; + capabilitiesH264.profileLevels[1].level = CodecProfileLevel.AVCLevel42; + return ImmutableList.of( + MediaCodecInfo.newInstance( + /* name= */ "h264-codec", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ capabilitiesH264, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)); + case MimeTypes.VIDEO_H265: + CodecCapabilities capabilitiesH265 = new CodecCapabilities(); + capabilitiesH265.profileLevels = + new CodecProfileLevel[] {new CodecProfileLevel(), new CodecProfileLevel()}; + capabilitiesH265.profileLevels[0].profile = CodecProfileLevel.HEVCProfileMain; + capabilitiesH265.profileLevels[0].level = CodecProfileLevel.HEVCMainTierLevel41; + capabilitiesH265.profileLevels[1].profile = CodecProfileLevel.HEVCProfileMain10; + capabilitiesH265.profileLevels[1].level = CodecProfileLevel.HEVCHighTierLevel51; + return ImmutableList.of( + MediaCodecInfo.newInstance( + /* name= */ "h265-codec", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ capabilitiesH265, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)); + default: + return ImmutableList.of(); + } + }; + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + @Capabilities + int capabilitiesDvheDtrFallbackToH265 = renderer.supportsFormat(formatDvheDtrFallbackToH265); + @Capabilities + int capabilitiesDvheStFallbackToH265 = renderer.supportsFormat(formatDvheStFallbackToH265); + @Capabilities + int capabilitiesDvavSeFallbackToH264 = renderer.supportsFormat(formatDvavSeFallbackToH264); + @Capabilities + int capabilitiesNoFallbackPossible = renderer.supportsFormat(formatNoFallbackPossible); + + assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvheDtrFallbackToH265)) + .isEqualTo(C.FORMAT_HANDLED); + assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvheStFallbackToH265)) + .isEqualTo(C.FORMAT_HANDLED); + assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvavSeFallbackToH264)) + .isEqualTo(C.FORMAT_HANDLED); + assertThat(RendererCapabilities.getFormatSupport(capabilitiesNoFallbackPossible)) + .isEqualTo(C.FORMAT_UNSUPPORTED_SUBTYPE); + } }