From d05d2fcea90d7e9e29900eb561e56b0fb197b72c Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Fri, 17 Aug 2018 00:56:09 -0700 Subject: [PATCH] Support seamless adaptation of xHE-AAC audio streams ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=209113673 --- RELEASENOTES.md | 1 + .../audio/MediaCodecAudioRenderer.java | 78 +++++++++---------- .../exoplayer2/mediacodec/MediaCodecInfo.java | 62 ++++++++++++++- .../video/MediaCodecVideoRenderer.java | 26 ++----- 4 files changed, 102 insertions(+), 65 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 770d1ae377..bdb4cff892 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -33,6 +33,7 @@ each encoding for passthrough playbacks ([#3803](https://github.com/google/ExoPlayer/issues/3803)). * Add support for attaching auxiliary audio effects to the `AudioTrack`. + * Add support for seamless adaptation while playing xHE-AAC streams. * Video: * Add callback to `VideoListener` to notify of surface size changes. * Scale up the initial video decoder maximum input size so playlist item diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 96c7eb4bdf..1197cb5a71 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -76,8 +76,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private boolean passthroughEnabled; private boolean codecNeedsDiscardChannelsWorkaround; private android.media.MediaFormat passthroughMediaFormat; - @C.Encoding - private int pcmEncoding; + private @C.Encoding int pcmEncoding; private int channelCount; private int encoderDelay; private int encoderPadding; @@ -288,8 +287,12 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media // Check capabilities for the first decoder in the list, which takes priority. MediaCodecInfo decoderInfo = decoderInfos.get(0); boolean isFormatSupported = decoderInfo.isFormatSupported(format); + int adaptiveSupport = + isFormatSupported && decoderInfo.isSeamlessAdaptationSupported(format) + ? ADAPTIVE_SEAMLESS + : ADAPTIVE_NOT_SEAMLESS; int formatSupport = isFormatSupported ? FORMAT_HANDLED : FORMAT_EXCEEDS_CAPABILITIES; - return ADAPTIVE_NOT_SEAMLESS | tunnelingSupport | formatSupport; + return adaptiveSupport | tunnelingSupport | formatSupport; } @Override @@ -344,13 +347,16 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override protected @KeepCodecResult int canKeepCodec( MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { - return KEEP_CODEC_RESULT_NO; - // TODO: Determine when codecs can be safely kept. When doing so, also uncomment the commented - // out code in getCodecMaxInputSize. - // return getCodecMaxInputSize(codecInfo, newFormat) <= codecMaxInputSize - // && areAdaptationCompatible(oldFormat, newFormat) - // ? KEEP_CODEC_RESULT_YES_WITHOUT_RECONFIGURATION - // : KEEP_CODEC_RESULT_NO; + if (getCodecMaxInputSize(codecInfo, newFormat) <= codecMaxInputSize + && codecInfo.isSeamlessAdaptationSupported(oldFormat, newFormat) + && oldFormat.encoderDelay == 0 + && oldFormat.encoderPadding == 0 + && newFormat.encoderDelay == 0 + && newFormat.encoderPadding == 0) { + return KEEP_CODEC_RESULT_YES_WITHOUT_RECONFIGURATION; + } else { + return KEEP_CODEC_RESULT_NO; + } } @Override @@ -361,9 +367,16 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override protected float getCodecOperatingRate( float operatingRate, Format format, Format[] streamFormats) { - return format.sampleRate == Format.NO_VALUE - ? CODEC_OPERATING_RATE_UNSET - : (format.sampleRate * operatingRate); + // Use the highest known stream sample-rate up front, to avoid having to reconfigure the codec + // should an adaptive switch to that stream occur. + int maxSampleRate = -1; + for (Format streamFormat : streamFormats) { + int streamSampleRate = streamFormat.sampleRate; + if (streamSampleRate != Format.NO_VALUE) { + maxSampleRate = Math.max(maxSampleRate, streamSampleRate); + } + } + return maxSampleRate == -1 ? CODEC_OPERATING_RATE_UNSET : (maxSampleRate * operatingRate); } @Override @@ -603,16 +616,16 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media protected int getCodecMaxInputSize( MediaCodecInfo codecInfo, Format format, Format[] streamFormats) { int maxInputSize = getCodecMaxInputSize(codecInfo, format); - // if (streamFormats.length == 1) { - // // The single entry in streamFormats must correspond to the format for which the codec is - // // being configured. - // return maxInputSize; - // } - // for (Format streamFormat : streamFormats) { - // if (areAdaptationCompatible(format, streamFormat)) { - // maxInputSize = Math.max(maxInputSize, getCodecMaxInputSize(codecInfo, streamFormat)); - // } - // } + if (streamFormats.length == 1) { + // The single entry in streamFormats must correspond to the format for which the codec is + // being configured. + return maxInputSize; + } + for (Format streamFormat : streamFormats) { + if (codecInfo.isSeamlessAdaptationSupported(format, streamFormat)) { + maxInputSize = Math.max(maxInputSize, getCodecMaxInputSize(codecInfo, streamFormat)); + } + } return maxInputSize; } @@ -689,25 +702,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } } - /** - * Returns whether a codec with suitable maximum input size will support adaptation between two - * {@link Format}s. - * - * @param first The first format. - * @param second The second format. - * @return Whether the codec will support adaptation between the two {@link Format}s. - */ - private static boolean areAdaptationCompatible(Format first, Format second) { - return first.sampleMimeType.equals(second.sampleMimeType) - && first.channelCount == second.channelCount - && first.sampleRate == second.sampleRate - && first.encoderDelay == 0 - && first.encoderPadding == 0 - && second.encoderDelay == 0 - && second.encoderPadding == 0 - && first.initializationDataEquals(second); - } - /** * Returns whether the decoder is known to output six audio channels when provided with input with * fewer than six channels. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java index ca3d99edc6..727dfaf1d5 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecInfo.java @@ -30,10 +30,9 @@ import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; -/** - * Information about a {@link MediaCodec} for a given mime type. - */ +/** Information about a {@link MediaCodec} for a given mime type. */ @TargetApi(16) +@SuppressWarnings("InlinedApi") public final class MediaCodecInfo { public static final String TAG = "MediaCodecInfo"; @@ -259,6 +258,63 @@ public final class MediaCodecInfo { return false; } + /** + * Returns whether it may be possible to adapt to playing a different format when the codec is + * configured to play media in the specified {@code format}. For adaptation to succeed, the codec + * must also be configured with appropriate maximum values and {@link + * #isSeamlessAdaptationSupported(Format, Format)} must return {@code true} for the old/new + * formats. + * + * @param format The format of media for which the decoder will be configured. + * @return Whether adaptation may be possible + */ + public boolean isSeamlessAdaptationSupported(Format format) { + if (isVideo) { + return adaptive; + } else { + Pair codecProfileLevel = + MediaCodecUtil.getCodecProfileAndLevel(format.codecs); + return codecProfileLevel != null && codecProfileLevel.first == CodecProfileLevel.AACObjectXHE; + } + } + + /** + * Returns whether it is possible to adapt the decoder seamlessly from {@code oldFormat} to {@code + * newFormat}. + * + * @param oldFormat The format being decoded. + * @param newFormat The new format. + * @return Whether it is possible to adapt the decoder seamlessly. + */ + public boolean isSeamlessAdaptationSupported(Format oldFormat, Format newFormat) { + if (isVideo) { + return oldFormat.sampleMimeType.equals(newFormat.sampleMimeType) + && oldFormat.rotationDegrees == newFormat.rotationDegrees + && (adaptive + || (oldFormat.width == newFormat.width && oldFormat.height == newFormat.height)) + && Util.areEqual(oldFormat.colorInfo, newFormat.colorInfo); + } else { + if (!MimeTypes.AUDIO_AAC.equals(mimeType) + || !oldFormat.sampleMimeType.equals(newFormat.sampleMimeType) + || oldFormat.channelCount != newFormat.channelCount + || oldFormat.sampleRate != newFormat.sampleRate) { + return false; + } + // Check the codec profile levels support adaptation. + Pair oldCodecProfileLevel = + MediaCodecUtil.getCodecProfileAndLevel(oldFormat.codecs); + Pair newCodecProfileLevel = + MediaCodecUtil.getCodecProfileAndLevel(newFormat.codecs); + if (oldCodecProfileLevel == null || newCodecProfileLevel == null) { + return false; + } + int oldProfile = oldCodecProfileLevel.first; + int newProfile = newCodecProfileLevel.first; + return oldProfile == CodecProfileLevel.AACObjectXHE + && newProfile == CodecProfileLevel.AACObjectXHE; + } + } + /** * Whether the decoder supports video with a given width, height and frame rate. *

diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 1fa9fbd8fb..181232b7b2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -266,7 +266,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // Check capabilities for the first decoder in the list, which takes priority. MediaCodecInfo decoderInfo = decoderInfos.get(0); boolean isFormatSupported = decoderInfo.isFormatSupported(format); - int adaptiveSupport = decoderInfo.adaptive ? ADAPTIVE_SEAMLESS : ADAPTIVE_NOT_SEAMLESS; + int adaptiveSupport = + decoderInfo.isSeamlessAdaptationSupported(format) + ? ADAPTIVE_SEAMLESS + : ADAPTIVE_NOT_SEAMLESS; int tunnelingSupport = decoderInfo.tunneling ? TUNNELING_SUPPORTED : TUNNELING_NOT_SUPPORTED; int formatSupport = isFormatSupported ? FORMAT_HANDLED : FORMAT_EXCEEDS_CAPABILITIES; return adaptiveSupport | tunnelingSupport | formatSupport; @@ -473,7 +476,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected @KeepCodecResult int canKeepCodec( MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { - if (areAdaptationCompatible(codecInfo.adaptive, oldFormat, newFormat) + if (codecInfo.isSeamlessAdaptationSupported(oldFormat, newFormat) && newFormat.width <= codecMaxValues.width && newFormat.height <= codecMaxValues.height && getMaxInputSize(codecInfo, newFormat) <= codecMaxValues.inputSize) { @@ -1049,7 +1052,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } boolean haveUnknownDimensions = false; for (Format streamFormat : streamFormats) { - if (areAdaptationCompatible(codecInfo.adaptive, format, streamFormat)) { + if (codecInfo.isSeamlessAdaptationSupported(format, streamFormat)) { haveUnknownDimensions |= (streamFormat.width == Format.NO_VALUE || streamFormat.height == Format.NO_VALUE); maxWidth = Math.max(maxWidth, streamFormat.width); @@ -1196,23 +1199,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return (maxPixels * 3) / (2 * minCompressionRatio); } - /** - * Returns whether a codec with suitable {@link CodecMaxValues} will support adaptation between - * two {@link Format}s. - * - * @param codecIsAdaptive Whether the codec supports seamless resolution switches. - * @param first The first format. - * @param second The second format. - * @return Whether the codec will support adaptation between the two {@link Format}s. - */ - private static boolean areAdaptationCompatible( - boolean codecIsAdaptive, Format first, Format second) { - return first.sampleMimeType.equals(second.sampleMimeType) - && first.rotationDegrees == second.rotationDegrees - && (codecIsAdaptive || (first.width == second.width && first.height == second.height)) - && Util.areEqual(first.colorInfo, second.colorInfo); - } - /** * Returns whether the device is known to enable frame-rate conversion logic that negatively * impacts ExoPlayer.