diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java index 081e502ccf..cdfe5fa86a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java @@ -22,12 +22,16 @@ import static com.google.android.exoplayer2.util.Util.SDK_INT; import android.annotation.SuppressLint; import android.media.MediaCodec; +import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; +import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.mediacodec.MediaCodecUtil; import com.google.android.exoplayer2.util.MediaFormatUtil; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; import java.io.IOException; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -36,6 +40,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* package */ final class DefaultCodecFactory implements Codec.DecoderFactory, Codec.EncoderFactory { + // TODO(b/210591626) Fall back adaptively to H265 if possible. + private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264; + private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface; + private static final int DEFAULT_FRAME_RATE = 60; + private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1; + @Override public Codec createForAudioDecoding(Format format) throws TransformationException { MediaFormat mediaFormat = @@ -91,20 +101,34 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public Codec createForVideoEncoding(Format format) throws TransformationException { + checkArgument(format.sampleMimeType != null); checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); // According to interface Javadoc, format.rotationDegrees should be 0. The video should always // be in landscape orientation. checkArgument(format.height < format.width); checkArgument(format.rotationDegrees == 0); + // Checking again to silence null checker warning. + checkNotNull(format.sampleMimeType); + format = getVideoEncoderSupportedFormat(format); MediaFormat mediaFormat = MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); - mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface); - mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); - mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); - mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000); + mediaFormat.setFloat(MediaFormat.KEY_FRAME_RATE, format.frameRate); + mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate); + + @Nullable + Pair codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format); + if (codecProfileAndLevel != null) { + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfileAndLevel.first); + if (SDK_INT >= 23) { + mediaFormat.setInteger(MediaFormat.KEY_LEVEL, codecProfileAndLevel.second); + } + } + + mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, DEFAULT_COLOR_FORMAT); + mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL_SECS); return createCodec( format, @@ -168,6 +192,85 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TraceUtil.endSection(); } + @RequiresNonNull("#1.sampleMimeType") + private static Format getVideoEncoderSupportedFormat(Format requestedFormat) + throws TransformationException { + String mimeType = requestedFormat.sampleMimeType; + Format.Builder formatBuilder = requestedFormat.buildUpon(); + + // TODO(b/210591626) Implement encoder filtering. + if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + mimeType = DEFAULT_FALLBACK_MIME_TYPE; + if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + throw createTransformationException( + new IllegalArgumentException( + "No encoder is found for requested MIME type " + requestedFormat.sampleMimeType), + requestedFormat, + /* isVideo= */ true, + /* isDecoder= */ false, + /* mediaCodecName= */ null); + } + } + + formatBuilder.setSampleMimeType(mimeType); + MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0); + + int width = requestedFormat.width; + int height = requestedFormat.height; + @Nullable + Pair encoderSupportedResolution = + EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height); + if (encoderSupportedResolution == null) { + throw createTransformationException( + new IllegalArgumentException( + "Cannot find fallback resolution for resolution " + width + " x " + height), + requestedFormat, + /* isVideo= */ true, + /* isDecoder= */ false, + /* mediaCodecName= */ null); + } + width = encoderSupportedResolution.first; + height = encoderSupportedResolution.second; + formatBuilder.setWidth(width).setHeight(height); + + // The frameRate does not affect the resulting frame rate. It affects the encoder's rate control + // algorithm. Setting it too high may lead to video quality degradation. + float frameRate = + requestedFormat.frameRate != Format.NO_VALUE + ? requestedFormat.frameRate + : DEFAULT_FRAME_RATE; + int bitrate = + EncoderUtil.getClosestSupportedBitrate( + encoderInfo, + mimeType, + /* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE + ? requestedFormat.averageBitrate + : getSuggestedBitrate(width, height, frameRate)); + formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate); + + @Nullable + Pair profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat); + if (profileLevel == null + // Transcoding to another MIME type. + || !requestedFormat.sampleMimeType.equals(mimeType) + || !EncoderUtil.isProfileLevelSupported( + encoderInfo, + mimeType, + /* profile= */ profileLevel.first, + /* level= */ profileLevel.second)) { + formatBuilder.setCodecs(null); + } + + return formatBuilder.build(); + } + + /** Computes the video bit rate using the Kush Gauge. */ + private static int getSuggestedBitrate(int width, int height, float frameRate) { + // TODO(b/210591626) Implement bitrate estimation. + // 1080p30 -> 6.2Mbps, 720p30 -> 2.7Mbps. + return (int) (width * height * frameRate * 0.1); + } + private static TransformationException createTransformationException( Exception cause, Format format, diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java new file mode 100644 index 0000000000..65e96efbc8 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java @@ -0,0 +1,156 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import static java.lang.Math.round; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.util.Pair; +import androidx.annotation.Nullable; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; + +/** Utility methods for {@link MediaCodec} encoders. */ +public final class EncoderUtil { + + private static final List encoders = new ArrayList<>(); + + /** + * Returns a list of {@link MediaCodecInfo encoders} that support the given {@code mimeType}, or + * an empty list if there is none. + */ + public static ImmutableList getSupportedEncoders(String mimeType) { + maybePopulateEncoderInfos(); + + ImmutableList.Builder availableEncoders = new ImmutableList.Builder<>(); + for (int i = 0; i < encoders.size(); i++) { + MediaCodecInfo encoderInfo = encoders.get(i); + String[] supportedMimeTypes = encoderInfo.getSupportedTypes(); + for (String supportedMimeType : supportedMimeTypes) { + if (Ascii.equalsIgnoreCase(supportedMimeType, mimeType)) { + availableEncoders.add(encoderInfo); + } + } + } + return availableEncoders.build(); + } + + /** + * Finds the {@link MediaCodecInfo encoder}'s closest supported resolution from the given + * resolution. + * + *

The input resolution is returned, if it is supported by the {@link MediaCodecInfo encoder}. + * + *

The resolution will be clamped to the {@link MediaCodecInfo encoder}'s range of supported + * resolutions, and adjusted to the {@link MediaCodecInfo encoder}'s size alignment. The + * adjustment process takes into account the original aspect ratio. But the fixed resolution may + * not preserve the original aspect ratio, depending on the encoder's required size alignment. + * + * @param encoderInfo The {@link MediaCodecInfo} of the encoder. + * @param mimeType The output MIME type. + * @param width The original width. + * @param height The original height. + * @return A {@link Pair} of width and height, or {@code null} if unable to find a fix. + */ + @Nullable + public static Pair getClosestSupportedResolution( + MediaCodecInfo encoderInfo, String mimeType, int width, int height) { + MediaCodecInfo.VideoCapabilities videoEncoderCapabilities = + encoderInfo.getCapabilitiesForType(mimeType).getVideoCapabilities(); + + if (videoEncoderCapabilities.isSizeSupported(width, height)) { + return Pair.create(width, height); + } + + // Fix frame being too wide or too tall. + int adjustedHeight = videoEncoderCapabilities.getSupportedHeights().clamp(height); + if (adjustedHeight != height) { + width = (int) round((double) width * adjustedHeight / height); + height = adjustedHeight; + } + + int adjustedWidth = videoEncoderCapabilities.getSupportedWidths().clamp(width); + if (adjustedWidth != width) { + height = (int) round((double) height * adjustedWidth / width); + width = adjustedWidth; + } + + // Fix pixel alignment. + width = alignResolution(width, videoEncoderCapabilities.getWidthAlignment()); + height = alignResolution(height, videoEncoderCapabilities.getHeightAlignment()); + + return videoEncoderCapabilities.isSizeSupported(width, height) + ? Pair.create(width, height) + : null; + } + + /** Returns whether the {@link MediaCodecInfo encoder} supports the given profile and level. */ + public static boolean isProfileLevelSupported( + MediaCodecInfo encoderInfo, String mimeType, int profile, int level) { + MediaCodecInfo.CodecProfileLevel[] profileLevels = + encoderInfo.getCapabilitiesForType(mimeType).profileLevels; + + for (MediaCodecInfo.CodecProfileLevel profileLevel : profileLevels) { + if (profileLevel.profile == profile && profileLevel.level == level) { + return true; + } + } + return false; + } + + /** + * Finds the {@link MediaCodecInfo encoder}'s closest supported bitrate from the given bitrate. + */ + public static int getClosestSupportedBitrate( + MediaCodecInfo encoderInfo, String mimeType, int bitrate) { + return encoderInfo + .getCapabilitiesForType(mimeType) + .getVideoCapabilities() + .getBitrateRange() + .clamp(bitrate); + } + + /** + * Align to the closest resolution that respects the encoder's supported alignment. + * + *

For example, size 35 will be aligned to 32 if the alignment is 16, and size 45 will be + * aligned to 48. + */ + private static int alignResolution(int size, int alignment) { + return alignment * Math.round((float) size / alignment); + } + + private static synchronized void maybePopulateEncoderInfos() { + if (encoders.isEmpty()) { + MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS); + MediaCodecInfo[] allCodecInfos = mediaCodecList.getCodecInfos(); + + for (MediaCodecInfo mediaCodecInfo : allCodecInfos) { + if (!mediaCodecInfo.isEncoder()) { + continue; + } + encoders.add(mediaCodecInfo); + } + } + } + + private EncoderUtil() {} +}