mirror of
https://github.com/samsonjs/media.git
synced 2026-04-09 11:55:46 +00:00
Use encoder output format for configuring the Encoder.
This CL implements fixing the input format to the encoder spec. Fixed parameters include: - MIME type - Profile & level - Resolution - frame rate, and - bitrate PiperOrigin-RevId: 422513738
This commit is contained in:
claincly
Ian Baker
parent
25a362ec67
commit
9b3483ec5f
2 changed files with 263 additions and 4 deletions
|
|
@ -22,12 +22,16 @@ import static com.google.android.exoplayer2.util.Util.SDK_INT;
|
|||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaCodecInfo.CodecCapabilities;
|
||||
import android.media.MediaFormat;
|
||||
import android.util.Pair;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import com.google.android.exoplayer2.Format;
|
||||
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
|
||||
import com.google.android.exoplayer2.util.MediaFormatUtil;
|
||||
import com.google.android.exoplayer2.util.MimeTypes;
|
||||
import com.google.android.exoplayer2.util.TraceUtil;
|
||||
import java.io.IOException;
|
||||
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
|
|
@ -36,6 +40,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
|||
/* package */ final class DefaultCodecFactory
|
||||
implements Codec.DecoderFactory, Codec.EncoderFactory {
|
||||
|
||||
// TODO(b/210591626) Fall back adaptively to H265 if possible.
|
||||
private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264;
|
||||
private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface;
|
||||
private static final int DEFAULT_FRAME_RATE = 60;
|
||||
private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1;
|
||||
|
||||
@Override
|
||||
public Codec createForAudioDecoding(Format format) throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
|
|
@ -91,20 +101,34 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
|||
|
||||
@Override
|
||||
public Codec createForVideoEncoding(Format format) throws TransformationException {
|
||||
checkArgument(format.sampleMimeType != null);
|
||||
checkArgument(format.width != Format.NO_VALUE);
|
||||
checkArgument(format.height != Format.NO_VALUE);
|
||||
// According to interface Javadoc, format.rotationDegrees should be 0. The video should always
|
||||
// be in landscape orientation.
|
||||
checkArgument(format.height < format.width);
|
||||
checkArgument(format.rotationDegrees == 0);
|
||||
// Checking again to silence null checker warning.
|
||||
checkNotNull(format.sampleMimeType);
|
||||
format = getVideoEncoderSupportedFormat(format);
|
||||
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createVideoFormat(
|
||||
checkNotNull(format.sampleMimeType), format.width, format.height);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000);
|
||||
mediaFormat.setFloat(MediaFormat.KEY_FRAME_RATE, format.frameRate);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate);
|
||||
|
||||
@Nullable
|
||||
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
|
||||
if (codecProfileAndLevel != null) {
|
||||
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfileAndLevel.first);
|
||||
if (SDK_INT >= 23) {
|
||||
mediaFormat.setInteger(MediaFormat.KEY_LEVEL, codecProfileAndLevel.second);
|
||||
}
|
||||
}
|
||||
|
||||
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, DEFAULT_COLOR_FORMAT);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL_SECS);
|
||||
|
||||
return createCodec(
|
||||
format,
|
||||
|
|
@ -168,6 +192,85 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
|||
TraceUtil.endSection();
|
||||
}
|
||||
|
||||
@RequiresNonNull("#1.sampleMimeType")
|
||||
private static Format getVideoEncoderSupportedFormat(Format requestedFormat)
|
||||
throws TransformationException {
|
||||
String mimeType = requestedFormat.sampleMimeType;
|
||||
Format.Builder formatBuilder = requestedFormat.buildUpon();
|
||||
|
||||
// TODO(b/210591626) Implement encoder filtering.
|
||||
if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
|
||||
mimeType = DEFAULT_FALLBACK_MIME_TYPE;
|
||||
if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
|
||||
throw createTransformationException(
|
||||
new IllegalArgumentException(
|
||||
"No encoder is found for requested MIME type " + requestedFormat.sampleMimeType),
|
||||
requestedFormat,
|
||||
/* isVideo= */ true,
|
||||
/* isDecoder= */ false,
|
||||
/* mediaCodecName= */ null);
|
||||
}
|
||||
}
|
||||
|
||||
formatBuilder.setSampleMimeType(mimeType);
|
||||
MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0);
|
||||
|
||||
int width = requestedFormat.width;
|
||||
int height = requestedFormat.height;
|
||||
@Nullable
|
||||
Pair<Integer, Integer> encoderSupportedResolution =
|
||||
EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height);
|
||||
if (encoderSupportedResolution == null) {
|
||||
throw createTransformationException(
|
||||
new IllegalArgumentException(
|
||||
"Cannot find fallback resolution for resolution " + width + " x " + height),
|
||||
requestedFormat,
|
||||
/* isVideo= */ true,
|
||||
/* isDecoder= */ false,
|
||||
/* mediaCodecName= */ null);
|
||||
}
|
||||
width = encoderSupportedResolution.first;
|
||||
height = encoderSupportedResolution.second;
|
||||
formatBuilder.setWidth(width).setHeight(height);
|
||||
|
||||
// The frameRate does not affect the resulting frame rate. It affects the encoder's rate control
|
||||
// algorithm. Setting it too high may lead to video quality degradation.
|
||||
float frameRate =
|
||||
requestedFormat.frameRate != Format.NO_VALUE
|
||||
? requestedFormat.frameRate
|
||||
: DEFAULT_FRAME_RATE;
|
||||
int bitrate =
|
||||
EncoderUtil.getClosestSupportedBitrate(
|
||||
encoderInfo,
|
||||
mimeType,
|
||||
/* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE
|
||||
? requestedFormat.averageBitrate
|
||||
: getSuggestedBitrate(width, height, frameRate));
|
||||
formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate);
|
||||
|
||||
@Nullable
|
||||
Pair<Integer, Integer> profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat);
|
||||
if (profileLevel == null
|
||||
// Transcoding to another MIME type.
|
||||
|| !requestedFormat.sampleMimeType.equals(mimeType)
|
||||
|| !EncoderUtil.isProfileLevelSupported(
|
||||
encoderInfo,
|
||||
mimeType,
|
||||
/* profile= */ profileLevel.first,
|
||||
/* level= */ profileLevel.second)) {
|
||||
formatBuilder.setCodecs(null);
|
||||
}
|
||||
|
||||
return formatBuilder.build();
|
||||
}
|
||||
|
||||
/** Computes the video bit rate using the Kush Gauge. */
|
||||
private static int getSuggestedBitrate(int width, int height, float frameRate) {
|
||||
// TODO(b/210591626) Implement bitrate estimation.
|
||||
// 1080p30 -> 6.2Mbps, 720p30 -> 2.7Mbps.
|
||||
return (int) (width * height * frameRate * 0.1);
|
||||
}
|
||||
|
||||
private static TransformationException createTransformationException(
|
||||
Exception cause,
|
||||
Format format,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
/*
|
||||
* Copyright 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.google.android.exoplayer2.transformer;
|
||||
|
||||
import static java.lang.Math.round;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaCodecList;
|
||||
import android.util.Pair;
|
||||
import androidx.annotation.Nullable;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** Utility methods for {@link MediaCodec} encoders. */
|
||||
public final class EncoderUtil {
|
||||
|
||||
private static final List<MediaCodecInfo> encoders = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* Returns a list of {@link MediaCodecInfo encoders} that support the given {@code mimeType}, or
|
||||
* an empty list if there is none.
|
||||
*/
|
||||
public static ImmutableList<MediaCodecInfo> getSupportedEncoders(String mimeType) {
|
||||
maybePopulateEncoderInfos();
|
||||
|
||||
ImmutableList.Builder<MediaCodecInfo> availableEncoders = new ImmutableList.Builder<>();
|
||||
for (int i = 0; i < encoders.size(); i++) {
|
||||
MediaCodecInfo encoderInfo = encoders.get(i);
|
||||
String[] supportedMimeTypes = encoderInfo.getSupportedTypes();
|
||||
for (String supportedMimeType : supportedMimeTypes) {
|
||||
if (Ascii.equalsIgnoreCase(supportedMimeType, mimeType)) {
|
||||
availableEncoders.add(encoderInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
return availableEncoders.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the {@link MediaCodecInfo encoder}'s closest supported resolution from the given
|
||||
* resolution.
|
||||
*
|
||||
* <p>The input resolution is returned, if it is supported by the {@link MediaCodecInfo encoder}.
|
||||
*
|
||||
* <p>The resolution will be clamped to the {@link MediaCodecInfo encoder}'s range of supported
|
||||
* resolutions, and adjusted to the {@link MediaCodecInfo encoder}'s size alignment. The
|
||||
* adjustment process takes into account the original aspect ratio. But the fixed resolution may
|
||||
* not preserve the original aspect ratio, depending on the encoder's required size alignment.
|
||||
*
|
||||
* @param encoderInfo The {@link MediaCodecInfo} of the encoder.
|
||||
* @param mimeType The output MIME type.
|
||||
* @param width The original width.
|
||||
* @param height The original height.
|
||||
* @return A {@link Pair} of width and height, or {@code null} if unable to find a fix.
|
||||
*/
|
||||
@Nullable
|
||||
public static Pair<Integer, Integer> getClosestSupportedResolution(
|
||||
MediaCodecInfo encoderInfo, String mimeType, int width, int height) {
|
||||
MediaCodecInfo.VideoCapabilities videoEncoderCapabilities =
|
||||
encoderInfo.getCapabilitiesForType(mimeType).getVideoCapabilities();
|
||||
|
||||
if (videoEncoderCapabilities.isSizeSupported(width, height)) {
|
||||
return Pair.create(width, height);
|
||||
}
|
||||
|
||||
// Fix frame being too wide or too tall.
|
||||
int adjustedHeight = videoEncoderCapabilities.getSupportedHeights().clamp(height);
|
||||
if (adjustedHeight != height) {
|
||||
width = (int) round((double) width * adjustedHeight / height);
|
||||
height = adjustedHeight;
|
||||
}
|
||||
|
||||
int adjustedWidth = videoEncoderCapabilities.getSupportedWidths().clamp(width);
|
||||
if (adjustedWidth != width) {
|
||||
height = (int) round((double) height * adjustedWidth / width);
|
||||
width = adjustedWidth;
|
||||
}
|
||||
|
||||
// Fix pixel alignment.
|
||||
width = alignResolution(width, videoEncoderCapabilities.getWidthAlignment());
|
||||
height = alignResolution(height, videoEncoderCapabilities.getHeightAlignment());
|
||||
|
||||
return videoEncoderCapabilities.isSizeSupported(width, height)
|
||||
? Pair.create(width, height)
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Returns whether the {@link MediaCodecInfo encoder} supports the given profile and level. */
|
||||
public static boolean isProfileLevelSupported(
|
||||
MediaCodecInfo encoderInfo, String mimeType, int profile, int level) {
|
||||
MediaCodecInfo.CodecProfileLevel[] profileLevels =
|
||||
encoderInfo.getCapabilitiesForType(mimeType).profileLevels;
|
||||
|
||||
for (MediaCodecInfo.CodecProfileLevel profileLevel : profileLevels) {
|
||||
if (profileLevel.profile == profile && profileLevel.level == level) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the {@link MediaCodecInfo encoder}'s closest supported bitrate from the given bitrate.
|
||||
*/
|
||||
public static int getClosestSupportedBitrate(
|
||||
MediaCodecInfo encoderInfo, String mimeType, int bitrate) {
|
||||
return encoderInfo
|
||||
.getCapabilitiesForType(mimeType)
|
||||
.getVideoCapabilities()
|
||||
.getBitrateRange()
|
||||
.clamp(bitrate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Align to the closest resolution that respects the encoder's supported alignment.
|
||||
*
|
||||
* <p>For example, size 35 will be aligned to 32 if the alignment is 16, and size 45 will be
|
||||
* aligned to 48.
|
||||
*/
|
||||
private static int alignResolution(int size, int alignment) {
|
||||
return alignment * Math.round((float) size / alignment);
|
||||
}
|
||||
|
||||
private static synchronized void maybePopulateEncoderInfos() {
|
||||
if (encoders.isEmpty()) {
|
||||
MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS);
|
||||
MediaCodecInfo[] allCodecInfos = mediaCodecList.getCodecInfos();
|
||||
|
||||
for (MediaCodecInfo mediaCodecInfo : allCodecInfos) {
|
||||
if (!mediaCodecInfo.isEncoder()) {
|
||||
continue;
|
||||
}
|
||||
encoders.add(mediaCodecInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private EncoderUtil() {}
|
||||
}
|
||||
Loading…
Reference in a new issue