From 815f5da91a4b0ac33f2fc9d1fbc42cfeb3a3ce24 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 9 Dec 2021 18:41:30 +0000 Subject: [PATCH] Keep orientation information during the transformation. The input rotation is used to rotate the video during decoding, the video is rotated so that it is in landscape orientation before encoding and a rotation is added to the output format where necessary so that the output video has the same orientation as the input. PiperOrigin-RevId: 415301328 --- .../transformer/MediaCodecAdapterWrapper.java | 9 +++++- .../transformer/VideoSamplePipeline.java | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java index e41174a4ee..6fcffa3a8b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import android.annotation.SuppressLint; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; import android.media.MediaCodecInfo.CodecCapabilities; @@ -133,6 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @return A configured and started decoder wrapper. * @throws IOException If the underlying codec cannot be created. */ + @SuppressLint("InlinedApi") public static MediaCodecAdapterWrapper createForVideoDecoding(Format format, Surface surface) throws IOException { @Nullable MediaCodecAdapter adapter = null; @@ -140,6 +142,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaFormat mediaFormat = MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); + MediaFormatUtil.maybeSetInteger( + mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees); MediaFormatUtil.maybeSetInteger( mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); @@ -201,7 +205,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param format The {@link Format} (of the output data) used to determine the underlying {@link * MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link * Format#width} and {@link Format#height} must be set to those of the desired output video - * format. + * format. {@link Format#rotationDegrees} should be 0. The video should always be in landscape + * orientation. * @param additionalEncoderConfig A map of {@link MediaFormat}'s integer settings, where the keys * are from {@code MediaFormat.KEY_*} constants. Its values will override those in {@code * format}. @@ -212,6 +217,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Format format, Map additionalEncoderConfig) throws IOException { checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); + checkArgument(format.height < format.width); + checkArgument(format.rotationDegrees == 0); @Nullable MediaCodecAdapter adapter = null; try { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 8fe846e9ab..7e7c4e2ecd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -37,6 +37,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String TAG = "VideoSamplePipeline"; + private final int outputRotationDegrees; private final DecoderInputBuffer decoderInputBuffer; private final MediaCodecAdapterWrapper decoder; @@ -59,6 +60,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; if (transformation.outputHeight != Format.NO_VALUE @@ -67,12 +69,29 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputHeight = transformation.outputHeight; } + if (inputFormat.height > inputFormat.width) { + // The encoder may not support encoding in portrait orientation, so the decoded video is + // rotated to landscape orientation and a rotation is added back later to the output format. + outputRotationDegrees = (inputFormat.rotationDegrees + 90) % 360; + int temp = outputWidth; + outputWidth = outputHeight; + outputHeight = temp; + } else { + outputRotationDegrees = inputFormat.rotationDegrees; + } + // The decoder rotates videos to their intended display orientation. The frameEditor rotates + // them back for improved encoder compatibility. + // TODO(internal b/201293185): After fragment shader transformations are implemented, put + // postrotation in a later vertex shader. + transformation.transformationMatrix.postRotate(outputRotationDegrees); + try { encoder = MediaCodecAdapterWrapper.createForVideoEncoding( new Format.Builder() .setWidth(outputWidth) .setHeight(outputHeight) + .setRotationDegrees(0) .setSampleMimeType( transformation.videoMimeType != null ? transformation.videoMimeType @@ -84,7 +103,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw createRendererException( e, rendererIndex, inputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED); } - if (inputFormat.height != outputHeight || !transformation.transformationMatrix.isIdentity()) { + if (inputFormat.height != outputHeight + || inputFormat.width != outputWidth + || !transformation.transformationMatrix.isIdentity()) { frameEditor = FrameEditor.create( context, @@ -150,7 +171,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override @Nullable public Format getOutputFormat() { - return encoder.getOutputFormat(); + Format format = encoder.getOutputFormat(); + return format == null + ? null + : format.buildUpon().setRotationDegrees(outputRotationDegrees).build(); } @Override