From 40c8dae55b6005d1240a15a2e3595cb1ca260114 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 24 Mar 2022 15:42:12 +0000 Subject: [PATCH] Transformer: Always use FrameProcessorChain when decoding. This allows us to bypass many device-specific issues, that only occur when decoding directly to an encoder surface, without OpenGL. This also allows us to maintain fewer code branches, which require additional testing to verify correctness. PiperOrigin-RevId: 437003138 --- .../transformer/TransformerEndToEndTest.java | 30 ------- .../PresentationFrameProcessor.java | 27 +----- .../transformer/ScaleToFitFrameProcessor.java | 12 --- .../VideoTranscodingSamplePipeline.java | 88 ++++++------------- .../PresentationFrameProcessorTest.java | 4 - .../ScaleToFitFrameProcessorTest.java | 6 -- 6 files changed, 31 insertions(+), 136 deletions(-) diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index 35cb731792..69f85e13ed 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -21,7 +21,6 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.android.exoplayer2.util.MimeTypes; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,37 +31,8 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public class TransformerEndToEndTest { - private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm"; private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4"; - @Test - public void videoTranscoding_completesWithConsistentFrameCount() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); - FrameCountingMuxer.Factory muxerFactory = - new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory()); - Transformer transformer = - new Transformer.Builder(context) - .setTransformationRequest( - new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build()) - .setMuxerFactory(muxerFactory) - .setEncoderFactory( - new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false)) - .build(); - // Result of the following command: - // ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm - int expectedFrameCount = 82; - - new TransformerAndroidTestRunner.Builder(context, transformer) - .build() - .run( - /* testId= */ "videoTranscoding_completesWithConsistentFrameCount", - VP9_VIDEO_URI_STRING); - - FrameCountingMuxer frameCountingMuxer = - checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated()); - assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount); - } - @Test public void videoEditing_completesWithConsistentFrameCount() throws Exception { Context context = ApplicationProvider.getApplicationContext(); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java index f51860b7a1..b4e9935b7a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessor.java @@ -81,7 +81,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; private int inputWidth; private int inputHeight; - private int outputHeight; private int outputRotationDegrees; private @MonotonicNonNull Matrix transformationMatrix; @@ -97,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputWidth = C.LENGTH_UNSET; inputHeight = C.LENGTH_UNSET; - outputHeight = C.LENGTH_UNSET; outputRotationDegrees = C.LENGTH_UNSET; } @@ -113,18 +111,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return outputRotationDegrees; } - /** - * Returns whether this {@code PresentationFrameProcessor} will apply any changes on a frame. - * - *

The {@code PresentationFrameProcessor} should only be used if this returns true. - * - *

This method can only be called after {@link #configureOutputSize(int, int)}. - */ - public boolean shouldProcess() { - checkStateNotNull(transformationMatrix); - return inputHeight != outputHeight || !transformationMatrix.isIdentity(); - } - @Override public Size configureOutputSize(int inputWidth, int inputHeight) { this.inputWidth = inputWidth; @@ -140,24 +126,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; displayHeight = requestedHeight; } - int outputWidth; // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded // frame before encoding, so the encoded frame's width >= height, and set // outputRotationDegrees to ensure the frame is displayed in the correct orientation. if (displayHeight > displayWidth) { outputRotationDegrees = 90; - outputWidth = displayHeight; - outputHeight = displayWidth; - // TODO(b/201293185): After fragment shader transformations are implemented, put postRotate in - // a later GlFrameProcessor. + // TODO(b/201293185): After fragment shader transformations are implemented, put + // postRotate in a later GlFrameProcessor. transformationMatrix.postRotate(outputRotationDegrees); + return new Size(displayHeight, displayWidth); } else { outputRotationDegrees = 0; - outputWidth = displayWidth; - outputHeight = displayHeight; + return new Size(displayWidth, displayHeight); } - - return new Size(outputWidth, outputHeight); } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java index 5aa6c5aa0c..aa79f41b5a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java @@ -122,18 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputHeight = C.LENGTH_UNSET; } - /** - * Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame. - * - *

The ScaleToFitFrameProcessor should only be used if this returns true. - * - *

This method can only be called after {@link #configureOutputSize(int, int)}. - */ - public boolean shouldProcess() { - checkStateNotNull(adjustedTransformationMatrix); - return !transformationMatrix.isIdentity(); - } - @Override public Size configureOutputSize(int inputWidth, int inputHeight) { this.inputWidth = inputWidth; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 6a8fb1c4f0..a38c3e47bc 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -42,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure; private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; - @Nullable private final FrameProcessorChain frameProcessorChain; + private final FrameProcessorChain frameProcessorChain; private final Codec encoder; private final DecoderInputBuffer encoderOutputBuffer; @@ -70,6 +70,7 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. ScaleToFitFrameProcessor scaleToFitFrameProcessor = new ScaleToFitFrameProcessor.Builder(context) .setScale(transformationRequest.scaleX, transformationRequest.scaleY) @@ -109,36 +110,24 @@ import org.checkerframework.dataflow.qual.Pure; requestedEncoderFormat, encoderSupportedFormat)); - if (transformationRequest.enableHdrEditing - || inputFormat.height != encoderSupportedFormat.height - || inputFormat.width != encoderSupportedFormat.width - || scaleToFitFrameProcessor.shouldProcess() - || presentationFrameProcessor.shouldProcess() - || shouldAlwaysUseFrameProcessorChain()) { - // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size - // has to change due to encoder fallback or append another GlFrameProcessor. - frameProcessorSizes.set( - frameProcessorSizes.size() - 1, - new Size(encoderSupportedFormat.width, encoderSupportedFormat.height)); - frameProcessorChain = - FrameProcessorChain.create( - context, - inputFormat.pixelWidthHeightRatio, - frameProcessors, - frameProcessorSizes, - /* outputSurface= */ encoder.getInputSurface(), - transformationRequest.enableHdrEditing, - debugViewProvider); - } else { - frameProcessorChain = null; - } + // TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size + // has to change due to encoder fallback or append another GlFrameProcessor. + frameProcessorSizes.set( + frameProcessorSizes.size() - 1, + new Size(encoderSupportedFormat.width, encoderSupportedFormat.height)); + frameProcessorChain = + FrameProcessorChain.create( + context, + inputFormat.pixelWidthHeightRatio, + frameProcessors, + frameProcessorSizes, + /* outputSurface= */ encoder.getInputSurface(), + transformationRequest.enableHdrEditing, + debugViewProvider); decoder = decoderFactory.createForVideoDecoding( - inputFormat, - frameProcessorChain == null - ? encoder.getInputSurface() - : frameProcessorChain.createInputSurface()); + inputFormat, frameProcessorChain.createInputSurface()); } @Override @@ -154,15 +143,13 @@ import org.checkerframework.dataflow.qual.Pure; @Override public boolean processData() throws TransformationException { - if (frameProcessorChain != null) { - frameProcessorChain.getAndRethrowBackgroundExceptions(); - if (frameProcessorChain.isEnded()) { - if (!signaledEndOfStreamToEncoder) { - encoder.signalEndOfInputStream(); - signaledEndOfStreamToEncoder = true; - } - return false; + frameProcessorChain.getAndRethrowBackgroundExceptions(); + if (frameProcessorChain.isEnded()) { + if (!signaledEndOfStreamToEncoder) { + encoder.signalEndOfInputStream(); + signaledEndOfStreamToEncoder = true; } + return false; } if (decoder.isEnded()) { return false; @@ -178,13 +165,7 @@ import org.checkerframework.dataflow.qual.Pure; canProcessMoreDataImmediately = processDataDefault(); } if (decoder.isEnded()) { - if (frameProcessorChain != null) { - frameProcessorChain.signalEndOfInputStream(); - } else { - encoder.signalEndOfInputStream(); - signaledEndOfStreamToEncoder = true; - return false; - } + frameProcessorChain.signalEndOfInputStream(); } return canProcessMoreDataImmediately; } @@ -215,7 +196,7 @@ import org.checkerframework.dataflow.qual.Pure; */ private boolean processDataDefault() throws TransformationException { // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. - if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) { + if (frameProcessorChain.hasPendingFrames()) { return false; } return maybeProcessDecoderOutput(); @@ -255,9 +236,7 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void release() { - if (frameProcessorChain != null) { - frameProcessorChain.release(); - } + frameProcessorChain.release(); decoder.release(); encoder.release(); } @@ -292,17 +271,6 @@ import org.checkerframework.dataflow.qual.Pure; .build(); } - /** Always use {@link FrameProcessorChain} to work around device-specific encoder issues. */ - private static boolean shouldAlwaysUseFrameProcessorChain() { - switch (Util.MODEL) { - case "XT1635-02": - case "Nexus 5": - return true; - default: - return false; - } - } - /** * Feeds at most one decoder output frame to the next step of the pipeline. * @@ -314,9 +282,7 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - if (frameProcessorChain != null) { - frameProcessorChain.registerInputFrame(); - } + frameProcessorChain.registerInputFrame(); decoder.releaseOutputBuffer(/* render= */ true); return true; } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java index 14b003d714..4336fa76ed 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/PresentationFrameProcessorTest.java @@ -42,7 +42,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -57,7 +56,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -72,7 +70,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); - assertThat(presentationFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputWidth); } @@ -90,7 +87,6 @@ public final class PresentationFrameProcessorTest { Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(presentationFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java index 81c606d651..5cf7b6aa4f 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java @@ -42,7 +42,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -69,7 +68,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f)); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -85,7 +83,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -101,7 +98,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2); } @@ -117,7 +113,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(inputHeight); assertThat(outputSize.getHeight()).isEqualTo(inputWidth); } @@ -134,7 +129,6 @@ public final class ScaleToFitFrameProcessorTest { Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); - assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue(); assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight); assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight); }