From 47f4d90515eb447ea1e6955e9ecf7b75f75e355a Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 29 Dec 2021 19:10:12 +0000 Subject: [PATCH] Use TransformationException for GL errors. PiperOrigin-RevId: 418820557 --- .../FrameEditorDataProcessingTest.java | 236 +++++++++++++++++ .../transformer/FrameEditorTest.java | 238 +++--------------- .../exoplayer2/transformer/FrameEditor.java | 138 +++++----- .../transformer/TransformationException.java | 14 +- .../transformer/VideoSamplePipeline.java | 10 +- 5 files changed, 372 insertions(+), 264 deletions(-) create mode 100644 library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java new file mode 100644 index 0000000000..a2762ed080 --- /dev/null +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java @@ -0,0 +1,236 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; +import static java.lang.Math.abs; +import static java.lang.Math.max; + +import android.content.res.AssetFileDescriptor; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Matrix; +import android.graphics.PixelFormat; +import android.media.Image; +import android.media.ImageReader; +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import androidx.annotation.Nullable; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.util.MimeTypes; +import java.io.InputStream; +import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Test for frame processing via {@link FrameEditor#processData()}. */ +@RunWith(AndroidJUnit4.class) +public final class FrameEditorDataProcessingTest { + + private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; + private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame.png"; + /** + * Maximum allowed average pixel difference between the expected and actual edited images for the + * test to pass. The value is chosen so that differences in decoder behavior across emulator + * versions shouldn't affect whether the test passes, but substantial distortions introduced by + * changes in the behavior of the frame editor will cause the test to fail. + */ + private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f; + /** Timeout for dequeueing buffers from the codec, in microseconds. */ + private static final int DEQUEUE_TIMEOUT_US = 5_000_000; + /** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */ + private static final int SURFACE_WAIT_MS = 1000; + /** The ratio of width over height, for each pixel in a frame. */ + private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; + + private @MonotonicNonNull FrameEditor frameEditor; + private @MonotonicNonNull ImageReader frameEditorOutputImageReader; + private @MonotonicNonNull MediaFormat mediaFormat; + + @Before + public void setUp() throws Exception { + // Set up the extractor to read the first video frame and get its format. + MediaExtractor mediaExtractor = new MediaExtractor(); + @Nullable MediaCodec mediaCodec = null; + try (AssetFileDescriptor afd = + getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) { + mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); + for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { + if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { + mediaFormat = mediaExtractor.getTrackFormat(i); + mediaExtractor.selectTrack(i); + break; + } + } + + int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); + int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + frameEditorOutputImageReader = + ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + Matrix identityMatrix = new Matrix(); + frameEditor = + FrameEditor.create( + getApplicationContext(), + width, + height, + PIXEL_WIDTH_HEIGHT_RATIO, + identityMatrix, + frameEditorOutputImageReader.getSurface(), + Transformer.DebugViewProvider.NONE); + + // Queue the first video frame from the extractor. + String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); + mediaCodec = MediaCodec.createDecoderByType(mimeType); + mediaCodec.configure( + mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0); + mediaCodec.start(); + int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); + assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); + ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]); + int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0); + mediaCodec.queueInputBuffer( + inputBufferIndex, + /* offset= */ 0, + sampleSize, + mediaExtractor.getSampleTime(), + mediaExtractor.getSampleFlags()); + + // Queue an end-of-stream buffer to force the codec to produce output. + inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); + assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); + mediaCodec.queueInputBuffer( + inputBufferIndex, + /* offset= */ 0, + /* size= */ 0, + /* presentationTimeUs= */ 0, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + + // Dequeue and render the output video frame. + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + int outputBufferIndex; + do { + outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US); + assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); + } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED + || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED); + mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true); + + // Sleep to give time for the surface texture to be populated. + Thread.sleep(SURFACE_WAIT_MS); + assertThat(frameEditor.hasInputData()).isTrue(); + } finally { + mediaExtractor.release(); + if (mediaCodec != null) { + mediaCodec.release(); + } + } + } + + @After + public void tearDown() { + if (frameEditor != null) { + frameEditor.release(); + } + } + + @Test + public void processData_noEdits_producesExpectedOutput() throws Exception { + Bitmap expectedBitmap; + try (InputStream inputStream = + getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) { + expectedBitmap = BitmapFactory.decodeStream(inputStream); + } + + checkNotNull(frameEditor).processData(); + Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); + Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); + + // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + /** + * Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per + * component image. + */ + private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) { + int width = image.getWidth(); + int height = image.getHeight(); + assertThat(image.getPlanes()).hasLength(1); + assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888); + Image.Plane plane = image.getPlanes()[0]; + ByteBuffer buffer = plane.getBuffer(); + int[] colors = new int[width * height]; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int offset = y * plane.getRowStride() + x * plane.getPixelStride(); + int r = buffer.get(offset) & 0xFF; + int g = buffer.get(offset + 1) & 0xFF; + int b = buffer.get(offset + 2) & 0xFF; + int a = buffer.get(offset + 3) & 0xFF; + colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b; + } + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); + } + + /** + * Returns the sum of the absolute differences between the expected and actual bitmaps, calculated + * using the maximum difference across all color channels for each pixel, then divided by the + * total number of pixels in the image. The bitmap resolutions must match and they must use + * configuration {@link Bitmap.Config#ARGB_8888}. + */ + private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) { + int width = actual.getWidth(); + int height = actual.getHeight(); + assertThat(width).isEqualTo(expected.getWidth()); + assertThat(height).isEqualTo(expected.getHeight()); + assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888); + long sumMaximumAbsoluteDifferences = 0; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int color = actual.getPixel(x, y); + int expectedColor = expected.getPixel(x, y); + int maximumAbsoluteDifference = 0; + maximumAbsoluteDifference = + max( + maximumAbsoluteDifference, + abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF))); + maximumAbsoluteDifference = + max( + maximumAbsoluteDifference, + abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF))); + maximumAbsoluteDifference = + max( + maximumAbsoluteDifference, + abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF))); + maximumAbsoluteDifference = + max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF))); + sumMaximumAbsoluteDifferences += maximumAbsoluteDifference; + } + } + return (float) sumMaximumAbsoluteDifferences / (width * height); + } +} diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java index b47205947f..a30ffc51c2 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java @@ -16,221 +16,55 @@ package com.google.android.exoplayer2.transformer; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; -import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertThat; -import static java.lang.Math.abs; -import static java.lang.Math.max; +import static org.junit.Assert.assertThrows; -import android.content.res.AssetFileDescriptor; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; +import android.content.Context; import android.graphics.Matrix; -import android.graphics.PixelFormat; -import android.media.Image; -import android.media.ImageReader; -import android.media.MediaCodec; -import android.media.MediaExtractor; -import android.media.MediaFormat; -import androidx.annotation.Nullable; +import android.graphics.SurfaceTexture; +import android.view.Surface; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.android.exoplayer2.util.MimeTypes; -import java.io.InputStream; -import java.nio.ByteBuffer; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -/** Test for frame processing via {@link FrameEditor}. */ +/** + * Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface, + * Transformer.DebugViewProvider) creating} a {@link FrameEditor}. + */ @RunWith(AndroidJUnit4.class) public final class FrameEditorTest { + // TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a + // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20. - private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; - private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING = - "media/bitmap/sample_mp4_first_frame.png"; - /** - * Maximum allowed average pixel difference between the expected and actual edited images for the - * test to pass. The value is chosen so that differences in decoder behavior across emulator - * versions shouldn't affect whether the test passes, but substantial distortions introduced by - * changes in the behavior of the frame editor will cause the test to fail. - */ - private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f; - /** Timeout for dequeueing buffers from the codec, in microseconds. */ - private static final int DEQUEUE_TIMEOUT_US = 5_000_000; - /** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */ - private static final int SURFACE_WAIT_MS = 1000; - /** The ratio of width over height, for each pixel in a frame. */ - private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; - - private @MonotonicNonNull FrameEditor frameEditor; - private @MonotonicNonNull ImageReader frameEditorOutputImageReader; - private @MonotonicNonNull MediaFormat mediaFormat; - - @Before - public void setUp() throws Exception { - // Set up the extractor to read the first video frame and get its format. - MediaExtractor mediaExtractor = new MediaExtractor(); - @Nullable MediaCodec mediaCodec = null; - try (AssetFileDescriptor afd = - getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) { - mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); - for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { - if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { - mediaFormat = mediaExtractor.getTrackFormat(i); - mediaExtractor.selectTrack(i); - break; - } - } - - int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); - int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - frameEditorOutputImageReader = - ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - Matrix identityMatrix = new Matrix(); - frameEditor = - FrameEditor.create( - getApplicationContext(), - width, - height, - PIXEL_WIDTH_HEIGHT_RATIO, - identityMatrix, - frameEditorOutputImageReader.getSurface(), - Transformer.DebugViewProvider.NONE); - - // Queue the first video frame from the extractor. - String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); - mediaCodec = MediaCodec.createDecoderByType(mimeType); - mediaCodec.configure( - mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0); - mediaCodec.start(); - int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); - assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]); - int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0); - mediaCodec.queueInputBuffer( - inputBufferIndex, - /* offset= */ 0, - sampleSize, - mediaExtractor.getSampleTime(), - mediaExtractor.getSampleFlags()); - - // Queue an end-of-stream buffer to force the codec to produce output. - inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); - assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - mediaCodec.queueInputBuffer( - inputBufferIndex, - /* offset= */ 0, - /* size= */ 0, - /* presentationTimeUs= */ 0, - MediaCodec.BUFFER_FLAG_END_OF_STREAM); - - // Dequeue and render the output video frame. - MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - int outputBufferIndex; - do { - outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US); - assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED - || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED); - mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true); - - // Sleep to give time for the surface texture to be populated. - Thread.sleep(SURFACE_WAIT_MS); - assertThat(frameEditor.hasInputData()).isTrue(); - } finally { - mediaExtractor.release(); - if (mediaCodec != null) { - mediaCodec.release(); - } - } - } - - @After - public void tearDown() { - if (frameEditor != null) { - frameEditor.release(); - } + @Test + public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully() + throws TransformationException { + FrameEditor.create( + getApplicationContext(), + /* outputWidth= */ 200, + /* outputHeight= */ 100, + /* pixelWidthHeightRatio= */ 1, + new Matrix(), + new Surface(new SurfaceTexture(false)), + Transformer.DebugViewProvider.NONE); } @Test - public void processData_noEdits_producesExpectedOutput() throws Exception { - Bitmap expectedBitmap; - try (InputStream inputStream = - getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) { - expectedBitmap = BitmapFactory.decodeStream(inputStream); - } + public void create_withUnsupportedPixelWidthHeightRatio_throwsException() { + TransformationException exception = + assertThrows( + TransformationException.class, + () -> + FrameEditor.create( + getApplicationContext(), + /* outputWidth= */ 200, + /* outputHeight= */ 100, + /* pixelWidthHeightRatio= */ 2, + new Matrix(), + new Surface(new SurfaceTexture(false)), + Transformer.DebugViewProvider.NONE)); - checkNotNull(frameEditor).processData(); - Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); - Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); - - // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. - float averagePixelAbsoluteDifference = - getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); - assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); - } - - /** - * Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per - * component image. - */ - private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) { - int width = image.getWidth(); - int height = image.getHeight(); - assertThat(image.getPlanes()).hasLength(1); - assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888); - Image.Plane plane = image.getPlanes()[0]; - ByteBuffer buffer = plane.getBuffer(); - int[] colors = new int[width * height]; - for (int y = 0; y < height; y++) { - for (int x = 0; x < width; x++) { - int offset = y * plane.getRowStride() + x * plane.getPixelStride(); - int r = buffer.get(offset) & 0xFF; - int g = buffer.get(offset + 1) & 0xFF; - int b = buffer.get(offset + 2) & 0xFF; - int a = buffer.get(offset + 3) & 0xFF; - colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b; - } - } - return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); - } - - /** - * Returns the sum of the absolute differences between the expected and actual bitmaps, calculated - * using the maximum difference across all color channels for each pixel, then divided by the - * total number of pixels in the image. The bitmap resolutions must match and they must use - * configuration {@link Bitmap.Config#ARGB_8888}. - */ - private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) { - int width = actual.getWidth(); - int height = actual.getHeight(); - assertThat(width).isEqualTo(expected.getWidth()); - assertThat(height).isEqualTo(expected.getHeight()); - assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888); - long sumMaximumAbsoluteDifferences = 0; - for (int y = 0; y < height; y++) { - for (int x = 0; x < width; x++) { - int color = actual.getPixel(x, y); - int expectedColor = expected.getPixel(x, y); - int maximumAbsoluteDifference = 0; - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF))); - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF))); - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF))); - maximumAbsoluteDifference = - max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF))); - sumMaximumAbsoluteDifferences += maximumAbsoluteDifference; - } - } - return (float) sumMaximumAbsoluteDifferences / (width * height); + assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); + assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java index 1846cd956e..29e0d50a8e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java @@ -52,6 +52,9 @@ import java.util.concurrent.atomic.AtomicInteger; * @param outputSurface The {@link Surface}. * @param debugViewProvider Provider for optional debug views to show intermediate output. * @return A configured {@code FrameEditor}. + * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader + * files fails, or an OpenGL error occurs while creating and configuring the OpenGL + * components. */ public static FrameEditor create( Context context, @@ -63,30 +66,71 @@ import java.util.concurrent.atomic.AtomicInteger; Transformer.DebugViewProvider debugViewProvider) throws TransformationException { if (pixelWidthHeightRatio != 1.0f) { - // TODO(http://b/211782176): Consider implementing support for non-square pixels. - throw new TransformationException( - "FrameEditor Error", - new IllegalArgumentException( + // TODO(b/211782176): Consider implementing support for non-square pixels. + throw TransformationException.createForFrameEditor( + new UnsupportedOperationException( "Transformer's frame editor currently does not support frame edits on non-square" + " pixels. The pixelWidthHeightRatio is: " + pixelWidthHeightRatio), TransformationException.ERROR_CODE_GL_INIT_FAILED); } - EGLDisplay eglDisplay = GlUtil.createEglDisplay(); - EGLContext eglContext = GlUtil.createEglContext(eglDisplay); - EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - int textureId = GlUtil.createExternalTexture(); + @Nullable + SurfaceView debugSurfaceView = + debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); + + EGLDisplay eglDisplay; + EGLContext eglContext; + EGLSurface eglSurface; + int textureId; GlUtil.Program glProgram; + @Nullable EGLSurface debugPreviewEglSurface; try { - // TODO(internal b/205002913): check the loaded program is consistent with the attributes - // and uniforms expected in the code. - glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); - } catch (IOException e) { - throw new IllegalStateException(e); + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); + GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + textureId = GlUtil.createExternalTexture(); + glProgram = configureGlProgram(context, transformationMatrix, textureId); + debugPreviewEglSurface = + debugSurfaceView == null + ? null + : GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } catch (IOException | GlUtil.GlException e) { + throw TransformationException.createForFrameEditor( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); } + int debugPreviewWidth; + int debugPreviewHeight; + if (debugSurfaceView != null) { + debugPreviewWidth = debugSurfaceView.getWidth(); + debugPreviewHeight = debugSurfaceView.getHeight(); + } else { + debugPreviewWidth = C.LENGTH_UNSET; + debugPreviewHeight = C.LENGTH_UNSET; + } + + return new FrameEditor( + eglDisplay, + eglContext, + eglSurface, + textureId, + glProgram, + outputWidth, + outputHeight, + debugPreviewEglSurface, + debugPreviewWidth, + debugPreviewHeight); + } + + private static GlUtil.Program configureGlProgram( + Context context, Matrix transformationMatrix, int textureId) throws IOException { + // TODO(b/205002913): check the loaded program is consistent with the attributes + // and uniforms expected in the code. + GlUtil.Program glProgram = + new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); + glProgram.setBufferAttribute( "aPosition", new float[] { @@ -109,34 +153,7 @@ import java.util.concurrent.atomic.AtomicInteger; float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix); glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray); - - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); - @Nullable EGLSurface debugPreviewEglSurface; - int debugPreviewWidth; - int debugPreviewHeight; - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - debugPreviewWidth = debugSurfaceView.getWidth(); - debugPreviewHeight = debugSurfaceView.getHeight(); - } else { - debugPreviewEglSurface = null; - debugPreviewWidth = C.LENGTH_UNSET; - debugPreviewHeight = C.LENGTH_UNSET; - } - return new FrameEditor( - eglDisplay, - eglContext, - eglSurface, - textureId, - glProgram, - outputWidth, - outputHeight, - debugPreviewEglSurface, - debugPreviewWidth, - debugPreviewHeight); + return glProgram; } /** @@ -240,22 +257,31 @@ import java.util.concurrent.atomic.AtomicInteger; return pendingInputFrameCount.get() > 0; } - /** Processes pending input frame. */ - public void processData() { - inputSurfaceTexture.updateTexImage(); - inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); - glProgram.bindAttributesAndUniforms(); + /** + * Processes pending input frame. + * + * @throws TransformationException If an OpenGL error occurs while processing the data. + */ + public void processData() throws TransformationException { + try { + inputSurfaceTexture.updateTexImage(); + inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); + glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix); + glProgram.bindAttributesAndUniforms(); - focusAndDrawQuad(eglSurface, outputWidth, outputHeight); - long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); - EGL14.eglSwapBuffers(eglDisplay, eglSurface); - pendingInputFrameCount.decrementAndGet(); + focusAndDrawQuad(eglSurface, outputWidth, outputHeight); + long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + pendingInputFrameCount.decrementAndGet(); - if (debugPreviewEglSurface != null) { - focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); - EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); + if (debugPreviewEglSurface != null) { + focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); + EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); + } + } catch (GlUtil.GlException e) { + throw TransformationException.createForFrameEditor( + e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java index 0e56881c72..c5a4107b43 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java @@ -229,7 +229,7 @@ public final class TransformationException extends Exception { } /** - * Creates an instance for an audio processing related exception. + * Creates an instance for an {@link AudioProcessor} related exception. * * @param cause The cause of the failure. * @param componentName The name of the {@link AudioProcessor} used. @@ -243,6 +243,18 @@ public final class TransformationException extends Exception { componentName + " error, audio_format = " + audioFormat, cause, errorCode); } + /** + * Creates an instance for a {@link FrameEditor} related exception. + * + * @param cause The cause of the failure. + * @param errorCode See {@link #errorCode}. + * @return The created instance. + */ + /* package */ static TransformationException createForFrameEditor( + Throwable cause, int errorCode) { + return new TransformationException("FrameEditor error", cause, errorCode); + } + /** * Creates an instance for a muxer related exception. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 74f9fb237a..0da5b88b7d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -81,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } // The decoder rotates videos to their intended display orientation. The frameEditor rotates // them back for improved encoder compatibility. - // TODO(internal b/201293185): After fragment shader transformations are implemented, put + // TODO(b/201293185): After fragment shader transformations are implemented, put // postrotation in a later vertex shader. transformationRequest.transformationMatrix.postRotate(outputRotationDegrees); @@ -129,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public boolean processData() { + public boolean processData() throws TransformationException { if (decoder.isEnded()) { return false; } @@ -155,7 +155,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Transformer}, using this method requires API level 29 or higher. */ @RequiresApi(29) - private boolean processDataV29() { + private boolean processDataV29() throws TransformationException { if (frameEditor != null) { while (frameEditor.hasInputData()) { // Processes as much frames in one invocation: FrameEditor's output surface will block @@ -170,7 +170,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (decoder.isEnded()) { - // TODO(internal b/208986865): Handle possible last frame drop. + // TODO(b/208986865): Handle possible last frame drop. encoder.signalEndOfInputStream(); return false; } @@ -179,7 +179,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } /** Processes input data. */ - private boolean processDataDefault() { + private boolean processDataDefault() throws TransformationException { if (frameEditor != null) { if (frameEditor.hasInputData()) { waitingForFrameEditorInput = false;