Use TransformationException for GL errors.

PiperOrigin-RevId: 418820557

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java

@@ -0,0 +1,236 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.common.truth.Truth.assertThat;
+import static java.lang.Math.abs;
+import static java.lang.Math.max;
+
+import android.content.res.AssetFileDescriptor;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Matrix;
+import android.graphics.PixelFormat;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.MediaCodec;
+import android.media.MediaExtractor;
+import android.media.MediaFormat;
+import androidx.annotation.Nullable;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.util.MimeTypes;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Test for frame processing via {@link FrameEditor#processData()}. */
+@RunWith(AndroidJUnit4.class)
+public final class FrameEditorDataProcessingTest {
+
+  private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
+  private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING =
+      "media/bitmap/sample_mp4_first_frame.png";
+  /**
+   * Maximum allowed average pixel difference between the expected and actual edited images for the
+   * test to pass. The value is chosen so that differences in decoder behavior across emulator
+   * versions shouldn't affect whether the test passes, but substantial distortions introduced by
+   * changes in the behavior of the frame editor will cause the test to fail.
+   */
+  private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
+  /** Timeout for dequeueing buffers from the codec, in microseconds. */
+  private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
+  /** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */
+  private static final int SURFACE_WAIT_MS = 1000;
+  /** The ratio of width over height, for each pixel in a frame. */
+  private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
+
+  private @MonotonicNonNull FrameEditor frameEditor;
+  private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
+  private @MonotonicNonNull MediaFormat mediaFormat;
+
+  @Before
+  public void setUp() throws Exception {
+    // Set up the extractor to read the first video frame and get its format.
+    MediaExtractor mediaExtractor = new MediaExtractor();
+    @Nullable MediaCodec mediaCodec = null;
+    try (AssetFileDescriptor afd =
+        getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
+      mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
+      for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
+        if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) {
+          mediaFormat = mediaExtractor.getTrackFormat(i);
+          mediaExtractor.selectTrack(i);
+          break;
+        }
+      }
+
+      int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
+      int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
+      frameEditorOutputImageReader =
+          ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
+      Matrix identityMatrix = new Matrix();
+      frameEditor =
+          FrameEditor.create(
+              getApplicationContext(),
+              width,
+              height,
+              PIXEL_WIDTH_HEIGHT_RATIO,
+              identityMatrix,
+              frameEditorOutputImageReader.getSurface(),
+              Transformer.DebugViewProvider.NONE);
+
+      // Queue the first video frame from the extractor.
+      String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
+      mediaCodec = MediaCodec.createDecoderByType(mimeType);
+      mediaCodec.configure(
+          mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
+      mediaCodec.start();
+      int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
+      assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+      ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]);
+      int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0);
+      mediaCodec.queueInputBuffer(
+          inputBufferIndex,
+          /* offset= */ 0,
+          sampleSize,
+          mediaExtractor.getSampleTime(),
+          mediaExtractor.getSampleFlags());
+
+      // Queue an end-of-stream buffer to force the codec to produce output.
+      inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
+      assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+      mediaCodec.queueInputBuffer(
+          inputBufferIndex,
+          /* offset= */ 0,
+          /* size= */ 0,
+          /* presentationTimeUs= */ 0,
+          MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+
+      // Dequeue and render the output video frame.
+      MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+      int outputBufferIndex;
+      do {
+        outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US);
+        assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+      } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
+          || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
+      mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
+
+      // Sleep to give time for the surface texture to be populated.
+      Thread.sleep(SURFACE_WAIT_MS);
+      assertThat(frameEditor.hasInputData()).isTrue();
+    } finally {
+      mediaExtractor.release();
+      if (mediaCodec != null) {
+        mediaCodec.release();
+      }
+    }
+  }
+
+  @After
+  public void tearDown() {
+    if (frameEditor != null) {
+      frameEditor.release();
+    }
+  }
+
+  @Test
+  public void processData_noEdits_producesExpectedOutput() throws Exception {
+    Bitmap expectedBitmap;
+    try (InputStream inputStream =
+        getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) {
+      expectedBitmap = BitmapFactory.decodeStream(inputStream);
+    }
+
+    checkNotNull(frameEditor).processData();
+    Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
+    Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
+
+    // TODO(internal b/207848601): switch to using proper tooling for testing against golden data.
+    float averagePixelAbsoluteDifference =
+        getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap);
+    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+  }
+
+  /**
+   * Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per
+   * component image.
+   */
+  private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) {
+    int width = image.getWidth();
+    int height = image.getHeight();
+    assertThat(image.getPlanes()).hasLength(1);
+    assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888);
+    Image.Plane plane = image.getPlanes()[0];
+    ByteBuffer buffer = plane.getBuffer();
+    int[] colors = new int[width * height];
+    for (int y = 0; y < height; y++) {
+      for (int x = 0; x < width; x++) {
+        int offset = y * plane.getRowStride() + x * plane.getPixelStride();
+        int r = buffer.get(offset) & 0xFF;
+        int g = buffer.get(offset + 1) & 0xFF;
+        int b = buffer.get(offset + 2) & 0xFF;
+        int a = buffer.get(offset + 3) & 0xFF;
+        colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b;
+      }
+    }
+    return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
+  }
+
+  /**
+   * Returns the sum of the absolute differences between the expected and actual bitmaps, calculated
+   * using the maximum difference across all color channels for each pixel, then divided by the
+   * total number of pixels in the image. The bitmap resolutions must match and they must use
+   * configuration {@link Bitmap.Config#ARGB_8888}.
+   */
+  private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) {
+    int width = actual.getWidth();
+    int height = actual.getHeight();
+    assertThat(width).isEqualTo(expected.getWidth());
+    assertThat(height).isEqualTo(expected.getHeight());
+    assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888);
+    long sumMaximumAbsoluteDifferences = 0;
+    for (int y = 0; y < height; y++) {
+      for (int x = 0; x < width; x++) {
+        int color = actual.getPixel(x, y);
+        int expectedColor = expected.getPixel(x, y);
+        int maximumAbsoluteDifference = 0;
+        maximumAbsoluteDifference =
+            max(
+                maximumAbsoluteDifference,
+                abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF)));
+        maximumAbsoluteDifference =
+            max(
+                maximumAbsoluteDifference,
+                abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF)));
+        maximumAbsoluteDifference =
+            max(
+                maximumAbsoluteDifference,
+                abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF)));
+        maximumAbsoluteDifference =
+            max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF)));
+        sumMaximumAbsoluteDifferences += maximumAbsoluteDifference;
+      }
+    }
+    return (float) sumMaximumAbsoluteDifferences / (width * height);
+  }
+}

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java

@@ -16,221 +16,55 @@
 package com.google.android.exoplayer2.transformer;
 
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
-import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.common.truth.Truth.assertThat;
-import static java.lang.Math.abs;
+import static org.junit.Assert.assertThrows;
-import static java.lang.Math.max;
 
-import android.content.res.AssetFileDescriptor;
+import android.content.Context;
-import android.graphics.Bitmap;
-import android.graphics.BitmapFactory;
 import android.graphics.Matrix;
-import android.graphics.PixelFormat;
+import android.graphics.SurfaceTexture;
-import android.media.Image;
+import android.view.Surface;
-import android.media.ImageReader;
-import android.media.MediaCodec;
-import android.media.MediaExtractor;
-import android.media.MediaFormat;
-import androidx.annotation.Nullable;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
-import com.google.android.exoplayer2.util.MimeTypes;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
-/** Test for frame processing via {@link FrameEditor}. */
+/**
+ * Test for {@link FrameEditor#create(Context, int, int, float, Matrix, Surface,
+ * Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
+ */
 @RunWith(AndroidJUnit4.class)
 public final class FrameEditorTest {
+  // TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
+  // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
 
-  private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
+  @Test
-  private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING =
+  public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
-      "media/bitmap/sample_mp4_first_frame.png";
+      throws TransformationException {
-  /**
-   * Maximum allowed average pixel difference between the expected and actual edited images for the
-   * test to pass. The value is chosen so that differences in decoder behavior across emulator
-   * versions shouldn't affect whether the test passes, but substantial distortions introduced by
-   * changes in the behavior of the frame editor will cause the test to fail.
-   */
-  private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
-  /** Timeout for dequeueing buffers from the codec, in microseconds. */
-  private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
-  /** Time to wait for the frame editor's input to be populated by the decoder, in milliseconds. */
-  private static final int SURFACE_WAIT_MS = 1000;
-  /** The ratio of width over height, for each pixel in a frame. */
-  private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
-
-  private @MonotonicNonNull FrameEditor frameEditor;
-  private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
-  private @MonotonicNonNull MediaFormat mediaFormat;
-
-  @Before
-  public void setUp() throws Exception {
-    // Set up the extractor to read the first video frame and get its format.
-    MediaExtractor mediaExtractor = new MediaExtractor();
-    @Nullable MediaCodec mediaCodec = null;
-    try (AssetFileDescriptor afd =
-        getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
-      mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
-      for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
-        if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) {
-          mediaFormat = mediaExtractor.getTrackFormat(i);
-          mediaExtractor.selectTrack(i);
-          break;
-        }
-      }
-
-      int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
-      int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
-      frameEditorOutputImageReader =
-          ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
-      Matrix identityMatrix = new Matrix();
-      frameEditor =
     FrameEditor.create(
         getApplicationContext(),
-              width,
+        /* outputWidth= */ 200,
-              height,
+        /* outputHeight= */ 100,
-              PIXEL_WIDTH_HEIGHT_RATIO,
+        /* pixelWidthHeightRatio= */ 1,
-              identityMatrix,
+        new Matrix(),
-              frameEditorOutputImageReader.getSurface(),
+        new Surface(new SurfaceTexture(false)),
         Transformer.DebugViewProvider.NONE);
-
-      // Queue the first video frame from the extractor.
-      String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
-      mediaCodec = MediaCodec.createDecoderByType(mimeType);
-      mediaCodec.configure(
-          mediaFormat, frameEditor.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
-      mediaCodec.start();
-      int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
-      assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
-      ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]);
-      int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0);
-      mediaCodec.queueInputBuffer(
-          inputBufferIndex,
-          /* offset= */ 0,
-          sampleSize,
-          mediaExtractor.getSampleTime(),
-          mediaExtractor.getSampleFlags());
-
-      // Queue an end-of-stream buffer to force the codec to produce output.
-      inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
-      assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
-      mediaCodec.queueInputBuffer(
-          inputBufferIndex,
-          /* offset= */ 0,
-          /* size= */ 0,
-          /* presentationTimeUs= */ 0,
-          MediaCodec.BUFFER_FLAG_END_OF_STREAM);
-
-      // Dequeue and render the output video frame.
-      MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
-      int outputBufferIndex;
-      do {
-        outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US);
-        assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
-      } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
-          || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
-      mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
-
-      // Sleep to give time for the surface texture to be populated.
-      Thread.sleep(SURFACE_WAIT_MS);
-      assertThat(frameEditor.hasInputData()).isTrue();
-    } finally {
-      mediaExtractor.release();
-      if (mediaCodec != null) {
-        mediaCodec.release();
-      }
-    }
-  }
-
-  @After
-  public void tearDown() {
-    if (frameEditor != null) {
-      frameEditor.release();
-    }
   }
 
   @Test
-  public void processData_noEdits_producesExpectedOutput() throws Exception {
+  public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
-    Bitmap expectedBitmap;
+    TransformationException exception =
-    try (InputStream inputStream =
+        assertThrows(
-        getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) {
+            TransformationException.class,
-      expectedBitmap = BitmapFactory.decodeStream(inputStream);
+            () ->
-    }
+                FrameEditor.create(
+                    getApplicationContext(),
+                    /* outputWidth= */ 200,
+                    /* outputHeight= */ 100,
+                    /* pixelWidthHeightRatio= */ 2,
+                    new Matrix(),
+                    new Surface(new SurfaceTexture(false)),
+                    Transformer.DebugViewProvider.NONE));
 
-    checkNotNull(frameEditor).processData();
+    assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
-    Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
+    assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
-    Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage);
-
-    // TODO(internal b/207848601): switch to using proper tooling for testing against golden data.
-    float averagePixelAbsoluteDifference =
-        getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap);
-    assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
-  }
-
-  /**
-   * Returns a bitmap with the same information as the provided alpha/red/green/blue 8-bits per
-   * component image.
-   */
-  private static Bitmap getArgb8888BitmapForRgba8888Image(Image image) {
-    int width = image.getWidth();
-    int height = image.getHeight();
-    assertThat(image.getPlanes()).hasLength(1);
-    assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888);
-    Image.Plane plane = image.getPlanes()[0];
-    ByteBuffer buffer = plane.getBuffer();
-    int[] colors = new int[width * height];
-    for (int y = 0; y < height; y++) {
-      for (int x = 0; x < width; x++) {
-        int offset = y * plane.getRowStride() + x * plane.getPixelStride();
-        int r = buffer.get(offset) & 0xFF;
-        int g = buffer.get(offset + 1) & 0xFF;
-        int b = buffer.get(offset + 2) & 0xFF;
-        int a = buffer.get(offset + 3) & 0xFF;
-        colors[y * width + x] = (a << 24) + (r << 16) + (g << 8) + b;
-      }
-    }
-    return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
-  }
-
-  /**
-   * Returns the sum of the absolute differences between the expected and actual bitmaps, calculated
-   * using the maximum difference across all color channels for each pixel, then divided by the
-   * total number of pixels in the image. The bitmap resolutions must match and they must use
-   * configuration {@link Bitmap.Config#ARGB_8888}.
-   */
-  private static float getAveragePixelAbsoluteDifferenceArgb8888(Bitmap expected, Bitmap actual) {
-    int width = actual.getWidth();
-    int height = actual.getHeight();
-    assertThat(width).isEqualTo(expected.getWidth());
-    assertThat(height).isEqualTo(expected.getHeight());
-    assertThat(actual.getConfig()).isEqualTo(Bitmap.Config.ARGB_8888);
-    long sumMaximumAbsoluteDifferences = 0;
-    for (int y = 0; y < height; y++) {
-      for (int x = 0; x < width; x++) {
-        int color = actual.getPixel(x, y);
-        int expectedColor = expected.getPixel(x, y);
-        int maximumAbsoluteDifference = 0;
-        maximumAbsoluteDifference =
-            max(
-                maximumAbsoluteDifference,
-                abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF)));
-        maximumAbsoluteDifference =
-            max(
-                maximumAbsoluteDifference,
-                abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF)));
-        maximumAbsoluteDifference =
-            max(
-                maximumAbsoluteDifference,
-                abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF)));
-        maximumAbsoluteDifference =
-            max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF)));
-        sumMaximumAbsoluteDifferences += maximumAbsoluteDifference;
-      }
-    }
-    return (float) sumMaximumAbsoluteDifferences / (width * height);
   }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java

@@ -52,6 +52,9 @@ import java.util.concurrent.atomic.AtomicInteger;
    * @param outputSurface The {@link Surface}.
    * @param debugViewProvider Provider for optional debug views to show intermediate output.
    * @return A configured {@code FrameEditor}.
+   * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
+   *     files fails, or an OpenGL error occurs while creating and configuring the OpenGL
+   *     components.
    */
   public static FrameEditor create(
       Context context,
@@ -63,30 +66,71 @@ import java.util.concurrent.atomic.AtomicInteger;
       Transformer.DebugViewProvider debugViewProvider)
       throws TransformationException {
     if (pixelWidthHeightRatio != 1.0f) {
-      // TODO(http://b/211782176): Consider implementing support for non-square pixels.
+      // TODO(b/211782176): Consider implementing support for non-square pixels.
-      throw new TransformationException(
+      throw TransformationException.createForFrameEditor(
-          "FrameEditor Error",
+          new UnsupportedOperationException(
-          new IllegalArgumentException(
               "Transformer's frame editor currently does not support frame edits on non-square"
                   + " pixels. The pixelWidthHeightRatio is: "
                   + pixelWidthHeightRatio),
           TransformationException.ERROR_CODE_GL_INIT_FAILED);
     }
 
-    EGLDisplay eglDisplay = GlUtil.createEglDisplay();
+    @Nullable
-    EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
+    SurfaceView debugSurfaceView =
-    EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
+        debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
-    GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
+
-    int textureId = GlUtil.createExternalTexture();
+    EGLDisplay eglDisplay;
+    EGLContext eglContext;
+    EGLSurface eglSurface;
+    int textureId;
     GlUtil.Program glProgram;
+    @Nullable EGLSurface debugPreviewEglSurface;
     try {
-      // TODO(internal b/205002913): check the loaded program is consistent with the attributes
+      eglDisplay = GlUtil.createEglDisplay();
-      // and uniforms expected in the code.
+      eglContext = GlUtil.createEglContext(eglDisplay);
-      glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
+      eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
-    } catch (IOException e) {
+      GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
-      throw new IllegalStateException(e);
+      textureId = GlUtil.createExternalTexture();
+      glProgram = configureGlProgram(context, transformationMatrix, textureId);
+      debugPreviewEglSurface =
+          debugSurfaceView == null
+              ? null
+              : GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
+    } catch (IOException | GlUtil.GlException e) {
+      throw TransformationException.createForFrameEditor(
+          e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
     }
 
+    int debugPreviewWidth;
+    int debugPreviewHeight;
+    if (debugSurfaceView != null) {
+      debugPreviewWidth = debugSurfaceView.getWidth();
+      debugPreviewHeight = debugSurfaceView.getHeight();
+    } else {
+      debugPreviewWidth = C.LENGTH_UNSET;
+      debugPreviewHeight = C.LENGTH_UNSET;
+    }
+
+    return new FrameEditor(
+        eglDisplay,
+        eglContext,
+        eglSurface,
+        textureId,
+        glProgram,
+        outputWidth,
+        outputHeight,
+        debugPreviewEglSurface,
+        debugPreviewWidth,
+        debugPreviewHeight);
+  }
+
+  private static GlUtil.Program configureGlProgram(
+      Context context, Matrix transformationMatrix, int textureId) throws IOException {
+    // TODO(b/205002913): check the loaded program is consistent with the attributes
+    // and uniforms expected in the code.
+    GlUtil.Program glProgram =
+        new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
+
     glProgram.setBufferAttribute(
         "aPosition",
         new float[] {
@@ -109,34 +153,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
     float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
     glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrixArray);
-
+    return glProgram;
-    @Nullable
-    SurfaceView debugSurfaceView =
-        debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
-    @Nullable EGLSurface debugPreviewEglSurface;
-    int debugPreviewWidth;
-    int debugPreviewHeight;
-    if (debugSurfaceView != null) {
-      debugPreviewEglSurface =
-          GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
-      debugPreviewWidth = debugSurfaceView.getWidth();
-      debugPreviewHeight = debugSurfaceView.getHeight();
-    } else {
-      debugPreviewEglSurface = null;
-      debugPreviewWidth = C.LENGTH_UNSET;
-      debugPreviewHeight = C.LENGTH_UNSET;
-    }
-    return new FrameEditor(
-        eglDisplay,
-        eglContext,
-        eglSurface,
-        textureId,
-        glProgram,
-        outputWidth,
-        outputHeight,
-        debugPreviewEglSurface,
-        debugPreviewWidth,
-        debugPreviewHeight);
   }
 
   /**
@@ -240,8 +257,13 @@ import java.util.concurrent.atomic.AtomicInteger;
     return pendingInputFrameCount.get() > 0;
   }
 
-  /** Processes pending input frame. */
+  /**
-  public void processData() {
+   * Processes pending input frame.
+   *
+   * @throws TransformationException If an OpenGL error occurs while processing the data.
+   */
+  public void processData() throws TransformationException {
+    try {
       inputSurfaceTexture.updateTexImage();
       inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
       glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
@@ -257,6 +279,10 @@ import java.util.concurrent.atomic.AtomicInteger;
         focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
         EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
       }
+    } catch (GlUtil.GlException e) {
+      throw TransformationException.createForFrameEditor(
+          e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
+    }
   }
 
   /** Releases all resources. */

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java

@@ -229,7 +229,7 @@ public final class TransformationException extends Exception {
   }
 
   /**
-   * Creates an instance for an audio processing related exception.
+   * Creates an instance for an {@link AudioProcessor} related exception.
    *
    * @param cause The cause of the failure.
    * @param componentName The name of the {@link AudioProcessor} used.
@@ -243,6 +243,18 @@ public final class TransformationException extends Exception {
         componentName + " error, audio_format = " + audioFormat, cause, errorCode);
   }
 
+  /**
+   * Creates an instance for a {@link FrameEditor} related exception.
+   *
+   * @param cause The cause of the failure.
+   * @param errorCode See {@link #errorCode}.
+   * @return The created instance.
+   */
+  /* package */ static TransformationException createForFrameEditor(
+      Throwable cause, int errorCode) {
+    return new TransformationException("FrameEditor error", cause, errorCode);
+  }
+
   /**
    * Creates an instance for a muxer related exception.
    *

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java

@@ -81,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
     // The decoder rotates videos to their intended display orientation. The frameEditor rotates
     // them back for improved encoder compatibility.
-    // TODO(internal b/201293185): After fragment shader transformations are implemented, put
+    // TODO(b/201293185): After fragment shader transformations are implemented, put
     // postrotation in a later vertex shader.
     transformationRequest.transformationMatrix.postRotate(outputRotationDegrees);
 
@@ -129,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   @Override
-  public boolean processData() {
+  public boolean processData() throws TransformationException {
     if (decoder.isEnded()) {
       return false;
     }
@@ -155,7 +155,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    * Transformer}, using this method requires API level 29 or higher.
    */
   @RequiresApi(29)
-  private boolean processDataV29() {
+  private boolean processDataV29() throws TransformationException {
     if (frameEditor != null) {
       while (frameEditor.hasInputData()) {
         // Processes as much frames in one invocation: FrameEditor's output surface will block
@@ -170,7 +170,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
 
     if (decoder.isEnded()) {
-      // TODO(internal b/208986865): Handle possible last frame drop.
+      // TODO(b/208986865): Handle possible last frame drop.
       encoder.signalEndOfInputStream();
       return false;
     }
@@ -179,7 +179,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   /** Processes input data. */
-  private boolean processDataDefault() {
+  private boolean processDataDefault() throws TransformationException {
     if (frameEditor != null) {
       if (frameEditor.hasInputData()) {
         waitingForFrameEditorInput = false;