Add Open GL step to Transformer

PiperOrigin-RevId: 394708737

library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java

@@ -223,6 +223,9 @@ public final class GlUtil {
     }
   }
 
+  /** Represents an unset texture ID. */
+  public static final int TEXTURE_ID_UNSET = -1;
+
   /** Whether to throw a {@link GlException} in case of an OpenGL error. */
   public static boolean glAssertionsEnabled = false;
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java

@@ -343,8 +343,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    * be available until the previous has been released.
    */
   public void releaseOutputBuffer() {
+    releaseOutputBuffer(/* render= */ false);
+  }
+
+  /**
+   * Releases the current output buffer. If the {@link MediaCodec} was configured with an output
+   * surface, setting {@code render} to {@code true} will first send the buffer to the output
+   * surface. The surface will release the buffer back to the codec once it is no longer
+   * used/displayed.
+   *
+   * <p>This should be called after the buffer has been processed. The next output buffer will not
+   * be available until the previous has been released.
+   */
+  public void releaseOutputBuffer(boolean render) {
     outputBuffer = null;
-    codec.releaseOutputBuffer(outputBufferIndex, /* render= */ false);
+    codec.releaseOutputBuffer(outputBufferIndex, render);
     outputBufferIndex = C.INDEX_UNSET;
   }
 
@@ -359,18 +372,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     codec.release();
   }
 
-  /** Returns {@code true} if a buffer is successfully obtained, rendered and released. */
-  public boolean maybeDequeueRenderAndReleaseOutputBuffer() {
-    if (!maybeDequeueOutputBuffer()) {
-      return false;
-    }
-
-    codec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
-    outputBuffer = null;
-    outputBufferIndex = C.INDEX_UNSET;
-    return true;
-  }
-
   /**
    * Tries obtaining an output buffer and sets {@link #outputBuffer} to the obtained output buffer.
    *

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java

@@ -20,6 +20,7 @@ import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
 import com.google.android.exoplayer2.BaseRenderer;
 import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.ExoPlaybackException;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.RendererCapabilities;
 import com.google.android.exoplayer2.util.MediaClock;
@@ -75,7 +76,7 @@ import com.google.android.exoplayer2.util.MimeTypes;
   }
 
   @Override
-  protected final void onStarted() {
+  protected void onStarted() throws ExoPlaybackException {
     isRendererStarted = true;
   }
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java

@@ -17,8 +17,18 @@
 package com.google.android.exoplayer2.transformer;
 
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.android.exoplayer2.util.Assertions.checkState;
 
+import android.content.Context;
+import android.graphics.SurfaceTexture;
 import android.media.MediaCodec;
+import android.opengl.EGL14;
+import android.opengl.EGLContext;
+import android.opengl.EGLDisplay;
+import android.opengl.EGLExt;
+import android.opengl.EGLSurface;
+import android.opengl.GLES20;
+import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
 import com.google.android.exoplayer2.C;
@@ -28,34 +38,57 @@ import com.google.android.exoplayer2.FormatHolder;
 import com.google.android.exoplayer2.PlaybackException;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.source.SampleStream;
+import com.google.android.exoplayer2.util.GlUtil;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
 @RequiresApi(18)
 /* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer {
 
+  static {
+    GlUtil.glAssertionsEnabled = true;
+  }
+
   private static final String TAG = "TransformerTranscodingVideoRenderer";
 
-  private final DecoderInputBuffer decoderInputBuffer;
+  private final Context context;
   /** The format the encoder is configured to output, may differ from the actual output format. */
   private final Format encoderConfigurationOutputFormat;
 
+  private final DecoderInputBuffer decoderInputBuffer;
+  private final float[] decoderTextureTransformMatrix;
+
+  @Nullable private EGLDisplay eglDisplay;
+  @Nullable private EGLContext eglContext;
+  @Nullable private EGLSurface eglSurface;
+
+  private int decoderTextureId;
+  @Nullable private SurfaceTexture decoderSurfaceTexture;
+  @Nullable private Surface decoderSurface;
   @Nullable private MediaCodecAdapterWrapper decoder;
+  private volatile boolean isDecoderSurfacePopulated;
+  private boolean waitingForPopulatedDecoderSurface;
+  @Nullable private GlUtil.Uniform decoderTextureTransformUniform;
+
   @Nullable private MediaCodecAdapterWrapper encoder;
+  private long nextEncoderTimeUs;
   /** Whether encoder's actual output format is obtained. */
   private boolean hasEncoderActualOutputFormat;
 
   private boolean muxerWrapperTrackEnded;
 
   public TransformerTranscodingVideoRenderer(
+      Context context,
       MuxerWrapper muxerWrapper,
       TransformerMediaClock mediaClock,
       Transformation transformation,
       Format encoderConfigurationOutputFormat) {
     super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
-
+    this.context = context;
-    decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
     this.encoderConfigurationOutputFormat = encoderConfigurationOutputFormat;
+    decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
+    decoderTextureTransformMatrix = new float[16];
+    decoderTextureId = GlUtil.TEXTURE_ID_UNSET;
   }
 
   @Override
@@ -64,12 +97,15 @@ import java.nio.ByteBuffer;
   }
 
   @Override
-  public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+  protected void onStarted() throws ExoPlaybackException {
-    if (!isRendererStarted || isEnded()) {
+    super.onStarted();
-      return;
+    ensureEncoderConfigured();
-    }
+    ensureOpenGlConfigured();
+  }
 
-    if (!ensureEncoderConfigured() || !ensureDecoderConfigured()) {
+  @Override
+  public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+    if (!isRendererStarted || isEnded() || !ensureDecoderConfigured()) {
       return;
     }
 
@@ -87,10 +123,28 @@ import java.nio.ByteBuffer;
   protected void onReset() {
     decoderInputBuffer.clear();
     decoderInputBuffer.data = null;
+    GlUtil.destroyEglContext(eglDisplay, eglContext);
+    eglDisplay = null;
+    eglContext = null;
+    eglSurface = null;
+    if (decoderTextureId != GlUtil.TEXTURE_ID_UNSET) {
+      GlUtil.deleteTexture(decoderTextureId);
+    }
+    if (decoderSurfaceTexture != null) {
+      decoderSurfaceTexture.release();
+      decoderSurfaceTexture = null;
+    }
+    if (decoderSurface != null) {
+      decoderSurface.release();
+      decoderSurface = null;
+    }
     if (decoder != null) {
       decoder.release();
       decoder = null;
     }
+    isDecoderSurfacePopulated = false;
+    waitingForPopulatedDecoderSurface = false;
+    decoderTextureTransformUniform = null;
     if (encoder != null) {
       encoder.release();
       encoder = null;
@@ -99,6 +153,94 @@ import java.nio.ByteBuffer;
     muxerWrapperTrackEnded = false;
   }
 
+  private void ensureEncoderConfigured() throws ExoPlaybackException {
+    if (encoder != null) {
+      return;
+    }
+
+    try {
+      encoder = MediaCodecAdapterWrapper.createForVideoEncoding(encoderConfigurationOutputFormat);
+    } catch (IOException e) {
+      throw createRendererException(
+          // TODO(claincly): should be "ENCODER_INIT_FAILED"
+          e,
+          checkNotNull(this.decoder).getOutputFormat(),
+          PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
+    }
+  }
+
+  private void ensureOpenGlConfigured() {
+    if (eglDisplay != null) {
+      return;
+    }
+
+    eglDisplay = GlUtil.createEglDisplay();
+    EGLContext eglContext;
+    try {
+      eglContext = GlUtil.createEglContext(eglDisplay);
+      this.eglContext = eglContext;
+    } catch (GlUtil.UnsupportedEglVersionException e) {
+      throw new IllegalStateException("EGL version is unsupported", e);
+    }
+    eglSurface =
+        GlUtil.getEglSurface(eglDisplay, checkNotNull(checkNotNull(encoder).getInputSurface()));
+    GlUtil.focusSurface(
+        eglDisplay,
+        eglContext,
+        eglSurface,
+        encoderConfigurationOutputFormat.width,
+        encoderConfigurationOutputFormat.height);
+    decoderTextureId = GlUtil.createExternalTexture();
+    String vertexShaderCode;
+    String fragmentShaderCode;
+    try {
+      vertexShaderCode = GlUtil.loadAsset(context, "shaders/blit_vertex_shader.glsl");
+      fragmentShaderCode = GlUtil.loadAsset(context, "shaders/copy_external_fragment_shader.glsl");
+    } catch (IOException e) {
+      throw new IllegalStateException(e);
+    }
+    int copyProgram = GlUtil.compileProgram(vertexShaderCode, fragmentShaderCode);
+    GLES20.glUseProgram(copyProgram);
+    GlUtil.Attribute[] copyAttributes = GlUtil.getAttributes(copyProgram);
+    checkState(copyAttributes.length == 2, "Expected program to have two vertex attributes.");
+    for (GlUtil.Attribute copyAttribute : copyAttributes) {
+      if (copyAttribute.name.equals("a_position")) {
+        copyAttribute.setBuffer(
+            new float[] {
+              -1.0f, -1.0f, 0.0f, 1.0f,
+              1.0f, -1.0f, 0.0f, 1.0f,
+              -1.0f, 1.0f, 0.0f, 1.0f,
+              1.0f, 1.0f, 0.0f, 1.0f,
+            },
+            /* size= */ 4);
+      } else if (copyAttribute.name.equals("a_texcoord")) {
+        copyAttribute.setBuffer(
+            new float[] {
+              0.0f, 0.0f, 0.0f, 1.0f,
+              1.0f, 0.0f, 0.0f, 1.0f,
+              0.0f, 1.0f, 0.0f, 1.0f,
+              1.0f, 1.0f, 0.0f, 1.0f,
+            },
+            /* size= */ 4);
+      } else {
+        throw new IllegalStateException("Unexpected attribute name.");
+      }
+      copyAttribute.bind();
+    }
+    GlUtil.Uniform[] copyUniforms = GlUtil.getUniforms(copyProgram);
+    checkState(copyUniforms.length == 2, "Expected program to have two uniforms.");
+    for (GlUtil.Uniform copyUniform : copyUniforms) {
+      if (copyUniform.name.equals("tex_sampler")) {
+        copyUniform.setSamplerTexId(decoderTextureId, 0);
+        copyUniform.bind();
+      } else if (copyUniform.name.equals("tex_transform")) {
+        decoderTextureTransformUniform = copyUniform;
+      } else {
+        throw new IllegalStateException("Unexpected uniform name.");
+      }
+    }
+  }
+
   private boolean ensureDecoderConfigured() throws ExoPlaybackException {
     if (decoder != null) {
       return true;
@@ -114,11 +256,13 @@ import java.nio.ByteBuffer;
     }
 
     Format inputFormat = checkNotNull(formatHolder.format);
-    MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder);
+    checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET);
+    decoderSurfaceTexture = new SurfaceTexture(decoderTextureId);
+    decoderSurfaceTexture.setOnFrameAvailableListener(
+        surfaceTexture -> isDecoderSurfacePopulated = true);
+    decoderSurface = new Surface(decoderSurfaceTexture);
     try {
-      decoder =
+      decoder = MediaCodecAdapterWrapper.createForVideoDecoding(inputFormat, decoderSurface);
-          MediaCodecAdapterWrapper.createForVideoDecoding(
-              inputFormat, checkNotNull(encoder.getInputSurface()));
     } catch (IOException e) {
       throw createRendererException(
           e, formatHolder.format, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
@@ -126,23 +270,6 @@ import java.nio.ByteBuffer;
     return true;
   }
 
-  private boolean ensureEncoderConfigured() throws ExoPlaybackException {
-    if (encoder != null) {
-      return true;
-    }
-
-    try {
-      encoder = MediaCodecAdapterWrapper.createForVideoEncoding(encoderConfigurationOutputFormat);
-    } catch (IOException e) {
-      throw createRendererException(
-          // TODO(claincly): should be "ENCODER_INIT_FAILED"
-          e,
-          checkNotNull(this.decoder).getOutputFormat(),
-          PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
-    }
-    return true;
-  }
-
   private boolean feedDecoderFromInput() {
     MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
     if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) {
@@ -174,14 +301,35 @@ import java.nio.ByteBuffer;
       return false;
     }
 
-    // Rendering the decoder output queues input to the encoder because they share the same surface.
+    if (!isDecoderSurfacePopulated) {
-    boolean hasProcessedOutputBuffer = decoder.maybeDequeueRenderAndReleaseOutputBuffer();
+      if (!waitingForPopulatedDecoderSurface) {
-    if (decoder.isEnded()) {
+        if (decoder.getOutputBuffer() != null) {
-      checkNotNull(encoder).signalEndOfInputStream();
+          nextEncoderTimeUs = checkNotNull(decoder.getOutputBufferInfo()).presentationTimeUs;
-      // All decoded frames have been rendered to the encoder's input surface.
+          decoder.releaseOutputBuffer(/* render= */ true);
+          waitingForPopulatedDecoderSurface = true;
+        }
+        if (decoder.isEnded()) {
+          checkNotNull(encoder).signalEndOfInputStream();
+        }
+      }
       return false;
     }
-    return hasProcessedOutputBuffer;
+
+    waitingForPopulatedDecoderSurface = false;
+    SurfaceTexture decoderSurfaceTexture = checkNotNull(this.decoderSurfaceTexture);
+    decoderSurfaceTexture.updateTexImage();
+    decoderSurfaceTexture.getTransformMatrix(decoderTextureTransformMatrix);
+    GlUtil.Uniform decoderTextureTransformUniform =
+        checkNotNull(this.decoderTextureTransformUniform);
+    decoderTextureTransformUniform.setFloats(decoderTextureTransformMatrix);
+    decoderTextureTransformUniform.bind();
+    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
+    EGLDisplay eglDisplay = checkNotNull(this.eglDisplay);
+    EGLSurface eglSurface = checkNotNull(this.eglSurface);
+    EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nextEncoderTimeUs * 1000L);
+    EGL14.eglSwapBuffers(eglDisplay, eglSurface);
+    isDecoderSurfacePopulated = false;
+    return true;
   }
 
   private boolean feedMuxerFromEncoder() {