From 457f446114823831b940bf46fb256daa0ea00815 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 8 Jun 2022 09:44:25 +0000 Subject: [PATCH] Implement default GlTextureProcessor in SingleFrameGlTextureProcessor. SingleFrameGlTextureProcessor is now an abstract class containing a default implementation of the more flexible GlTextureProcessor interface while still exposing the same simple abstract methods for single frame processing it previously did. FrameProcessorChain and GlEffect will be changed to use GlTextureProcessor in follow-ups. PiperOrigin-RevId: 453633000 --- .../transformer/BitmapOverlayProcessor.java | 3 +- .../PeriodicVignetteProcessor.java | 3 +- .../demo/transformer/MediaPipeProcessor.java | 3 +- .../androidx/media3/common/util/GlUtil.java | 35 +++++- .../transformer/FrameProcessorChainTest.java | 5 +- .../transformer/ExternalTextureProcessor.java | 3 +- .../transformer/FrameProcessorChain.java | 13 +-- .../MatrixTransformationProcessor.java | 3 +- .../SingleFrameGlTextureProcessor.java | 109 +++++++++++++++--- 9 files changed, 140 insertions(+), 37 deletions(-) diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java index 67dff8aa95..b9c0efac37 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java @@ -44,7 +44,7 @@ import java.util.Locale; */ // TODO(b/227625365): Delete this class and use a texture processor from the Transformer library, // once overlaying a bitmap and text is supported in Transformer. -/* package */ final class BitmapOverlayProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class BitmapOverlayProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; } @@ -147,6 +147,7 @@ import java.util.Locale; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java index 42aec157e0..fb00e5f888 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java @@ -30,7 +30,7 @@ import java.io.IOException; * A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are * darker the further they are away from the frame center. */ -/* package */ final class PeriodicVignetteProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class PeriodicVignetteProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; } @@ -108,6 +108,7 @@ import java.io.IOException; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 65a8666634..7d6ddfae5b 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -40,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that * can immediately produce one output frame per input frame. */ -/* package */ final class MediaPipeProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor { private static final LibraryLoader LOADER = new LibraryLoader("mediapipe_jni") { @@ -160,6 +160,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void release() { + super.release(); checkStateNotNull(frameProcessor).close(); } } diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index 2cf1fead3e..485f2db986 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -343,6 +343,13 @@ public final class GlUtil { } } + /** Fills the pixels in the current output render target with (r=0, g=0, b=0, a=0). */ + public static void clearOutputFrame() { + GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GlUtil.checkGlError(); + } + /** * Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by * {@code height} pixels. @@ -369,6 +376,22 @@ public final class GlUtil { Api17.focusRenderTarget(eglDisplay, eglContext, eglSurface, framebuffer, width, height); } + /** + * Makes the specified {@code framebuffer} the render target, using a viewport of {@code width} by + * {@code height} pixels. + * + *

The caller must ensure that there is a current OpenGL context before calling this method. + * + * @param framebuffer The identifier of the framebuffer object to bind as the output render + * target. + * @param width The viewport width, in pixels. + * @param height The viewport height, in pixels. + */ + @RequiresApi(17) + public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) { + Api17.focusFramebufferUsingCurrentContext(framebuffer, width, height); + } + /** * Deletes a GL texture. * @@ -613,14 +636,22 @@ public final class GlUtil { int framebuffer, int width, int height) { + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext); + checkEglException("Error making context current"); + focusFramebufferUsingCurrentContext(framebuffer, width, height); + } + + @DoNotInline + public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) { + checkEglException( + !Util.areEqual(EGL14.eglGetCurrentContext(), EGL14.EGL_NO_CONTEXT), "No current context"); + int[] boundFramebuffer = new int[1]; GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, boundFramebuffer, /* offset= */ 0); if (boundFramebuffer[0] != framebuffer) { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebuffer); } checkGlError(); - EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext); - checkEglException("Error making context current"); GLES20.glViewport(/* x= */ 0, /* y= */ 0, width, height); checkGlError(); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java index 92a0a5395b..5dfe72e5b1 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -135,7 +135,7 @@ public final class FrameProcessorChainTest { /* enableExperimentalHdrEditing= */ false); } - private static class FakeTextureProcessor implements SingleFrameGlTextureProcessor { + private static class FakeTextureProcessor extends SingleFrameGlTextureProcessor { private final Size outputSize; @@ -150,8 +150,5 @@ public final class FrameProcessorChainTest { @Override public void drawFrame(int inputTexId, long presentationTimeNs) {} - - @Override - public void release() {} } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java index 2d68dd63d9..823b883668 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java @@ -26,7 +26,7 @@ import androidx.media3.common.util.GlUtil; import java.io.IOException; /** Copies frames from an external texture and applies color transformations for HDR if needed. */ -/* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor { +/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -115,6 +115,7 @@ import java.io.IOException; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 9ea96e3a9b..c4ded83fd9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -29,7 +29,6 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; -import android.opengl.GLES20; import android.util.Size; import android.view.Surface; import android.view.SurfaceHolder; @@ -517,12 +516,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTexture.fboId, outputTexture.width, outputTexture.height); - clearOutputFrame(); + GlUtil.clearOutputFrame(); textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs); inputTexId = outputTexture.texId; } GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight); - clearOutputFrame(); + GlUtil.clearOutputFrame(); getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs); EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs); @@ -533,7 +532,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int finalInputTexId = inputTexId; debugSurfaceViewWrapper.maybeRenderToSurfaceView( () -> { - clearOutputFrame(); + GlUtil.clearOutputFrame(); try { getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs); } catch (FrameProcessingException e) { @@ -553,12 +552,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static void clearOutputFrame() { - GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - GlUtil.checkGlError(); - } - /** * Releases the {@link SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} and destroys * the OpenGL context. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index c8d214fdf4..1288ff6ba0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -41,7 +41,7 @@ import java.util.Arrays; */ @UnstableApi @SuppressWarnings("FunctionalInterfaceClash") // b/228192298 -/* package */ final class MatrixTransformationProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -171,6 +171,7 @@ import java.util.Arrays; @Override public void release() { + super.release(); glProgram.delete(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java index 873ebfe130..5ec5522ddd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java @@ -16,38 +16,42 @@ package androidx.media3.transformer; import android.util.Size; +import androidx.annotation.CallSuper; +import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels * into an output frame, with changes to pixels specific to the implementation. * - *

Methods must be called in the following order: - * - *

    - *
  1. {@link #configure(int, int)}, to configure the frame processor based on the input - * dimensions. - *
  2. {@link #drawFrame(int, long)}, to process one frame. - *
  3. {@link #release()}, upon conclusion of processing. - *
+ *

{@code SingleFrameGlTextureProcessor} implementations must produce exactly one output frame + * per input frame with the same presentation timestamp. For more flexibility, implement {@link + * GlTextureProcessor} directly. * *

All methods in this class must be called on the thread that owns the OpenGL context. */ @UnstableApi -// TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an -// abstract class with a default implementation of GlTextureProcessor methods. -public interface SingleFrameGlTextureProcessor { +public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor { + + private @MonotonicNonNull Listener listener; + private int inputWidth; + private int inputHeight; + private @MonotonicNonNull TextureInfo outputTexture; + private boolean outputTextureInUse; /** * Configures the texture processor based on the input dimensions. * - *

This method can be called multiple times. + *

This method must be called before {@linkplain #drawFrame(int,long) drawing} the first frame + * and before drawing subsequent frames with different input dimensions. * * @param inputWidth The input width, in pixels. * @param inputHeight The input height, in pixels. * @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}. */ - Size configure(int inputWidth, int inputHeight); + public abstract Size configure(int inputWidth, int inputHeight); /** * Draws one frame. @@ -63,8 +67,81 @@ public interface SingleFrameGlTextureProcessor { * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @throws FrameProcessingException If an error occurs while processing or drawing the frame. */ - void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException; + public abstract void drawFrame(int inputTexId, long presentationTimeUs) + throws FrameProcessingException; - /** Releases all resources. */ - void release(); + @Override + public final void setListener(Listener listener) { + this.listener = listener; + } + + @Override + public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + if (outputTextureInUse) { + return false; + } + + try { + if (outputTexture == null + || inputTexture.width != inputWidth + || inputTexture.height != inputHeight) { + configureOutputTexture(inputTexture.width, inputTexture.height); + } + outputTextureInUse = true; + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.fboId, outputTexture.width, outputTexture.height); + GlUtil.clearOutputFrame(); + drawFrame(inputTexture.texId, presentationTimeUs); + if (listener != null) { + listener.onInputFrameProcessed(inputTexture); + listener.onOutputFrameAvailable(outputTexture, presentationTimeUs); + } + } catch (FrameProcessingException | RuntimeException e) { + if (listener != null) { + listener.onFrameProcessingError( + e instanceof FrameProcessingException + ? (FrameProcessingException) e + : new FrameProcessingException(e)); + } + } + return true; + } + + @EnsuresNonNull("outputTexture") + private void configureOutputTexture(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + Size outputSize = configure(inputWidth, inputHeight); + if (outputTexture == null + || outputSize.getWidth() != outputTexture.width + || outputSize.getHeight() != outputTexture.height) { + if (outputTexture != null) { + GlUtil.deleteTexture(outputTexture.texId); + } + int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); + int outputFboId = GlUtil.createFboForTexture(outputTexId); + outputTexture = + new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight()); + } + } + + @Override + public final void releaseOutputFrame(TextureInfo outputTexture) { + outputTextureInUse = false; + } + + @Override + public final void signalEndOfInputStream() { + if (listener != null) { + listener.onOutputStreamEnded(); + } + } + + @Override + @CallSuper + public void release() { + if (outputTexture != null) { + GlUtil.deleteTexture(outputTexture.texId); + } + } }