From 6edd1d2a2ff982b187364305b77228a8f6d780fb Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 22 Jun 2022 17:16:54 +0100 Subject: [PATCH] Use GlTextureProcessor to avoid redundant copy in MediaPipeProcessor. After this change GlEffects can use any GlTextureProcessor not just SingleFrameGlTextureProcessor. MediaPipeProcessor now implements GlTextureProcessor directly which allows it to reuse MediaPipe's output texture for its output texture and avoids an extra copy shader step. PiperOrigin-RevId: 456530718 (cherry picked from commit e25bf811959baf1066ba18adc37e8bbdaad4791a) --- .../transformerdemo/TransformerActivity.java | 4 +- .../demo/transformer/MediaPipeProcessor.java | 167 +++++++++--------- .../FrameProcessorChainPixelTest.java | 2 +- ...lMatrixTransformationProcessorWrapper.java | 3 + .../exoplayer2/transformer/GlEffect.java | 10 +- .../exoplayer2/transformer/TextureInfo.java | 2 +- 6 files changed, 97 insertions(+), 91 deletions(-) diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index 545f43f4bc..547ad3770a 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -41,8 +41,8 @@ import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; import com.google.android.exoplayer2.transformer.EncoderSelector; import com.google.android.exoplayer2.transformer.GlEffect; +import com.google.android.exoplayer2.transformer.GlTextureProcessor; import com.google.android.exoplayer2.transformer.ProgressHolder; -import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor; import com.google.android.exoplayer2.transformer.TransformationException; import com.google.android.exoplayer2.transformer.TransformationRequest; import com.google.android.exoplayer2.transformer.TransformationResult; @@ -282,7 +282,7 @@ public final class TransformerActivity extends AppCompatActivity { effects.add( (Context context) -> { try { - return (SingleFrameGlTextureProcessor) + return (GlTextureProcessor) constructor.newInstance( context, /* graphName= */ "edge_detector_mediapipe_graph.binarypb", diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 424a76b2be..01e7242620 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -20,27 +20,24 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.EGL14; -import android.opengl.GLES20; -import android.util.Size; +import android.os.Build; +import androidx.annotation.ChecksSdkIntAtLeast; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.transformer.FrameProcessingException; -import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor; -import com.google.android.exoplayer2.util.ConditionVariable; -import com.google.android.exoplayer2.util.GlProgram; -import com.google.android.exoplayer2.util.GlUtil; +import com.google.android.exoplayer2.transformer.GlTextureProcessor; +import com.google.android.exoplayer2.transformer.TextureInfo; import com.google.android.exoplayer2.util.LibraryLoader; +import com.google.android.exoplayer2.util.Util; import com.google.mediapipe.components.FrameProcessor; -import com.google.mediapipe.framework.AndroidAssetUtil; import com.google.mediapipe.framework.AppTextureFrame; import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.glutil.EglManager; -import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** - * Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that - * can immediately produce one output frame per input frame. - */ -/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor { +/** Runs a MediaPipe graph on input frames. */ +/* package */ final class MediaPipeProcessor implements GlTextureProcessor { private static final LibraryLoader LOADER = new LibraryLoader("mediapipe_jni") { @@ -60,17 +57,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl"; - private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl"; - - private final ConditionVariable frameProcessorConditionVariable; private final FrameProcessor frameProcessor; - private final GlProgram glProgram; - - private int inputWidth; - private int inputHeight; - private @MonotonicNonNull TextureFrame outputFrame; - private @MonotonicNonNull RuntimeException frameProcessorPendingError; + private volatile GlTextureProcessor.@MonotonicNonNull Listener listener; + private volatile boolean acceptedFrame; + // Only available from API 23 and above. + @Nullable private final ConcurrentHashMap outputFrames; + // Used instead for API 21 and 22. + @Nullable private volatile TextureInfo outputTexture; + @Nullable private volatile TextureFrame outputFrame; /** * Creates a new texture processor that wraps a MediaPipe graph. @@ -79,92 +73,103 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param graphName Name of a MediaPipe graph asset to load. * @param inputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph. - * @throws FrameProcessingException If a problem occurs while reading shader files or initializing - * MediaPipe resources. */ + @SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23. public MediaPipeProcessor( - Context context, String graphName, String inputStreamName, String outputStreamName) - throws FrameProcessingException { + Context context, String graphName, String inputStreamName, String outputStreamName) { checkState(LOADER.isAvailable()); - - frameProcessorConditionVariable = new ConditionVariable(); - AndroidAssetUtil.initializeNativeAssetManager(context); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); frameProcessor = new FrameProcessor( context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); - // Unblock drawFrame when there is an output frame or an error. + outputFrames = areMultipleOutputFramesSupported() ? new ConcurrentHashMap<>() : null; frameProcessor.setConsumer( frame -> { - outputFrame = frame; - frameProcessorConditionVariable.open(); + TextureInfo texture = + new TextureInfo( + frame.getTextureName(), + /* fboId= */ C.INDEX_UNSET, + frame.getWidth(), + frame.getHeight()); + if (areMultipleOutputFramesSupported()) { + checkStateNotNull(outputFrames).put(texture, frame); + } else { + outputFrame = frame; + outputTexture = texture; + } + if (listener != null) { + listener.onOutputFrameAvailable(texture, frame.getTimestamp()); + } }); frameProcessor.setAsynchronousErrorListener( error -> { - frameProcessorPendingError = error; - frameProcessorConditionVariable.open(); + if (listener != null) { + listener.onFrameProcessingError(new FrameProcessingException(error)); + } }); - try { - glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME); - } catch (IOException | GlUtil.GlException e) { - throw new FrameProcessingException(e); + frameProcessor.setOnWillAddFrameListener((long timestamp) -> acceptedFrame = true); + } + + @Override + public void setListener(GlTextureProcessor.Listener listener) { + this.listener = listener; + } + + @Override + public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + if (!areMultipleOutputFramesSupported() && outputTexture != null) { + return false; } - } - @Override - public Size configure(int inputWidth, int inputHeight) { - this.inputWidth = inputWidth; - this.inputHeight = inputHeight; - return new Size(inputWidth, inputHeight); - } - - @Override - public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { - frameProcessorConditionVariable.close(); - - // Pass the input frame to MediaPipe. - AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexId, inputWidth, inputHeight); + acceptedFrame = false; + AppTextureFrame appTextureFrame = + new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height); appTextureFrame.setTimestamp(presentationTimeUs); checkStateNotNull(frameProcessor).onNewFrame(appTextureFrame); - - // Wait for output to be passed to the consumer. try { - frameProcessorConditionVariable.block(); + appTextureFrame.waitUntilReleasedWithGpuSync(); } catch (InterruptedException e) { - // Propagate the interrupted flag so the next blocking operation will throw. - // TODO(b/230469581): The next processor that runs will not have valid input due to returning - // early here. This could be fixed by checking for interruption in the outer loop that runs - // through the texture processors. Thread.currentThread().interrupt(); - return; + if (listener != null) { + listener.onFrameProcessingError(new FrameProcessingException(e)); + } } - - if (frameProcessorPendingError != null) { - throw new FrameProcessingException(frameProcessorPendingError); + if (listener != null) { + listener.onInputFrameProcessed(inputTexture); } + return acceptedFrame; + } - // Copy from MediaPipe's output texture to the current output. - try { - checkStateNotNull(glProgram).use(); - glProgram.setSamplerTexIdUniform( - "uTexSampler", checkStateNotNull(outputFrame).getTextureName(), /* texUnitIndex= */ 0); - glProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - glProgram.bindAttributesAndUniforms(); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); - } catch (GlUtil.GlException e) { - throw new FrameProcessingException(e, presentationTimeUs); - } finally { + @Override + public void releaseOutputFrame(TextureInfo outputTexture) { + if (areMultipleOutputFramesSupported()) { + checkStateNotNull(checkStateNotNull(outputFrames).get(outputTexture)).release(); + } else { + checkState(Util.areEqual(outputTexture, this.outputTexture)); + this.outputTexture = null; checkStateNotNull(outputFrame).release(); + outputFrame = null; } } @Override - public void release() throws FrameProcessingException { - super.release(); + public void release() { checkStateNotNull(frameProcessor).close(); } + + @Override + public final void signalEndOfInputStream() { + frameProcessor.waitUntilIdle(); + if (listener != null) { + listener.onOutputStreamEnded(); + } + } + + @ChecksSdkIntAtLeast(api = Build.VERSION_CODES.M) + private static boolean areMultipleOutputFramesSupported() { + // Android devices running Lollipop (API 21/22) have a bug in ConcurrentHashMap that can result + // in lost updates, so we only allow one output frame to be pending at a time to avoid using + // ConcurrentHashMap. + return Util.SDK_INT >= 23; + } } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java index 80ae0a7922..54955779ea 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java @@ -491,7 +491,7 @@ public final class FrameProcessorChainPixelTest { } @Override - public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) + public GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException { return effect.toGlTextureProcessor(context); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java index d9eec44936..3c1bed8393 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java @@ -23,6 +23,7 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; +import android.opengl.GLES20; import android.util.Size; import android.view.Surface; import android.view.SurfaceHolder; @@ -314,6 +315,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); renderingTask.run(); EGL14.eglSwapBuffers(eglDisplay, eglSurface); + // Prevents white flashing on the debug SurfaceView when frames are rendered too fast. + GLES20.glFinish(); } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java index 0a470e75af..c7f2c13971 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java @@ -18,16 +18,14 @@ package com.google.android.exoplayer2.transformer; import android.content.Context; /** - * Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. + * Interface for a video frame effect with a {@link GlTextureProcessor} implementation. * *

Implementations contain information specifying the effect and can be {@linkplain - * #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which - * applies the effect. + * #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the + * effect. */ public interface GlEffect { /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ - // TODO(b/227625423): use GlTextureProcessor here once this interface exists. - SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) - throws FrameProcessingException; + GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java index 50eaf1fcc4..b61c3b6534 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java @@ -16,7 +16,7 @@ package com.google.android.exoplayer2.transformer; /** Contains information describing an OpenGL texture. */ -/* package */ final class TextureInfo { +public final class TextureInfo { /** The OpenGL texture identifier. */ public final int texId; /** Identifier of a framebuffer object associated with the texture. */