diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index 66da72cc58..f9f43c52b6 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -862,6 +862,133 @@ public final class GlUtil { checkGlError(); } + /** + * Creates a pixel buffer object with a data store of the given size and usage {@link + * GLES30#GL_DYNAMIC_READ}. + * + *

The buffer is suitable for repeated modification by OpenGL and reads by the application. + * + * @param size The size of the buffer object's data store. + * @return The pixel buffer object. + */ + public static int createPixelBufferObject(int size) throws GlException { + int[] ids = new int[1]; + GLES30.glGenBuffers(/* n= */ 1, ids, /* offset= */ 0); + GlUtil.checkGlError(); + + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, ids[0]); + GlUtil.checkGlError(); + + GLES30.glBufferData( + GLES30.GL_PIXEL_PACK_BUFFER, /* size= */ size, /* data= */ null, GLES30.GL_DYNAMIC_READ); + GlUtil.checkGlError(); + + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0); + GlUtil.checkGlError(); + return ids[0]; + } + + /** + * Reads pixel data from the {@link GLES30#GL_COLOR_ATTACHMENT0} attachment of a framebuffer into + * the data store of a pixel buffer object. + * + *

The texture backing the color attachment of {@code readFboId} and the buffer store of {@code + * bufferId} must hold an image of the given {@code width} and {@code height} with format {@link + * GLES30#GL_RGBA} and type {@link GLES30#GL_UNSIGNED_BYTE}. + * + *

This a non-blocking call which reads the data asynchronously. + * + *

HDR support is not yet implemented. + * + * @param readFboId The framebuffer that holds pixel data. + * @param width The image width. + * @param height The image height. + * @param bufferId The pixel buffer object to read into. + */ + public static void schedulePixelBufferRead(int readFboId, int width, int height, int bufferId) + throws GlException { + focusFramebufferUsingCurrentContext(readFboId, width, height); + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId); + GlUtil.checkGlError(); + + GLES30.glReadBuffer(GLES30.GL_COLOR_ATTACHMENT0); + GLES30.glReadPixels( + /* x= */ 0, + /* y= */ 0, + width, + height, + GLES30.GL_RGBA, + GLES30.GL_UNSIGNED_BYTE, + /* offset= */ 0); + GlUtil.checkGlError(); + + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0); + GlUtil.checkGlError(); + } + + /** + * Maps the pixel buffer object's data store of a given size and returns a {@link ByteBuffer} of + * OpenGL managed memory. + * + *

The application must not write into the returned {@link ByteBuffer}. + * + *

The pixel buffer object should have a {@linkplain #schedulePixelBufferRead previously + * scheduled pixel buffer read}. + * + *

When the application no longer needs to access the returned buffer, call {@link + * #unmapPixelBufferObject}. + * + *

This call blocks until the pixel buffer data from the last {@link #schedulePixelBufferRead} + * call is available. + * + * @param bufferId The pixel buffer object. + * @param size The size of the pixel buffer object's data store to be mapped. + * @return The {@link ByteBuffer} that holds pixel data. + */ + public static ByteBuffer mapPixelBufferObject(int bufferId, int size) throws GlException { + GLES20.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId); + checkGlError(); + ByteBuffer mappedPixelBuffer = + (ByteBuffer) + GLES30.glMapBufferRange( + GLES30.GL_PIXEL_PACK_BUFFER, + /* offset= */ 0, + /* length= */ size, + GLES30.GL_MAP_READ_BIT); + GlUtil.checkGlError(); + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0); + GlUtil.checkGlError(); + return mappedPixelBuffer; + } + + /** + * Unmaps the pixel buffer object {@code bufferId}'s data store. + * + *

The pixel buffer object should be previously {@linkplain #mapPixelBufferObject mapped}. + * + *

After this method returns, accessing data inside a previously {@linkplain + * #mapPixelBufferObject mapped} {@link ByteBuffer} results in undefined behaviour. + * + *

When this method returns, the pixel buffer object {@code bufferId} can be reused by {@link + * #schedulePixelBufferRead}. + * + * @param bufferId The pixel buffer object. + */ + public static void unmapPixelBufferObject(int bufferId) throws GlException { + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, bufferId); + GlUtil.checkGlError(); + GLES30.glUnmapBuffer(GLES30.GL_PIXEL_PACK_BUFFER); + GlUtil.checkGlError(); + GLES30.glBindBuffer(GLES30.GL_PIXEL_PACK_BUFFER, /* buffer= */ 0); + GlUtil.checkGlError(); + } + + /** Deletes a buffer object, or silently ignores the method call if {@code bufferId} is unused. */ + public static void deleteBuffer(int bufferId) throws GlException { + GLES20.glDeleteBuffers(/* n= */ 1, new int[] {bufferId}, /* offset= */ 0); + checkGlError(); + } + /** * Throws a {@link GlException} with the given message if {@code expression} evaluates to {@code * false}. diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java index a4d838022a..126239f1a5 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java @@ -177,5 +177,14 @@ public class QueuingGlShaderProgramTest { checkState(result == presentationTimeUs); events.add(Pair.create("finishProcessingAndBlend", presentationTimeUs)); } + + @Override + public void signalEndOfCurrentInputStream() {} + + @Override + public void flush() {} + + @Override + public void release() {} } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java index c038954767..7cb0a7d465 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java @@ -15,18 +15,22 @@ */ package androidx.media3.effect; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import android.graphics.Rect; -import android.opengl.GLES20; -import android.opengl.GLES30; import androidx.media3.common.C; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; +import com.google.common.util.concurrent.SettableFuture; import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.Future; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -44,6 +48,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final int BYTES_PER_PIXEL = 4; private final ByteBufferGlEffect.Processor processor; + private final int pendingPixelBufferQueueSize; + private final Queue unmappedPixelBuffers; + private final Queue mappedPixelBuffers; + private final PixelBufferObjectProvider pixelBufferObjectProvider; private int inputWidth; private int inputHeight; @@ -52,10 +60,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Creates an instance. * + * @param pendingPixelBufferQueueSize The maximum number of scheduled but not yet completed + * texture to {@linkplain ByteBuffer pixel buffer} transfers. * @param processor The {@linkplain ByteBufferGlEffect.Processor effect}. */ - public ByteBufferConcurrentEffect(ByteBufferGlEffect.Processor processor) { + public ByteBufferConcurrentEffect( + int pendingPixelBufferQueueSize, ByteBufferGlEffect.Processor processor) { this.processor = processor; + this.pendingPixelBufferQueueSize = pendingPixelBufferQueueSize; + unmappedPixelBuffers = new ArrayDeque<>(); + mappedPixelBuffers = new ArrayDeque<>(); + pixelBufferObjectProvider = new PixelBufferObjectProvider(); inputWidth = C.LENGTH_UNSET; inputHeight = C.LENGTH_UNSET; } @@ -64,9 +79,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public Future queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) { try { + while (unmappedPixelBuffers.size() >= pendingPixelBufferQueueSize) { + checkState(mapOnePixelBuffer()); + } + if (effectInputTexture == null || textureInfo.width != inputWidth || textureInfo.height != inputHeight) { + while (mapOnePixelBuffer()) {} inputWidth = textureInfo.width; inputHeight = textureInfo.height; Size effectInputSize = processor.configure(inputWidth, inputHeight); @@ -90,20 +110,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; new Rect( /* left= */ 0, /* top= */ 0, effectInputTexture.width, effectInputTexture.height)); - GlUtil.focusFramebufferUsingCurrentContext( - effectInputTexture.fboId, effectInputTexture.width, effectInputTexture.height); - ByteBuffer pixelBuffer = - ByteBuffer.allocateDirect(texturePixelBufferSize(effectInputTexture)); - GLES20.glReadPixels( - /* x= */ 0, - /* y= */ 0, - effectInputTexture.width, - effectInputTexture.height, - GLES30.GL_RGBA, - GLES30.GL_UNSIGNED_BYTE, - pixelBuffer); - GlUtil.checkGlError(); - return processor.processPixelBuffer(pixelBuffer, presentationTimeUs); + TexturePixelBuffer texturePixelBuffer = new TexturePixelBuffer(effectInputTexture); + unmappedPixelBuffers.add(texturePixelBuffer); + return Util.transformFutureAsync( + texturePixelBuffer.byteBufferSettableFuture, + (pixelBuffer) -> processor.processPixelBuffer(pixelBuffer, presentationTimeUs)); } catch (GlUtil.GlException | VideoFrameProcessingException e) { return immediateFailedFuture(e); } @@ -112,10 +123,145 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void finishProcessingAndBlend(GlTextureInfo textureInfo, long presentationTimeUs, T result) throws VideoFrameProcessingException { + try { + TexturePixelBuffer oldestRunningFrame = checkNotNull(mappedPixelBuffers.poll()); + oldestRunningFrame.unmapAndRecycle(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + processor.finishProcessingAndBlend(textureInfo, presentationTimeUs, result); } + @Override + public void signalEndOfCurrentInputStream() throws VideoFrameProcessingException { + try { + while (mapOnePixelBuffer()) {} + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + @Override + public void flush() throws VideoFrameProcessingException { + try { + unmapAndRecyclePixelBuffers(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + @Override + public void release() throws VideoFrameProcessingException { + try { + unmapAndRecyclePixelBuffers(); + pixelBufferObjectProvider.release(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + private static int texturePixelBufferSize(GlTextureInfo textureInfo) { return textureInfo.width * textureInfo.height * BYTES_PER_PIXEL; } + + private void unmapAndRecyclePixelBuffers() throws GlUtil.GlException { + TexturePixelBuffer texturePixelBuffer; + while ((texturePixelBuffer = unmappedPixelBuffers.poll()) != null) { + texturePixelBuffer.unmapAndRecycle(); + } + while ((texturePixelBuffer = mappedPixelBuffers.poll()) != null) { + texturePixelBuffer.unmapAndRecycle(); + } + } + + private boolean mapOnePixelBuffer() throws GlUtil.GlException { + TexturePixelBuffer texturePixelBuffer = unmappedPixelBuffers.poll(); + if (texturePixelBuffer == null) { + return false; + } + texturePixelBuffer.map(); + mappedPixelBuffers.add(texturePixelBuffer); + return true; + } + + /** + * Manages the lifecycle of a {@link PixelBufferObjectInfo} which is mapped to a {@link + * GlTextureInfo}. + */ + private final class TexturePixelBuffer { + public final PixelBufferObjectInfo pixelBufferObjectInfo; + public final SettableFuture byteBufferSettableFuture; + + private boolean mapped; + + public TexturePixelBuffer(GlTextureInfo textureInfo) throws GlUtil.GlException { + int pixelBufferSize = texturePixelBufferSize(textureInfo); + pixelBufferObjectInfo = pixelBufferObjectProvider.getPixelBufferObject(pixelBufferSize); + GlUtil.schedulePixelBufferRead( + textureInfo.fboId, textureInfo.width, textureInfo.height, pixelBufferObjectInfo.id); + byteBufferSettableFuture = SettableFuture.create(); + } + + public void map() throws GlUtil.GlException { + ByteBuffer byteBuffer = + GlUtil.mapPixelBufferObject(pixelBufferObjectInfo.id, pixelBufferObjectInfo.size); + byteBufferSettableFuture.set(byteBuffer); + mapped = true; + } + + public void unmapAndRecycle() throws GlUtil.GlException { + if (mapped) { + GlUtil.unmapPixelBufferObject(pixelBufferObjectInfo.id); + } + pixelBufferObjectProvider.recycle(pixelBufferObjectInfo); + } + } + + /** One pixel buffer object with a data store. */ + private static final class PixelBufferObjectInfo { + public final int id; + public final int size; + + public PixelBufferObjectInfo(int size) throws GlUtil.GlException { + this.size = size; + id = GlUtil.createPixelBufferObject(size); + } + + public void release() throws GlUtil.GlException { + GlUtil.deleteBuffer(id); + } + } + + /** Provider for {@link PixelBufferObjectInfo} objects. */ + private static final class PixelBufferObjectProvider { + private final Queue availablePixelBufferObjects; + + public PixelBufferObjectProvider() { + availablePixelBufferObjects = new ArrayDeque<>(); + } + + private PixelBufferObjectInfo getPixelBufferObject(int pixelBufferSize) + throws GlUtil.GlException { + PixelBufferObjectInfo pixelBufferObjectInfo; + while ((pixelBufferObjectInfo = availablePixelBufferObjects.poll()) != null) { + if (pixelBufferObjectInfo.size == pixelBufferSize) { + return pixelBufferObjectInfo; + } + GlUtil.deleteBuffer(pixelBufferObjectInfo.id); + } + return new PixelBufferObjectInfo(pixelBufferSize); + } + + private void recycle(PixelBufferObjectInfo pixelBufferObjectInfo) { + availablePixelBufferObjects.add(pixelBufferObjectInfo); + } + + public void release() throws GlUtil.GlException { + PixelBufferObjectInfo pixelBufferObjectInfo; + while ((pixelBufferObjectInfo = availablePixelBufferObjects.poll()) != null) { + pixelBufferObjectInfo.release(); + } + } + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java index 3efc23027a..70319edae7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java @@ -39,6 +39,7 @@ import java.util.concurrent.Future; /* package */ class ByteBufferGlEffect implements GlEffect { private static final int DEFAULT_QUEUE_SIZE = 6; + private static final int DEFAULT_PENDING_PIXEL_BUFFER_QUEUE_SIZE = 1; /** * A processor that takes in {@link ByteBuffer ByteBuffers} that represent input image data, and @@ -134,6 +135,7 @@ import java.util.concurrent.Future; return new QueuingGlShaderProgram<>( /* useHighPrecisionColorComponents= */ useHdr, /* queueSize= */ DEFAULT_QUEUE_SIZE, - new ByteBufferConcurrentEffect<>(processor)); + new ByteBufferConcurrentEffect<>( + /* pendingPixelBufferQueueSize= */ DEFAULT_PENDING_PIXEL_BUFFER_QUEUE_SIZE, processor)); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java index c64f14f220..242ad1a3ac 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java @@ -110,6 +110,32 @@ import java.util.concurrent.TimeUnit; */ void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result) throws VideoFrameProcessingException; + + /** + * Notifies the {@code ConcurrentEffect} that no further input frames belonging to the current + * input stream will be queued. + * + *

Can block until the {@code ConcurrentEffect} finishes processing pending frames. + * + * @throws VideoFrameProcessingException If an error occurs while processing pending frames. + */ + void signalEndOfCurrentInputStream() throws VideoFrameProcessingException; + + /** + * Flushes the {@code ConcurrentEffect}. + * + *

The {@code ConcurrentEffect} should reclaim the ownership of any allocated resources. + * + * @throws VideoFrameProcessingException If an error occurs while reclaiming resources. + */ + void flush() throws VideoFrameProcessingException; + + /** + * Releases all resources. + * + * @throws VideoFrameProcessingException If an error occurs while releasing resources. + */ + void release() throws VideoFrameProcessingException; } private final ConcurrentEffect concurrentEffect; @@ -222,6 +248,11 @@ import java.util.concurrent.TimeUnit; @Override public void signalEndOfCurrentInputStream() { + try { + concurrentEffect.signalEndOfCurrentInputStream(); + } catch (VideoFrameProcessingException e) { + onError(e); + } while (outputOneFrame()) {} outputListener.onCurrentOutputStreamEnded(); } @@ -229,6 +260,11 @@ import java.util.concurrent.TimeUnit; @Override @CallSuper public void flush() { + try { + concurrentEffect.flush(); + } catch (VideoFrameProcessingException e) { + onError(e); + } cancelProcessingOfPendingFrames(); outputTexturePool.freeAllTextures(); inputListener.onFlush(); @@ -242,6 +278,7 @@ import java.util.concurrent.TimeUnit; public void release() throws VideoFrameProcessingException { try { cancelProcessingOfPendingFrames(); + concurrentEffect.release(); outputTexturePool.deleteAllTextures(); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e);