diff --git a/demos/transformer/src/withMediaPipe/java/com/google/android/exoplayer2/transformerdemo/MediaPipeShaderProgram.java b/demos/transformer/src/withMediaPipe/java/com/google/android/exoplayer2/transformerdemo/MediaPipeShaderProgram.java index b998fde4c8..ff81086f12 100644 --- a/demos/transformer/src/withMediaPipe/java/com/google/android/exoplayer2/transformerdemo/MediaPipeShaderProgram.java +++ b/demos/transformer/src/withMediaPipe/java/com/google/android/exoplayer2/transformerdemo/MediaPipeShaderProgram.java @@ -25,7 +25,7 @@ import android.opengl.EGL14; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.effect.GlShaderProgram; -import com.google.android.exoplayer2.effect.TextureInfo; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.LibraryLoader; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.VideoFrameProcessingException; @@ -68,7 +68,7 @@ import java.util.concurrent.Future; } private final FrameProcessor frameProcessor; - private final ConcurrentHashMap outputFrames; + private final ConcurrentHashMap outputFrames; private final boolean isSingleFrameGraph; @Nullable private final ExecutorService singleThreadExecutorService; private final Queue> futures; @@ -137,10 +137,11 @@ import java.util.concurrent.Future; this.outputListener = outputListener; frameProcessor.setConsumer( frame -> { - TextureInfo texture = - new TextureInfo( + GlTextureInfo texture = + new GlTextureInfo( frame.getTextureName(), /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, frame.getWidth(), frame.getHeight()); outputFrames.put(texture, frame); @@ -159,7 +160,7 @@ import java.util.concurrent.Future; } @Override - public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { AppTextureFrame appTextureFrame = new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height); // TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist. @@ -183,7 +184,7 @@ import java.util.concurrent.Future; } private boolean maybeQueueInputFrameSynchronous( - AppTextureFrame appTextureFrame, TextureInfo inputTexture) { + AppTextureFrame appTextureFrame, GlTextureInfo inputTexture) { acceptedFrame = false; frameProcessor.onNewFrame(appTextureFrame); try { @@ -200,7 +201,7 @@ import java.util.concurrent.Future; } private void queueInputFrameAsynchronous( - AppTextureFrame appTextureFrame, TextureInfo inputTexture) { + AppTextureFrame appTextureFrame, GlTextureInfo inputTexture) { removeFinishedFutures(); futures.add( checkStateNotNull(singleThreadExecutorService) @@ -222,7 +223,7 @@ import java.util.concurrent.Future; } @Override - public void releaseOutputFrame(TextureInfo outputTexture) { + public void releaseOutputFrame(GlTextureInfo outputTexture) { checkStateNotNull(outputFrames.get(outputTexture)).release(); if (isSingleFrameGraph) { inputListener.onReadyToAcceptInputFrame(); diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlObjectsProvider.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlObjectsProvider.java new file mode 100644 index 0000000000..5f196ba765 --- /dev/null +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlObjectsProvider.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.util; + +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import androidx.annotation.IntRange; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.util.GlUtil.GlException; + +// TODO(271433904): Expand this class to cover more methods in GlUtil. +/** Provider to customize the creation and maintenance of GL objects. */ +public interface GlObjectsProvider { + /** + * Provider for GL objects that configures a GL context with 8-bit RGB or 10-bit RGB attributes, + * and no depth buffer or render buffers. + */ + GlObjectsProvider DEFAULT = + new GlObjectsProvider() { + @Override + @RequiresApi(17) + public EGLContext createEglContext( + EGLDisplay eglDisplay, int openGlVersion, int[] configAttributes) throws GlException { + return GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes); + } + + @Override + public GlTextureInfo createBuffersForTexture(int texId, int width, int height) + throws GlException { + int fboId = GlUtil.createFboForTexture(texId); + return new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, width, height); + } + + @Override + public void clearOutputFrame() throws GlException { + GlUtil.clearOutputFrame(); + } + }; + + /** + * Creates a new {@link EGLContext} for the specified {@link EGLDisplay}. + * + * @param eglDisplay The {@link EGLDisplay} to create an {@link EGLContext} for. + * @param openGlVersion The version of OpenGL ES to configure. Accepts either {@code 2}, for + * OpenGL ES 2.0, or {@code 3}, for OpenGL ES 3.0. + * @param configAttributes The attributes to configure EGL with. + * @throws GlException If an error occurs during creation. + */ + @RequiresApi(17) + EGLContext createEglContext( + EGLDisplay eglDisplay, @IntRange(from = 2, to = 3) int openGlVersion, int[] configAttributes) + throws GlException; + + /** + * Returns a {@link GlTextureInfo} containing the identifiers of the newly created buffers. + * + * @param texId The identifier of the texture to attach to the buffers. + * @param width The width of the texture in pixels. + * @param height The height of the texture in pixels. + * @throws GlException If an error occurs during creation. + */ + GlTextureInfo createBuffersForTexture(int texId, int width, int height) throws GlException; + + /** + * Clears the current render target. + * + * @throws GlException If an error occurs during clearing. + */ + void clearOutputFrame() throws GlException; +} diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlTextureInfo.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlTextureInfo.java new file mode 100644 index 0000000000..f5ae662779 --- /dev/null +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlTextureInfo.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.util; + +/** Contains information describing an OpenGL texture. */ +public final class GlTextureInfo { + + /** A {@link GlTextureInfo} instance with all fields unset. */ + public static final GlTextureInfo UNSET = + new GlTextureInfo( + /* texId= */ C.INDEX_UNSET, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + /* width= */ C.LENGTH_UNSET, + /* height= */ C.LENGTH_UNSET); + + /** The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. */ + public final int texId; + /** + * Identifier of a framebuffer object associated with the texture, or {@link C#INDEX_UNSET} if not + * specified. + */ + public final int fboId; + /** + * Identifier of a renderbuffer object attached with the framebuffer, or {@link C#INDEX_UNSET} if + * not specified. + */ + public final int rboId; + /** The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */ + public final int width; + /** The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */ + public final int height; + + /** + * Creates a new instance. + * + * @param texId The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. + * @param fboId Identifier of a framebuffer object associated with the texture, or {@link + * C#INDEX_UNSET} if not specified. + * @param rboId Identifier of a renderbuffer object associated with the texture, or {@link + * C#INDEX_UNSET} if not specified. + * @param width The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. + * @param height The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. + */ + public GlTextureInfo(int texId, int fboId, int rboId, int width, int height) { + this.texId = texId; + this.fboId = fboId; + this.rboId = rboId; + this.width = width; + this.height = height; + } +} diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java index d1efb6c0cb..c7fa124c11 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java @@ -25,8 +25,10 @@ import android.media.Image; import android.media.ImageReader; import androidx.annotation.Nullable; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.DebugViewProvider; import com.google.android.exoplayer2.util.FrameInfo; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.SurfaceInfo; import com.google.android.exoplayer2.util.Util; @@ -373,7 +375,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { /** Produces blank frames with the given timestamps. */ private static final class BlankFrameProducer implements GlShaderProgram { - private @MonotonicNonNull TextureInfo blankTexture; + private @MonotonicNonNull GlTextureInfo blankTexture; private @MonotonicNonNull OutputListener outputListener; public void configureGlObjects() throws VideoFrameProcessingException { @@ -381,7 +383,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { int texId = GlUtil.createTexture(WIDTH, HEIGHT, /* useHighPrecisionColorComponents= */ false); int fboId = GlUtil.createFboForTexture(texId); - blankTexture = new TextureInfo(texId, fboId, WIDTH, HEIGHT); + blankTexture = new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, WIDTH, HEIGHT); GlUtil.focusFramebufferUsingCurrentContext(fboId, WIDTH, HEIGHT); GlUtil.clearOutputFrame(); } catch (GlUtil.GlException e) { @@ -409,13 +411,13 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { public void setErrorListener(Executor executor, ErrorListener errorListener) {} @Override - public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { // No input is queued in these tests. The BlankFrameProducer is used to produce frames. throw new UnsupportedOperationException(); } @Override - public void releaseOutputFrame(TextureInfo outputTexture) {} + public void releaseOutputFrame(GlTextureInfo outputTexture) {} @Override public void signalEndOfCurrentInputStream() { diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListener.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListener.java index 2bbac88fe0..1de4dc51f3 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListener.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListener.java @@ -21,6 +21,7 @@ import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.effect.GlShaderProgram.InputListener; import com.google.android.exoplayer2.effect.GlShaderProgram.OutputListener; +import com.google.android.exoplayer2.util.GlTextureInfo; import java.util.ArrayDeque; import java.util.Queue; @@ -38,7 +39,7 @@ import java.util.Queue; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @GuardedBy("this") - private final Queue> availableFrames; + private final Queue> availableFrames; @GuardedBy("this") private int consumingGlShaderProgramInputCapacity; @@ -67,7 +68,7 @@ import java.util.Queue; @Override public synchronized void onReadyToAcceptInputFrame() { - @Nullable Pair pendingFrame = availableFrames.poll(); + @Nullable Pair pendingFrame = availableFrames.poll(); if (pendingFrame == null) { consumingGlShaderProgramInputCapacity++; return; @@ -86,7 +87,7 @@ import java.util.Queue; } @Override - public void onInputFrameProcessed(TextureInfo inputTexture) { + public void onInputFrameProcessed(GlTextureInfo inputTexture) { videoFrameProcessingTaskExecutor.submit( () -> producingGlShaderProgram.releaseOutputFrame(inputTexture)); } @@ -100,7 +101,7 @@ import java.util.Queue; @Override public synchronized void onOutputFrameAvailable( - TextureInfo outputTexture, long presentationTimeUs) { + GlTextureInfo outputTexture, long presentationTimeUs) { if (consumingGlShaderProgramInputCapacity > 0) { videoFrameProcessingTaskExecutor.submit( () -> @@ -115,7 +116,7 @@ import java.util.Queue; @Override public synchronized void onCurrentOutputStreamEnded() { if (!availableFrames.isEmpty()) { - availableFrames.add(new Pair<>(TextureInfo.UNSET, C.TIME_END_OF_SOURCE)); + availableFrames.add(new Pair<>(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE)); } else { videoFrameProcessingTaskExecutor.submit( consumingGlShaderProgram::signalEndOfCurrentInputStream); diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java index 6526ad0abe..eb33d293fe 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java @@ -23,6 +23,7 @@ import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.effect.GlShaderProgram.InputListener; import com.google.android.exoplayer2.util.FrameInfo; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.VideoFrameProcessingException; import com.google.android.exoplayer2.util.VideoFrameProcessor; @@ -126,7 +127,7 @@ import java.util.concurrent.atomic.AtomicInteger; } @Override - public void onInputFrameProcessed(TextureInfo inputTexture) { + public void onInputFrameProcessed(GlTextureInfo inputTexture) { videoFrameProcessingTaskExecutor.submit( () -> { currentFrame = null; @@ -234,8 +235,12 @@ import java.util.concurrent.atomic.AtomicInteger; // Correct the presentation time so that GlShaderPrograms don't see the stream offset. long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs - streamOffsetUs; externalShaderProgram.queueInputFrame( - new TextureInfo( - externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height), + new GlTextureInfo( + externalTexId, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + currentFrame.width, + currentFrame.height), presentationTimeUs); checkStateNotNull(pendingFrames.remove()); diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java index 1bc21e239d..24a3acb204 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java @@ -34,6 +34,7 @@ import androidx.annotation.GuardedBy; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.DebugViewProvider; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Size; @@ -79,7 +80,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final VideoFrameProcessor.Listener videoFrameProcessorListener; private final float[] textureTransformMatrix; private final Queue streamOffsetUsQueue; - private final Queue> availableFrames; + private final Queue> availableFrames; private int inputWidth; private int inputHeight; @@ -176,7 +177,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Methods that must be called on the GL thread. @Override - public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { long streamOffsetUs = checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified."); long offsetPresentationTimeUs = presentationTimeUs + streamOffsetUs; @@ -192,14 +193,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void releaseOutputFrame(TextureInfo outputTexture) { + public void releaseOutputFrame(GlTextureInfo outputTexture) { // The final shader program writes to a surface so there is no texture to release. throw new UnsupportedOperationException(); } public void releaseOutputFrame(long releaseTimeNs) { checkState(!releaseFramesAutomatically); - Pair oldestAvailableFrame = availableFrames.remove(); + Pair oldestAvailableFrame = availableFrames.remove(); renderFrameToSurfaces( /* inputTexture= */ oldestAvailableFrame.first, /* presentationTimeUs= */ oldestAvailableFrame.second, @@ -272,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private void renderFrameToSurfaces( - TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) { + GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) { try { maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs); } catch (VideoFrameProcessingException | GlUtil.GlException e) { @@ -286,7 +287,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private synchronized void maybeRenderFrameToOutputSurface( - TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) + GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) throws VideoFrameProcessingException, GlUtil.GlException { if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME || !ensureConfigured(inputTexture.width, inputTexture.height)) { @@ -436,7 +437,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return defaultShaderProgram; } - private void maybeRenderFrameToDebugSurface(TextureInfo inputTexture, long presentationTimeUs) { + private void maybeRenderFrameToDebugSurface(GlTextureInfo inputTexture, long presentationTimeUs) { if (debugSurfaceViewWrapper == null || this.defaultShaderProgram == null) { return; } diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java index 611126dd14..24197217a8 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java @@ -19,7 +19,9 @@ import static com.google.android.exoplayer2.util.Assertions.checkState; import android.content.Context; import android.opengl.GLES20; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.GlProgram; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.VideoFrameProcessingException; import com.google.common.collect.Iterables; @@ -31,7 +33,7 @@ import java.util.NoSuchElementException; import java.util.concurrent.Executor; /** - * Manages a pool of {@linkplain TextureInfo textures}, and caches the input frame. + * Manages a pool of {@linkplain GlTextureInfo textures}, and caches the input frame. * *

Implements {@link FrameCache}. */ @@ -41,8 +43,8 @@ import java.util.concurrent.Executor; private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH = "shaders/fragment_shader_transformation_es2.glsl"; - private final ArrayDeque freeOutputTextures; - private final ArrayDeque inUseOutputTextures; + private final ArrayDeque freeOutputTextures; + private final ArrayDeque inUseOutputTextures; private final GlProgram copyProgram; private final int capacity; private final boolean useHdr; @@ -112,12 +114,12 @@ import java.util.concurrent.Executor; } @Override - public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { try { configureAllOutputTextures(inputTexture.width, inputTexture.height); // Focus on the next free buffer. - TextureInfo outputTexture = freeOutputTextures.remove(); + GlTextureInfo outputTexture = freeOutputTextures.remove(); inUseOutputTextures.add(outputTexture); // Copy frame to fbo. @@ -144,7 +146,7 @@ import java.util.concurrent.Executor; } @Override - public void releaseOutputFrame(TextureInfo outputTexture) { + public void releaseOutputFrame(GlTextureInfo outputTexture) { checkState(inUseOutputTextures.contains(outputTexture)); inUseOutputTextures.remove(outputTexture); freeOutputTextures.add(outputTexture); @@ -177,13 +179,13 @@ import java.util.concurrent.Executor; private void configureAllOutputTextures(int inputWidth, int inputHeight) throws GlUtil.GlException { - Iterator allTextures = getIteratorToAllTextures(); + Iterator allTextures = getIteratorToAllTextures(); if (!allTextures.hasNext()) { createAllOutputTextures(inputWidth, inputHeight); return; } - TextureInfo outputTextureInfo = allTextures.next(); - if (outputTextureInfo.width != inputWidth || outputTextureInfo.height != inputHeight) { + GlTextureInfo outputGlTextureInfo = allTextures.next(); + if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) { deleteAllOutputTextures(); createAllOutputTextures(inputWidth, inputHeight); } @@ -195,15 +197,16 @@ import java.util.concurrent.Executor; for (int i = 0; i < capacity; i++) { int outputTexId = GlUtil.createTexture(width, height, useHdr); int outputFboId = GlUtil.createFboForTexture(outputTexId); - TextureInfo outputTexture = new TextureInfo(outputTexId, outputFboId, width, height); + GlTextureInfo outputTexture = + new GlTextureInfo(outputTexId, outputFboId, /* rboId= */ C.INDEX_UNSET, width, height); freeOutputTextures.add(outputTexture); } } private void deleteAllOutputTextures() throws GlUtil.GlException { - Iterator allTextures = getIteratorToAllTextures(); + Iterator allTextures = getIteratorToAllTextures(); while (allTextures.hasNext()) { - TextureInfo textureInfo = allTextures.next(); + GlTextureInfo textureInfo = allTextures.next(); GlUtil.deleteTexture(textureInfo.texId); GlUtil.deleteFbo(textureInfo.fboId); } @@ -211,7 +214,7 @@ import java.util.concurrent.Executor; inUseOutputTextures.clear(); } - private Iterator getIteratorToAllTextures() { + private Iterator getIteratorToAllTextures() { return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator(); } } diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlShaderProgram.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlShaderProgram.java index d0c69e051c..c68d351dc8 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlShaderProgram.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlShaderProgram.java @@ -15,6 +15,7 @@ */ package com.google.android.exoplayer2.effect; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.VideoFrameProcessingException; import java.util.concurrent.Executor; @@ -22,14 +23,14 @@ import java.util.concurrent.Executor; * Processes frames from one OpenGL 2D texture to another. * *

The {@code GlShaderProgram} consumes input frames it accepts via {@link - * #queueInputFrame(TextureInfo, long)} and surrenders each texture back to the caller via its - * {@linkplain InputListener#onInputFrameProcessed(TextureInfo) listener} once the texture's + * #queueInputFrame(GlTextureInfo, long)} and surrenders each texture back to the caller via its + * {@linkplain InputListener#onInputFrameProcessed(GlTextureInfo) listener} once the texture's * contents have been processed. * *

The {@code GlShaderProgram} produces output frames asynchronously and notifies its owner when - * they are available via its {@linkplain OutputListener#onOutputFrameAvailable(TextureInfo, long) + * they are available via its {@linkplain OutputListener#onOutputFrameAvailable(GlTextureInfo, long) * listener}. The {@code GlShaderProgram} instance's owner must surrender the texture back to the - * {@code GlShaderProgram} via {@link #releaseOutputFrame(TextureInfo)} when it has finished + * {@code GlShaderProgram} via {@link #releaseOutputFrame(GlTextureInfo)} when it has finished * processing it. * *

{@code GlShaderProgram} implementations can choose to produce output frames before receiving @@ -53,7 +54,7 @@ public interface GlShaderProgram { /** * Called when the {@link GlShaderProgram} is ready to accept another input frame. * - *

For each time this method is called, {@link #queueInputFrame(TextureInfo, long)} can be + *

For each time this method is called, {@link #queueInputFrame(GlTextureInfo, long)} can be * called once. */ default void onReadyToAcceptInputFrame() {} @@ -64,10 +65,10 @@ public interface GlShaderProgram { *

The implementation shall not assume the {@link GlShaderProgram} is {@linkplain * #onReadyToAcceptInputFrame ready to accept another input frame} when this method is called. * - * @param inputTexture The {@link TextureInfo} that was used to {@linkplain - * #queueInputFrame(TextureInfo, long) queue} the input frame. + * @param inputTexture The {@link GlTextureInfo} that was used to {@linkplain + * #queueInputFrame(GlTextureInfo, long) queue} the input frame. */ - default void onInputFrameProcessed(TextureInfo inputTexture) {} + default void onInputFrameProcessed(GlTextureInfo inputTexture) {} /** * Called when the {@link GlShaderProgram} has been flushed. @@ -88,15 +89,15 @@ public interface GlShaderProgram { * Called when the {@link GlShaderProgram} has produced an output frame. * *

After the listener's owner has processed the output frame, it must call {@link - * #releaseOutputFrame(TextureInfo)}. The output frame should be released as soon as possible, + * #releaseOutputFrame(GlTextureInfo)}. The output frame should be released as soon as possible, * as there is no guarantee that the {@link GlShaderProgram} will produce further output frames * before this output frame is released. * - * @param outputTexture A {@link TextureInfo} describing the texture containing the output + * @param outputTexture A {@link GlTextureInfo} describing the texture containing the output * frame. * @param presentationTimeUs The presentation timestamp of the output frame, in microseconds. */ - default void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) {} + default void onOutputFrameAvailable(GlTextureInfo outputTexture, long presentationTimeUs) {} /** * Called when the {@link GlShaderProgram} will not produce further output frames belonging to @@ -149,22 +150,22 @@ public interface GlShaderProgram { * Processes an input frame if possible. * *

The {@code GlShaderProgram} owns the accepted frame until it calls {@link - * InputListener#onInputFrameProcessed(TextureInfo)}. The caller should not overwrite or release + * InputListener#onInputFrameProcessed(GlTextureInfo)}. The caller should not overwrite or release * the texture before the {@code GlShaderProgram} has finished processing it. * *

This method must only be called when the {@code GlShaderProgram} can {@linkplain * InputListener#onReadyToAcceptInputFrame() accept an input frame}. * - * @param inputTexture A {@link TextureInfo} describing the texture containing the input frame. + * @param inputTexture A {@link GlTextureInfo} describing the texture containing the input frame. * @param presentationTimeUs The presentation timestamp of the input frame, in microseconds. */ - void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs); + void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs); /** * Notifies the {@code GlShaderProgram} that the frame on the given output texture is no longer * used and can be overwritten. */ - void releaseOutputFrame(TextureInfo outputTexture); + void releaseOutputFrame(GlTextureInfo outputTexture); /** * Notifies the {@code GlShaderProgram} that no further input frames belonging to the current diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java index d5f5ce8dd2..536a1d3762 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java @@ -22,6 +22,7 @@ import android.graphics.Bitmap; import android.opengl.GLES20; import android.opengl.GLUtils; import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.VideoFrameProcessingException; import com.google.android.exoplayer2.util.VideoFrameProcessor; @@ -41,7 +42,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // The queue holds all bitmaps with one or more frames pending to be sent downstream. private final Queue pendingBitmaps; - private @MonotonicNonNull TextureInfo currentTextureInfo; + private @MonotonicNonNull GlTextureInfo currentGlTextureInfo; private int downstreamShaderProgramCapacity; private int framesToQueueForCurrentBitmap; private long currentPresentationTimeUs; @@ -124,8 +125,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames; int currentTexId; try { - if (currentTextureInfo != null) { - GlUtil.deleteTexture(currentTextureInfo.texId); + if (currentGlTextureInfo != null) { + GlUtil.deleteTexture(currentGlTextureInfo.texId); } currentTexId = GlUtil.createTexture( @@ -138,15 +139,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } catch (GlUtil.GlException e) { throw VideoFrameProcessingException.from(e); } - currentTextureInfo = - new TextureInfo( - currentTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight()); + currentGlTextureInfo = + new GlTextureInfo( + currentTexId, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + bitmap.getWidth(), + bitmap.getHeight()); } framesToQueueForCurrentBitmap--; downstreamShaderProgramCapacity--; - shaderProgram.queueInputFrame(checkNotNull(currentTextureInfo), currentPresentationTimeUs); + shaderProgram.queueInputFrame(checkNotNull(currentGlTextureInfo), currentPresentationTimeUs); currentPresentationTimeUs += currentBitmapInfo.frameDurationUs; if (framesToQueueForCurrentBitmap == 0) { diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/SingleFrameGlShaderProgram.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/SingleFrameGlShaderProgram.java index ad07ca7248..3bdd716d61 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/SingleFrameGlShaderProgram.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/SingleFrameGlShaderProgram.java @@ -18,6 +18,8 @@ package com.google.android.exoplayer2.effect; import static com.google.android.exoplayer2.util.Assertions.checkState; import androidx.annotation.CallSuper; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.Size; import com.google.android.exoplayer2.util.VideoFrameProcessingException; @@ -46,7 +48,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { private Executor errorListenerExecutor; private int inputWidth; private int inputHeight; - private @MonotonicNonNull TextureInfo outputTexture; + private @MonotonicNonNull GlTextureInfo outputTexture; private boolean outputTextureInUse; /** @@ -114,7 +116,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { } @Override - public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + public final void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { checkState( !outputTextureInUse, "The shader program does not currently accept input frames. Release prior output frames" @@ -159,12 +161,17 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr); int outputFboId = GlUtil.createFboForTexture(outputTexId); outputTexture = - new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight()); + new GlTextureInfo( + outputTexId, + outputFboId, + /* rboId= */ C.INDEX_UNSET, + outputSize.getWidth(), + outputSize.getHeight()); } } @Override - public final void releaseOutputFrame(TextureInfo outputTexture) { + public final void releaseOutputFrame(GlTextureInfo outputTexture) { outputTextureInUse = false; inputListener.onReadyToAcceptInputFrame(); } diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureInfo.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureInfo.java deleted file mode 100644 index e0d2eeb6b4..0000000000 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureInfo.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.effect; - -import com.google.android.exoplayer2.C; - -/** Contains information describing an OpenGL texture. */ -public final class TextureInfo { - - /** A {@link TextureInfo} instance with all fields unset. */ - public static final TextureInfo UNSET = - new TextureInfo(C.INDEX_UNSET, C.INDEX_UNSET, C.LENGTH_UNSET, C.LENGTH_UNSET); - - /** The OpenGL texture identifier. */ - public final int texId; - /** Identifier of a framebuffer object associated with the texture. */ - public final int fboId; - /** The width of the texture, in pixels. */ - public final int width; - /** The height of the texture, in pixels. */ - public final int height; - - /** - * Creates a new instance. - * - * @param texId The OpenGL texture identifier. - * @param fboId Identifier of a framebuffer object associated with the texture. - * @param width The width of the texture, in pixels. - * @param height The height of the texture, in pixels. - */ - public TextureInfo(int texId, int fboId, int width, int height) { - this.texId = texId; - this.fboId = fboId; - this.width = width; - this.height = height; - } -} diff --git a/library/effect/src/test/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListenerTest.java b/library/effect/src/test/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListenerTest.java index 2dbfe6cf28..b61d823040 100644 --- a/library/effect/src/test/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListenerTest.java +++ b/library/effect/src/test/java/com/google/android/exoplayer2/effect/ChainingGlShaderProgramListenerTest.java @@ -19,6 +19,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.VideoFrameProcessor; import org.junit.After; @@ -51,8 +53,13 @@ public final class ChainingGlShaderProgramListenerTest { @Test public void onInputFrameProcessed_surrendersFrameToPreviousGlShaderProgram() throws InterruptedException { - TextureInfo texture = - new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + GlTextureInfo texture = + new GlTextureInfo( + /* texId= */ 1, + /* fboId= */ 1, + /* rboId= */ C.INDEX_UNSET, + /* width= */ 100, + /* height= */ 100); chainingGlShaderProgramListener.onInputFrameProcessed(texture); Thread.sleep(EXECUTOR_WAIT_TIME_MS); @@ -63,8 +70,13 @@ public final class ChainingGlShaderProgramListenerTest { @Test public void onOutputFrameAvailable_afterAcceptsInputFrame_passesFrameToNextGlShaderProgram() throws InterruptedException { - TextureInfo texture = - new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + GlTextureInfo texture = + new GlTextureInfo( + /* texId= */ 1, + /* fboId= */ 1, + /* rboId= */ C.INDEX_UNSET, + /* width= */ 100, + /* height= */ 100); long presentationTimeUs = 123; chainingGlShaderProgramListener.onReadyToAcceptInputFrame(); @@ -77,8 +89,13 @@ public final class ChainingGlShaderProgramListenerTest { @Test public void onOutputFrameAvailable_beforeAcceptsInputFrame_passesFrameToNextGlShaderProgram() throws InterruptedException { - TextureInfo texture = - new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + GlTextureInfo texture = + new GlTextureInfo( + /* texId= */ 1, + /* fboId= */ 1, + /* rboId= */ C.INDEX_UNSET, + /* width= */ 100, + /* height= */ 100); long presentationTimeUs = 123; chainingGlShaderProgramListener.onOutputFrameAvailable(texture, presentationTimeUs); @@ -91,11 +108,21 @@ public final class ChainingGlShaderProgramListenerTest { @Test public void onOutputFrameAvailable_twoFrames_passesFirstBeforeSecondToNextGlShaderProgram() throws InterruptedException { - TextureInfo firstTexture = - new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + GlTextureInfo firstTexture = + new GlTextureInfo( + /* texId= */ 1, + /* fboId= */ 1, + /* rboId= */ C.INDEX_UNSET, + /* width= */ 100, + /* height= */ 100); long firstPresentationTimeUs = 123; - TextureInfo secondTexture = - new TextureInfo(/* texId= */ 2, /* fboId= */ 2, /* width= */ 100, /* height= */ 100); + GlTextureInfo secondTexture = + new GlTextureInfo( + /* texId= */ 2, + /* fboId= */ 2, + /* rboId= */ C.INDEX_UNSET, + /* width= */ 100, + /* height= */ 100); long secondPresentationTimeUs = 567; chainingGlShaderProgramListener.onOutputFrameAvailable(firstTexture, firstPresentationTimeUs);