diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index a5e67d4022..c7239a7b49 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -123,7 +123,8 @@ public interface VideoFrameProcessor { void onOutputSizeChanged(int width, int height); /** - * Called when an output frame with the given {@code presentationTimeUs} becomes available. + * Called when an output frame with the given {@code presentationTimeUs} becomes available for + * rendering. * * @param presentationTimeUs The presentation time of the frame, in microseconds. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 77463e4e64..adda1683eb 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -114,6 +114,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * *

If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via * {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to. + * + *

If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will + * be be adjusted as needed, to match the output surface's output. */ @VisibleForTesting @CanIgnoreReturnValue diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 4ae6ae502b..5cdda92e6c 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -48,12 +48,13 @@ import com.google.common.collect.ImmutableList; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * Wrapper around a {@link DefaultShaderProgram} that writes to the provided output surface and if - * provided, the optional debug surface view or output texture. + * Wrapper around a {@link DefaultShaderProgram} that renders to the provided output surface or + * texture. + * + *

Also renders to a debug surface, if provided. * *

The wrapped {@link DefaultShaderProgram} applies the {@link GlMatrixTransformation} and {@link * RgbMatrix} instances passed to the constructor, followed by any transformations needed to convert @@ -92,15 +93,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private InputListener inputListener; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; @Nullable private SurfaceView debugSurfaceView; - private @MonotonicNonNull GlTextureInfo outputTexture; + @Nullable private GlTextureInfo outputTexture; private boolean frameProcessingStarted; - private volatile boolean outputChanged; + private volatile boolean outputSurfaceInfoChanged; @GuardedBy("this") @Nullable private SurfaceInfo outputSurfaceInfo; + /** Wraps the {@link Surface} in {@link #outputSurfaceInfo}. */ @GuardedBy("this") @Nullable private EGLSurface outputEglSurface; @@ -240,6 +242,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } try { if (outputTexture != null) { + GlTextureInfo outputTexture = checkNotNull(this.outputTexture); GlUtil.deleteTexture(outputTexture.texId); GlUtil.deleteFbo(outputTexture.fboId); } @@ -270,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } this.outputEglSurface = null; } - outputChanged = + outputSurfaceInfoChanged = this.outputSurfaceInfo == null || outputSurfaceInfo == null || this.outputSurfaceInfo.width != outputSurfaceInfo.width @@ -279,14 +282,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.outputSurfaceInfo = outputSurfaceInfo; } - private void renderFrame( + private synchronized void renderFrame( GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) { try { - maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs); - if (textureOutputListener != null && defaultShaderProgram != null) { + if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME + || !ensureConfigured(inputTexture.width, inputTexture.height)) { + inputListener.onInputFrameProcessed(inputTexture); + return; // Drop frames when requested, or there is no output surface. + } + if (outputSurfaceInfo != null) { + renderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs); + } + if (textureOutputListener != null) { renderFrameToOutputTexture(inputTexture, presentationTimeUs); } - } catch (VideoFrameProcessingException | GlUtil.GlException e) { videoFrameProcessorListenerExecutor.execute( () -> @@ -300,17 +309,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputListener.onInputFrameProcessed(inputTexture); } - private synchronized void maybeRenderFrameToOutputSurface( + private synchronized void renderFrameToOutputSurface( GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) throws VideoFrameProcessingException, GlUtil.GlException { - if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME - || !ensureConfigured(inputTexture.width, inputTexture.height)) { - return; // Drop frames when requested, or there is no output surface. - } - - EGLSurface outputEglSurface = this.outputEglSurface; - SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; - DefaultShaderProgram defaultShaderProgram = this.defaultShaderProgram; + EGLSurface outputEglSurface = checkNotNull(this.outputEglSurface); + SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo); + DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram); GlUtil.focusEglSurface( eglDisplay, @@ -332,7 +336,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs) throws GlUtil.GlException, VideoFrameProcessingException { - checkNotNull(outputTexture); + GlTextureInfo outputTexture = checkNotNull(this.outputTexture); GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); GlUtil.clearOutputFrame(); @@ -345,12 +349,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * *

Returns {@code false} if {@code outputSurfaceInfo} is unset. */ - @EnsuresNonNullIf( - expression = {"outputSurfaceInfo", "outputEglSurface", "defaultShaderProgram"}, - result = true) private synchronized boolean ensureConfigured(int inputWidth, int inputHeight) throws VideoFrameProcessingException, GlUtil.GlException { - + // Clear extra or outdated resources. boolean inputSizeChanged = this.inputWidth != inputWidth || this.inputHeight != inputHeight @@ -370,20 +371,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputSizeBeforeSurfaceTransformation.getHeight())); } } + checkNotNull(outputSizeBeforeSurfaceTransformation); if (outputSurfaceInfo == null) { + GlUtil.destroyEglSurface(eglDisplay, outputEglSurface); + outputEglSurface = null; + } + if (outputSurfaceInfo == null && textureOutputListener == null) { if (defaultShaderProgram != null) { defaultShaderProgram.release(); defaultShaderProgram = null; } - GlUtil.destroyEglSurface(eglDisplay, outputEglSurface); - outputEglSurface = null; return false; } - SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; - @Nullable EGLSurface outputEglSurface = this.outputEglSurface; - if (outputEglSurface == null) { + int outputWidth = + outputSurfaceInfo == null + ? outputSizeBeforeSurfaceTransformation.getWidth() + : outputSurfaceInfo.width; + int outputHeight = + outputSurfaceInfo == null + ? outputSizeBeforeSurfaceTransformation.getHeight() + : outputSurfaceInfo.height; + + // Allocate or update resources. + if (outputSurfaceInfo != null && outputEglSurface == null) { outputEglSurface = glObjectsProvider.createEglSurface( eglDisplay, @@ -391,67 +403,58 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputColorInfo.colorTransfer, // Frames are only released automatically when outputting to an encoder. /* isEncoderInputSurface= */ releaseFramesAutomatically); - - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView( - outputSurfaceInfo.width, outputSurfaceInfo.height); - if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) { - debugSurfaceViewWrapper = - new SurfaceViewWrapper( - eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer); - } - this.debugSurfaceView = debugSurfaceView; } - if (defaultShaderProgram != null && (outputChanged || inputSizeChanged)) { + @Nullable + SurfaceView debugSurfaceView = + debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); + if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) { + debugSurfaceViewWrapper = + new SurfaceViewWrapper( + eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer); + } + this.debugSurfaceView = debugSurfaceView; + + if (textureOutputListener != null) { + int outputTexId = + GlUtil.createTexture( + outputWidth, + outputHeight, + /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo)); + outputTexture = + glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight); + } + + if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) { defaultShaderProgram.release(); defaultShaderProgram = null; - outputChanged = false; + outputSurfaceInfoChanged = false; } if (defaultShaderProgram == null) { - DefaultShaderProgram defaultShaderProgram = - createDefaultShaderProgramForOutputSurface(outputSurfaceInfo); - if (textureOutputListener != null) { - configureOutputTexture( - checkNotNull(outputSizeBeforeSurfaceTransformation).getWidth(), - checkNotNull(outputSizeBeforeSurfaceTransformation).getHeight()); - } - this.defaultShaderProgram = defaultShaderProgram; + defaultShaderProgram = + createDefaultShaderProgram( + outputSurfaceInfo == null ? 0 : outputSurfaceInfo.orientationDegrees, + outputWidth, + outputHeight); } - this.outputSurfaceInfo = outputSurfaceInfo; - this.outputEglSurface = outputEglSurface; return true; } - private void configureOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { - if (outputTexture != null) { - GlUtil.deleteTexture(outputTexture.texId); - GlUtil.deleteFbo(outputTexture.fboId); - } - int outputTexId = - GlUtil.createTexture( - outputWidth, - outputHeight, - /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo)); - outputTexture = - glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight); - } - - private DefaultShaderProgram createDefaultShaderProgramForOutputSurface( - SurfaceInfo outputSurfaceInfo) throws VideoFrameProcessingException { + private synchronized DefaultShaderProgram createDefaultShaderProgram( + int outputOrientationDegrees, int outputWidth, int outputHeight) + throws VideoFrameProcessingException { ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder().addAll(matrixTransformations); - if (outputSurfaceInfo.orientationDegrees != 0) { + if (outputOrientationDegrees != 0) { matrixTransformationListBuilder.add( new ScaleAndRotateTransformation.Builder() - .setRotationDegrees(outputSurfaceInfo.orientationDegrees) + .setRotationDegrees(outputOrientationDegrees) .build()); } matrixTransformationListBuilder.add( Presentation.createForWidthAndHeight( - outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); + outputWidth, outputHeight, Presentation.LAYOUT_SCALE_TO_FIT)); DefaultShaderProgram defaultShaderProgram; ImmutableList expandedMatrixTransformations = @@ -488,8 +491,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix); Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight); - checkState(outputSize.getWidth() == outputSurfaceInfo.width); - checkState(outputSize.getHeight() == outputSurfaceInfo.height); + if (outputSurfaceInfo != null) { + SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo); + checkState(outputSize.getWidth() == outputSurfaceInfo.width); + checkState(outputSize.getHeight() == outputSurfaceInfo.height); + } return defaultShaderProgram; } diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 6549e8d747..acc663f664 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -281,14 +281,17 @@ public final class VideoFrameProcessorTestRunner { new VideoFrameProcessor.Listener() { @Override public void onOutputSizeChanged(int width, int height) { + @Nullable Surface outputSurface = bitmapReader.getSurface( width, height, /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr( outputColorInfo)); - checkNotNull(videoFrameProcessor) - .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height)); + if (outputSurface != null) { + checkNotNull(videoFrameProcessor) + .setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height)); + } } @Override @@ -368,7 +371,8 @@ public final class VideoFrameProcessorTestRunner { /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */ public interface BitmapReader { - /** Returns the {@link VideoFrameProcessor} output {@link Surface}. */ + /** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */ + @Nullable Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents); /** Returns the output {@link Bitmap}. */ @@ -388,6 +392,7 @@ public final class VideoFrameProcessorTestRunner { @Override @SuppressLint("WrongConstant") + @Nullable public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index 6925b3de7a..9860c87de6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -31,7 +31,6 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.graphics.Bitmap; -import android.graphics.SurfaceTexture; import android.view.Surface; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; @@ -52,6 +51,7 @@ import androidx.media3.transformer.EncoderUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; @@ -306,16 +306,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { private @MonotonicNonNull Bitmap outputBitmap; @Override + @Nullable public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; - int texId; - try { - texId = GlUtil.createExternalTexture(); - } catch (GlUtil.GlException e) { - throw new RuntimeException(e); - } - SurfaceTexture surfaceTexture = new SurfaceTexture(texId); - return new Surface(surfaceTexture); + return null; } @Override