diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java index c967e0f225..b59d11f549 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java @@ -83,10 +83,11 @@ public class GaussianBlurTest { // Golden images for these tests were generated on an API 33 emulator. API 26 emulators have a // different text rendering implementation that leads to a larger pixel difference. + @Test @RequiresNonNull({"textureBitmapReader", "testId"}) public void gaussianBlur_blursFrame() throws Exception { - ImmutableList frameTimesUs = ImmutableList.of(32_000L); + ImmutableList frameTimesUs = ImmutableList.of(22_000L); ImmutableList actualPresentationTimesUs = generateAndProcessFrames( BLANK_FRAME_WIDTH, @@ -96,31 +97,7 @@ public class GaussianBlurTest { textureBitmapReader, TEXT_SPAN_CONSUMER); - assertThat(actualPresentationTimesUs).containsExactly(32_000L); - getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); - } - - @Test - @RequiresNonNull({"textureBitmapReader", "testId"}) - public void gaussianBlur_sigmaChangesWithTime_differentFramesHaveDifferentBlurs() - throws Exception { - ImmutableList frameTimesUs = ImmutableList.of(32_000L, 71_000L); - ImmutableList actualPresentationTimesUs = - generateAndProcessFrames( - BLANK_FRAME_WIDTH, - BLANK_FRAME_HEIGHT, - frameTimesUs, - new SeparableConvolution() { - @Override - public ConvolutionFunction1D getConvolution(long presentationTimeUs) { - return new GaussianFunction( - presentationTimeUs < 40_000L ? 5f : 20f, /* numStandardDeviations= */ 2.0f); - } - }, - textureBitmapReader, - TEXT_SPAN_CONSUMER); - - assertThat(actualPresentationTimesUs).containsExactly(32_000L, 71_000L); + assertThat(actualPresentationTimesUs).containsExactly(22_000L); getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); } } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java new file mode 100644 index 0000000000..83ceff86c7 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.BackgroundColorSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link GaussianBlur}. */ +@RunWith(AndroidJUnit4.class) +public class GaussianBlurWithFrameOverlaidTest { + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "media/bitmap/GaussianBlurWithFrameOverlaidTest"; + private static final int BLANK_FRAME_WIDTH = 200; + private static final int BLANK_FRAME_HEIGHT = 100; + private static final Consumer TEXT_SPAN_CONSUMER = + (text) -> { + text.setSpan( + new BackgroundColorSpan(Color.BLUE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new ForegroundColorSpan(Color.WHITE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 100), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + + private @MonotonicNonNull String testId; + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + + @EnsuresNonNull({"textureBitmapReader", "testId"}) + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + // Golden images for these tests were generated on an API 33 emulator. API 26 emulators have a + // different text rendering implementation that leads to a larger pixel difference. + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void gaussianBlurWithFrameOverlaid_blursFrameAndOverlaysSharpImage() throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(32_000L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new GaussianBlurWithFrameOverlaid( + /* sigma= */ 5f, /* scaleSharpX= */ 0.5f, /* scaleSharpY= */ 1f), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactly(32_000L); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void gaussianBlurWithFrameOverlaid_sigmaChangesWithTime_differentFramesHaveDifferentBlurs() + throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(32_000L, 71_000L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new SeparableConvolution() { + @Override + public ConvolutionFunction1D getConvolution(long presentationTimeUs) { + return new GaussianFunction( + presentationTimeUs < 40_000L ? 5f : 20f, /* numStandardDeviations= */ 2.0f); + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new SharpSeparableConvolutionShaderProgram( + context, + useHdr, + /* convolution= */ this, + /* scaleFactor= */ + /* scaleSharpX= */ 0.5f, + /* scaleSharpY= */ 1f); + } + }, + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactly(32_000L, 71_000L); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java new file mode 100644 index 0000000000..0e8f9a1795 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.annotation.FloatRange; +import androidx.annotation.RequiresApi; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** + * A {@link SeparableConvolution} to apply a Gaussian blur on image data. + * + *

The width of the blur is specified in pixels and applied symmetrically. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolution. +public final class GaussianBlurWithFrameOverlaid extends SeparableConvolution { + private final float sigma; + private final float numStandardDeviations; + private final float scaleSharpX; + private final float scaleSharpY; + + /** + * Creates an instance. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + * @param numStandardDeviations The size of function domain, measured in the number of standard + * deviations. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + */ + public GaussianBlurWithFrameOverlaid( + @FloatRange(from = 0.0, fromInclusive = false) float sigma, + @FloatRange(from = 0.0, fromInclusive = false) float numStandardDeviations, + float scaleSharpX, + float scaleSharpY) { + this.sigma = sigma; + this.numStandardDeviations = numStandardDeviations; + this.scaleSharpX = scaleSharpX; + this.scaleSharpY = scaleSharpY; + } + + /** + * Creates an instance with {@code numStandardDeviations} set to {@code 2.0f}. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + */ + public GaussianBlurWithFrameOverlaid(float sigma, float scaleSharpX, float scaleSharpY) { + this(sigma, /* numStandardDeviations= */ 2.0f, scaleSharpX, scaleSharpY); + } + + @Override + public ConvolutionFunction1D getConvolution(long presentationTimeUs) { + return new GaussianFunction(sigma, numStandardDeviations); + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new SharpSeparableConvolutionShaderProgram( + context, useHdr, /* convolution= */ this, scaleSharpX, scaleSharpY); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java index 2cf51f5db6..ef28f02bde 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java @@ -29,21 +29,25 @@ import androidx.media3.common.util.UnstableApi; @UnstableApi @RequiresApi(26) // See SeparableConvolutionShaderProgram. public abstract class SeparableConvolution implements GlEffect { - private final float scaleFactor; + private final float scaleWidth; + private final float scaleHeight; - /** Creates an instance with a {@code scaleFactor} of {@code 1}. */ + /** Creates an instance with {@code scaleWidth} and {@code scaleHeight} set to {@code 1.0f}. */ public SeparableConvolution() { - this(/* scaleFactor= */ 1.0f); + this(/* scaleWidth= */ 1.0f, /* scaleHeight= */ 1.0f); } /** * Creates an instance. * - * @param scaleFactor The scaling factor used to determine the size of the output relative to the - * input. The aspect ratio remains constant. + * @param scaleWidth The scaling factor used to determine the width of the output relative to the + * input. + * @param scaleHeight The scaling factor used to determine the height of the output relative to + * the input. */ - public SeparableConvolution(float scaleFactor) { - this.scaleFactor = scaleFactor; + public SeparableConvolution(float scaleWidth, float scaleHeight) { + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; } /** @@ -56,6 +60,7 @@ public abstract class SeparableConvolution implements GlEffect { @Override public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws VideoFrameProcessingException { - return new SeparableConvolutionShaderProgram(context, useHdr, this, scaleFactor); + return new SeparableConvolutionShaderProgram( + context, useHdr, /* convolution= */ this, scaleWidth, scaleHeight); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java index 1b2b7aeab9..743f07f055 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java @@ -15,13 +15,11 @@ */ package androidx.media3.effect; -import static androidx.media3.effect.MatrixUtils.getGlMatrixArray; - import android.content.Context; import android.graphics.Bitmap; -import android.graphics.Matrix; import android.opengl.GLES20; import android.opengl.GLUtils; +import androidx.annotation.CallSuper; import androidx.annotation.RequiresApi; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; @@ -30,6 +28,7 @@ import androidx.media3.common.util.Assertions; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.nio.ShortBuffer; @@ -43,7 +42,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * on a second pass. */ @RequiresApi(26) // Uses Bitmap.Config.RGBA_F16. -/* package */ final class SeparableConvolutionShaderProgram implements GlShaderProgram { +@UnstableApi +public class SeparableConvolutionShaderProgram implements GlShaderProgram { private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_separable_convolution_es2.glsl"; @@ -67,27 +67,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // END FP16 copied code. private final GlProgram glProgram; - private final GlProgram sharpTransformGlProgram; - private final float[] sharpTransformMatrixValues; private final boolean useHdr; private final SeparableConvolution convolution; - private final float scaleFactor; + private final float scaleWidth; + private final float scaleHeight; private GlShaderProgram.InputListener inputListener; private GlShaderProgram.OutputListener outputListener; private GlShaderProgram.ErrorListener errorListener; private Executor errorListenerExecutor; - private Size outputSize; - private Size lastInputSize; - private Size intermediateSize; - private GlTextureInfo outputTexture; private boolean outputTextureInUse; + private GlTextureInfo outputTexture; private GlTextureInfo intermediateTexture; private GlTextureInfo functionLutTexture; // Values for the function LUT as a texture. private float functionLutTexelStep; private float functionLutCenterX; private float functionLutDomainStart; private float functionLutWidth; + private Size outputSize; + private Size lastInputSize; + private Size intermediateSize; private @MonotonicNonNull ConvolutionFunction1D lastConvolutionFunction; /** @@ -97,20 +96,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. * @param convolution The {@link SeparableConvolution} to apply in each direction. - * @param scaleFactor The scaling factor used to determine the size of the output relative to the - * input. The aspect ratio remains constant. + * @param scaleWidth The scaling factor used to determine the width of the output relative to the + * input. + * @param scaleHeight The scaling factor used to determine the height of the output relative to + * the input. * @throws VideoFrameProcessingException If a problem occurs while reading shader files. */ public SeparableConvolutionShaderProgram( - Context context, boolean useHdr, SeparableConvolution convolution, float scaleFactor) + Context context, + boolean useHdr, + SeparableConvolution convolution, + float scaleWidth, + float scaleHeight) throws VideoFrameProcessingException { this.useHdr = useHdr; this.convolution = convolution; - this.scaleFactor = scaleFactor; + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; inputListener = new InputListener() {}; outputListener = new OutputListener() {}; errorListener = (frameProcessingException) -> {}; errorListenerExecutor = MoreExecutors.directExecutor(); + functionLutTexture = GlTextureInfo.UNSET; + intermediateTexture = GlTextureInfo.UNSET; + outputTexture = GlTextureInfo.UNSET; lastInputSize = Size.ZERO; intermediateSize = Size.ZERO; outputSize = Size.ZERO; @@ -118,25 +127,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - sharpTransformGlProgram = - new GlProgram( - context, - "shaders/vertex_shader_transformation_es2.glsl", - "shaders/fragment_shader_copy_es2.glsl"); } catch (IOException | GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } - - Matrix sharpTransformMatrix = new Matrix(); - sharpTransformMatrix.setScale(/* sx= */ .5f, /* sy= */ 1f); - sharpTransformMatrixValues = getGlMatrixArray(sharpTransformMatrix); - functionLutTexture = GlTextureInfo.UNSET; - intermediateTexture = GlTextureInfo.UNSET; - outputTexture = GlTextureInfo.UNSET; } @Override - public void setInputListener(InputListener inputListener) { + public final void setInputListener(InputListener inputListener) { this.inputListener = inputListener; if (!outputTextureInUse) { inputListener.onReadyToAcceptInputFrame(); @@ -144,18 +141,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void setOutputListener(OutputListener outputListener) { + public final void setOutputListener(OutputListener outputListener) { this.outputListener = outputListener; } @Override - public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { + public final void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { this.errorListenerExecutor = errorListenerExecutor; this.errorListener = errorListener; } @Override - public void queueInputFrame( + public final void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { Assertions.checkState( !outputTextureInUse, @@ -167,17 +164,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTextureInUse = true; renderHorizontal(inputTexture); renderVertical(); - float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); - sharpTransformGlProgram.use(); - sharpTransformGlProgram.setSamplerTexIdUniform( - "uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); - sharpTransformGlProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); - sharpTransformGlProgram.setFloatsUniform("uTransformationMatrix", sharpTransformMatrixValues); - sharpTransformGlProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - sharpTransformGlProgram.bindAttributesAndUniforms(); + + onBlurRendered(inputTexture); // The four-vertex triangle strip forms a quad. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* i1= */ 0, /* i2= */ 4); @@ -191,36 +179,48 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void releaseOutputFrame(GlTextureInfo outputTexture) { + public final void releaseOutputFrame(GlTextureInfo outputTexture) { outputTextureInUse = false; inputListener.onReadyToAcceptInputFrame(); } @Override - public void signalEndOfCurrentInputStream() { + public final void signalEndOfCurrentInputStream() { outputListener.onCurrentOutputStreamEnded(); } @Override - public void flush() { + public final void flush() { outputTextureInUse = false; inputListener.onFlush(); inputListener.onReadyToAcceptInputFrame(); } @Override + @CallSuper public void release() throws VideoFrameProcessingException { try { outputTexture.release(); intermediateTexture.release(); functionLutTexture.release(); glProgram.delete(); - sharpTransformGlProgram.delete(); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } } + /** + * Called when the blur has been rendered onto the frame. + * + *

The default implementation is a no-op. + * + * @param inputTexture The input texture. + * @throws GlUtil.GlException If an error occurs. + */ + protected void onBlurRendered(GlTextureInfo inputTexture) throws GlUtil.GlException { + // Do nothing. + } + private void renderOnePass(int inputTexId, boolean isHorizontal) throws GlUtil.GlException { int size = isHorizontal ? lastInputSize.getWidth() : intermediateSize.getHeight(); glProgram.use(); @@ -252,8 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); return new Size( - (int) (inputSize.getWidth() * scaleFactor * 2), - (int) (inputSize.getHeight() * scaleFactor)); + (int) (inputSize.getWidth() * scaleWidth), (int) (inputSize.getHeight() * scaleHeight)); } private void renderHorizontal(GlTextureInfo inputTexture) throws GlUtil.GlException { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java new file mode 100644 index 0000000000..3f17cd5a38 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.MatrixUtils.getGlMatrixArray; + +import android.content.Context; +import android.graphics.Matrix; +import androidx.annotation.RequiresApi; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import java.io.IOException; + +/** + * An extension of {@link SeparableConvolutionShaderProgram} that draws the sharp version of the + * input frame on top of the output convolution. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolutionShaderProgram. +/* package */ final class SharpSeparableConvolutionShaderProgram + extends SeparableConvolutionShaderProgram { + private final GlProgram sharpTransformGlProgram; + private final float[] sharpTransformMatrixValues; + + /** + * Creates an instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @param convolution The {@link SeparableConvolution} to apply in each direction. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + * @throws VideoFrameProcessingException If a problem occurs while reading shader files. + */ + public SharpSeparableConvolutionShaderProgram( + Context context, + boolean useHdr, + SeparableConvolution convolution, + float scaleSharpX, + float scaleSharpY) + throws VideoFrameProcessingException { + super( + context, + useHdr, + convolution, + /* scaleWidth= */ 1 / scaleSharpX, + /* scaleHeight= */ 1 / scaleSharpY); + try { + sharpTransformGlProgram = + new GlProgram( + context, + "shaders/vertex_shader_transformation_es2.glsl", + "shaders/fragment_shader_copy_es2.glsl"); + } catch (IOException | GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + Matrix sharpTransformMatrix = new Matrix(); + sharpTransformMatrix.setScale(scaleSharpX, scaleSharpY); + sharpTransformMatrixValues = getGlMatrixArray(sharpTransformMatrix); + } + + @Override + protected void onBlurRendered(GlTextureInfo inputTexture) throws GlUtil.GlException { + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + sharpTransformGlProgram.use(); + sharpTransformGlProgram.setSamplerTexIdUniform( + "uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); + sharpTransformGlProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + sharpTransformGlProgram.setFloatsUniform("uTransformationMatrix", sharpTransformMatrixValues); + sharpTransformGlProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + sharpTransformGlProgram.bindAttributesAndUniforms(); + } + + @Override + public void release() throws VideoFrameProcessingException { + super.release(); + try { + sharpTransformGlProgram.delete(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png new file mode 100644 index 0000000000..7b69b90068 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_32000.png similarity index 100% rename from libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png rename to libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_32000.png diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_71000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_71000.png similarity index 100% rename from libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_71000.png rename to libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_71000.png