From 0155ae998bbdbf941599f7f782b37647cc4dc4e2 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Mon, 15 Jan 2024 09:36:57 -0800 Subject: [PATCH] Gaussian Blur: support blurring without drawing sharp image on top PiperOrigin-RevId: 598626481 --- .../media3/effect/GaussianBlurTest.java | 29 +--- .../GaussianBlurWithFrameOverlaidTest.java | 142 ++++++++++++++++++ .../effect/GaussianBlurWithFrameOverlaid.java | 83 ++++++++++ .../media3/effect/SeparableConvolution.java | 21 ++- .../SeparableConvolutionShaderProgram.java | 95 ++++++------ ...harpSeparableConvolutionShaderProgram.java | 105 +++++++++++++ .../bitmap/GaussianBlurTest/pts_22000.png | Bin 0 -> 8610 bytes .../pts_32000.png | Bin .../pts_71000.png | Bin 9 files changed, 393 insertions(+), 82 deletions(-) create mode 100644 libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java create mode 100644 libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png rename libraries/test_data/src/test/assets/media/bitmap/{GaussianBlurTest => GaussianBlurWithFrameOverlaidTest}/pts_32000.png (100%) rename libraries/test_data/src/test/assets/media/bitmap/{GaussianBlurTest => GaussianBlurWithFrameOverlaidTest}/pts_71000.png (100%) diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java index c967e0f225..b59d11f549 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java @@ -83,10 +83,11 @@ public class GaussianBlurTest { // Golden images for these tests were generated on an API 33 emulator. API 26 emulators have a // different text rendering implementation that leads to a larger pixel difference. + @Test @RequiresNonNull({"textureBitmapReader", "testId"}) public void gaussianBlur_blursFrame() throws Exception { - ImmutableList frameTimesUs = ImmutableList.of(32_000L); + ImmutableList frameTimesUs = ImmutableList.of(22_000L); ImmutableList actualPresentationTimesUs = generateAndProcessFrames( BLANK_FRAME_WIDTH, @@ -96,31 +97,7 @@ public class GaussianBlurTest { textureBitmapReader, TEXT_SPAN_CONSUMER); - assertThat(actualPresentationTimesUs).containsExactly(32_000L); - getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); - } - - @Test - @RequiresNonNull({"textureBitmapReader", "testId"}) - public void gaussianBlur_sigmaChangesWithTime_differentFramesHaveDifferentBlurs() - throws Exception { - ImmutableList frameTimesUs = ImmutableList.of(32_000L, 71_000L); - ImmutableList actualPresentationTimesUs = - generateAndProcessFrames( - BLANK_FRAME_WIDTH, - BLANK_FRAME_HEIGHT, - frameTimesUs, - new SeparableConvolution() { - @Override - public ConvolutionFunction1D getConvolution(long presentationTimeUs) { - return new GaussianFunction( - presentationTimeUs < 40_000L ? 5f : 20f, /* numStandardDeviations= */ 2.0f); - } - }, - textureBitmapReader, - TEXT_SPAN_CONSUMER); - - assertThat(actualPresentationTimesUs).containsExactly(32_000L, 71_000L); + assertThat(actualPresentationTimesUs).containsExactly(22_000L); getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); } } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java new file mode 100644 index 0000000000..83ceff86c7 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurWithFrameOverlaidTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.BackgroundColorSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link GaussianBlur}. */ +@RunWith(AndroidJUnit4.class) +public class GaussianBlurWithFrameOverlaidTest { + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "media/bitmap/GaussianBlurWithFrameOverlaidTest"; + private static final int BLANK_FRAME_WIDTH = 200; + private static final int BLANK_FRAME_HEIGHT = 100; + private static final Consumer TEXT_SPAN_CONSUMER = + (text) -> { + text.setSpan( + new BackgroundColorSpan(Color.BLUE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new ForegroundColorSpan(Color.WHITE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 100), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + + private @MonotonicNonNull String testId; + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + + @EnsuresNonNull({"textureBitmapReader", "testId"}) + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + // Golden images for these tests were generated on an API 33 emulator. API 26 emulators have a + // different text rendering implementation that leads to a larger pixel difference. + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void gaussianBlurWithFrameOverlaid_blursFrameAndOverlaysSharpImage() throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(32_000L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new GaussianBlurWithFrameOverlaid( + /* sigma= */ 5f, /* scaleSharpX= */ 0.5f, /* scaleSharpY= */ 1f), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactly(32_000L); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void gaussianBlurWithFrameOverlaid_sigmaChangesWithTime_differentFramesHaveDifferentBlurs() + throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(32_000L, 71_000L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new SeparableConvolution() { + @Override + public ConvolutionFunction1D getConvolution(long presentationTimeUs) { + return new GaussianFunction( + presentationTimeUs < 40_000L ? 5f : 20f, /* numStandardDeviations= */ 2.0f); + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new SharpSeparableConvolutionShaderProgram( + context, + useHdr, + /* convolution= */ this, + /* scaleFactor= */ + /* scaleSharpX= */ 0.5f, + /* scaleSharpY= */ 1f); + } + }, + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactly(32_000L, 71_000L); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java new file mode 100644 index 0000000000..0e8f9a1795 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlurWithFrameOverlaid.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.annotation.FloatRange; +import androidx.annotation.RequiresApi; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** + * A {@link SeparableConvolution} to apply a Gaussian blur on image data. + * + *

The width of the blur is specified in pixels and applied symmetrically. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolution. +public final class GaussianBlurWithFrameOverlaid extends SeparableConvolution { + private final float sigma; + private final float numStandardDeviations; + private final float scaleSharpX; + private final float scaleSharpY; + + /** + * Creates an instance. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + * @param numStandardDeviations The size of function domain, measured in the number of standard + * deviations. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + */ + public GaussianBlurWithFrameOverlaid( + @FloatRange(from = 0.0, fromInclusive = false) float sigma, + @FloatRange(from = 0.0, fromInclusive = false) float numStandardDeviations, + float scaleSharpX, + float scaleSharpY) { + this.sigma = sigma; + this.numStandardDeviations = numStandardDeviations; + this.scaleSharpX = scaleSharpX; + this.scaleSharpY = scaleSharpY; + } + + /** + * Creates an instance with {@code numStandardDeviations} set to {@code 2.0f}. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + */ + public GaussianBlurWithFrameOverlaid(float sigma, float scaleSharpX, float scaleSharpY) { + this(sigma, /* numStandardDeviations= */ 2.0f, scaleSharpX, scaleSharpY); + } + + @Override + public ConvolutionFunction1D getConvolution(long presentationTimeUs) { + return new GaussianFunction(sigma, numStandardDeviations); + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new SharpSeparableConvolutionShaderProgram( + context, useHdr, /* convolution= */ this, scaleSharpX, scaleSharpY); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java index 2cf51f5db6..ef28f02bde 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java @@ -29,21 +29,25 @@ import androidx.media3.common.util.UnstableApi; @UnstableApi @RequiresApi(26) // See SeparableConvolutionShaderProgram. public abstract class SeparableConvolution implements GlEffect { - private final float scaleFactor; + private final float scaleWidth; + private final float scaleHeight; - /** Creates an instance with a {@code scaleFactor} of {@code 1}. */ + /** Creates an instance with {@code scaleWidth} and {@code scaleHeight} set to {@code 1.0f}. */ public SeparableConvolution() { - this(/* scaleFactor= */ 1.0f); + this(/* scaleWidth= */ 1.0f, /* scaleHeight= */ 1.0f); } /** * Creates an instance. * - * @param scaleFactor The scaling factor used to determine the size of the output relative to the - * input. The aspect ratio remains constant. + * @param scaleWidth The scaling factor used to determine the width of the output relative to the + * input. + * @param scaleHeight The scaling factor used to determine the height of the output relative to + * the input. */ - public SeparableConvolution(float scaleFactor) { - this.scaleFactor = scaleFactor; + public SeparableConvolution(float scaleWidth, float scaleHeight) { + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; } /** @@ -56,6 +60,7 @@ public abstract class SeparableConvolution implements GlEffect { @Override public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws VideoFrameProcessingException { - return new SeparableConvolutionShaderProgram(context, useHdr, this, scaleFactor); + return new SeparableConvolutionShaderProgram( + context, useHdr, /* convolution= */ this, scaleWidth, scaleHeight); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java index 1b2b7aeab9..743f07f055 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java @@ -15,13 +15,11 @@ */ package androidx.media3.effect; -import static androidx.media3.effect.MatrixUtils.getGlMatrixArray; - import android.content.Context; import android.graphics.Bitmap; -import android.graphics.Matrix; import android.opengl.GLES20; import android.opengl.GLUtils; +import androidx.annotation.CallSuper; import androidx.annotation.RequiresApi; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; @@ -30,6 +28,7 @@ import androidx.media3.common.util.Assertions; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.nio.ShortBuffer; @@ -43,7 +42,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * on a second pass. */ @RequiresApi(26) // Uses Bitmap.Config.RGBA_F16. -/* package */ final class SeparableConvolutionShaderProgram implements GlShaderProgram { +@UnstableApi +public class SeparableConvolutionShaderProgram implements GlShaderProgram { private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_separable_convolution_es2.glsl"; @@ -67,27 +67,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // END FP16 copied code. private final GlProgram glProgram; - private final GlProgram sharpTransformGlProgram; - private final float[] sharpTransformMatrixValues; private final boolean useHdr; private final SeparableConvolution convolution; - private final float scaleFactor; + private final float scaleWidth; + private final float scaleHeight; private GlShaderProgram.InputListener inputListener; private GlShaderProgram.OutputListener outputListener; private GlShaderProgram.ErrorListener errorListener; private Executor errorListenerExecutor; - private Size outputSize; - private Size lastInputSize; - private Size intermediateSize; - private GlTextureInfo outputTexture; private boolean outputTextureInUse; + private GlTextureInfo outputTexture; private GlTextureInfo intermediateTexture; private GlTextureInfo functionLutTexture; // Values for the function LUT as a texture. private float functionLutTexelStep; private float functionLutCenterX; private float functionLutDomainStart; private float functionLutWidth; + private Size outputSize; + private Size lastInputSize; + private Size intermediateSize; private @MonotonicNonNull ConvolutionFunction1D lastConvolutionFunction; /** @@ -97,20 +96,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. * @param convolution The {@link SeparableConvolution} to apply in each direction. - * @param scaleFactor The scaling factor used to determine the size of the output relative to the - * input. The aspect ratio remains constant. + * @param scaleWidth The scaling factor used to determine the width of the output relative to the + * input. + * @param scaleHeight The scaling factor used to determine the height of the output relative to + * the input. * @throws VideoFrameProcessingException If a problem occurs while reading shader files. */ public SeparableConvolutionShaderProgram( - Context context, boolean useHdr, SeparableConvolution convolution, float scaleFactor) + Context context, + boolean useHdr, + SeparableConvolution convolution, + float scaleWidth, + float scaleHeight) throws VideoFrameProcessingException { this.useHdr = useHdr; this.convolution = convolution; - this.scaleFactor = scaleFactor; + this.scaleWidth = scaleWidth; + this.scaleHeight = scaleHeight; inputListener = new InputListener() {}; outputListener = new OutputListener() {}; errorListener = (frameProcessingException) -> {}; errorListenerExecutor = MoreExecutors.directExecutor(); + functionLutTexture = GlTextureInfo.UNSET; + intermediateTexture = GlTextureInfo.UNSET; + outputTexture = GlTextureInfo.UNSET; lastInputSize = Size.ZERO; intermediateSize = Size.ZERO; outputSize = Size.ZERO; @@ -118,25 +127,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - sharpTransformGlProgram = - new GlProgram( - context, - "shaders/vertex_shader_transformation_es2.glsl", - "shaders/fragment_shader_copy_es2.glsl"); } catch (IOException | GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } - - Matrix sharpTransformMatrix = new Matrix(); - sharpTransformMatrix.setScale(/* sx= */ .5f, /* sy= */ 1f); - sharpTransformMatrixValues = getGlMatrixArray(sharpTransformMatrix); - functionLutTexture = GlTextureInfo.UNSET; - intermediateTexture = GlTextureInfo.UNSET; - outputTexture = GlTextureInfo.UNSET; } @Override - public void setInputListener(InputListener inputListener) { + public final void setInputListener(InputListener inputListener) { this.inputListener = inputListener; if (!outputTextureInUse) { inputListener.onReadyToAcceptInputFrame(); @@ -144,18 +141,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void setOutputListener(OutputListener outputListener) { + public final void setOutputListener(OutputListener outputListener) { this.outputListener = outputListener; } @Override - public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { + public final void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { this.errorListenerExecutor = errorListenerExecutor; this.errorListener = errorListener; } @Override - public void queueInputFrame( + public final void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { Assertions.checkState( !outputTextureInUse, @@ -167,17 +164,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTextureInUse = true; renderHorizontal(inputTexture); renderVertical(); - float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); - sharpTransformGlProgram.use(); - sharpTransformGlProgram.setSamplerTexIdUniform( - "uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); - sharpTransformGlProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); - sharpTransformGlProgram.setFloatsUniform("uTransformationMatrix", sharpTransformMatrixValues); - sharpTransformGlProgram.setBufferAttribute( - "aFramePosition", - GlUtil.getNormalizedCoordinateBounds(), - GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); - sharpTransformGlProgram.bindAttributesAndUniforms(); + + onBlurRendered(inputTexture); // The four-vertex triangle strip forms a quad. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* i1= */ 0, /* i2= */ 4); @@ -191,36 +179,48 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void releaseOutputFrame(GlTextureInfo outputTexture) { + public final void releaseOutputFrame(GlTextureInfo outputTexture) { outputTextureInUse = false; inputListener.onReadyToAcceptInputFrame(); } @Override - public void signalEndOfCurrentInputStream() { + public final void signalEndOfCurrentInputStream() { outputListener.onCurrentOutputStreamEnded(); } @Override - public void flush() { + public final void flush() { outputTextureInUse = false; inputListener.onFlush(); inputListener.onReadyToAcceptInputFrame(); } @Override + @CallSuper public void release() throws VideoFrameProcessingException { try { outputTexture.release(); intermediateTexture.release(); functionLutTexture.release(); glProgram.delete(); - sharpTransformGlProgram.delete(); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } } + /** + * Called when the blur has been rendered onto the frame. + * + *

The default implementation is a no-op. + * + * @param inputTexture The input texture. + * @throws GlUtil.GlException If an error occurs. + */ + protected void onBlurRendered(GlTextureInfo inputTexture) throws GlUtil.GlException { + // Do nothing. + } + private void renderOnePass(int inputTexId, boolean isHorizontal) throws GlUtil.GlException { int size = isHorizontal ? lastInputSize.getWidth() : intermediateSize.getHeight(); glProgram.use(); @@ -252,8 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); return new Size( - (int) (inputSize.getWidth() * scaleFactor * 2), - (int) (inputSize.getHeight() * scaleFactor)); + (int) (inputSize.getWidth() * scaleWidth), (int) (inputSize.getHeight() * scaleHeight)); } private void renderHorizontal(GlTextureInfo inputTexture) throws GlUtil.GlException { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java new file mode 100644 index 0000000000..3f17cd5a38 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SharpSeparableConvolutionShaderProgram.java @@ -0,0 +1,105 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.MatrixUtils.getGlMatrixArray; + +import android.content.Context; +import android.graphics.Matrix; +import androidx.annotation.RequiresApi; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import java.io.IOException; + +/** + * An extension of {@link SeparableConvolutionShaderProgram} that draws the sharp version of the + * input frame on top of the output convolution. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolutionShaderProgram. +/* package */ final class SharpSeparableConvolutionShaderProgram + extends SeparableConvolutionShaderProgram { + private final GlProgram sharpTransformGlProgram; + private final float[] sharpTransformMatrixValues; + + /** + * Creates an instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @param convolution The {@link SeparableConvolution} to apply in each direction. + * @param scaleSharpX The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the horizontal direction. + * @param scaleSharpY The scaling factor used to determine the size of the sharp image in the + * output frame relative to the whole output frame in the vertical direction. + * @throws VideoFrameProcessingException If a problem occurs while reading shader files. + */ + public SharpSeparableConvolutionShaderProgram( + Context context, + boolean useHdr, + SeparableConvolution convolution, + float scaleSharpX, + float scaleSharpY) + throws VideoFrameProcessingException { + super( + context, + useHdr, + convolution, + /* scaleWidth= */ 1 / scaleSharpX, + /* scaleHeight= */ 1 / scaleSharpY); + try { + sharpTransformGlProgram = + new GlProgram( + context, + "shaders/vertex_shader_transformation_es2.glsl", + "shaders/fragment_shader_copy_es2.glsl"); + } catch (IOException | GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + Matrix sharpTransformMatrix = new Matrix(); + sharpTransformMatrix.setScale(scaleSharpX, scaleSharpY); + sharpTransformMatrixValues = getGlMatrixArray(sharpTransformMatrix); + } + + @Override + protected void onBlurRendered(GlTextureInfo inputTexture) throws GlUtil.GlException { + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + sharpTransformGlProgram.use(); + sharpTransformGlProgram.setSamplerTexIdUniform( + "uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); + sharpTransformGlProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + sharpTransformGlProgram.setFloatsUniform("uTransformationMatrix", sharpTransformMatrixValues); + sharpTransformGlProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + sharpTransformGlProgram.bindAttributesAndUniforms(); + } + + @Override + public void release() throws VideoFrameProcessingException { + super.release(); + try { + sharpTransformGlProgram.delete(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_22000.png new file mode 100644 index 0000000000000000000000000000000000000000..7b69b9006858be85dbd33ef00953fedd797d7884 GIT binary patch literal 8610 zcmV;TAzj{yP)Px#1am@3R0s$N2z&@+hyVZ}07*naRCt{2om+R>SQ3THMi}Ex(&?Fp`Tu`#rjvAR z?w$u_`+VBe5ds6Y31=-Wah1+4)xAnbHxbXzaUIuj9qG7z>j_-Pb-d?s6_D$=j-GK9 zkn6aPo^chB>$r}daTSp3xQ?E26_D$=j-JuH^#nAUWG3VBu>UTX+n=pguba;)#vvF# z3bu&L#CP)+br2_{h^Wu`{^lkp{BqfTr`Ozzz$E@#5S{bhg^}N_R_#?GKF6ldH9itP zUc2Y#cJA*HU&NU$NJc!p9*^7iqfyz9M(uan`&h5rO|Mt0vR^K{499UwVms3uijla4 zN$-g-!*a!UywTJqal8|h*CZ&3Ihhnql(_O{*hI-=}1o3dZ8+x=$K5_P?fSS$*m z=kq$TkBPe`{L^V+n7g~(7&U~Eg1^}mZ1Z`}Onr*sBjQ^w+wV_}uZFh~v!22iBvC0C zlSv`y-Cg(lrYVARds~hnldSRlTm(bJU9Gx8v0S$262X_th^MFH;>;LlIxRT+Fv|T9 zBhl@M$gzn@=JPHy&1N|x9rGIQi;qI`Uhxqlaq9DCI_>_bcEWuo(kB=q>ivEDo!--` z&^(Ko&p37z4bPP-TP}yqtP#({PLyi`^luylntGTpHu=tqE4qpU`S|bAqfX9 z2+gyItIahKT`UR6Y}TIZ@v(icxaaZlFhbLnkOFvk*j5DG=#o$gP{Yy%aTik8g4$Ks zCX-eH=(w|4d)(QqxRm=a(MQ2IpO^T!`u z;!05N@4G_ceyqD#?p~3|6cEkdEy&YTyUD#8j|*Y3Wzk2EQZdZvG<)2D4^K%ZTfGE)K(xs~)F zmKyf{zQyP64U-!_`k3f<#5b9=_$VYDxBt&S3sXsQHL>xyxFquXm<3W+AtDzJ5mZFh zYg!TtNMH7&CLlE;OTaVj<`<5~bbKT(iA?6vCcI)Ox*}p8rnlWfW(<^RSt=h(6u zQpQBTGd}ljI+o~P*^7(U@f%IA!sbO6}Z*Iyw+y!wa%G^jL9Wt#Qv8@0YQ)X83 zcf=>2^l;!sK)!y>Q=$rr$V{wUyL8Hxl3PU~acko4Ri<@FP~UUZgk&-)Y28e-`LP(w zkl0z`L1tN-m`Ho#^dyAvxnEwx7LjM{7K=hVYFAoaGLZf!E9nYg7d{Hgd&U>hy2|_D zBY)B|PA0zuL@6~{4`*6U%1t3pE`M}7ZHXz%@%i(%1#z?N=OHnvb^3^{-*Aq!Y|M=5 z!ntrczKC}3eyg+%8AwE?hIJ+23c#@tV~vsIN|JHpmuCez{YTg$x~?c)k4wSFgwrm3 z!X}<&2r!xaia~Y>hlYx9(nOM2W~5}M`YZ@DS3FOaD|fXO;HR(d?GRn zM+)SGgG}QLApFwBrT+;tJRAntjili2Yk{%?mpJs0&X0{NTzZ8R4&;$JaMsHw)2%c64AaNjgE_tmievmsiCrl z&n@JZfGif zg)quWDmU&JJsZMYx@%UysMLDvb=OMjf1W5I-H6S#Yb`6pYsff_b&uZ-UuGd+T&zF- z$TLV;uZ*-PLAzvTf|6A!Gr+82(5EDErD<3x9*>I<=>oWEcK0nKu6%0EQ}`qxUUenD zTF~llsyF3HNTmqKK+L|}-ECV?kHfMS951}FKaJPb@%i<;bbPMR<&HJ?_qhv_F|Aw= zby4VvLJOa)2MbBw za>2G4T1YWZtr?Gur1??1DlXh?+C43$`|%k*PvNsOR!o-RW4FU4<8xm1KEW3Op_JAt zo9w<-I?e+iZ)9VpL*|s)K<8lZjrKaPEEb8YG@wkWexu>;)@W2Ljboz#tBXJvPuVbP zA7Z2u@M0&G4cmCsS*$%3f#y!;5k_iYXCS$mn@#z>@Tpmog;TkZ5k<)i#g};iZ-H;I zDA(%{d|5YuSMD20t%m!NWFT$I^2i$S-T+FU7Xm39A}WRB@v+_1g*uycO{FXZtM=1r z3Ct)Qw2HJQ<8jd{yg)u*DFLM&v)?jZ?dfq@NYYVFk&ln1fl^FFjMT2WknY1f6Dcke zK3ylxQ%ocbi$$3qu@GAgpP2SN;Bz6_2cPpbmB0(nD$g%%AhRTkL8g(`rgoKp_~tc; zOyjt4{QOxq5?8T^3y55y<+2-tWEn07_nH?mkudse(Gx8+@vVPGcV zkgJpRIb{VvuRX~8OE`Y~i1_E9c5~vYC@CV{-L=G}`{nV8&q2g@A)&D7b14D&#w0!o zoe}lsW;@Fwm*erVU=+^B$1cX3o8s2d#T0%oe)7DIEmK_d`-P7#EAiFZJU${mzkaOy zQNhvcXlTB8^4cbbSgp1-a&{sHa@xxz&t^qHM9}Zw+x?Fpt%2q&(iX5ZUtyH+ zGY#%>9$9+DS$qFx~J&9;LBV%D)a~l2pK0!vt`8_A6akA?QOSP zbG6#ms1g#nS$zHX-`zVI1#r16nn(oY?ZLDj+BoMveAHMH5O*_abv(|>D)np#C3a#I zPMt&6OTz#3)C#{V0d^XAKua$mbC-1vkH;LJXm&~XY6^}mPK`5+Z~h$*rr~WO@=GNg z^5r+1qD>_jSz3_U5H8W96S-i@cIh<*L;^rT zA#r65C>%1mdQD+)i{TBBo=wZJN;oo2mBnRL9z2(8Sus&o{iW55C>BzRL&2<)5?KpJ zrUlW-3NVs?)urveyEVNToy|r0r7xU zgrc9$mStwJG_ub$vN`iRe&&M2??`-qoe`MFcP*=n&3jJxy<=;dF7r79x`uU}wYbvp zh+#bm^BR0UxA%(Au}O)t&#TFpgIc(zk#!JBI5R*L4haj3N;1Q&cgW02(5OYS;pt=% ztrjC?!tT~E(^5KCCM225nQ2+YUb^7axMT)rA_bcIzY7+b$LCJ>I7@9e4g73Nb^b2+ zJUGv?bImTH$Xpm|3#I5H%E|N_`F#`+ub#7*Bnv`KlnF+}cKXkYS(%2&F0i|16nc&9 zxV7U)lnay5k6KkYYlH5J|DS%jD=*E-=|CK{tkALWkScD6O^|?^WMkp;<3%)WR(=V*!>oey=n8;nXjDbkwJ_ZsK zMRfgfKMu947>IMCm33_!v8@1VOtcR^GFsnYYK z+voOC)1um*pLa8lE0`hpIJftRk2ooOV$eeMU(lQB|_iTZ|qm&Jnq83+Az$002G ze5iYMDfsw5Ve+{zml2J-Ow1)*X+x8B8aaS*_*DDJ0$x#akwQg~r-S8zKUI zV;6QWQVEkTPM>S#Bg>fa0z?egu9~a(Z+TU1!i9nNai}UD zQP#kXDEn+=6>?e)7Y@ZA5(F+shOP`RF;QI_+2*nXYN0wPTM9U6!vGn=A<4`O47Wn$cmVkJ7 zEqkN%oh*k}yFKG{i00tPf^sg1?Q0m(#iI2f(9C*J0SKR}zfCjTtPdT|07EZU=B_y7 z9D5uh9Q(!}8t2Bh54J{blBZmK&mJkJWV1Gc*Nj?FPT|lpk)F|_fUt#t^_Q$I+wplo zETSF~s1MD=3>{uH8#4L=(?hlFN5ohAO*UREh+B!Lr*8X)_e#-EKc-+rYzO8303I%c zy=%2Uf%Mh^Dg)vozKHGafK!ZPFx4T)IEE{Au5nC!hu~~Pva8sHQ&B<8E8H)%kB6ohf3)l+j>wq zCX?>6^m_bP-*XP#da##_%S){0GL#8R^wY_k_WH7lNDoZAJ-s?*1AhsnhDk6}GrIM|UZq4@G;+qJ5#q^^30tViZnaVa4!EmU(Iv#~FMYt3~2n|%l4 ztbJ-0L&iSLb1r;C#;FRWH#fgrEWcS^9l8<>Zzp-pV3mA4ZW*2#s_fyPS;bu4!dWq} zw(b^PE57z+X?*wvIP`*eck{l-@#e|w^**C^{$t@AI(W;r#wqJG``~N7ee1TLQ!uhh z`D?6)SIKLIK(`!xE*@J+bjMiDt&&jqVssRW%yp!j)kn-ji&zoePUBr8>x|a1vX0!o zQELWt7wgdK_@T^v3VgMVzpc%?V?nZe&1)^_hmKm6oLQ>Oie#>pyH(j@5ZRKZ6c`ph z={jnj5#1nfsIT7FU8*5X$#V@I`!*H!`QkOpI7Bc$GCqG@JO7J-NC0-VpdUK=3W2qU z)k}bergO6eqJ242ZOqCIiK~WmlxCW4h@ZaC$nJ2g`RhZNyvAJF|N8_#!d2Fd_xEEi7AhXuVz(hxHJcCjo-I%>;}Y$ zdX=1QB&>P(!|J_5H!B3X)zIS;C>(OV{`n`rX-lsald%`8&u=Gz`I?)An~O+9?B|B~ z1sv**aK5~qRP*n(uAIxwgz!4AJK`EBT)A)!eKMA8)|gg-;kFx^o|o zBMwZ=?H*nwmjLj!#~rjBzQ&aXS4Ht>j8HHXlxMY%A#n#6EXy=hR;Bd|SzC{Z8cMY4 zqhUmM70H^G-6bSGf7iCPl}XNxuO=wj*cYF_7B1~?Z;RX4G$#^}S__Ha^yC8KkL0EA z9nlpIe|#H-!Jly6kJ4$;MrMMlaPCv&qL z+k3#53Cq6tWJMgG-|9>uap4}1BMwN!^^eT0VwGIMc-BHf>*1@C45@W6_Mk=SyFIuz zGRDaWodtcaO}S;OkME+VFRkj9gnN$6`nk|Ylx25(Rl{@PY;#zB4-?1gdkA zmVBI;uTJjk5NECk{qdSMjcEU$S;=GI)BOAJ$*nmjK6jsmNkel!nt%+gl4r4pHy<)6 zXYsNo9E>??&*|Q$*r}pvZM|3*lo|v1g(edz`JhLS)mD^banv6?SzDw#qV9M5n#&P| zfoh6?Zpgm)$pUrjLhxocS$5M5+vbgf9Q_$*t| zUNM`J7W$7b(48?*jd3`4(&y%;)PCH0{toygBwokaY$BR3Uvh@owYb}I2|ukV0`l-s zD&lml^qGVti#xcM70spJ%;h(Asku@>T@Y%qN3Cl>YhuyGVt`L#*IW_0AHIZ<=B{V+ z$ye6#_1n*%^EHUQ-QO3T~c{qOdR z>?R?dzyUfFPN!vQP{kBX<@PPOARkp(3raQb49wQh6)^;k0q=TUZk^R|jE;{)<#9-O zZmx;wE7SGh^S@`a7P~s9CDfx)Ii7rYDwb@4ylf3(F;J#moq4DrwD7pEa{re2=w_V) zA2EuDm|niQiDK$)tNhCzEa*C&lAdm%%LYG2O{I z^SOaMpWhSo9`N}p{incJizQz$%7Tb#xMx0jxh#8W2WbOkV?1l+*((ps{g&DFx+^G| zuSgK%ak&ViQTrV&N=-04-6x_EpN^^D^8#c-BEe)KO)iKutdt7fK?(I_QsypfzAjj; z^`GtseRCIRVvr@)Jls;(nCQ~*Wf&@-0NY4hII>3-c--MV zKVGDgwb1z@+GEgEQV@LTIMj&Lc68(Dj(Hs7gcehH&YnK{-CbcIUIbZR2C}!eC0O-^ zx@(Jc`&>O(<$Ss4HFOvDcfpq_I?ccN*T1?9L`^Nm(*gMC0}qPt?z$Ev+uVc-Jd0A^ z6c^F;cg47wJC#*^Ga-@B={_s*$x71ABH>ABcL!Jn7Hw)7lkvFwq%LY$`Sqk}?asQB}QfjwW!6e*o|71Ou$uZ zoS8K4?#l1Ds?G0!Z!#&0Zn-SHHyR;G_kI=+Oc#r8+D%dyQG#SX$i^EF`83+>&Oi~) z+ZbdeqL+PtC&{goNz*5#xIQ2bi3fW4M zuF$Ae{b5L4t%k9r&h_+EQneXAGVMphr>+I2qFH2SZQ9CDHh)g(k35sVxgw6e3(L(=W059SapI=ld4l;yi zNyJpru4Eu72mOZU;(6F{KxI~Y2-0Fvx(md(5nZ8wdMeD6;nNU4VR`TPSU(d!e=1qi zblX~91cVlZnJX1`=ne_wV$r^rDfQ=-*P87-ZAYSd5y~~vA(lhKEy!}&ZJ5()#+I7iV!Q~Do>>{?(#k*DHq|VBI|qhi8B%d=exUZ$HFP_xquuC zAB)Dj@JT2fU#;HKJUnbOkA%ccj{ipHj~c($C*;Jf4Q5C@8}b_AAVyuRti6SbL9tdu zp^0KJ?=_;OEM|11)0Z%!tbV(cGYhq+SlGPA_h7qzRYqA!N=7x zGkmjI7vJM!7a#dtLwwR`6?0GTlDS=y@9s)YTY{2Vh^#$;|KVh^u2I!CG@%egZHGEV&EDF-yaljb?J+9IBWZW!W8_|_&3hN<*afP&$ z>L{6}_8^g&J7(HEIOqA4-Khoe+I#JW%%;~_50YXiF^{k6;;To>1tAvg6{Nenlet&# z9iOsYyYOX-oXWBxU}tnrJ`;i(0}>!pbJT!t&bU!fsCB$fa>(EX>%-|WlDA$&YzvcU zAWTvUS(eM`p2cLOYnEkXR*lTVC635M^3A;!L$0J!+?j9?pU>mi_&j5EN_;LHqU%fj zN0RN#&4L;Wd1PFFo%Jj!`$eYl>Zl(-i#5b}hgy#sMi)q*hg(7IBeq@1j203qne(i- zC=+_m%w;ro*Ta`R;(hsij9zUmw5O+bjz52P-K0y!m+>GI0<&2}qd#Xg0&nb6R)@VS zoQP@^0>wyale;biEG}e@h0)KB&cP7f*4tH5;nLVphTZ>o= zS17~7;5DMeV5?R88zx=OfsXm|$e^JjO$bR&GSSy6`mc*c-q zL>A*%M9a)A-ydtZk4rz-E3s@}(EKS_x?d!s+ay`L&|Waj(%{)#$vix6?YWZKot9ip z(gRlwzJGk|()|+g5u3+6od?7I_)#Y8R>Wfo&lIpysiIyMm+1dlR>X;xz0TerGB`i) z1obqctO_j&vm?Uo0-s^mIh=vq$9D}|)7rFp^t=7B-4u4ZLVZKutLF3G@M-Zs;$v2B zC_V{;1}!!5qyj>zVycZ2tx4%WY6J1$O%tizW1RV=ry4akk{5&(S`uHvLG8n_>q7gj zq^vL0YDQTwT&pN^Aw9VxtP7BNT)S$>6_LMw4t(UZL-F|<1c%^bR!|L=O%v?{bov-5 z3!12rSQnth(Rbbrni1@rBf}>*O8C5>CiluO053$1k?Df4*Qk#knKjRj$!0wX^?Y7@ zaxvEs=Ft!n>3{DT-#)@o!?#?PTgKH`Nrm40{dbu(-6L5Mnl5%p>NA17y)6f&*;O-| zGFlAK&n+aL#;skL?4l?DteJ6rm&I=M&ue~qp9_*PWyVCb?%8~IF^>i{EkCaRp@?X?f#^Rr$jSX(1YX5SJzw*WgZR3r(>Q29}^?zz~{%(Or6wrFP@DWBSn;3OQ}_P zPkuyB6Bk~&H9pT%G0rZ03U)66Uv1p{=RYO6TFW8#F$lk7@hqYUhJViAKRG`BF6EwBhQ&Brp&6HlX3d+@%=}|r*U)o>ce$J oY%^5tJ0CTH_3z$#Tq?f*1EiJt)w_}PY5)KL07*qoM6N<$f_7BM(f|Me literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_32000.png similarity index 100% rename from libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png rename to libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_32000.png diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_71000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_71000.png similarity index 100% rename from libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_71000.png rename to libraries/test_data/src/test/assets/media/bitmap/GaussianBlurWithFrameOverlaidTest/pts_71000.png