diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java new file mode 100644 index 0000000000..496f0f576d --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java @@ -0,0 +1,211 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.text.SpannableString; +import androidx.annotation.Nullable; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.Consumer; +import androidx.media3.common.util.NullableType; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import androidx.media3.test.utils.TextureBitmapReader; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +/** Utilities for effects tests. */ +@UnstableApi +/* package */ class EffectsTestUtil { + + /** + * Gets the {@link Bitmap}s (generated at the timestamps in {@code presentationTimesUs}) from the + * {@link TextureBitmapReader}, and asserts that they are equal to files stored in the {@code + * goldenFileAssetPath} with the same {@code testId}. + * + *

Tries to save the {@link Bitmap}s as PNGs to the {@link Context#getCacheDir() cache + * directory}. + */ + public static void getAndAssertOutputBitmaps( + TextureBitmapReader textureBitmapReader, + List presentationTimesUs, + String testId, + String goldenFileAssetPath) + throws IOException { + for (int i = 0; i < presentationTimesUs.size(); i++) { + long presentationTimeUs = presentationTimesUs.get(i); + Bitmap actualBitmap = textureBitmapReader.getBitmapAtPresentationTimeUs(presentationTimeUs); + Bitmap expectedBitmap = + readBitmap( + Util.formatInvariant("%s/pts_%d.png", goldenFileAssetPath, presentationTimeUs)); + maybeSaveTestBitmap( + testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + } + + /** + * Generates and processes a frame for each timestamp in {@code presentationTimesUs} through a + * {@link DefaultVideoFrameProcessor}, applying the given {@link GlEffect}, and outputting the + * resulting frame to the provided {@link TextureBitmapReader}. + * + *

The generated frames have their associated timestamps overlaid on them. + * + * @param frameWidth The width of the generated frames. + * @param frameHeight The height of the generated frames. + * @param presentationTimesUs The timestamps of the generated frames, in microseconds. + * @param glEffect The effect to apply to the frames. + * @param textSpanConsumer A {@link Consumer} used to set the spans that styles the text overlaid + * onto the frames. + */ + // MoreExecutors.directExecutor() pattern is consistent with our codebase. + @SuppressWarnings("StaticImportPreferred") + public static ImmutableList generateAndProcessFrames( + int frameWidth, + int frameHeight, + List presentationTimesUs, + GlEffect glEffect, + TextureBitmapReader textureBitmapReader, + Consumer textSpanConsumer) + throws Exception { + ImmutableList.Builder actualPresentationTimesUs = new ImmutableList.Builder<>(); + @Nullable DefaultVideoFrameProcessor defaultVideoFrameProcessor = null; + + try { + AtomicReference<@NullableType VideoFrameProcessingException> + videoFrameProcessingExceptionReference = new AtomicReference<>(); + BlankFrameProducer blankFrameProducer = new BlankFrameProducer(frameWidth, frameHeight); + CountDownLatch videoFrameProcessorReadyCountDownLatch = new CountDownLatch(1); + CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1); + + defaultVideoFrameProcessor = + checkNotNull( + new DefaultVideoFrameProcessor.Factory.Builder() + .setTextureOutput( + (textureProducer, outputTexture, presentationTimeUs, token) -> { + checkNotNull(textureBitmapReader) + .readBitmap(outputTexture, presentationTimeUs); + textureProducer.releaseOutputTexture(presentationTimeUs); + }, + /* textureOutputCapacity= */ 1) + .build() + .create( + getApplicationContext(), + DebugViewProvider.NONE, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* renderFramesAutomatically= */ true, + MoreExecutors.directExecutor(), + new VideoFrameProcessor.Listener() { + @Override + public void onInputStreamRegistered( + @VideoFrameProcessor.InputType int inputType, + List effects, + FrameInfo frameInfo) { + videoFrameProcessorReadyCountDownLatch.countDown(); + } + + @Override + public void onOutputSizeChanged(int width, int height) {} + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + actualPresentationTimesUs.add(presentationTimeUs); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + videoFrameProcessingExceptionReference.set(exception); + videoFrameProcessorReadyCountDownLatch.countDown(); + videoFrameProcessingEndedCountDownLatch.countDown(); + } + + @Override + public void onEnded() { + videoFrameProcessingEndedCountDownLatch.countDown(); + } + })); + + defaultVideoFrameProcessor.getTaskExecutor().submit(blankFrameProducer::configureGlObjects); + // A frame needs to be registered despite not queuing any external input to ensure + // that the video frame processor knows about the stream offset. + checkNotNull(defaultVideoFrameProcessor) + .registerInputStream( + INPUT_TYPE_SURFACE, + /* effects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> blankFrameProducer, + // Use an overlay effect to generate bitmaps with timestamps on it. + new OverlayEffect( + ImmutableList.of( + new TextOverlay() { + @Override + public SpannableString getText(long presentationTimeUs) { + SpannableString text = + new SpannableString(String.valueOf(presentationTimeUs)); + textSpanConsumer.accept(text); + return text; + } + })), + glEffect), + new FrameInfo.Builder(ColorInfo.SDR_BT709_LIMITED, frameWidth, frameHeight).build()); + videoFrameProcessorReadyCountDownLatch.await(); + checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference); + blankFrameProducer.produceBlankFrames(presentationTimesUs); + defaultVideoFrameProcessor.signalEndOfInput(); + videoFrameProcessingEndedCountDownLatch.await(); + checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference); + } finally { + if (defaultVideoFrameProcessor != null) { + defaultVideoFrameProcessor.release(); + } + } + return actualPresentationTimesUs.build(); + } + + private static void checkNoVideoFrameProcessingExceptionIsThrown( + AtomicReference<@NullableType VideoFrameProcessingException> + videoFrameProcessingExceptionReference) + throws Exception { + @Nullable + Exception videoFrameProcessingException = videoFrameProcessingExceptionReference.get(); + if (videoFrameProcessingException != null) { + throw videoFrameProcessingException; + } + } + + private EffectsTestUtil() {} +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java index 0a3d067fa4..2be1e91bac 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java @@ -15,43 +15,24 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; -import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; -import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; -import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; -import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; import static com.google.common.truth.Truth.assertThat; -import android.graphics.Bitmap; import android.graphics.Color; import android.text.Spannable; import android.text.SpannableString; import android.text.style.AbsoluteSizeSpan; import android.text.style.ForegroundColorSpan; import android.text.style.TypefaceSpan; -import androidx.annotation.Nullable; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.FrameInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.util.NullableType; -import androidx.media3.common.util.Util; +import androidx.media3.common.util.Consumer; import androidx.media3.test.utils.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.MoreExecutors; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -64,12 +45,9 @@ public class FrameDropTest { @Rule public final TestName testName = new TestName(); private static final String ASSET_PATH = "media/bitmap/FrameDropTest"; - private static final int BLANK_FRAME_WIDTH = 100; - private static final int BLANK_FRAME_HEIGHT = 50; - private @MonotonicNonNull String testId; private @MonotonicNonNull TextureBitmapReader textureBitmapReader; - private @MonotonicNonNull DefaultVideoFrameProcessor defaultVideoFrameProcessor; + private @MonotonicNonNull String testId; @EnsuresNonNull({"textureBitmapReader", "testId"}) @Before @@ -78,24 +56,20 @@ public class FrameDropTest { testId = testName.getMethodName(); } - @After - public void tearDown() { - checkNotNull(defaultVideoFrameProcessor).release(); - } - @Test @RequiresNonNull({"textureBitmapReader", "testId"}) public void frameDrop_withDefaultStrategy_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { ImmutableList frameTimesUs = ImmutableList.of(0L, 16_000L, 32_000L, 48_000L, 58_000L, 71_000L, 86_000L); + FrameDropEffect frameDropEffect = + FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30); ImmutableList actualPresentationTimesUs = - processFramesToEndOfStream( - frameTimesUs, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)); + generateAndProcessBlackTimeStampedFrames(frameTimesUs, frameDropEffect); assertThat(actualPresentationTimesUs).containsExactly(0L, 32_000L, 71_000L).inOrder(); - getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); } @Test @@ -104,162 +78,60 @@ public class FrameDropTest { throws Exception { ImmutableList frameTimesUs = ImmutableList.of(0L, 250_000L, 500_000L, 750_000L, 1_000_000L, 1_500_000L); + FrameDropEffect frameDropEffect = + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2); ImmutableList actualPresentationTimesUs = - processFramesToEndOfStream( - frameTimesUs, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)); + generateAndProcessBlackTimeStampedFrames(frameTimesUs, frameDropEffect); assertThat(actualPresentationTimesUs).containsExactly(0L, 750_000L).inOrder(); - getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); } @Test @RequiresNonNull({"textureBitmapReader", "testId"}) public void frameDrop_withSimpleStrategy_outputsAllFrames() throws Exception { ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); + FrameDropEffect frameDropEffect = + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3); ImmutableList actualPresentationTimesUs = - processFramesToEndOfStream( - frameTimesUs, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3)); + generateAndProcessBlackTimeStampedFrames(frameTimesUs, frameDropEffect); assertThat(actualPresentationTimesUs).containsExactly(0L, 333_333L, 666_667L).inOrder(); - getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); } - private static void getAndAssertOutputBitmaps( - TextureBitmapReader textureBitmapReader, List presentationTimesUs, String testId) - throws IOException { - for (int i = 0; i < presentationTimesUs.size(); i++) { - long presentationTimeUs = presentationTimesUs.get(i); - Bitmap actualBitmap = textureBitmapReader.getBitmapAtPresentationTimeUs(presentationTimeUs); - Bitmap expectedBitmap = - readBitmap(Util.formatInvariant("%s/pts_%d.png", ASSET_PATH, presentationTimeUs)); - maybeSaveTestBitmap( - testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); - float averagePixelAbsoluteDifference = - getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); - assertThat(averagePixelAbsoluteDifference) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); - } - } - - @EnsuresNonNull("defaultVideoFrameProcessor") - private ImmutableList processFramesToEndOfStream( - List inputPresentationTimesUs, FrameDropEffect frameDropEffect) throws Exception { - AtomicReference<@NullableType VideoFrameProcessingException> - videoFrameProcessingExceptionReference = new AtomicReference<>(); - BlankFrameProducer blankFrameProducer = - new BlankFrameProducer(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT); - CountDownLatch videoFrameProcessorReadyCountDownLatch = new CountDownLatch(1); - CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1); - ImmutableList.Builder actualPresentationTimesUs = new ImmutableList.Builder<>(); - - defaultVideoFrameProcessor = - checkNotNull( - new DefaultVideoFrameProcessor.Factory.Builder() - .setTextureOutput( - (textureProducer, outputTexture, presentationTimeUs, token) -> { - checkNotNull(textureBitmapReader) - .readBitmap(outputTexture, presentationTimeUs); - textureProducer.releaseOutputTexture(presentationTimeUs); - }, - /* textureOutputCapacity= */ 1) - .build() - .create( - getApplicationContext(), - DebugViewProvider.NONE, - /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, - /* renderFramesAutomatically= */ true, - MoreExecutors.directExecutor(), - new VideoFrameProcessor.Listener() { - @Override - public void onInputStreamRegistered( - @VideoFrameProcessor.InputType int inputType, - List effects, - FrameInfo frameInfo) { - videoFrameProcessorReadyCountDownLatch.countDown(); - } - - @Override - public void onOutputSizeChanged(int width, int height) {} - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - actualPresentationTimesUs.add(presentationTimeUs); - } - - @Override - public void onError(VideoFrameProcessingException exception) { - videoFrameProcessingExceptionReference.set(exception); - videoFrameProcessorReadyCountDownLatch.countDown(); - videoFrameProcessingEndedCountDownLatch.countDown(); - } - - @Override - public void onEnded() { - videoFrameProcessingEndedCountDownLatch.countDown(); - } - })); - - defaultVideoFrameProcessor.getTaskExecutor().submit(blankFrameProducer::configureGlObjects); - // A frame needs to be registered despite not queuing any external input to ensure - // that the video frame processor knows about the stream offset. - checkNotNull(defaultVideoFrameProcessor) - .registerInputStream( - INPUT_TYPE_SURFACE, - /* effects= */ ImmutableList.of( - (GlEffect) (context, useHdr) -> blankFrameProducer, - // Use an overlay effect to generate bitmaps with timestamps on it. - new OverlayEffect( - ImmutableList.of( - new TextOverlay() { - @Override - public SpannableString getText(long presentationTimeUs) { - SpannableString text = - new SpannableString(String.valueOf(presentationTimeUs)); - text.setSpan( - new ForegroundColorSpan(Color.BLACK), - /* start= */ 0, - text.length(), - Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); - text.setSpan( - new AbsoluteSizeSpan(/* size= */ 24), - /* start= */ 0, - text.length(), - Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); - text.setSpan( - new TypefaceSpan(/* family= */ "sans-serif"), - /* start= */ 0, - text.length(), - Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); - return text; - } - })), - frameDropEffect), - new FrameInfo.Builder( - ColorInfo.SDR_BT709_LIMITED, BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT) - .build()); - videoFrameProcessorReadyCountDownLatch.await(); - checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference); - blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); - defaultVideoFrameProcessor.signalEndOfInput(); - videoFrameProcessingEndedCountDownLatch.await(); - checkNoVideoFrameProcessingExceptionIsThrown(videoFrameProcessingExceptionReference); - return actualPresentationTimesUs.build(); - } - - private static void checkNoVideoFrameProcessingExceptionIsThrown( - AtomicReference<@NullableType VideoFrameProcessingException> - videoFrameProcessingExceptionReference) - throws Exception { - @Nullable - Exception videoFrameProcessingException = videoFrameProcessingExceptionReference.get(); - if (videoFrameProcessingException != null) { - throw videoFrameProcessingException; - } + private ImmutableList generateAndProcessBlackTimeStampedFrames( + ImmutableList frameTimesUs, FrameDropEffect frameDropEffect) throws Exception { + int blankFrameWidth = 100; + int blankFrameHeight = 50; + Consumer textSpanConsumer = + (text) -> { + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + return generateAndProcessFrames( + blankFrameWidth, + blankFrameHeight, + frameTimesUs, + frameDropEffect, + checkNotNull(textureBitmapReader), + textSpanConsumer); } } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java new file mode 100644 index 0000000000..0ab5cfc083 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GaussianBlurTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.BackgroundColorSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link GaussianBlur}. */ +@RunWith(AndroidJUnit4.class) +public class GaussianBlurTest { + @Rule public final TestName testName = new TestName(); + + // Golden images were generated on an API 33 emulator. API 26 emulators have a different text + // rendering implementation that leads to a larger pixel difference. + private static final String ASSET_PATH = "media/bitmap/GaussianBlurTest"; + private static final int BLANK_FRAME_WIDTH = 200; + private static final int BLANK_FRAME_HEIGHT = 100; + private static final Consumer TEXT_SPAN_CONSUMER = + (text) -> { + text.setSpan( + new BackgroundColorSpan(Color.BLUE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new ForegroundColorSpan(Color.WHITE), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 100), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + + private @MonotonicNonNull String testId; + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + + @EnsuresNonNull({"textureBitmapReader", "testId"}) + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void gaussianBlur_blursFrame() throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(32_000L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new GaussianBlur(/* sigma= */ 5f), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactly(32_000L); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } +} diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_separable_convolution_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_separable_convolution_es2.glsl new file mode 100644 index 0000000000..2093c54aae --- /dev/null +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_separable_convolution_es2.glsl @@ -0,0 +1,105 @@ +#version 100 +// Copyright 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +precision highp float; +varying vec2 vTexSamplingCoord; +uniform sampler2D uTexSampler; +// 1D function LUT, only 2D due to OpenGL ES 2.0 limitations. +uniform int uIsHorizontal; +// Size of one texel in the source image, along the axis of interest. +// To properly leverage the bilinear texture sampling for efficient weighted +// lookup, it is important to know exactly where texels are centered. +uniform float uSourceTexelSize; +// Size of source texture in texels. +uniform float uSourceFullSize; +// Starting point of the convolution, in units of the source texels. +uniform float uConvStartTexels; +// Width of the convolution, in units of the source texels. +uniform float uConvWidthTexels; +// Convolution function has a different resolution than the source texture. +// Need to be able convert steps in source texels to steps in the function +// lookup texture. +uniform float uFunctionLookupStepSize; +// Center position of the function in the lookup texture. +uniform vec2 uFunctionLookupCenter; +uniform sampler2D uFunctionLookupSampler; + +// Reference Implementation: +void main() { + const float epsilon = 0.0001; + vec2 singleTexelStep; + float centerPositionTexels; + if (uIsHorizontal > 0) { + singleTexelStep = vec2(uSourceTexelSize, 0.0); + centerPositionTexels = vTexSamplingCoord.x * uSourceFullSize; + } else { + singleTexelStep = vec2(0.0, uSourceTexelSize); + centerPositionTexels = vTexSamplingCoord.y * uSourceFullSize; + } + + float supportStartEdgeTexels = + max(0.0, centerPositionTexels + uConvStartTexels); + + // Perform calculations at texel centers. + // Find first texel center > supportStartEdge. + // Texels are centered at 1/2 pixel offsets. + float startSampleTexels = floor(supportStartEdgeTexels + 0.5 - epsilon) + 0.5; + // Make use of bilinear sampling below, so each step is actually 2 samples. + int numSteps = int(ceil(uConvWidthTexels / 2.0)); + + // Loop through, leveraging linear texture sampling to perform 2 texel + // samples at once. + vec4 accumulatedRgba = vec4(0.0, 0.0, 0.0, 0.0); + float accumulatedWeight = 0.0; + + vec2 functionLookupStepPerTexel = vec2(uFunctionLookupStepSize, 0.0); + + for (int i = 0; i < numSteps; ++i) { + float sample0Texels = startSampleTexels + float(2 * i); + + float sample0OffsetTexels = sample0Texels - centerPositionTexels; + float sample1OffsetTexels = sample0OffsetTexels + 1.0; + + vec2 function0Coord = uFunctionLookupCenter + + sample0OffsetTexels * functionLookupStepPerTexel; + vec2 function1Coord = uFunctionLookupCenter + + sample1OffsetTexels * functionLookupStepPerTexel; + + float sample0Weight = texture2D(uFunctionLookupSampler, function0Coord).x; + float sample1Weight = texture2D(uFunctionLookupSampler, function1Coord).x; + float totalSampleWeight = sample0Weight + sample1Weight; + + // Skip samples with very low weight to avoid unnecessary lookups and + // avoid dividing by 0. + if (abs(totalSampleWeight) > epsilon) { + // Select a coordinate so that a linear sample at that location + // intrinsically includes the relative sampling weights. + float sampleOffsetTexels = (sample0OffsetTexels * sample0Weight + + sample1OffsetTexels * sample1Weight) / + totalSampleWeight; + + vec2 textureSamplePos = + vTexSamplingCoord + sampleOffsetTexels * singleTexelStep; + + vec4 textureSampleColor = texture2D(uTexSampler, textureSamplePos); + accumulatedRgba += textureSampleColor * totalSampleWeight; + accumulatedWeight += totalSampleWeight; + } + } + + if (accumulatedWeight > 0.0) { + gl_FragColor = accumulatedRgba / accumulatedWeight; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ConvolutionFunction1D.java b/libraries/effect/src/main/java/androidx/media3/effect/ConvolutionFunction1D.java new file mode 100644 index 0000000000..f1ef2cd222 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ConvolutionFunction1D.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import androidx.media3.common.util.UnstableApi; + +/** + * An interface for 1 dimensional convolution functions. + * + *

The domain defines the region over which the function operates, in pixels. + */ +@UnstableApi +public interface ConvolutionFunction1D { + + /** Returns the start of the domain. */ + float domainStart(); + + /** Returns the end of the domain. */ + float domainEnd(); + + /** Returns the width of the domain. */ + default float width() { + return domainEnd() - domainStart(); + } + + /** Returns the value of the function at the {@code samplePosition}. */ + float value(float samplePosition); +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlur.java b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlur.java new file mode 100644 index 0000000000..d92bd6d015 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/GaussianBlur.java @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import androidx.annotation.FloatRange; +import androidx.annotation.RequiresApi; +import androidx.media3.common.util.UnstableApi; + +/** + * A {@link SeparableConvolution} to apply a Gaussian blur on image data. + * + *

The width of the blur is specified in pixels and applied symmetrically. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolution. +public final class GaussianBlur extends SeparableConvolution { + private final float sigma; + private final float numStandardDeviations; + + /** + * Creates an instance. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + * @param numStandardDeviations The size of function domain, measured in the number of standard + * deviations. + */ + public GaussianBlur( + @FloatRange(from = 0.0, fromInclusive = false) float sigma, + @FloatRange(from = 0.0, fromInclusive = false) float numStandardDeviations) { + this.sigma = sigma; + this.numStandardDeviations = numStandardDeviations; + } + + /** + * Creates an instance with {@code numStandardDeviations} set to {@code 2.0f}. + * + * @param sigma The half-width of 1 standard deviation, in pixels. + */ + public GaussianBlur(float sigma) { + this.sigma = sigma; + this.numStandardDeviations = 2.0f; + } + + @Override + public ConvolutionFunction1D getConvolution() { + return new GaussianFunction(sigma, numStandardDeviations); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GaussianFunction.java b/libraries/effect/src/main/java/androidx/media3/effect/GaussianFunction.java new file mode 100644 index 0000000000..5eeec38439 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/GaussianFunction.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static java.lang.Math.PI; +import static java.lang.Math.exp; +import static java.lang.Math.sqrt; + +import androidx.annotation.FloatRange; +import androidx.media3.common.util.UnstableApi; + +/** + * Implementation of a symmetric Gaussian function with a limited domain. + * + *

The half-width of the domain is {@code sigma} times {@code numStdDev}. Values strictly outside + * of that range are zero. + */ +@UnstableApi +public final class GaussianFunction implements ConvolutionFunction1D { + private final float sigma; + private final float numStdDev; + + /** + * Creates an instance. + * + * @param sigma The one standard deviation, in pixels. + * @param numStandardDeviations The half-width of function domain, measured in the number of + * standard deviations. + */ + public GaussianFunction( + @FloatRange(from = 0.0, fromInclusive = false) float sigma, + @FloatRange(from = 0.0, fromInclusive = false) float numStandardDeviations) { + checkArgument(sigma > 0 && numStandardDeviations > 0); + this.sigma = sigma; + this.numStdDev = numStandardDeviations; + } + + @Override + public float domainStart() { + return -numStdDev * sigma; + } + + @Override + public float domainEnd() { + return numStdDev * sigma; + } + + @Override + public float value(float samplePosition) { + if (Math.abs(samplePosition) > numStdDev * sigma) { + return 0.0f; + } + float samplePositionOverSigma = samplePosition / sigma; + return (float) + (exp(-samplePositionOverSigma * samplePositionOverSigma / 2) / sqrt(2 * PI) / sigma); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java new file mode 100644 index 0000000000..1fd760bd96 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolution.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.annotation.RequiresApi; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** + * A {@link GlEffect} for performing separable convolutions. + * + *

A single 1D convolution function is applied horizontally on a first pass and vertically on a + * second pass. + */ +@UnstableApi +@RequiresApi(26) // See SeparableConvolutionShaderProgram. +public abstract class SeparableConvolution implements GlEffect { + private final float scaleFactor; + + /** Creates an instance with a {@code scaleFactor} of {@code 1}. */ + public SeparableConvolution() { + this(/* scaleFactor= */ 1.0f); + } + + /** + * Creates an instance. + * + * @param scaleFactor The scaling factor used to determine the size of the output relative to the + * input. The aspect ratio remains constant. + */ + public SeparableConvolution(float scaleFactor) { + this.scaleFactor = scaleFactor; + } + + /** Returns a {@linkplain ConvolutionFunction1D 1D convolution function}. */ + public abstract ConvolutionFunction1D getConvolution(); + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new SeparableConvolutionShaderProgram(context, useHdr, this, scaleFactor); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java new file mode 100644 index 0000000000..70ace03a9c --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SeparableConvolutionShaderProgram.java @@ -0,0 +1,446 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.effect.MatrixUtils.getGlMatrixArray; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Matrix; +import android.opengl.GLES20; +import android.opengl.GLUtils; +import androidx.annotation.RequiresApi; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.Assertions; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.nio.ShortBuffer; +import java.util.concurrent.Executor; + +/** + * A {@link GlShaderProgram} for performing separable convolutions. + * + *

A single {@link ConvolutionFunction1D} is applied horizontally on a first pass and vertically + * on a second pass. + */ +@RequiresApi(26) // Uses Bitmap.Config.RGBA_F16. +/* package */ final class SeparableConvolutionShaderProgram implements GlShaderProgram { + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = + "shaders/fragment_shader_separable_convolution_es2.glsl"; + + // Constants specifically for fp16FromFloat(). + // TODO (b/282767994): Fix TAP hanging issue and update samples per texel. + private static final int RASTER_SAMPLES_PER_TEXEL = 5; + // Apply some padding in the function LUT to avoid any issues from GL sampling off the texture. + private static final int FUNCTION_LUT_PADDING = RASTER_SAMPLES_PER_TEXEL; + + // BEGIN COPIED FP16 code. + // Source: libcore/luni/src/main/java/libcore/util/FP16.java + private static final int FP16_EXPONENT_BIAS = 15; + private static final int FP16_SIGN_SHIFT = 15; + private static final int FP16_EXPONENT_SHIFT = 10; + private static final int FP32_SIGN_SHIFT = 31; + private static final int FP32_EXPONENT_SHIFT = 23; + private static final int FP32_SHIFTED_EXPONENT_MASK = 0xff; + private static final int FP32_SIGNIFICAND_MASK = 0x7fffff; + private static final int FP32_EXPONENT_BIAS = 127; + // END FP16 copied code. + + private final GlProgram glProgram; + private final GlProgram sharpTransformGlProgram; + private final float[] sharpTransformMatrixValues; + private final boolean useHdr; + private final SeparableConvolution convolution; + private final float scaleFactor; + + private GlShaderProgram.InputListener inputListener; + private GlShaderProgram.OutputListener outputListener; + private GlShaderProgram.ErrorListener errorListener; + private Executor errorListenerExecutor; + private Size outputSize; + private Size lastInputSize; + private Size intermediateSize; + private GlTextureInfo outputTexture; + private boolean outputTextureInUse; + private GlTextureInfo intermediateTexture; + private GlTextureInfo functionLutTexture; // Values for the function LUT as a texture. + private float functionLutTexelStep; + private float functionLutCenterX; + private float functionLutDomainStart; + private float functionLutWidth; + + /** + * Creates an instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @param convolution The {@link SeparableConvolution} to apply in each direction. + * @param scaleFactor The scaling factor used to determine the size of the output relative to the + * input. The aspect ratio remains constant. + * @throws VideoFrameProcessingException If a problem occurs while reading shader files. + */ + public SeparableConvolutionShaderProgram( + Context context, boolean useHdr, SeparableConvolution convolution, float scaleFactor) + throws VideoFrameProcessingException { + this.useHdr = useHdr; + this.convolution = convolution; + this.scaleFactor = scaleFactor; + inputListener = new InputListener() {}; + outputListener = new OutputListener() {}; + errorListener = (frameProcessingException) -> {}; + + errorListenerExecutor = MoreExecutors.directExecutor(); + lastInputSize = Size.ZERO; + intermediateSize = Size.ZERO; + outputSize = Size.ZERO; + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + sharpTransformGlProgram = + new GlProgram( + context, + "shaders/vertex_shader_transformation_es2.glsl", + "shaders/fragment_shader_copy_es2.glsl"); + } catch (IOException | GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + + Matrix sharpTransformMatrix = new Matrix(); + sharpTransformMatrix.setScale(/* sx= */ .5f, /* sy= */ 1f); + sharpTransformMatrixValues = getGlMatrixArray(sharpTransformMatrix); + functionLutTexture = GlTextureInfo.UNSET; + intermediateTexture = GlTextureInfo.UNSET; + outputTexture = GlTextureInfo.UNSET; + } + + @Override + public void setInputListener(InputListener inputListener) { + this.inputListener = inputListener; + if (!outputTextureInUse) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { + this.errorListenerExecutor = errorListenerExecutor; + this.errorListener = errorListener; + } + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + Assertions.checkState( + !outputTextureInUse, + "The shader program does not currently accept input frames. Release prior output frames" + + " first."); + try { + ensureTexturesAreConfigured( + glObjectsProvider, new Size(inputTexture.width, inputTexture.height)); + outputTextureInUse = true; + renderHorizontal(inputTexture); + renderVertical(); + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + sharpTransformGlProgram.use(); + sharpTransformGlProgram.setSamplerTexIdUniform( + "uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); + sharpTransformGlProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + sharpTransformGlProgram.setFloatsUniform("uTransformationMatrix", sharpTransformMatrixValues); + sharpTransformGlProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + sharpTransformGlProgram.bindAttributesAndUniforms(); + + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* i1= */ 0, /* i2= */ 4); + GlUtil.checkGlError(); + inputListener.onInputFrameProcessed(inputTexture); + outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); + } catch (GlUtil.GlException e) { + errorListenerExecutor.execute( + () -> errorListener.onError(VideoFrameProcessingException.from(e, presentationTimeUs))); + } + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + outputTextureInUse = false; + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void signalEndOfCurrentInputStream() { + outputListener.onCurrentOutputStreamEnded(); + } + + @Override + public void flush() { + outputTextureInUse = false; + inputListener.onFlush(); + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void release() throws VideoFrameProcessingException { + try { + outputTexture.release(); + intermediateTexture.release(); + functionLutTexture.release(); + glProgram.delete(); + sharpTransformGlProgram.delete(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + private void renderOnePass(int inputTexId, boolean isHorizontal) throws GlUtil.GlException { + int size = isHorizontal ? lastInputSize.getWidth() : intermediateSize.getHeight(); + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setIntUniform("uIsHorizontal", isHorizontal ? 1 : 0); + glProgram.setFloatUniform("uSourceTexelSize", 1.0f / size); + glProgram.setFloatUniform("uSourceFullSize", (float) size); + glProgram.setFloatUniform("uConvStartTexels", functionLutDomainStart); + glProgram.setFloatUniform("uConvWidthTexels", functionLutWidth); + glProgram.setFloatUniform("uFunctionLookupStepSize", functionLutTexelStep); + glProgram.setFloatsUniform("uFunctionLookupCenter", new float[] {functionLutCenterX, 0.5f}); + glProgram.setSamplerTexIdUniform( + "uFunctionLookupSampler", functionLutTexture.texId, /* texUnitIndex= */ 1); + glProgram.bindAttributesAndUniforms(); + + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GlUtil.checkGlError(); + } + + private Size configure(Size inputSize) { + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + glProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + + return new Size( + (int) (inputSize.getWidth() * scaleFactor * 2), + (int) (inputSize.getHeight() * scaleFactor)); + } + + private void renderHorizontal(GlTextureInfo inputTexture) throws GlUtil.GlException { + // Render horizontal reads from the input texture and renders to the intermediate texture. + GlUtil.focusFramebufferUsingCurrentContext( + intermediateTexture.fboId, intermediateTexture.width, intermediateTexture.height); + GlUtil.clearFocusedBuffers(); + renderOnePass(inputTexture.texId, /* isHorizontal= */ true); + } + + private void renderVertical() throws GlUtil.GlException { + // Render vertical reads from the intermediate and renders to the output texture. + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.fboId, outputTexture.width, outputTexture.height); + GlUtil.clearFocusedBuffers(); + renderOnePass(intermediateTexture.texId, /* isHorizontal= */ false); + } + + private void ensureTexturesAreConfigured(GlObjectsProvider glObjectsProvider, Size inputSize) + throws GlUtil.GlException { + // Always update the function texture, as it could change on each render cycle. + updateFunctionTexture(glObjectsProvider); + + // Only update intermediate and output textures if the size changes. + if (inputSize.equals(lastInputSize)) { + return; + } + + outputSize = configure(inputSize); + // If there is a size change with the filtering (for example, a scaling operation), the first + // pass is applied horizontally. As a result, width of the intermediate texture will match the + // output size, while the height will be unchanged from the input + intermediateSize = new Size(outputSize.getWidth(), inputSize.getHeight()); + intermediateTexture = + configurePixelTexture(glObjectsProvider, intermediateTexture, intermediateSize); + outputTexture = configurePixelTexture(glObjectsProvider, outputTexture, outputSize); + + this.lastInputSize = inputSize; + } + + /** + * Creates a function lookup table for the convolution, and stores it in a 16b floating point + * texture for GPU access. + */ + private void updateFunctionTexture(GlObjectsProvider glObjectsProvider) + throws GlUtil.GlException { + + ConvolutionFunction1D convolutionFunction = convolution.getConvolution(); + + int lutRasterSize = + (int) + Math.ceil( + convolutionFunction.width() * RASTER_SAMPLES_PER_TEXEL + 2 * FUNCTION_LUT_PADDING); + + // The function LUT is mapped to [0, 1] texture coords. We need to calculate what change + // in texture coordinated corresponds exactly with a size of 1 texel (or pixel) in the function. + // This is basically 1 / function_width, but due to the ceil() call above, it needs to be + // calculated based on the actual raster size. + this.functionLutTexelStep = 1.0f / ((float) lutRasterSize / RASTER_SAMPLES_PER_TEXEL); + + // The function values are stored in an FP16 texture. Setting FP16 values in a Bitmap requires + // multiple steps. For each step, calculate the function value as a Float, and then use the + // Half class to convert to FP16 and then read the value as a Short int + ShortBuffer functionShortBuffer = ShortBuffer.allocate(lutRasterSize * 4); + float rasterSampleStep = 1.0f / RASTER_SAMPLES_PER_TEXEL; + float functionDomainStart = convolutionFunction.domainStart(); + int index = 0; + + for (int i = 0; i < lutRasterSize; i++) { + float sampleValue = 0.0f; + int unpaddedI = i - FUNCTION_LUT_PADDING; + float samplePosition = functionDomainStart + unpaddedI * rasterSampleStep; + + if (unpaddedI >= 0 && i <= lutRasterSize - FUNCTION_LUT_PADDING) { + sampleValue = convolutionFunction.value(samplePosition); + } + + // Convert float to half (fp16) and read out the bits as a short. + // Texture for Bitmap is RGBA_F16, so we store the function value in RGB channels and 1.0 + // in A. + short shortEncodedValue = fp16FromFloat(sampleValue); + + // Set RGB + functionShortBuffer.put(index++, shortEncodedValue); + functionShortBuffer.put(index++, shortEncodedValue); + functionShortBuffer.put(index++, shortEncodedValue); + + // Set Alpha + functionShortBuffer.put(index++, fp16FromFloat(1.0f)); + } + + // Calculate the center of the function in the raster. The formula below is a slight + // adjustment on (value - min) / (max - min), where value = 0 at center and + // rasterSampleStep * lutRasterSize is equal to (max - min) over the range of the raster + // samples, which may be slightly different than the difference between the function's max + // and min domain values. + // To find the value associated at position 0 in the texture, is the value corresponding with + // the leading edge position of the first sample. This needs to account for the padding and + // the 1/2 texel offsets used in texture lookups (index 0 is centered at 0.5 / numTexels). + float minValueWithPadding = + functionDomainStart - rasterSampleStep * (FUNCTION_LUT_PADDING + 0.5f); + this.functionLutCenterX = -minValueWithPadding / (rasterSampleStep * lutRasterSize); + this.functionLutDomainStart = convolutionFunction.domainStart(); + this.functionLutWidth = convolutionFunction.width(); + + // TODO(b/276982847): Use alternative to Bitmap to create function LUT texture. + Bitmap functionLookupBitmap = + Bitmap.createBitmap(lutRasterSize, /* height= */ 1, Bitmap.Config.RGBA_F16); + functionLookupBitmap.copyPixelsFromBuffer(functionShortBuffer); + + // Create new GL texture if needed. + if (functionLutTexture == GlTextureInfo.UNSET || functionLutTexture.width != lutRasterSize) { + functionLutTexture.release(); + + // Need to use high precision to force 16FP color. + int functionLutTextureId = + GlUtil.createTexture( + lutRasterSize, /* height= */ 1, /* useHighPrecisionColorComponents= */ true); + + functionLutTexture = + glObjectsProvider.createBuffersForTexture( + functionLutTextureId, lutRasterSize, /* height= */ 1); + } + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, functionLookupBitmap, /* border= */ 0); + GlUtil.checkGlError(); + } + + private GlTextureInfo configurePixelTexture( + GlObjectsProvider glObjectsProvider, GlTextureInfo existingTexture, Size size) + throws GlUtil.GlException { + if (size.getWidth() == existingTexture.width && size.getHeight() == existingTexture.height) { + return existingTexture; + } + + existingTexture.release(); + int texId = GlUtil.createTexture(size.getWidth(), size.getHeight(), useHdr); + + return glObjectsProvider.createBuffersForTexture(texId, size.getWidth(), size.getHeight()); + } + + // BEGIN COPIED FP16 code. + // Source: libcore/luni/src/main/java/libcore/util/FP16.java + // Float to half float conversion, copied from FP16. This code is introduced in API26, so the + // one required method is copied here. + private static short fp16FromFloat(float f) { + int bits = Float.floatToRawIntBits(f); + int s = bits >>> FP32_SIGN_SHIFT; + int e = (bits >>> FP32_EXPONENT_SHIFT) & FP32_SHIFTED_EXPONENT_MASK; + int m = bits & FP32_SIGNIFICAND_MASK; + int outE = 0; + int outM = 0; + if (e == 0xff) { // Infinite or NaN + outE = 0x1f; + outM = (m != 0) ? 0x200 : 0; + } else { + e = e - FP32_EXPONENT_BIAS + FP16_EXPONENT_BIAS; + if (e >= 0x1f) { // Overflow + outE = 0x1f; + } else if (e <= 0) { // Underflow + if (e >= -10) { + // The fp32 value is a normalized float less than MIN_NORMAL, + // we convert to a denorm fp16 + m |= 0x800000; + int shift = 14 - e; + outM = m >>> shift; + int lowm = m & ((1 << shift) - 1); + int hway = 1 << (shift - 1); + // if above halfway or exactly halfway and outM is odd + if (lowm + (outM & 1) > hway) { + // Round to nearest even + // Can overflow into exponent bit, which surprisingly is OK. + // This increment relies on the +outM in the return statement below + outM++; + } + } + } else { + outE = e; + outM = m >>> 13; + // if above halfway or exactly halfway and outM is odd + if ((m & 0x1fff) + (outM & 0x1) > 0x1000) { + // Round to nearest even + // Can overflow into exponent bit, which surprisingly is OK. + // This increment relies on the +outM in the return statement below + outM++; + } + } + } + // The outM is added here as the +1 increments for outM above can + // cause an overflow in the exponent bit which is OK. + return (short) ((s << FP16_SIGN_SHIFT) | ((outE << FP16_EXPONENT_SHIFT) + outM)); + } + // END FP16 copied code. +} diff --git a/libraries/effect/src/test/java/androidx/media3/effect/GaussianFunctionTest.java b/libraries/effect/src/test/java/androidx/media3/effect/GaussianFunctionTest.java new file mode 100644 index 0000000000..d181a743c2 --- /dev/null +++ b/libraries/effect/src/test/java/androidx/media3/effect/GaussianFunctionTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link GaussianFunction}. */ +@RunWith(AndroidJUnit4.class) +public class GaussianFunctionTest { + + private final GaussianFunction function = + new GaussianFunction(/* sigma= */ 2.55f, /* numStandardDeviations= */ 4.45f); + + @Test + public void value_samplePositionAboveRange_returnsZero() { + assertThat(function.value(/* samplePosition= */ function.domainEnd() + .00001f)).isEqualTo(0); + } + + @Test + public void value_samplePositionBelowRange_returnsZero() { + assertThat(function.value(/* samplePosition= */ -10000000000000f)).isEqualTo(0); + } + + @Test + public void value_samplePositionInRange_returnsSymmetricGaussianFunction() { + assertThat(function.value(/* samplePosition= */ 9.999f)).isEqualTo(7.1712595E-5f); + assertThat(function.value(/* samplePosition= */ -9.999f)).isEqualTo(7.1712595E-5f); + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png new file mode 100644 index 0000000000..1d6008eaa7 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/GaussianBlurTest/pts_32000.png differ