Must be called before {@link #create}. + * + *
The default value is {@code false}. + */ + @VisibleForTesting + public Factory setOutputToTexture(boolean outputToTexture) { + this.outputToTexture = outputToTexture; + return this; + } + /** * {@inheritDoc} * @@ -153,7 +168,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { singleThreadExecutorService, listenerExecutor, listener, - glObjectsProvider)); + glObjectsProvider, + outputToTexture)); try { return defaultVideoFrameProcessorFuture.get(); @@ -226,7 +242,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { /** Returns the task executor that runs video frame processing tasks. */ @VisibleForTesting - /* package */ VideoFrameProcessingTaskExecutor getTaskExecutor() { + public VideoFrameProcessingTaskExecutor getTaskExecutor() { return videoFrameProcessingTaskExecutor; } @@ -289,6 +305,20 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo); } + /** + * Gets the output {@link GlTextureInfo}. + * + *
Should only be called if {@code outputToTexture} is true, and after a frame is available, as
+ * reported by the output {@linkplain #setOutputSurfaceInfo surface}'s {@link
+ * SurfaceTexture#setOnFrameAvailableListener}. Returns {@code null} if an output texture is not
+ * yet available.
+ */
+ @VisibleForTesting
+ @Nullable
+ public GlTextureInfo getOutputTextureInfo() {
+ return finalShaderProgramWrapper.getOutputTextureInfo();
+ }
+
@Override
public void releaseOutputFrame(long releaseTimeNs) {
checkState(
@@ -382,7 +412,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ExecutorService singleThreadExecutorService,
Executor executor,
Listener listener,
- GlObjectsProvider glObjectsProvider)
+ GlObjectsProvider glObjectsProvider,
+ boolean outputToTexture)
throws GlUtil.GlException, VideoFrameProcessingException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
@@ -425,7 +456,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
releaseFramesAutomatically,
executor,
listener,
- glObjectsProvider);
+ glObjectsProvider,
+ outputToTexture);
setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
@@ -464,7 +496,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
boolean releaseFramesAutomatically,
Executor executor,
Listener listener,
- GlObjectsProvider glObjectsProvider)
+ GlObjectsProvider glObjectsProvider,
+ boolean outputToTexture)
throws VideoFrameProcessingException {
ImmutableList.Builder The wrapped {@link DefaultShaderProgram} applies the {@link GlMatrixTransformation} and {@link
* RgbMatrix} instances passed to the constructor, followed by any transformations needed to convert
@@ -82,6 +83,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final float[] textureTransformMatrix;
private final Queue To run pixel difference-based tests on physical devices, please use a value of 5f, rather
- * than 0.5f. This higher value will ignore some very small errors, but will allow for some
- * differences caused by graphics implementations to be ignored. When the difference is close to
- * the threshold, manually inspect expected/actual bitmaps to confirm failure, as it's possible
- * this is caused by a difference in the codec or graphics implementation as opposed to an issue
- * in the tested component.
+ * The value is chosen so that differences in decoder behavior across emulator versions don't
+ * affect whether the test passes, but substantial distortions introduced by changes in tested
+ * components will cause the test to fail.
+ *
+ * When the difference is close to the threshold, manually inspect expected/actual bitmaps to
+ * confirm failure, as it's possible this is caused by a difference in the codec or graphics
+ * implementation as opposed to an issue in the tested component.
*/
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 1.f;
+ /**
+ * Maximum allowed average pixel difference between the expected and actual edited images in pixel
+ * difference-based tests, between devices, or devices and emulators.
+ *
+ * The value is chosen so that differences in decoder behavior across devices don't affect
+ * whether the test passes, but substantial distortions introduced by changes in tested components
+ * will cause the test to fail.
+ *
+ * When the difference is close to the threshold, manually inspect expected/actual bitmaps to
+ * confirm failure, as it's possible this is caused by a difference in the codec or graphics
+ * implementation as opposed to an issue in the tested component.
+ *
+ * This value is larger than {@link #MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} to support the
+ * larger variance in decoder outputs between different physical devices and emulators.
+ */
+ public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE = 5.f;
+
/**
* Reads a bitmap from the specified asset location.
*
diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
index 0dbfe640d8..f1f7125a80 100644
--- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
+++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
@@ -28,6 +28,7 @@ import android.graphics.PixelFormat;
import android.media.Image;
import android.media.ImageReader;
import android.media.MediaFormat;
+import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.ColorInfo;
@@ -57,6 +58,7 @@ public final class VideoFrameProcessorTestRunner {
private @MonotonicNonNull String testId;
private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
+ private BitmapReader.@MonotonicNonNull Factory bitmapReaderFactory;
private @MonotonicNonNull String videoAssetPath;
private @MonotonicNonNull String outputFileLabel;
private @MonotonicNonNull ImmutableList The default value is {@link SurfaceBitmapReader.Factory}.
+ */
+ @CanIgnoreReturnValue
+ public Builder setBitmapReaderFactory(BitmapReader.Factory bitmapReaderFactory) {
+ this.bitmapReaderFactory = bitmapReaderFactory;
+ return this;
+ }
+
/**
* Sets the input video asset path.
*
@@ -205,6 +218,7 @@ public final class VideoFrameProcessorTestRunner {
return new VideoFrameProcessorTestRunner(
testId,
videoFrameProcessorFactory,
+ bitmapReaderFactory == null ? new SurfaceBitmapReader.Factory() : bitmapReaderFactory,
videoAssetPath,
outputFileLabel == null ? "" : outputFileLabel,
effects == null ? ImmutableList.of() : effects,
@@ -227,15 +241,16 @@ public final class VideoFrameProcessorTestRunner {
private final String outputFileLabel;
private final float pixelWidthHeightRatio;
private final AtomicReference Reads from a {@link Surface}. Only supports SDR input.
+ */
+ public static final class SurfaceBitmapReader
+ implements VideoFrameProcessorTestRunner.BitmapReader {
+ public static final class Factory
+ implements VideoFrameProcessorTestRunner.BitmapReader.Factory {
+ @Override
+ public SurfaceBitmapReader create(
+ VideoFrameProcessor videoFrameProcessor, int width, int height) {
+ return new SurfaceBitmapReader(width, height);
+ }
+ }
+
+ // ImageReader only supports SDR input.
+ private final ImageReader imageReader;
+
+ @SuppressLint("WrongConstant")
+ private SurfaceBitmapReader(int width, int height) {
+ imageReader =
+ ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
+ }
+
+ @Override
+ public Surface getSurface() {
+ return imageReader.getSurface();
+ }
+
+ @Override
+ public Bitmap getBitmap() {
+ Image outputImage = checkNotNull(imageReader).acquireLatestImage();
+ Bitmap outputBitmap = createArgb8888BitmapFromRgba8888Image(outputImage);
+ outputImage.close();
+ return outputBitmap;
+ }
+ }
}
diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
new file mode 100644
index 0000000000..83b49a1479
--- /dev/null
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer.mh;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.graphics.Bitmap;
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+import androidx.media3.common.GlTextureInfo;
+import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.effect.BitmapOverlay;
+import androidx.media3.effect.DefaultVideoFrameProcessor;
+import androidx.media3.effect.OverlayEffect;
+import androidx.media3.test.utils.BitmapPixelTestUtil;
+import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Pixel test for video frame processing, outputting to a texture, via {@link
+ * DefaultVideoFrameProcessor}.
+ *
+ * Uses a {@link DefaultVideoFrameProcessor} to process one frame, and checks that the actual
+ * output matches expected output, either from a golden file or from another edit.
+ */
+// TODO(b/263395272): Move this test to effects/mh tests, and remove @TestOnly dependencies.
+@RunWith(AndroidJUnit4.class)
+public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
+ private static final String ORIGINAL_PNG_ASSET_PATH =
+ "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
+ private static final String BITMAP_OVERLAY_PNG_ASSET_PATH =
+ "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png";
+ private static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png";
+ /** Input video of which we only use the first frame. */
+ private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
+
+ private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner;
+
+ @After
+ public void release() {
+ checkNotNull(videoFrameProcessorTestRunner).release();
+ }
+
+ @Test
+ public void noEffects_matchesGoldenFile() throws Exception {
+ String testId = "noEffects_matchesGoldenFile";
+ videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
+ Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
+
+ Bitmap actualBitmap = videoFrameProcessorTestRunner.processFirstFrameAndEnd();
+
+ // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+ float averagePixelAbsoluteDifference =
+ getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+ assertThat(averagePixelAbsoluteDifference)
+ .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
+ }
+
+ @Test
+ public void bitmapOverlay_matchesGoldenFile() throws Exception {
+ String testId = "bitmapOverlay_matchesGoldenFile";
+ Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
+ BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap);
+ videoFrameProcessorTestRunner =
+ getDefaultFrameProcessorTestRunnerBuilder(testId)
+ .setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay)))
+ .build();
+ Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH);
+
+ Bitmap actualBitmap = videoFrameProcessorTestRunner.processFirstFrameAndEnd();
+
+ // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
+ float averagePixelAbsoluteDifference =
+ getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
+ assertThat(averagePixelAbsoluteDifference)
+ .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
+ }
+
+ // TODO(b/227624622): Add a test for HDR input after BitmapPixelTestUtil can read HDR bitmaps,
+ // using GlEffectWrapper to ensure usage of intermediate textures.
+
+ private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
+ String testId) {
+ DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
+ new DefaultVideoFrameProcessor.Factory().setOutputToTexture(true);
+ return new VideoFrameProcessorTestRunner.Builder()
+ .setTestId(testId)
+ .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
+ .setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING)
+ .setBitmapReaderFactory(new TextureBitmapReader.Factory());
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * Reads from an OpenGL texture. Only for use on physical devices.
+ */
+ private static final class TextureBitmapReader
+ implements VideoFrameProcessorTestRunner.BitmapReader {
+ // TODO(b/239172735): This outputs an incorrect black output image on emulators.
+ public static final class Factory
+ implements VideoFrameProcessorTestRunner.BitmapReader.Factory {
+ @Override
+ public TextureBitmapReader create(
+ VideoFrameProcessor videoFrameProcessor, int width, int height) {
+ return new TextureBitmapReader((DefaultVideoFrameProcessor) videoFrameProcessor);
+ }
+ }
+
+ private final DefaultVideoFrameProcessor defaultVideoFrameProcessor;
+ private @MonotonicNonNull Bitmap outputBitmap;
+
+ private TextureBitmapReader(DefaultVideoFrameProcessor defaultVideoFrameProcessor) {
+ this.defaultVideoFrameProcessor = defaultVideoFrameProcessor;
+ }
+
+ @Override
+ public Surface getSurface() {
+ int texId;
+ try {
+ texId = GlUtil.createExternalTexture();
+ } catch (GlUtil.GlException e) {
+ throw new RuntimeException(e);
+ }
+ SurfaceTexture surfaceTexture = new SurfaceTexture(texId);
+ surfaceTexture.setOnFrameAvailableListener(this::onSurfaceTextureFrameAvailable);
+ return new Surface(surfaceTexture);
+ }
+
+ @Override
+ public Bitmap getBitmap() {
+ return checkStateNotNull(outputBitmap);
+ }
+
+ private void onSurfaceTextureFrameAvailable(SurfaceTexture surfaceTexture) {
+ defaultVideoFrameProcessor
+ .getTaskExecutor()
+ .submitWithHighPriority(this::getBitmapFromTexture);
+ }
+
+ private void getBitmapFromTexture() throws GlUtil.GlException {
+ GlTextureInfo outputTexture = checkNotNull(defaultVideoFrameProcessor.getOutputTextureInfo());
+
+ GlUtil.focusFramebufferUsingCurrentContext(
+ outputTexture.fboId, outputTexture.width, outputTexture.height);
+ outputBitmap =
+ BitmapPixelTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
+ outputTexture.width, outputTexture.height);
+ }
+ }
+}