diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png new file mode 100644 index 0000000000..908e00ef00 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png new file mode 100644 index 0000000000..07f121ecf2 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png differ diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index ed00d9c5cb..b4d0ffb632 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -368,10 +368,12 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessor.queueInputTexture(inputTexture.getTexId(), pts); } + /** {@link #endFrameProcessing(long)} with {@link #VIDEO_FRAME_PROCESSING_WAIT_MS} applied. */ public void endFrameProcessing() throws InterruptedException { endFrameProcessing(VIDEO_FRAME_PROCESSING_WAIT_MS); } + /** Have the {@link VideoFrameProcessor} finish processing. */ public void endFrameProcessing(long videoFrameProcessingWaitTime) throws InterruptedException { videoFrameProcessor.signalEndOfInput(); Thread.sleep(videoFrameProcessingWaitTime); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java new file mode 100644 index 0000000000..abbc3c6f07 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.util.Set; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Tests for {@link DefaultVideoFrameProcessor} texture output. + * + *

Confirms that the output timestamps are correct for each frame, and that the output pixels are + * correct for the first frame of each bitmap. + */ +@RunWith(AndroidJUnit4.class) +public class DefaultVideoFrameProcessorMultipleTextureOutputPixelTest { + private static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; + private static final String MEDIA3_TEST_PNG_ASSET_PATH = + "media/bitmap/input_images/media3test.png"; + private static final String SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png"; + private static final String SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png"; + + private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; + + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + + @After + public void release() { + checkNotNull(videoFrameProcessorTestRunner).release(); + } + + @Test + public void textureOutput_queueBitmap_matchesGoldenFile() throws Exception { + String testId = "textureOutput_queueBitmap_matchesGoldenFile"; + videoFrameProcessorTestRunner = getFrameProcessorTestRunnerBuilder(testId).build(); + + long offsetUs = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ 3 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.endFrameProcessing(); + + Set outputTimestamps = checkNotNull(textureBitmapReader).getOutputTimestamps(); + assertThat(outputTimestamps) + .containsExactly( + offsetUs, offsetUs + C.MICROS_PER_SECOND, offsetUs + 2 * C.MICROS_PER_SECOND); + Bitmap actualBitmap = checkNotNull(textureBitmapReader).getBitmap(offsetUs); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888( + readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); + } + + @Test + public void textureOutput_queueTwoBitmaps_matchesGoldenFiles() throws Exception { + String testId = "textureOutput_queueTwoBitmaps_matchesGoldenFiles"; + videoFrameProcessorTestRunner = getFrameProcessorTestRunnerBuilder(testId).build(); + + long offsetUs1 = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs1, + /* frameRate= */ 2); + long offsetUs2 = 2_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(MEDIA3_TEST_PNG_ASSET_PATH), + /* durationUs= */ 3 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs2, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.endFrameProcessing(); + + Set outputTimestamps = checkNotNull(textureBitmapReader).getOutputTimestamps(); + assertThat(outputTimestamps) + .containsExactly( + offsetUs1, + offsetUs1 + C.MICROS_PER_SECOND / 2, + offsetUs2, + offsetUs2 + C.MICROS_PER_SECOND, + offsetUs2 + 2 * C.MICROS_PER_SECOND); + Bitmap actualBitmap1 = checkNotNull(textureBitmapReader).getBitmap(offsetUs1); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual1", actualBitmap1, /* path= */ null); + Bitmap actualBitmap2 = checkNotNull(textureBitmapReader).getBitmap(offsetUs2); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual2", actualBitmap2, /* path= */ null); + float averagePixelAbsoluteDifference1 = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888( + readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), actualBitmap1, testId); + assertThat(averagePixelAbsoluteDifference1) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); + float averagePixelAbsoluteDifference2 = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888( + readBitmap(SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH), actualBitmap2, testId); + assertThat(averagePixelAbsoluteDifference2) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); + } + + private VideoFrameProcessorTestRunner.Builder getFrameProcessorTestRunnerBuilder(String testId) { + textureBitmapReader = new TextureBitmapReader(); + VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = + new DefaultVideoFrameProcessor.Factory.Builder() + .setTextureOutput( + textureBitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1) + .build(); + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) + .setInputType(INPUT_TYPE_BITMAP) + .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) + .setBitmapReader(textureBitmapReader); + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index f900f803af..e936606aea 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -17,8 +17,6 @@ package androidx.media3.transformer.mh; import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16; import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; @@ -33,7 +31,6 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.graphics.Bitmap; -import android.view.Surface; import androidx.media3.common.ColorInfo; import androidx.media3.common.Effect; import androidx.media3.common.Format; @@ -42,7 +39,6 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; -import androidx.media3.common.util.Util; import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; @@ -52,14 +48,12 @@ import androidx.media3.effect.OverlayEffect; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.test.utils.BitmapPixelTestUtil; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner.BitmapReader; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.EncoderUtil; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; @@ -579,57 +573,6 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .setBitmapReader(textureBitmapReader); } - /** - * {@inheritDoc} - * - *

Reads from an OpenGL texture. Only for use on physical devices. - */ - private static final class TextureBitmapReader implements BitmapReader { - // TODO(b/239172735): This outputs an incorrect black output image on emulators. - private boolean useHighPrecisionColorComponents; - private @MonotonicNonNull Bitmap outputBitmap; - - @Nullable - @Override - public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { - this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; - return null; - } - - @Override - public Bitmap getBitmap() { - return checkStateNotNull(outputBitmap); - } - - public void readBitmapFromTexture( - GlTextureInfo outputTexture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback) - throws VideoFrameProcessingException { - try { - GlUtil.focusFramebufferUsingCurrentContext( - outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight()); - outputBitmap = - createBitmapFromCurrentGlFrameBuffer( - outputTexture.getWidth(), - outputTexture.getHeight(), - useHighPrecisionColorComponents); - } catch (GlUtil.GlException e) { - throw new VideoFrameProcessingException(e); - } - releaseOutputTextureCallback.release(presentationTimeUs); - } - - private static Bitmap createBitmapFromCurrentGlFrameBuffer( - int width, int height, boolean useHighPrecisionColorComponents) throws GlUtil.GlException { - if (!useHighPrecisionColorComponents) { - return BitmapPixelTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); - } - checkState(Util.SDK_INT > 26, "useHighPrecisionColorComponents only supported on API 26+"); - return BitmapPixelTestUtil.createFp16BitmapFromCurrentGlFramebuffer(width, height); - } - } - private static boolean deviceSupportsHdrEditing(Format format) { return !EncoderUtil.getSupportedEncodersForHdrEditing( checkNotNull(checkNotNull(format).sampleMimeType), format.colorInfo) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TextureBitmapReader.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TextureBitmapReader.java new file mode 100644 index 0000000000..a7b76f6529 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TextureBitmapReader.java @@ -0,0 +1,97 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.graphics.Bitmap; +import android.view.Surface; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Util; +import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * {@inheritDoc} + * + *

Reads from an OpenGL texture. Only for use on physical devices. + */ +public final class TextureBitmapReader implements VideoFrameProcessorTestRunner.BitmapReader { + // TODO(b/239172735): This outputs an incorrect black output image on emulators. + private final Map outputTimestampsToBitmaps; + private boolean useHighPrecisionColorComponents; + private @MonotonicNonNull Bitmap outputBitmap; + + public TextureBitmapReader() { + outputTimestampsToBitmaps = new ConcurrentHashMap<>(); + } + + @Nullable + @Override + public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { + this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; + return null; + } + + @Override + public Bitmap getBitmap() { + return checkStateNotNull(outputBitmap); + } + + public Bitmap getBitmap(long presentationTimeUs) { + return checkStateNotNull(outputTimestampsToBitmaps.get(presentationTimeUs)); + } + + public Set getOutputTimestamps() { + return outputTimestampsToBitmaps.keySet(); + } + + public void readBitmapFromTexture( + GlTextureInfo outputTexture, + long presentationTimeUs, + DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback) + throws VideoFrameProcessingException { + try { + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight()); + outputBitmap = + createBitmapFromCurrentGlFrameBuffer( + outputTexture.getWidth(), outputTexture.getHeight(), useHighPrecisionColorComponents); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + outputTimestampsToBitmaps.put(presentationTimeUs, outputBitmap); + releaseOutputTextureCallback.release(presentationTimeUs); + } + + private static Bitmap createBitmapFromCurrentGlFrameBuffer( + int width, int height, boolean useHighPrecisionColorComponents) throws GlUtil.GlException { + if (!useHighPrecisionColorComponents) { + return BitmapPixelTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); + } + checkState(Util.SDK_INT > 26, "useHighPrecisionColorComponents only supported on API 26+"); + return BitmapPixelTestUtil.createFp16BitmapFromCurrentGlFramebuffer(width, height); + } +}