diff --git a/libraries/test_data/src/test/assets/media/bitmap/input_images/tokyo.jpg b/libraries/test_data/src/test/assets/media/bitmap/input_images/tokyo.jpg new file mode 100644 index 0000000000..b02c16251c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/input_images/tokyo.jpg differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_0.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_0.png new file mode 100644 index 0000000000..675f50fa77 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_0.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_1.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_1.png new file mode 100644 index 0000000000..bde2d35af5 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem_1.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_0.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_0.png new file mode 100644 index 0000000000..ac71715ca9 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_0.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_1.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_1.png new file mode 100644 index 0000000000..c6c8af04bc Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_1.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_2.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_2.png new file mode 100644 index 0000000000..757c4fe0f4 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_2.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_3.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_3.png new file mode 100644 index 0000000000..cec12db625 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndNoVideoEffects_3.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_0.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_0.png new file mode 100644 index 0000000000..e4aead4264 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_0.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_1.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_1.png new file mode 100644 index 0000000000..8f8e592ff8 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_1.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_2.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_2.png new file mode 100644 index 0000000000..5d097dfe81 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_2.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_3.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_3.png new file mode 100644 index 0000000000..2e7e25704c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withCompositionPresentationAndWithPerMediaItemEffects_3.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_0.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_0.png new file mode 100644 index 0000000000..675f50fa77 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_0.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_1.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_1.png new file mode 100644 index 0000000000..be6c02f3fd Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_1.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_2.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_2.png new file mode 100644 index 0000000000..0cbb9403a9 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_2.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_3.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_3.png new file mode 100644 index 0000000000..3beb194b64 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_sequence_effect_test/export_withNoCompositionPresentationAndWithPerMediaItemEffects_3.png differ diff --git a/libraries/test_data/src/test/assets/media/mp4/sample_portrait.mp4 b/libraries/test_data/src/test/assets/media/mp4/sample_portrait.mp4 new file mode 100644 index 0000000000..03f3f97f3b Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/sample_portrait.mp4 differ diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java index e8bf984a0b..10961a3676 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java @@ -25,6 +25,7 @@ import android.content.Context; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.Color; +import android.graphics.ImageFormat; import android.graphics.Matrix; import android.graphics.PixelFormat; import android.media.Image; @@ -145,6 +146,34 @@ public class BitmapPixelTestUtil { return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); } + /** + * Returns a grayscale bitmap from the Luma channel in the {@link ImageFormat#YUV_420_888} image. + */ + @RequiresApi(19) + public static Bitmap createGrayscaleArgb8888BitmapFromYuv420888Image(Image image) { + int width = image.getWidth(); + int height = image.getHeight(); + assertThat(image.getPlanes()).hasLength(3); + assertThat(image.getFormat()).isEqualTo(ImageFormat.YUV_420_888); + Image.Plane lumaPlane = image.getPlanes()[0]; + ByteBuffer lumaBuffer = lumaPlane.getBuffer(); + int[] colors = new int[width * height]; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int offset = y * lumaPlane.getRowStride() + x * lumaPlane.getPixelStride(); + int lumaValue = lumaBuffer.get(offset) & 0xFF; + colors[y * width + x] = + Color.argb( + /* alpha= */ 255, + /* red= */ lumaValue, + /* green= */ lumaValue, + /* blue= */ lumaValue); + } + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); + } + /** * Returns a solid {@link Bitmap} with every pixel having the same color. * diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 4d39ae2e50..a9af63f487 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -74,6 +74,8 @@ public final class AndroidTestUtil { public static final String PNG_ASSET_URI_STRING = "asset:///media/bitmap/input_images/media3test.png"; public static final String JPG_ASSET_URI_STRING = "asset:///media/bitmap/input_images/london.jpg"; + public static final String JPG_PORTRAIT_ASSET_URI_STRING = + "asset:///media/bitmap/input_images/tokyo.jpg"; public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4"; public static final Format MP4_ASSET_FORMAT = @@ -85,6 +87,17 @@ public final class AndroidTestUtil { .setCodecs("avc1.64001F") .build(); + public static final String MP4_PORTRAIT_ASSET_URI_STRING = + "asset:///media/mp4/sample_portrait.mp4"; + public static final Format MP4_PORTRAIT_ASSET_FORMAT = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setWidth(720) + .setHeight(1080) + .setFrameRate(29.97f) + .setCodecs("avc1.64001F") + .build(); + public static final String MP4_ASSET_AV1_VIDEO_URI_STRING = "asset:///media/mp4/sample_av1.mp4"; public static final Format MP4_ASSET_AV1_VIDEO_FORMAT = new Format.Builder() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java new file mode 100644 index 0000000000..d52982b9b5 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java @@ -0,0 +1,264 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.JPG_PORTRAIT_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_URI_STRING; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.media.Image; +import androidx.annotation.Nullable; +import androidx.media3.common.Effect; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.Util; +import androidx.media3.effect.BitmapOverlay; +import androidx.media3.effect.OverlayEffect; +import androidx.media3.effect.Presentation; +import androidx.media3.effect.RgbFilter; +import androidx.media3.effect.ScaleAndRotateTransformation; +import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Tests for using different {@linkplain Effect effects} for {@link MediaItem MediaItems} in one + * {@link EditedMediaItemSequence} . + */ +@RunWith(AndroidJUnit4.class) +public final class TransformerSequenceEffectTest { + + private static final ImmutableList NO_EFFECT = ImmutableList.of(); + private static final String PNG_ASSET_BASE_PATH = "media/bitmap/transformer_sequence_effect_test"; + private static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png"; + private static final int EXPORT_WIDTH = 360; + private static final int EXPORT_HEIGHT = 240; + + /** + * Allowing more difference than {@link + * BitmapPixelTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} for YUV to RGB conversion on + * emulators. + */ + private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = + 8 * BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; + + private final Context context = ApplicationProvider.getApplicationContext(); + + @Test + public void export_withNoCompositionPresentationAndWithPerMediaItemEffects() throws Exception { + String testId = "export_withNoCompositionPresentationAndWithPerMediaItemEffects"; + OverlayEffect overlayEffect = createOverlayEffect(); + Composition composition = + createComposition( + /* presentation= */ null, + oneFrameFromVideo( + MP4_ASSET_URI_STRING, + ImmutableList.of( + Presentation.createForWidthAndHeight( + EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT))), + oneFrameFromImage( + JPG_ASSET_URI_STRING, + ImmutableList.of( + new ScaleAndRotateTransformation.Builder().setRotationDegrees(72).build(), + overlayEffect)), + oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT), + // Transition to a different aspect ratio. + oneFrameFromImage( + JPG_ASSET_URI_STRING, + ImmutableList.of( + Presentation.createForWidthAndHeight( + EXPORT_WIDTH / 2, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT), + new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(), + overlayEffect))); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + assertBitmapsMatchExpected( + extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); + } + + @Test + public void export_withCompositionPresentationAndWithPerMediaItemEffects() throws Exception { + String testId = "export_withCompositionPresentationAndWithPerMediaItemEffects"; + Composition composition = + createComposition( + Presentation.createForWidthAndHeight( + EXPORT_WIDTH, /* height= */ EXPORT_WIDTH, Presentation.LAYOUT_SCALE_TO_FIT), + oneFrameFromImage( + JPG_ASSET_URI_STRING, + ImmutableList.of( + new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(), + Presentation.createForWidthAndHeight( + EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT))), + oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT), + oneFrameFromVideo( + MP4_ASSET_URI_STRING, ImmutableList.of(RgbFilter.createInvertedFilter())), + oneFrameFromVideo( + MP4_ASSET_URI_STRING, + ImmutableList.of( + Presentation.createForWidthAndHeight( + EXPORT_WIDTH / 2, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT), + createOverlayEffect()))); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + assertBitmapsMatchExpected( + extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); + } + + @Test + public void export_withCompositionPresentationAndNoVideoEffects() throws Exception { + String testId = "export_withCompositionPresentationAndNoVideoEffects"; + Composition composition = + createComposition( + Presentation.createForHeight(EXPORT_HEIGHT), + oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT), + oneFrameFromVideo(MP4_PORTRAIT_ASSET_URI_STRING, NO_EFFECT), + oneFrameFromVideo(MP4_ASSET_URI_STRING, NO_EFFECT), + oneFrameFromImage(JPG_PORTRAIT_ASSET_URI_STRING, NO_EFFECT)); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + assertBitmapsMatchExpected( + extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); + } + + @Test + public void export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem() + throws Exception { + String testId = "export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem"; + Composition composition = + createComposition( + Presentation.createForHeight(EXPORT_HEIGHT), + oneFrameFromVideo(MP4_ASSET_URI_STRING, NO_EFFECT), + oneFrameFromVideo( + MP4_PORTRAIT_ASSET_URI_STRING, ImmutableList.of(RgbFilter.createInvertedFilter()))); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + assertBitmapsMatchExpected( + extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); + } + + private static OverlayEffect createOverlayEffect() throws IOException { + return new OverlayEffect( + ImmutableList.of( + BitmapOverlay.createStaticBitmapOverlay(readBitmap(OVERLAY_PNG_ASSET_PATH)))); + } + + private static Composition createComposition( + @Nullable Presentation presentation, EditedMediaItem... editedMediaItems) { + Composition.Builder builder = + new Composition.Builder( + ImmutableList.of(new EditedMediaItemSequence(ImmutableList.copyOf(editedMediaItems)))); + if (presentation != null) { + builder.setEffects( + new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.of(presentation))); + } + return builder.build(); + } + + private static EditedMediaItem oneFrameFromVideo(String uri, List effects) { + return new EditedMediaItem.Builder( + MediaItem.fromUri(uri) + .buildUpon() + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + // Clip to only the first frame. + .setEndPositionMs(50) + .build()) + .build()) + .setRemoveAudio(true) + .setEffects( + new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects))) + .build(); + } + + private static EditedMediaItem oneFrameFromImage(String uri, List effects) { + return new EditedMediaItem.Builder(MediaItem.fromUri(uri)) + // 50ms for a 20-fps video is one frame. + .setFrameRate(20) + .setDurationUs(50_000) + .setEffects( + new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects))) + .build(); + } + + private static ImmutableList extractBitmapsFromVideo(Context context, String filePath) + throws IOException, InterruptedException { + ImmutableList.Builder bitmaps = new ImmutableList.Builder<>(); + try (VideoDecodingWrapper decodingWrapper = + new VideoDecodingWrapper( + context, filePath, /* comparisonInterval= */ 1, /* maxImagesAllowed= */ 1)) { + while (true) { + @Nullable Image image = decodingWrapper.runUntilComparisonFrameOrEnded(); + if (image == null) { + break; + } + bitmaps.add(BitmapPixelTestUtil.createGrayscaleArgb8888BitmapFromYuv420888Image(image)); + image.close(); + } + } + return bitmaps.build(); + } + + private static void assertBitmapsMatchExpected(List actualBitmaps, String testId) + throws IOException { + for (int i = 0; i < actualBitmaps.size(); i++) { + Bitmap actualBitmap = actualBitmaps.get(i); + String subTestId = testId + "_" + i; + Bitmap expectedBitmap = + readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId)); + + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + } +}