sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-25 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add test for MultiInputVideoGraph

Browse source 
This test composites the first frame from two video inputs.

PiperOrigin-RevId: 565090338
This commit is contained in:
claincly 2023-09-13 10:25:11 -07:00 committed by Copybara-Service
parent 5106f2f45a
commit 1d8135e563
6 changed files with 232 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesEachWithOneVideoMediaItem_succeeds_0.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 71 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesOneWithVideoOneWithImage_succeeds_0.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 78 KiB

7
libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java
View file

@ -118,6 +118,13 @@ public class BitmapPixelTestUtil {
*/
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16 = .01f;
/**
* Maximum allowed average pixel difference between bitmaps generated from luma values.
*
* @see #MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE
*/
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA = 8.0f;
/**
* Reads a bitmap from the specified asset location.
*

26
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View file

@ -22,9 +22,11 @@ import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Util.SDK_INT;
import static org.junit.Assume.assumeFalse;
import android.content.Context;
import android.graphics.Bitmap;
import android.media.Image;
import android.media.MediaFormat;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
@ -46,6 +48,9 @@ import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.media3.test.utils.VideoDecodingWrapper;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.FileWriter;
@ -600,6 +605,27 @@ public final class AndroidTestUtil {
writeTestSummaryToFile(context, testId, testJson);
}
public static ImmutableList<Bitmap> extractBitmapsFromVideo(Context context, String filePath)
throws IOException, InterruptedException {
// b/298599172 - runUntilComparisonFrameOrEnded fails on this device because reading decoder
// output as a bitmap doesn't work.
assumeFalse(Util.SDK_INT == 21 && Ascii.toLowerCase(Util.MODEL).contains("nexus"));
ImmutableList.Builder<Bitmap> bitmaps = new ImmutableList.Builder<>();
try (VideoDecodingWrapper decodingWrapper =
new VideoDecodingWrapper(
context, filePath, /* comparisonInterval= */ 1, /* maxImagesAllowed= */ 1)) {
while (true) {
@Nullable Image image = decodingWrapper.runUntilComparisonFrameOrEnded();
if (image == null) {
break;
}
bitmaps.add(BitmapPixelTestUtil.createGrayscaleArgb8888BitmapFromYuv420888Image(image));
image.close();
}
}
return bitmaps.build();
}
/** A customizable forwarding {@link Codec.EncoderFactory} that forces encoding. */
public static final class ForceEncodeEncoderFactory implements Codec.EncoderFactory {

196
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java Normal file
View file

@ -0,0 +1,196 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA;
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.extractBitmapsFromVideo;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import androidx.media3.effect.AlphaScale;
import androidx.media3.effect.Contrast;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Tests for using multiple {@link EditedMediaItemSequence} in a composition. */
@RunWith(AndroidJUnit4.class)
public final class TransformerMultiSequenceCompositionTest {
private static final String PNG_ASSET_BASE_PATH =
"media/bitmap/transformer_multi_sequence_composition_test";
// The duration of one frame of the 30 FPS test video, in milliseconds.
private static final long ONE_FRAME_DURATION_MS = 35;
private static final int EXPORT_WIDTH = 360;
private static final int EXPORT_HEIGHT = 240;
private final Context context = ApplicationProvider.getApplicationContext();
@Test
public void export_withTwoSequencesEachWithOneVideoMediaItem_succeeds() throws Exception {
String testId = "export_withTwoSequencesEachWithOneVideoMediaItem_succeeds";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_FORMAT,
/* outputFormat= */ MP4_ASSET_FORMAT)) {
return;
}
Composition composition =
createComposition(
/* compositionEffects= */ ImmutableList.of(
new Contrast(0.1f),
Presentation.createForWidthAndHeight(
EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT)),
/* firstSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING,
/* effects= */ ImmutableList.of(
new AlphaScale(0.5f),
new ScaleAndRotateTransformation.Builder()
.setRotationDegrees(180)
.build()))),
/* secondSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
.build()
.run(testId, composition);
assertThat(result.filePath).isNotNull();
assertBitmapsMatchExpected(
extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
}
@Test
public void export_withTwoSequencesOneWithVideoOneWithImage_succeeds() throws Exception {
String testId = "export_withTwoSequencesOneWithVideoOneWithImage_succeeds";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_FORMAT,
/* outputFormat= */ MP4_ASSET_FORMAT)) {
return;
}
Composition composition =
createComposition(
/* compositionEffects= */ ImmutableList.of(
new Contrast(0.1f),
Presentation.createForWidthAndHeight(
EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT)),
/* firstSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING,
/* effects= */ ImmutableList.of(
new AlphaScale(0.5f),
new ScaleAndRotateTransformation.Builder()
.setRotationDegrees(180)
.build()))),
/* secondSequenceMediaItems= */ ImmutableList.of(
editedMediaItemOfOneFrameImage(
JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
.build()
.run(testId, composition);
assertThat(result.filePath).isNotNull();
assertBitmapsMatchExpected(
extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
}
private static EditedMediaItem editedMediaItemByClippingVideo(String uri, List<Effect> effects) {
return new EditedMediaItem.Builder(
MediaItem.fromUri(uri)
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setEndPositionMs(ONE_FRAME_DURATION_MS)
.build())
.build())
.setRemoveAudio(true)
.setEffects(
new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects)))
.build();
}
private static EditedMediaItem editedMediaItemOfOneFrameImage(String uri, List<Effect> effects) {
return new EditedMediaItem.Builder(MediaItem.fromUri(uri))
.setRemoveAudio(true)
.setDurationUs(msToUs(ONE_FRAME_DURATION_MS))
.setFrameRate((int) (1000 / ONE_FRAME_DURATION_MS))
.setEffects(
new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects)))
.build();
}
private static Composition createComposition(
List<Effect> compositionEffects,
List<EditedMediaItem> firstSequenceMediaItems,
List<EditedMediaItem> secondSequenceMediaItems) {
return new Composition.Builder(
ImmutableList.of(
new EditedMediaItemSequence(firstSequenceMediaItems),
new EditedMediaItemSequence(secondSequenceMediaItems)))
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(), /* videoEffects= */ compositionEffects))
.build();
}
private static void assertBitmapsMatchExpected(List<Bitmap> actualBitmaps, String testId)
throws IOException {
for (int i = 0; i < actualBitmaps.size(); i++) {
Bitmap actualBitmap = actualBitmaps.get(i);
String subTestId = testId + "_" + i;
Bitmap expectedBitmap =
readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId));
maybeSaveTestBitmap(
testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA);
}
}
}

37
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java
View file

@ -18,6 +18,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA;
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
@ -26,12 +27,11 @@ import static androidx.media3.transformer.AndroidTestUtil.JPG_PORTRAIT_ASSET_URI
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.extractBitmapsFromVideo;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assume.assumeFalse;
import android.content.Context;
import android.graphics.Bitmap;
import android.media.Image;
import androidx.annotation.Nullable;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
@ -41,11 +41,8 @@ import androidx.media3.effect.OverlayEffect;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.media3.test.utils.VideoDecodingWrapper;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.List;
@ -65,13 +62,6 @@ public final class TransformerSequenceEffectTest {
private static final int EXPORT_WIDTH = 360;
private static final int EXPORT_HEIGHT = 240;
/**
* Maximum allowed average pixel difference between bitmaps generated from luma values.
*
* @see BitmapPixelTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE
*/
private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA = 8.0f;
private final Context context = ApplicationProvider.getApplicationContext();
@Test
@ -257,28 +247,7 @@ public final class TransformerSequenceEffectTest {
.build();
}
private static ImmutableList<Bitmap> extractBitmapsFromVideo(Context context, String filePath)
throws IOException, InterruptedException {
// b/298599172 - runUntilComparisonFrameOrEnded fails on this device because reading decoder
// output as a bitmap doesn't work.
assumeFalse(Util.SDK_INT == 21 && Ascii.toLowerCase(Util.MODEL).contains("nexus"));
ImmutableList.Builder<Bitmap> bitmaps = new ImmutableList.Builder<>();
try (VideoDecodingWrapper decodingWrapper =
new VideoDecodingWrapper(
context, filePath, /* comparisonInterval= */ 1, /* maxImagesAllowed= */ 1)) {
while (true) {
@Nullable Image image = decodingWrapper.runUntilComparisonFrameOrEnded();
if (image == null) {
break;
}
bitmaps.add(BitmapPixelTestUtil.createGrayscaleArgb8888BitmapFromYuv420888Image(image));
image.close();
}
}
return bitmaps.build();
}
private static void assertBitmapsMatchExpected(List<Bitmap> actualBitmaps, String testId)
static void assertBitmapsMatchExpected(List<Bitmap> actualBitmaps, String testId)
throws IOException {
for (int i = 0; i < actualBitmaps.size(); i++) {
Bitmap actualBitmap = actualBitmaps.get(i);