sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-16 13:05:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support transmux when both no op effects and regular rotations are set

Browse source 
PiperOrigin-RevId: 604319412
This commit is contained in:
tofunmi 2024-02-05 08:01:59 -08:00 committed by Copybara-Service
parent f8352580cb
commit 4da576f05f
5 changed files with 123 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View file

@ -65,6 +65,7 @@ import androidx.media3.effect.Contrast;
import androidx.media3.effect.DefaultGlObjectsProvider;
import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.FrameCache;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.ScaleAndRotateTransformation;
@ -87,6 +88,7 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class TransformerEndToEndTest {
private static final GlEffect NO_OP_EFFECT = new Contrast(0f);
private final Context context = ApplicationProvider.getApplicationContext();
private volatile @MonotonicNonNull TextureAssetLoader textureAssetLoader;
@ -449,6 +451,50 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}
@Test
public void
clippedAndRotatedMedia_withNoOpEffect_completesWithClippedDurationAndCorrectOrientation()
throws Exception {
String testId =
"clippedAndRotatedMedia_withNoOpEffect_completesWithClippedDurationAndCorrectOrientation";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
return;
}
Transformer transformer = new Transformer.Builder(context).build();
long clippingStartMs = 10_000;
long clippingEndMs = 11_000;
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(clippingStartMs)
.setEndPositionMs(clippingEndMs)
.build())
.build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
NO_OP_EFFECT);
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
Format format = FileUtil.retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
// The output video is portrait, but Transformer's default setup encodes videos landscape.
assertThat(format.rotationDegrees).isEqualTo(90);
}
@Test
public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
throws Exception {
@ -516,7 +562,8 @@ public class TransformerEndToEndTest {
.build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
NO_OP_EFFECT);
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
@ -642,10 +689,10 @@ public class TransformerEndToEndTest {
@Test
public void
clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()
clippedMediaAudioRemovedNoOpEffectAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()
throws Exception {
String testId =
"clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation";
"clippedMediaAudioRemovedNoOpEffectAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation";
if (!isRunningOnEmulator() || Util.SDK_INT != 33) {
// The trim optimization is only guaranteed to work on emulator for this (emulator-transcoded)
// file.
@ -667,7 +714,8 @@ public class TransformerEndToEndTest {
new Effects(
/* audioProcessors= */ ImmutableList.of(),
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
NO_OP_EFFECT));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).setEffects(effects).build();

4
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View file

@ -1440,7 +1440,9 @@ public final class Transformer {
}
Transformer.this.mediaItemInfo = mp4Info;
maybeSetMuxerWrapperAdditionalRotationDegrees(
remuxingMuxerWrapper, firstEditedMediaItem.effects.videoEffects);
remuxingMuxerWrapper,
firstEditedMediaItem.effects.videoEffects,
checkNotNull(mp4Info.videoFormat));
Composition trancodeComposition =
buildUponCompositionForTrimOptimization(
composition,

2
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View file

@ -558,7 +558,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
&& getProcessedTrackType(firstAssetLoaderInputFormat.sampleMimeType)
== TRACK_TYPE_VIDEO) {
maybeSetMuxerWrapperAdditionalRotationDegrees(
muxerWrapper, firstEditedMediaItem.effects.videoEffects);
muxerWrapper, firstEditedMediaItem.effects.videoEffects, firstAssetLoaderInputFormat);
}
assetLoaderInputTracker.setShouldTranscode(trackType, shouldTranscode);
return shouldTranscode;

82
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View file

@ -17,6 +17,7 @@
package androidx.media3.transformer;
import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR;
import static java.lang.Math.round;
import android.media.MediaCodec;
import androidx.annotation.Nullable;
@ -144,72 +145,67 @@ import com.google.common.collect.ImmutableList;
}
ImmutableList<Effect> videoEffects = firstEditedMediaItem.effects.videoEffects;
return !videoEffects.isEmpty()
&& !areVideoEffectsAllNoOp(videoEffects, inputFormat)
&& getRegularRotationDegrees(videoEffects) == -1;
&& maybeCalculateTotalRotationDegreesAppliedInEffects(videoEffects, inputFormat) == -1;
}
/**
* Returns whether the collection of {@code videoEffects} would be a {@linkplain
* GlEffect#isNoOp(int, int) no-op}, if queued samples of this {@link Format}.
* Returns the total rotation degrees of all the rotations in {@code videoEffects}, or {@code -1}
* if {@code videoEffects} contains any effects that are not no-ops or regular rotations.
*
* <p>If all the {@code videoEffects} are either noOps or regular rotations, then the rotations
* can be applied in the {@linkplain #maybeSetMuxerWrapperAdditionalRotationDegrees(MuxerWrapper,
* ImmutableList, Format) MuxerWrapper}.
*/
public static boolean areVideoEffectsAllNoOp(
private static float maybeCalculateTotalRotationDegreesAppliedInEffects(
ImmutableList<Effect> videoEffects, Format inputFormat) {
int decodedWidth =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
int width = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
int height = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
float totalRotationDegrees = 0;
for (int i = 0; i < videoEffects.size(); i++) {
Effect videoEffect = videoEffects.get(i);
if (!(videoEffect instanceof GlEffect)) {
// We cannot confirm whether Effect instances that are not GlEffect instances are
// no-ops.
return false;
return -1;
}
GlEffect glEffect = (GlEffect) videoEffect;
if (!glEffect.isNoOp(decodedWidth, decodedHeight)) {
return false;
if (videoEffect instanceof ScaleAndRotateTransformation) {
ScaleAndRotateTransformation scaleAndRotateTransformation =
(ScaleAndRotateTransformation) videoEffect;
if (scaleAndRotateTransformation.scaleX != 1f
|| scaleAndRotateTransformation.scaleY != 1f) {
return -1;
}
float rotationDegrees = scaleAndRotateTransformation.rotationDegrees;
if (rotationDegrees % 90f != 0) {
return -1;
}
totalRotationDegrees += rotationDegrees;
width = (totalRotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
height = (totalRotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
continue;
}
if (!glEffect.isNoOp(width, height)) {
return -1;
}
}
return true;
totalRotationDegrees %= 360;
return totalRotationDegrees % 90 == 0 ? totalRotationDegrees : -1;
}
private static float getRegularRotationDegrees(ImmutableList<Effect> videoEffects) {
if (videoEffects.size() != 1) {
return -1;
}
Effect videoEffect = videoEffects.get(0);
if (!(videoEffect instanceof ScaleAndRotateTransformation)) {
return -1;
}
ScaleAndRotateTransformation scaleAndRotateTransformation =
(ScaleAndRotateTransformation) videoEffect;
if (scaleAndRotateTransformation.scaleX != 1f || scaleAndRotateTransformation.scaleY != 1f) {
return -1;
}
float rotationDegrees = scaleAndRotateTransformation.rotationDegrees;
if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
return rotationDegrees;
}
return -1;
}
// TODO: b/322146331 - Support setting MuxerWrapper#setAdditionalRotationDegrees(int) for
// videoEffects lists that are a mix of no-ops and rotations.
/**
* Sets {@linkplain MuxerWrapper#setAdditionalRotationDegrees(int) the additionalRotationDegrees}
* on the given {@link MuxerWrapper} if the given {@code videoEffects} only contains one regular
* rotation effect. A regular rotation is a rotation divisible by 90 degrees.
* on the given {@link MuxerWrapper} if the given {@code videoEffects} only contains a mix of
* regular rotations and no-ops. A regular rotation is a rotation divisible by 90 degrees.
*/
public static void maybeSetMuxerWrapperAdditionalRotationDegrees(
MuxerWrapper muxerWrapper, ImmutableList<Effect> videoEffects) {
float rotationDegrees = getRegularRotationDegrees(videoEffects);
if (rotationDegrees == -1) {
return;
}
MuxerWrapper muxerWrapper, ImmutableList<Effect> videoEffects, Format inputFormat) {
float rotationDegrees =
maybeCalculateTotalRotationDegreesAppliedInEffects(videoEffects, inputFormat);
if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
// The MuxerWrapper rotation is clockwise while the ScaleAndRotateTransformation rotation
// is counterclockwise.
muxerWrapper.setAdditionalRotationDegrees(360 - Math.round(rotationDegrees));
muxerWrapper.setAdditionalRotationDegrees(360 - round(rotationDegrees));
}
}
}

28
libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java
View file

@ -66,6 +66,7 @@ import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.audio.SonicAudioProcessor;
import androidx.media3.effect.Contrast;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
@ -1098,6 +1099,33 @@ public final class MediaItemExportTest {
/* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
}
@Test
public void start_regularRotationsAndNoOps_transmuxes() throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
// Total rotation is 270.
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
new Contrast(0f),
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
Presentation.createForHeight(1080));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO))
.setEffects(new Effects(ImmutableList.of(), videoEffects))
.build();
transformer.start(editedMediaItem, outputDir.newFile().getPath());
TransformerTestRunner.runLooper(transformer);
// Video transcoding in unit tests is not supported.
DumpFileAsserts.assertOutput(
context,
muxerFactory.getCreatedMuxer(),
getDumpFileName(
/* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
}
@Test
public void getProgress_unknownDuration_returnsConsistentStates() throws Exception {
Transformer transformer =