Support transmux when both no op effects and regular rotations are set

PiperOrigin-RevId: 604319412

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -65,6 +65,7 @@ import androidx.media3.effect.Contrast;
 import androidx.media3.effect.DefaultGlObjectsProvider;
 import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.FrameCache;
+import androidx.media3.effect.GlEffect;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.RgbFilter;
 import androidx.media3.effect.ScaleAndRotateTransformation;
@@ -87,6 +88,7 @@ import org.junit.runner.RunWith;
 @RunWith(AndroidJUnit4.class)
 public class TransformerEndToEndTest {
 
+  private static final GlEffect NO_OP_EFFECT = new Contrast(0f);
   private final Context context = ApplicationProvider.getApplicationContext();
 
   private volatile @MonotonicNonNull TextureAssetLoader textureAssetLoader;
@@ -449,6 +451,50 @@ public class TransformerEndToEndTest {
     assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
   }
 
+  @Test
+  public void
+      clippedAndRotatedMedia_withNoOpEffect_completesWithClippedDurationAndCorrectOrientation()
+          throws Exception {
+    String testId =
+        "clippedAndRotatedMedia_withNoOpEffect_completesWithClippedDurationAndCorrectOrientation";
+    if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
+        /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
+      return;
+    }
+    Transformer transformer = new Transformer.Builder(context).build();
+    long clippingStartMs = 10_000;
+    long clippingEndMs = 11_000;
+    MediaItem mediaItem =
+        new MediaItem.Builder()
+            .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
+            .setClippingConfiguration(
+                new MediaItem.ClippingConfiguration.Builder()
+                    .setStartPositionMs(clippingStartMs)
+                    .setEndPositionMs(clippingEndMs)
+                    .build())
+            .build();
+    ImmutableList<Effect> videoEffects =
+        ImmutableList.of(
+            new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
+            NO_OP_EFFECT);
+    Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
+    Format format = FileUtil.retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
+    // The output video is portrait, but Transformer's default setup encodes videos landscape.
+    assertThat(format.rotationDegrees).isEqualTo(90);
+  }
+
   @Test
   public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
       throws Exception {
@@ -516,7 +562,8 @@ public class TransformerEndToEndTest {
             .build();
     ImmutableList<Effect> videoEffects =
         ImmutableList.of(
-            new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
+            new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
+            NO_OP_EFFECT);
     Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
     EditedMediaItem editedMediaItem =
         new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
@@ -642,10 +689,10 @@ public class TransformerEndToEndTest {
 
   @Test
   public void
-      clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()
+      clippedMediaAudioRemovedNoOpEffectAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()
           throws Exception {
     String testId =
-        "clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation";
+        "clippedMediaAudioRemovedNoOpEffectAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation";
     if (!isRunningOnEmulator() || Util.SDK_INT != 33) {
       // The trim optimization is only guaranteed to work on emulator for this (emulator-transcoded)
       // file.
@@ -667,7 +714,8 @@ public class TransformerEndToEndTest {
         new Effects(
             /* audioProcessors= */ ImmutableList.of(),
             ImmutableList.of(
-                new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
+                new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
+                NO_OP_EFFECT));
     EditedMediaItem editedMediaItem =
         new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).setEffects(effects).build();
 

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -1440,7 +1440,9 @@ public final class Transformer {
             }
             Transformer.this.mediaItemInfo = mp4Info;
             maybeSetMuxerWrapperAdditionalRotationDegrees(
-                remuxingMuxerWrapper, firstEditedMediaItem.effects.videoEffects);
+                remuxingMuxerWrapper,
+                firstEditedMediaItem.effects.videoEffects,
+                checkNotNull(mp4Info.videoFormat));
             Composition trancodeComposition =
                 buildUponCompositionForTrimOptimization(
                     composition,

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -558,7 +558,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             && getProcessedTrackType(firstAssetLoaderInputFormat.sampleMimeType)
                 == TRACK_TYPE_VIDEO) {
           maybeSetMuxerWrapperAdditionalRotationDegrees(
-              muxerWrapper, firstEditedMediaItem.effects.videoEffects);
+              muxerWrapper, firstEditedMediaItem.effects.videoEffects, firstAssetLoaderInputFormat);
         }
         assetLoaderInputTracker.setShouldTranscode(trackType, shouldTranscode);
         return shouldTranscode;

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java

@@ -17,6 +17,7 @@
 package androidx.media3.transformer;
 
 import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR;
+import static java.lang.Math.round;
 
 import android.media.MediaCodec;
 import androidx.annotation.Nullable;
@@ -144,72 +145,67 @@ import com.google.common.collect.ImmutableList;
     }
     ImmutableList<Effect> videoEffects = firstEditedMediaItem.effects.videoEffects;
     return !videoEffects.isEmpty()
-        && !areVideoEffectsAllNoOp(videoEffects, inputFormat)
+        && maybeCalculateTotalRotationDegreesAppliedInEffects(videoEffects, inputFormat) == -1;
-        && getRegularRotationDegrees(videoEffects) == -1;
   }
 
   /**
-   * Returns whether the collection of {@code videoEffects} would be a {@linkplain
+   * Returns the total rotation degrees of all the rotations in {@code videoEffects}, or {@code -1}
-   * GlEffect#isNoOp(int, int) no-op}, if queued samples of this {@link Format}.
+   * if {@code videoEffects} contains any effects that are not no-ops or regular rotations.
+   *
+   * <p>If all the {@code videoEffects} are either noOps or regular rotations, then the rotations
+   * can be applied in the {@linkplain #maybeSetMuxerWrapperAdditionalRotationDegrees(MuxerWrapper,
+   * ImmutableList, Format) MuxerWrapper}.
    */
-  public static boolean areVideoEffectsAllNoOp(
+  private static float maybeCalculateTotalRotationDegreesAppliedInEffects(
       ImmutableList<Effect> videoEffects, Format inputFormat) {
-    int decodedWidth =
+    int width = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
-        (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
+    int height = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
-    int decodedHeight =
+    float totalRotationDegrees = 0;
-        (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
     for (int i = 0; i < videoEffects.size(); i++) {
       Effect videoEffect = videoEffects.get(i);
       if (!(videoEffect instanceof GlEffect)) {
         // We cannot confirm whether Effect instances that are not GlEffect instances are
         // no-ops.
-        return false;
+        return -1;
       }
       GlEffect glEffect = (GlEffect) videoEffect;
-      if (!glEffect.isNoOp(decodedWidth, decodedHeight)) {
+      if (videoEffect instanceof ScaleAndRotateTransformation) {
-        return false;
+        ScaleAndRotateTransformation scaleAndRotateTransformation =
+            (ScaleAndRotateTransformation) videoEffect;
+        if (scaleAndRotateTransformation.scaleX != 1f
+            || scaleAndRotateTransformation.scaleY != 1f) {
+          return -1;
+        }
+        float rotationDegrees = scaleAndRotateTransformation.rotationDegrees;
+        if (rotationDegrees % 90f != 0) {
+          return -1;
+        }
+        totalRotationDegrees += rotationDegrees;
+        width = (totalRotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
+        height = (totalRotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
+        continue;
+      }
+      if (!glEffect.isNoOp(width, height)) {
+        return -1;
       }
     }
-    return true;
+    totalRotationDegrees %= 360;
+    return totalRotationDegrees % 90 == 0 ? totalRotationDegrees : -1;
   }
 
-  private static float getRegularRotationDegrees(ImmutableList<Effect> videoEffects) {
-    if (videoEffects.size() != 1) {
-      return -1;
-    }
-    Effect videoEffect = videoEffects.get(0);
-    if (!(videoEffect instanceof ScaleAndRotateTransformation)) {
-      return -1;
-    }
-    ScaleAndRotateTransformation scaleAndRotateTransformation =
-        (ScaleAndRotateTransformation) videoEffect;
-    if (scaleAndRotateTransformation.scaleX != 1f || scaleAndRotateTransformation.scaleY != 1f) {
-      return -1;
-    }
-    float rotationDegrees = scaleAndRotateTransformation.rotationDegrees;
-    if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
-      return rotationDegrees;
-    }
-    return -1;
-  }
-
-  // TODO: b/322146331 - Support setting MuxerWrapper#setAdditionalRotationDegrees(int) for
-  //  videoEffects lists that are a mix of no-ops and rotations.
   /**
    * Sets {@linkplain MuxerWrapper#setAdditionalRotationDegrees(int) the additionalRotationDegrees}
-   * on the given {@link MuxerWrapper} if the given {@code videoEffects} only contains one regular
+   * on the given {@link MuxerWrapper} if the given {@code videoEffects} only contains a mix of
-   * rotation effect. A regular rotation is a rotation divisible by 90 degrees.
+   * regular rotations and no-ops. A regular rotation is a rotation divisible by 90 degrees.
    */
   public static void maybeSetMuxerWrapperAdditionalRotationDegrees(
-      MuxerWrapper muxerWrapper, ImmutableList<Effect> videoEffects) {
+      MuxerWrapper muxerWrapper, ImmutableList<Effect> videoEffects, Format inputFormat) {
-    float rotationDegrees = getRegularRotationDegrees(videoEffects);
+    float rotationDegrees =
-    if (rotationDegrees == -1) {
+        maybeCalculateTotalRotationDegreesAppliedInEffects(videoEffects, inputFormat);
-      return;
-    }
     if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
       // The MuxerWrapper rotation is clockwise while the ScaleAndRotateTransformation rotation
       // is counterclockwise.
-      muxerWrapper.setAdditionalRotationDegrees(360 - Math.round(rotationDegrees));
+      muxerWrapper.setAdditionalRotationDegrees(360 - round(rotationDegrees));
     }
   }
 }

libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java

@@ -66,6 +66,7 @@ import androidx.media3.common.Format;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.audio.SonicAudioProcessor;
+import androidx.media3.effect.Contrast;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.ScaleAndRotateTransformation;
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
@@ -1098,6 +1099,33 @@ public final class MediaItemExportTest {
             /* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
   }
 
+  @Test
+  public void start_regularRotationsAndNoOps_transmuxes() throws Exception {
+    Transformer transformer =
+        createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
+    // Total rotation is 270.
+    ImmutableList<Effect> videoEffects =
+        ImmutableList.of(
+            new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
+            new Contrast(0f),
+            new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build(),
+            Presentation.createForHeight(1080));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO))
+            .setEffects(new Effects(ImmutableList.of(), videoEffects))
+            .build();
+
+    transformer.start(editedMediaItem, outputDir.newFile().getPath());
+    TransformerTestRunner.runLooper(transformer);
+
+    // Video transcoding in unit tests is not supported.
+    DumpFileAsserts.assertOutput(
+        context,
+        muxerFactory.getCreatedMuxer(),
+        getDumpFileName(
+            /* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
+  }
+
   @Test
   public void getProgress_unknownDuration_returnsConsistentStates() throws Exception {
     Transformer transformer =