Composition audio effects

Implement composition-level audio effects in AudioGraph.

PiperOrigin-RevId: 610689632

RELEASENOTES.md

@@ -16,6 +16,7 @@
         negative presentation timestamps before API 30.
     *   Relax trim optimization H.264 level checks.
     *   Add support for changing between SDR and HDR input media in a sequence.
+    *   Add support for composition-level audio effects.
 *   Track Selection:
 *   Extractors:
 *   Audio:

libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java

@@ -22,8 +22,11 @@ import static androidx.media3.common.util.Assertions.checkArgument;
 import android.util.SparseArray;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
+import androidx.media3.common.audio.AudioProcessingPipeline;
+import androidx.media3.common.audio.AudioProcessor;
 import androidx.media3.common.audio.AudioProcessor.AudioFormat;
 import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException;
+import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
@@ -31,17 +34,19 @@ import java.util.Objects;
 /* package */ final class AudioGraph {
   private final AudioMixer mixer;
   private final SparseArray<AudioGraphInput> inputs;
+  private final AudioProcessingPipeline audioProcessingPipeline;
 
-  private AudioFormat outputAudioFormat;
+  private AudioFormat mixerAudioFormat;
   private int finishedInputs;
-  private ByteBuffer currentOutput;
+  private ByteBuffer mixerOutput;
 
   /** Creates an instance. */
-  public AudioGraph(AudioMixer.Factory mixerFactory) {
+  public AudioGraph(AudioMixer.Factory mixerFactory, ImmutableList<AudioProcessor> effects) {
     mixer = mixerFactory.create();
     inputs = new SparseArray<>();
-    currentOutput = EMPTY_BUFFER;
-    outputAudioFormat = AudioFormat.NOT_SET;
+    audioProcessingPipeline = new AudioProcessingPipeline(effects);
+    mixerOutput = EMPTY_BUFFER;
+    mixerAudioFormat = AudioFormat.NOT_SET;
   }
 
   /** Returns whether an {@link AudioFormat} is valid as an input format. */
@@ -65,13 +70,15 @@ import java.util.Objects;
    *
    * <p>Should be called at most once, before {@link #registerInput registering input}.
    *
-   * @param requestedAudioFormat The {@link AudioFormat} requested for output from the mixer.
+   * @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer.
    * @throws UnhandledAudioFormatException If the audio format is not supported by the {@link
    *     AudioMixer}.
    */
-  public void configure(AudioFormat requestedAudioFormat) throws UnhandledAudioFormatException {
-    this.outputAudioFormat = requestedAudioFormat;
-    mixer.configure(requestedAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
+  public void configure(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException {
+    this.mixerAudioFormat = mixerAudioFormat;
+    mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0);
+    audioProcessingPipeline.configure(mixerAudioFormat);
+    audioProcessingPipeline.flush();
   }
 
   /**
@@ -85,9 +92,9 @@ import java.util.Objects;
     checkArgument(format.pcmEncoding != Format.NO_VALUE);
     try {
       AudioGraphInput audioGraphInput =
-          new AudioGraphInput(outputAudioFormat, editedMediaItem, format);
+          new AudioGraphInput(mixerAudioFormat, editedMediaItem, format);
 
-      if (Objects.equals(outputAudioFormat, AudioFormat.NOT_SET)) {
+      if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) {
         // Graph not configured, configure before doing anything else.
         configure(audioGraphInput.getOutputAudioFormat());
       }
@@ -105,7 +112,7 @@ import java.util.Objects;
    * AudioFormat#NOT_SET} if not {@linkplain #configure configured}.
    */
   public AudioFormat getOutputAudioFormat() {
-    return outputAudioFormat;
+    return audioProcessingPipeline.getOutputAudioFormat();
   }
 
   /**
@@ -117,11 +124,16 @@ import java.util.Objects;
     if (!mixer.isEnded()) {
       feedMixer();
     }
-    if (currentOutput.hasRemaining()) {
-      return currentOutput;
+    if (!mixerOutput.hasRemaining()) {
+      mixerOutput = mixer.getOutput();
     }
-    currentOutput = mixer.getOutput();
-    return currentOutput;
+
+    if (audioProcessingPipeline.isOperational()) {
+      feedProcessingPipelineFromMixer();
+      return audioProcessingPipeline.getOutput();
+    }
+
+    return mixerOutput;
   }
 
   /** Resets the graph to an unconfigured state, releasing any underlying resources. */
@@ -131,15 +143,31 @@ import java.util.Objects;
     }
     inputs.clear();
     mixer.reset();
+    audioProcessingPipeline.reset();
 
     finishedInputs = 0;
-    currentOutput = EMPTY_BUFFER;
-    outputAudioFormat = AudioFormat.NOT_SET;
+    mixerOutput = EMPTY_BUFFER;
+    mixerAudioFormat = AudioFormat.NOT_SET;
   }
 
   /** Returns whether the input has ended and all queued data has been output. */
   public boolean isEnded() {
-    return !currentOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded();
+    if (audioProcessingPipeline.isOperational()) {
+      return audioProcessingPipeline.isEnded();
+    }
+    return isMixerEnded();
+  }
+
+  private boolean isMixerEnded() {
+    return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded();
+  }
+
+  private void feedProcessingPipelineFromMixer() {
+    if (isMixerEnded()) {
+      audioProcessingPipeline.queueEndOfStream();
+      return;
+    }
+    audioProcessingPipeline.queueInput(mixerOutput);
   }
 
   private void feedMixer() throws ExportException {

libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java

@@ -24,9 +24,11 @@ import static java.lang.Math.min;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
+import androidx.media3.common.audio.AudioProcessor;
 import androidx.media3.common.audio.AudioProcessor.AudioFormat;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
+import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import org.checkerframework.dataflow.qual.Pure;
 
@@ -50,13 +52,14 @@ import org.checkerframework.dataflow.qual.Pure;
       Format firstInputFormat,
       TransformationRequest transformationRequest,
       EditedMediaItem firstEditedMediaItem,
+      ImmutableList<AudioProcessor> compositionAudioProcessors,
       AudioMixer.Factory mixerFactory,
       Codec.EncoderFactory encoderFactory,
       MuxerWrapper muxerWrapper,
       FallbackListener fallbackListener)
       throws ExportException {
     super(firstAssetLoaderTrackFormat, muxerWrapper);
-    audioGraph = new AudioGraph(mixerFactory);
+    audioGraph = new AudioGraph(mixerFactory, compositionAudioProcessors);
     this.firstInputFormat = firstInputFormat;
     firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat);
     encoderInputAudioFormat = audioGraph.getOutputAudioFormat();

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -950,8 +950,6 @@ public final class Transformer {
    * <p>This method is under development. A {@link Composition} must meet the following conditions:
    *
    * <ul>
-   *   <li>The {@linkplain Composition#effects composition effects} must contain no {@linkplain
-   *       Effects#audioProcessors audio effects}.
    *   <li>The video composition {@link Presentation} effect is applied after input streams are
    *       composited. Other composition effects are ignored.
    * </ul>
@@ -1539,7 +1537,6 @@ public final class Transformer {
       ComponentListener componentListener,
       long initialTimestampOffsetUs,
       boolean useDefaultAssetLoaderFactory) {
-    checkArgument(composition.effects.audioProcessors.isEmpty());
     checkState(transformerInternal == null, "There is already an export in progress.");
     TransformationRequest transformationRequest = this.transformationRequest;
     if (composition.hdrMode != Composition.HDR_MODE_KEEP_HDR) {

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -655,6 +655,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 /* firstInputFormat= */ assetLoaderOutputFormat,
                 transformationRequest,
                 firstEditedMediaItem,
+                composition.effects.audioProcessors,
                 audioMixerFactory,
                 encoderFactory,
                 muxerWrapper,

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java

@@ -93,6 +93,9 @@ import com.google.common.collect.ImmutableList;
     if (!firstEditedMediaItem.effects.audioProcessors.isEmpty()) {
       return true;
     }
+    if (!composition.effects.audioProcessors.isEmpty()) {
+      return true;
+    }
     return false;
   }
 

libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java

@@ -21,7 +21,9 @@ import static com.google.common.truth.Truth.assertThat;
 import androidx.media3.common.C;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.audio.AudioProcessor.AudioFormat;
+import androidx.media3.common.audio.SonicAudioProcessor;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -44,7 +46,7 @@ public class AudioGraphTest {
 
   @Test
   public void silentItem_outputsCorrectAmountOfBytes() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
     GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
     input.onMediaItemChanged(
@@ -56,16 +58,33 @@ public class AudioGraphTest {
     assertThat(bytesOutput).isEqualTo(3 * 50_000 * 2 * 6);
   }
 
+  @Test
+  public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception {
+    SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor();
+    changeTo100000Hz.setOutputSampleRateHz(100_000);
+    AudioGraph audioGraph =
+        new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(changeTo100000Hz));
+
+    GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
+    input.onMediaItemChanged(
+        FAKE_ITEM, /* durationUs= */ 3_000_000, /* trackFormat= */ null, /* isLast= */ true);
+    int bytesOutput = drainAudioGraph(audioGraph);
+
+    // 3 second stream with 100_000 frames per second.
+    // 16 bit PCM has 2 bytes per channel.
+    assertThat(bytesOutput).isEqualTo(3 * 100_000 * 2 * 6);
+  }
+
   @Test
   public void getOutputAudioFormat_afterInitialization_isNotSet() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
     assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET);
   }
 
   @Test
   public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
     audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000));
 
@@ -74,18 +93,18 @@ public class AudioGraphTest {
 
   @Test
   public void getOutputAudioFormat_afterConfigure_matchesConfiguredFormat() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
-    audioGraph.configure(SURROUND_50000);
+    audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
 
     assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(SURROUND_50000);
   }
 
   @Test
   public void registerInput_afterConfigure_doesNotChangeOutputFormat() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
-    audioGraph.configure(STEREO_44100);
+    audioGraph.configure(/* mixerAudioFormat= */ STEREO_44100);
     audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
     audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
 
@@ -94,7 +113,7 @@ public class AudioGraphTest {
 
   @Test
   public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception {
-    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory());
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
 
     audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
     audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
@@ -102,6 +121,56 @@ public class AudioGraphTest {
     assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(STEREO_48000);
   }
 
+  @Test
+  public void registerInput_afterReset_changesOutputFormat() throws Exception {
+    AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of());
+
+    audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000));
+    audioGraph.reset();
+    audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100));
+
+    assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100);
+  }
+
+  @Test
+  public void configure_withAudioProcessor_affectsOutputFormat() throws Exception {
+    SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
+    sonicAudioProcessor.setOutputSampleRateHz(48_000);
+    AudioGraph audioGraph =
+        new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(sonicAudioProcessor));
+
+    audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000);
+
+    assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
+  }
+
+  @Test
+  public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception {
+    SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
+    sonicAudioProcessor.setOutputSampleRateHz(48_000);
+    AudioGraph audioGraph =
+        new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(sonicAudioProcessor));
+
+    audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
+
+    assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
+  }
+
+  @Test
+  public void registerInput_withMultipleAudioProcessors_affectsOutputFormat() throws Exception {
+    SonicAudioProcessor changeTo96000Hz = new SonicAudioProcessor();
+    changeTo96000Hz.setOutputSampleRateHz(96_000);
+    SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor();
+    changeTo48000Hz.setOutputSampleRateHz(48_000);
+    AudioGraph audioGraph =
+        new AudioGraph(
+            new DefaultAudioMixer.Factory(), ImmutableList.of(changeTo96000Hz, changeTo48000Hz));
+
+    audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000));
+
+    assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000);
+  }
+
   /** Drains the graph and returns the number of bytes output. */
   private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException {
     int bytesOutput = 0;