From 477ace1be9bf687ea4aef03f4566b8368c83953d Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 27 Feb 2024 02:50:55 -0800 Subject: [PATCH] Composition audio effects Implement composition-level audio effects in AudioGraph. PiperOrigin-RevId: 610689632 --- RELEASENOTES.md | 1 + .../media3/transformer/AudioGraph.java | 66 +++++++++----- .../transformer/AudioSampleExporter.java | 5 +- .../media3/transformer/Transformer.java | 3 - .../transformer/TransformerInternal.java | 1 + .../media3/transformer/TransformerUtil.java | 3 + .../media3/transformer/AudioGraphTest.java | 85 +++++++++++++++++-- 7 files changed, 133 insertions(+), 31 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 971aa41cf9..ad682a22d2 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,6 +16,7 @@ negative presentation timestamps before API 30. * Relax trim optimization H.264 level checks. * Add support for changing between SDR and HDR input media in a sequence. + * Add support for composition-level audio effects. * Track Selection: * Extractors: * Audio: diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java index a11e8c27cb..ac180fecc5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraph.java @@ -22,8 +22,11 @@ import static androidx.media3.common.util.Assertions.checkArgument; import android.util.SparseArray; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.audio.AudioProcessingPipeline; +import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.util.Objects; @@ -31,17 +34,19 @@ import java.util.Objects; /* package */ final class AudioGraph { private final AudioMixer mixer; private final SparseArray inputs; + private final AudioProcessingPipeline audioProcessingPipeline; - private AudioFormat outputAudioFormat; + private AudioFormat mixerAudioFormat; private int finishedInputs; - private ByteBuffer currentOutput; + private ByteBuffer mixerOutput; /** Creates an instance. */ - public AudioGraph(AudioMixer.Factory mixerFactory) { + public AudioGraph(AudioMixer.Factory mixerFactory, ImmutableList effects) { mixer = mixerFactory.create(); inputs = new SparseArray<>(); - currentOutput = EMPTY_BUFFER; - outputAudioFormat = AudioFormat.NOT_SET; + audioProcessingPipeline = new AudioProcessingPipeline(effects); + mixerOutput = EMPTY_BUFFER; + mixerAudioFormat = AudioFormat.NOT_SET; } /** Returns whether an {@link AudioFormat} is valid as an input format. */ @@ -65,13 +70,15 @@ import java.util.Objects; * *

Should be called at most once, before {@link #registerInput registering input}. * - * @param requestedAudioFormat The {@link AudioFormat} requested for output from the mixer. + * @param mixerAudioFormat The {@link AudioFormat} requested for output from the mixer. * @throws UnhandledAudioFormatException If the audio format is not supported by the {@link * AudioMixer}. */ - public void configure(AudioFormat requestedAudioFormat) throws UnhandledAudioFormatException { - this.outputAudioFormat = requestedAudioFormat; - mixer.configure(requestedAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); + public void configure(AudioFormat mixerAudioFormat) throws UnhandledAudioFormatException { + this.mixerAudioFormat = mixerAudioFormat; + mixer.configure(mixerAudioFormat, /* bufferSizeMs= */ C.LENGTH_UNSET, /* startTimeUs= */ 0); + audioProcessingPipeline.configure(mixerAudioFormat); + audioProcessingPipeline.flush(); } /** @@ -85,9 +92,9 @@ import java.util.Objects; checkArgument(format.pcmEncoding != Format.NO_VALUE); try { AudioGraphInput audioGraphInput = - new AudioGraphInput(outputAudioFormat, editedMediaItem, format); + new AudioGraphInput(mixerAudioFormat, editedMediaItem, format); - if (Objects.equals(outputAudioFormat, AudioFormat.NOT_SET)) { + if (Objects.equals(mixerAudioFormat, AudioFormat.NOT_SET)) { // Graph not configured, configure before doing anything else. configure(audioGraphInput.getOutputAudioFormat()); } @@ -105,7 +112,7 @@ import java.util.Objects; * AudioFormat#NOT_SET} if not {@linkplain #configure configured}. */ public AudioFormat getOutputAudioFormat() { - return outputAudioFormat; + return audioProcessingPipeline.getOutputAudioFormat(); } /** @@ -117,11 +124,16 @@ import java.util.Objects; if (!mixer.isEnded()) { feedMixer(); } - if (currentOutput.hasRemaining()) { - return currentOutput; + if (!mixerOutput.hasRemaining()) { + mixerOutput = mixer.getOutput(); } - currentOutput = mixer.getOutput(); - return currentOutput; + + if (audioProcessingPipeline.isOperational()) { + feedProcessingPipelineFromMixer(); + return audioProcessingPipeline.getOutput(); + } + + return mixerOutput; } /** Resets the graph to an unconfigured state, releasing any underlying resources. */ @@ -131,15 +143,31 @@ import java.util.Objects; } inputs.clear(); mixer.reset(); + audioProcessingPipeline.reset(); finishedInputs = 0; - currentOutput = EMPTY_BUFFER; - outputAudioFormat = AudioFormat.NOT_SET; + mixerOutput = EMPTY_BUFFER; + mixerAudioFormat = AudioFormat.NOT_SET; } /** Returns whether the input has ended and all queued data has been output. */ public boolean isEnded() { - return !currentOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded(); + if (audioProcessingPipeline.isOperational()) { + return audioProcessingPipeline.isEnded(); + } + return isMixerEnded(); + } + + private boolean isMixerEnded() { + return !mixerOutput.hasRemaining() && finishedInputs >= inputs.size() && mixer.isEnded(); + } + + private void feedProcessingPipelineFromMixer() { + if (isMixerEnded()) { + audioProcessingPipeline.queueEndOfStream(); + return; + } + audioProcessingPipeline.queueInput(mixerOutput); } private void feedMixer() throws ExportException { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java index 1cf2a17bbf..6d434bd74c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java @@ -24,9 +24,11 @@ import static java.lang.Math.min; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.audio.AudioProcessor.AudioFormat; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import org.checkerframework.dataflow.qual.Pure; @@ -50,13 +52,14 @@ import org.checkerframework.dataflow.qual.Pure; Format firstInputFormat, TransformationRequest transformationRequest, EditedMediaItem firstEditedMediaItem, + ImmutableList compositionAudioProcessors, AudioMixer.Factory mixerFactory, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, FallbackListener fallbackListener) throws ExportException { super(firstAssetLoaderTrackFormat, muxerWrapper); - audioGraph = new AudioGraph(mixerFactory); + audioGraph = new AudioGraph(mixerFactory, compositionAudioProcessors); this.firstInputFormat = firstInputFormat; firstInput = audioGraph.registerInput(firstEditedMediaItem, firstInputFormat); encoderInputAudioFormat = audioGraph.getOutputAudioFormat(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index ded7d79d55..7a896ba810 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -950,8 +950,6 @@ public final class Transformer { *

This method is under development. A {@link Composition} must meet the following conditions: * *

    - *
  • The {@linkplain Composition#effects composition effects} must contain no {@linkplain - * Effects#audioProcessors audio effects}. *
  • The video composition {@link Presentation} effect is applied after input streams are * composited. Other composition effects are ignored. *
@@ -1539,7 +1537,6 @@ public final class Transformer { ComponentListener componentListener, long initialTimestampOffsetUs, boolean useDefaultAssetLoaderFactory) { - checkArgument(composition.effects.audioProcessors.isEmpty()); checkState(transformerInternal == null, "There is already an export in progress."); TransformationRequest transformationRequest = this.transformationRequest; if (composition.hdrMode != Composition.HDR_MODE_KEEP_HDR) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index b7d28f85fe..6381bbf2ea 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -655,6 +655,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* firstInputFormat= */ assetLoaderOutputFormat, transformationRequest, firstEditedMediaItem, + composition.effects.audioProcessors, audioMixerFactory, encoderFactory, muxerWrapper, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java index 00eb06e449..6bdeaa7488 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java @@ -93,6 +93,9 @@ import com.google.common.collect.ImmutableList; if (!firstEditedMediaItem.effects.audioProcessors.isEmpty()) { return true; } + if (!composition.effects.audioProcessors.isEmpty()) { + return true; + } return false; } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java index 8b19595cd8..4a8598eabb 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/AudioGraphTest.java @@ -21,7 +21,9 @@ import static com.google.common.truth.Truth.assertThat; import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.audio.SonicAudioProcessor; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import org.junit.Test; import org.junit.runner.RunWith; @@ -44,7 +46,7 @@ public class AudioGraphTest { @Test public void silentItem_outputsCorrectAmountOfBytes() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); input.onMediaItemChanged( @@ -56,16 +58,33 @@ public class AudioGraphTest { assertThat(bytesOutput).isEqualTo(3 * 50_000 * 2 * 6); } + @Test + public void silentItem_withSampleRateChange_outputsCorrectAmountOfBytes() throws Exception { + SonicAudioProcessor changeTo100000Hz = new SonicAudioProcessor(); + changeTo100000Hz.setOutputSampleRateHz(100_000); + AudioGraph audioGraph = + new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(changeTo100000Hz)); + + GraphInput input = audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); + input.onMediaItemChanged( + FAKE_ITEM, /* durationUs= */ 3_000_000, /* trackFormat= */ null, /* isLast= */ true); + int bytesOutput = drainAudioGraph(audioGraph); + + // 3 second stream with 100_000 frames per second. + // 16 bit PCM has 2 bytes per channel. + assertThat(bytesOutput).isEqualTo(3 * 100_000 * 2 * 6); + } + @Test public void getOutputAudioFormat_afterInitialization_isNotSet() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(AudioFormat.NOT_SET); } @Test public void getOutputAudioFormat_afterRegisterInput_matchesInputFormat() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_48000)); @@ -74,18 +93,18 @@ public class AudioGraphTest { @Test public void getOutputAudioFormat_afterConfigure_matchesConfiguredFormat() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); - audioGraph.configure(SURROUND_50000); + audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000); assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(SURROUND_50000); } @Test public void registerInput_afterConfigure_doesNotChangeOutputFormat() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); - audioGraph.configure(STEREO_44100); + audioGraph.configure(/* mixerAudioFormat= */ STEREO_44100); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); @@ -94,7 +113,7 @@ public class AudioGraphTest { @Test public void registerInput_afterRegisterInput_doesNotChangeOutputFormat() throws Exception { - AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory()); + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); @@ -102,6 +121,56 @@ public class AudioGraphTest { assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(STEREO_48000); } + @Test + public void registerInput_afterReset_changesOutputFormat() throws Exception { + AudioGraph audioGraph = new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of()); + + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(STEREO_48000)); + audioGraph.reset(); + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(MONO_44100)); + + assertThat(audioGraph.getOutputAudioFormat()).isEqualTo(MONO_44100); + } + + @Test + public void configure_withAudioProcessor_affectsOutputFormat() throws Exception { + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(48_000); + AudioGraph audioGraph = + new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(sonicAudioProcessor)); + + audioGraph.configure(/* mixerAudioFormat= */ SURROUND_50000); + + assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); + } + + @Test + public void registerInput_withAudioProcessor_affectsOutputFormat() throws Exception { + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(48_000); + AudioGraph audioGraph = + new AudioGraph(new DefaultAudioMixer.Factory(), ImmutableList.of(sonicAudioProcessor)); + + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); + + assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); + } + + @Test + public void registerInput_withMultipleAudioProcessors_affectsOutputFormat() throws Exception { + SonicAudioProcessor changeTo96000Hz = new SonicAudioProcessor(); + changeTo96000Hz.setOutputSampleRateHz(96_000); + SonicAudioProcessor changeTo48000Hz = new SonicAudioProcessor(); + changeTo48000Hz.setOutputSampleRateHz(48_000); + AudioGraph audioGraph = + new AudioGraph( + new DefaultAudioMixer.Factory(), ImmutableList.of(changeTo96000Hz, changeTo48000Hz)); + + audioGraph.registerInput(FAKE_ITEM, getPcmFormat(SURROUND_50000)); + + assertThat(audioGraph.getOutputAudioFormat().sampleRate).isEqualTo(48_000); + } + /** Drains the graph and returns the number of bytes output. */ private static int drainAudioGraph(AudioGraph audioGraph) throws ExportException { int bytesOutput = 0;