diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java index 26a0e7810c..8376c4d157 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java @@ -30,9 +30,6 @@ import androidx.media3.exoplayer.audio.AudioProcessor; import androidx.media3.exoplayer.audio.AudioProcessor.AudioFormat; import androidx.media3.exoplayer.audio.SonicAudioProcessor; import java.nio.ByteBuffer; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. @@ -42,9 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final String TAG = "AudioSamplePipeline"; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; - private final Format inputFormat; private final TransformationRequest transformationRequest; - private final Codec.EncoderFactory encoderFactory; private final Codec decoder; private final DecoderInputBuffer decoderInputBuffer; @@ -52,11 +47,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final SonicAudioProcessor sonicAudioProcessor; private final SpeedProvider speedProvider; + private final Codec encoder; + private final AudioFormat encoderInputAudioFormat; private final DecoderInputBuffer encoderInputBuffer; private final DecoderInputBuffer encoderOutputBuffer; - private @MonotonicNonNull AudioFormat encoderInputAudioFormat; - private @MonotonicNonNull Codec encoder; private long nextEncoderInputBufferTimeUs; private long encoderBufferDurationRemainder; @@ -70,20 +65,50 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory) throws TransformationException { - this.inputFormat = inputFormat; this.transformationRequest = transformationRequest; - this.encoderFactory = encoderFactory; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + + this.decoder = decoderFactory.createForAudioDecoding(inputFormat); + sonicAudioProcessor = new SonicAudioProcessor(); sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; speedProvider = new SegmentSpeedProvider(inputFormat); currentSpeed = speedProvider.getSpeed(0); - this.decoder = decoderFactory.createForAudioDecoding(inputFormat); + AudioFormat encoderInputAudioFormat = + new AudioFormat( + inputFormat.sampleRate, + inputFormat.channelCount, + // The decoder uses ENCODING_PCM_16BIT by default. + // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers + C.ENCODING_PCM_16BIT); + if (transformationRequest.flattenForSlowMotion) { + try { + encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat); + } catch (AudioProcessor.UnhandledAudioFormatException impossible) { + throw new IllegalStateException(impossible); + } + sonicAudioProcessor.setSpeed(currentSpeed); + sonicAudioProcessor.setPitch(currentSpeed); + sonicAudioProcessor.flush(); + } + + encoder = + encoderFactory.createForAudioEncoding( + new Format.Builder() + .setSampleMimeType( + transformationRequest.audioMimeType == null + ? inputFormat.sampleMimeType + : transformationRequest.audioMimeType) + .setSampleRate(encoderInputAudioFormat.sampleRate) + .setChannelCount(encoderInputAudioFormat.channelCount) + .setAverageBitrate(DEFAULT_ENCODER_BITRATE) + .build()); + this.encoderInputAudioFormat = encoderInputAudioFormat; } @Override @@ -98,10 +123,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public boolean processData() throws TransformationException { - if (!ensureEncoderAndAudioProcessingConfigured()) { - return false; - } + public boolean processData() { if (sonicAudioProcessor.isActive()) { return feedEncoderFromSonic() || feedSonicFromDecoder(); } else { @@ -152,7 +174,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to * pass more data immediately by calling this method again. */ - @RequiresNonNull({"encoderInputAudioFormat", "encoder"}) private boolean feedEncoderFromDecoder() { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { return false; @@ -182,7 +203,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Attempts to pass audio processor output data to the encoder, and returns whether it may be * possible to pass more data immediately by calling this method again. */ - @RequiresNonNull({"encoderInputAudioFormat", "encoder"}) private boolean feedEncoderFromSonic() { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { return false; @@ -247,7 +267,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Feeds as much data as possible between the current position and limit of the specified {@link * ByteBuffer} to the encoder, and advances its position by the number of bytes fed. */ - @RequiresNonNull({"encoder", "encoderInputAudioFormat"}) private void feedEncoder(ByteBuffer inputBuffer) { ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data); int bufferLimit = inputBuffer.limit(); @@ -264,7 +283,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; encoder.queueInputBuffer(encoderInputBuffer); } - @RequiresNonNull("encoder") private void queueEndOfStreamToEncoder() { checkState(checkNotNull(encoderInputBuffer.data).position() == 0); encoderInputBuffer.timeUs = nextEncoderInputBufferTimeUs; @@ -274,53 +292,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; encoder.queueInputBuffer(encoderInputBuffer); } - /** - * Attempts to configure the {@link #encoder} and Sonic (if applicable), if they have not been - * configured yet, and returns whether they have been configured. - */ - @EnsuresNonNullIf( - expression = {"encoder", "encoderInputAudioFormat"}, - result = true) - private boolean ensureEncoderAndAudioProcessingConfigured() throws TransformationException { - if (encoder != null && encoderInputAudioFormat != null) { - return true; - } - @Nullable Format decoderOutputFormat = decoder.getOutputFormat(); - if (decoderOutputFormat == null) { - return false; - } - AudioFormat outputAudioFormat = - new AudioFormat( - decoderOutputFormat.sampleRate, - decoderOutputFormat.channelCount, - decoderOutputFormat.pcmEncoding); - if (transformationRequest.flattenForSlowMotion) { - try { - outputAudioFormat = sonicAudioProcessor.configure(outputAudioFormat); - flushSonicAndSetSpeed(currentSpeed); - } catch (AudioProcessor.UnhandledAudioFormatException e) { - throw TransformationException.createForAudioProcessor( - e, - "Sonic", - outputAudioFormat, - TransformationException.ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED); - } - } - encoder = - encoderFactory.createForAudioEncoding( - new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) - .setSampleRate(outputAudioFormat.sampleRate) - .setChannelCount(outputAudioFormat.channelCount) - .setAverageBitrate(DEFAULT_ENCODER_BITRATE) - .build()); - encoderInputAudioFormat = outputAudioFormat; - return true; - } - private boolean isSpeedChanging(BufferInfo bufferInfo) { if (!transformationRequest.flattenForSlowMotion) { return false; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java index 56afb442d0..cf57953bc3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java @@ -73,7 +73,6 @@ public final class TransformationException extends Exception { ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED, ERROR_CODE_GL_INIT_FAILED, ERROR_CODE_GL_PROCESSING_FAILED, - ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED, }) public @interface ErrorCode {} @@ -156,12 +155,7 @@ public final class TransformationException extends Exception { /** Caused by a failure while using or releasing a GL program. */ public static final int ERROR_CODE_GL_PROCESSING_FAILED = 5002; - // Audio editing errors (6xxx). - - /** Caused by an audio processor initialization failure. */ - public static final int ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED = 6001; - - // Muxing errors (7xxx). + // Muxing errors (6xxx). /** * Caused by an output sample MIME type inferred from the input not being supported by the muxer. @@ -169,7 +163,7 @@ public final class TransformationException extends Exception { *

Use {@link TransformationRequest.Builder#setAudioMimeType(String)} or {@link * TransformationRequest.Builder#setVideoMimeType(String)} to transcode to a supported MIME type. */ - public static final int ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED = 7001; + public static final int ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED = 6001; private static final ImmutableBiMap NAME_TO_ERROR_CODE = new ImmutableBiMap.Builder() @@ -191,7 +185,6 @@ public final class TransformationException extends Exception { .put("ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED", ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED) .put("ERROR_CODE_GL_INIT_FAILED", ERROR_CODE_GL_INIT_FAILED) .put("ERROR_CODE_GL_PROCESSING_FAILED", ERROR_CODE_GL_PROCESSING_FAILED) - .put("ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED", ERROR_CODE_AUDIO_PROCESSOR_INIT_FAILED) .put( "ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED", ERROR_CODE_MUXER_SAMPLE_MIME_TYPE_UNSUPPORTED)