From 339205f42882a2a8847217ffdf4aa987a7a05dcf Mon Sep 17 00:00:00 2001 From: kimvde Date: Tue, 6 Dec 2022 14:50:21 +0000 Subject: [PATCH] Move progress updates to the AssetLoader This is necessary to move video slow motion flattening to the AssetLoader because this step can change the duration. As we use the duration before flattening to calculate the progress, we must also use the position before flattening. PiperOrigin-RevId: 493291990 --- .../AudioTranscodingSamplePipeline.java | 4 +- .../transformer/BaseSamplePipeline.java | 6 +- .../transformer/ExoPlayerAssetLoader.java | 32 ++++- .../PassthroughSamplePipeline.java | 4 +- .../media3/transformer/SamplePipeline.java | 19 --- .../media3/transformer/Transformer.java | 1 - .../transformer/TransformerInternal.java | 115 +++++++----------- .../VideoTranscodingSamplePipeline.java | 12 +- 8 files changed, 84 insertions(+), 109 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index cc0dbc531e..79ccd2342a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -64,7 +64,6 @@ import org.checkerframework.dataflow.qual.Pure; long forceSilentAudioDurationUs, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, - Listener listener, FallbackListener fallbackListener) throws TransformationException { super( @@ -72,8 +71,7 @@ import org.checkerframework.dataflow.qual.Pure; streamStartPositionUs, streamOffsetUs, transformationRequest.flattenForSlowMotion, - muxerWrapper, - listener); + muxerWrapper); if (forceSilentAudioDurationUs != C.TIME_UNSET) { silentAudioGenerator = diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java index 1fe8488f64..c80f677d1c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java @@ -33,7 +33,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final long streamStartPositionUs; private final long streamOffsetUs; private final MuxerWrapper muxerWrapper; - private final Listener listener; private final @C.TrackType int trackType; private final @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; @@ -45,12 +44,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; long streamStartPositionUs, long streamOffsetUs, boolean flattenForSlowMotion, - MuxerWrapper muxerWrapper, - Listener listener) { + MuxerWrapper muxerWrapper) { this.streamStartPositionUs = streamStartPositionUs; this.streamOffsetUs = streamOffsetUs; this.muxerWrapper = muxerWrapper; - this.listener = listener; trackType = MimeTypes.getTrackType(inputFormat.sampleMimeType); sefVideoSlowMotionFlattener = flattenForSlowMotion && trackType == C.TRACK_TYPE_VIDEO @@ -79,7 +76,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public void queueInputBuffer() throws TransformationException { DecoderInputBuffer inputBuffer = checkNotNull(this.inputBuffer); - listener.onInputBufferQueued(inputBuffer.timeUs - streamStartPositionUs); checkNotNull(inputBuffer.data); if (!shouldDropInputBuffer(inputBuffer)) { queueInputBufferInternal(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index e3f90abcfe..db2144d89e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -20,6 +20,11 @@ import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PL import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_MS; import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MAX_BUFFER_MS; import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_MIN_BUFFER_MS; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NO_TRANSFORMATION; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY; +import static java.lang.Math.min; import android.content.Context; import android.os.Handler; @@ -63,6 +68,8 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; private final MediaItem mediaItem; private final ExoPlayer player; + private @Transformer.ProgressState int progressState; + public ExoPlayerAssetLoader( Context context, MediaItem mediaItem, @@ -106,16 +113,29 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; player = playerBuilder.build(); player.addListener(new PlayerListener(listener)); + + progressState = PROGRESS_STATE_NO_TRANSFORMATION; } public void start() { player.setMediaItem(mediaItem); player.prepare(); player.play(); + progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY; + } + + public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { + if (progressState == PROGRESS_STATE_AVAILABLE) { + long durationMs = player.getDuration(); + long positionMs = player.getCurrentPosition(); + progressHolder.progress = min((int) (positionMs * 100 / durationMs), 99); + } + return progressState; } public void release() { player.release(); + progressState = PROGRESS_STATE_NO_TRANSFORMATION; } private static final class RenderersFactoryImpl implements RenderersFactory { @@ -167,7 +187,6 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; private final class PlayerListener implements Player.Listener { private final Listener listener; - private boolean hasSentDuration; public PlayerListener(Listener listener) { this.listener = listener; @@ -175,14 +194,21 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener; @Override public void onTimelineChanged(Timeline timeline, int reason) { - if (hasSentDuration) { + if (progressState != PROGRESS_STATE_WAITING_FOR_AVAILABILITY) { return; } Timeline.Window window = new Timeline.Window(); timeline.getWindow(/* windowIndex= */ 0, window); if (!window.isPlaceholder) { + long durationUs = window.durationUs; + // Make progress permanently unavailable if the duration is unknown, so that it doesn't jump + // to a high value at the end of the transformation if the duration is set once the media is + // entirely loaded. + progressState = + durationUs <= 0 || durationUs == C.TIME_UNSET + ? PROGRESS_STATE_UNAVAILABLE + : PROGRESS_STATE_AVAILABLE; listener.onDurationUs(window.durationUs); - hasSentDuration = true; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index de49af02dd..b83e65c9a6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -34,15 +34,13 @@ import androidx.media3.decoder.DecoderInputBuffer; long streamOffsetUs, TransformationRequest transformationRequest, MuxerWrapper muxerWrapper, - Listener listener, FallbackListener fallbackListener) { super( format, streamStartPositionUs, streamOffsetUs, transformationRequest.flattenForSlowMotion, - muxerWrapper, - listener); + muxerWrapper); this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); fallbackListener.onTransformationRequestFinalized(transformationRequest); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java index b2f4823571..f4058784ec 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java @@ -40,25 +40,6 @@ import androidx.media3.decoder.DecoderInputBuffer; void queueInputBuffer(); } - /** A listener for the sample pipeline events. */ - interface Listener { - - /** - * Called when an input buffer is {@linkplain #queueInputBuffer() queued}. - * - * @param positionUs The position of the buffer queued from the stream start position, in - * microseconds. - */ - void onInputBufferQueued(long positionUs); - - /** - * Called if an exception occurs in the sample pipeline. - * - * @param exception The {@link TransformationException} describing the exception. - */ - void onTransformationError(TransformationException exception); - } - /** * Returns whether the pipeline should be fed with decoded sample data. If false, encoded sample * data should be queued. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 1f55104046..2ff0236e78 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -776,7 +776,6 @@ public final class Transformer { encoderFactory, frameProcessorFactory, muxerFactory, - looper, transformerInternalListener, fallbackListener, debugViewProvider, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 01dc39bd35..b8fbec573a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -18,10 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.transformer.TransformationException.ERROR_CODE_MUXING_FAILED; -import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; -import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE; -import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY; -import static java.lang.Math.min; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NO_TRANSFORMATION; import static java.lang.annotation.ElementType.TYPE_USE; import android.content.Context; @@ -44,7 +41,6 @@ import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.Clock; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.HandlerWrapper; -import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.extractor.metadata.mp4.SlowMotionData; @@ -89,6 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final int MSG_QUEUE_INPUT = 3; private static final int MSG_DRAIN_PIPELINES = 4; private static final int MSG_END = 5; + private static final int MSG_UPDATE_PROGRESS = 6; private static final int DRAIN_PIPELINES_DELAY_MS = 50; @@ -103,21 +100,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Listener listener; private final DebugViewProvider debugViewProvider; private final Clock clock; - private final HandlerWrapper applicationHandler; private final HandlerThread internalHandlerThread; private final HandlerWrapper internalHandler; private final ExoPlayerAssetLoader exoPlayerAssetLoader; private final List samplePipelines; private final ConditionVariable dequeueBufferConditionVariable; private final MuxerWrapper muxerWrapper; - private final ConditionVariable cancellingConditionVariable; + private final ConditionVariable transformerConditionVariable; @Nullable private DecoderInputBuffer pendingInputBuffer; private boolean isDrainingPipelines; private int silentSamplePipelineIndex; private @Transformer.ProgressState int progressState; - private long progressPositionMs; - private long durationUs; private @MonotonicNonNull RuntimeException cancelException; private volatile boolean released; @@ -138,7 +132,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Codec.EncoderFactory encoderFactory, FrameProcessor.Factory frameProcessorFactory, Muxer.Factory muxerFactory, - Looper applicationLooper, Listener listener, FallbackListener fallbackListener, DebugViewProvider debugViewProvider, @@ -154,7 +147,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.listener = listener; this.debugViewProvider = debugViewProvider; this.clock = clock; - applicationHandler = clock.createHandler(applicationLooper, /* callback= */ null); internalHandlerThread = new HandlerThread("Transformer:Internal"); internalHandlerThread.start(); Looper internalLooper = internalHandlerThread.getLooper(); @@ -179,8 +171,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputParcelFileDescriptor, muxerFactory, /* errorConsumer= */ componentListener::onTransformationError); - cancellingConditionVariable = new ConditionVariable(); - progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY; + transformerConditionVariable = new ConditionVariable(); // It's safe to use "this" because we don't send a message before exiting the constructor. @SuppressWarnings("nullness:methodref.receiver.bound") HandlerWrapper internalHandler = @@ -193,9 +184,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { - if (progressState == PROGRESS_STATE_AVAILABLE) { - progressHolder.progress = min((int) (progressPositionMs * 100 / Util.usToMs(durationUs)), 99); + if (released) { + return PROGRESS_STATE_NO_TRANSFORMATION; } + internalHandler.obtainMessage(MSG_UPDATE_PROGRESS, progressHolder).sendToTarget(); + // TODO: figure out why calling clock.onThreadBlocked() here makes the tests fail. + transformerConditionVariable.blockUninterruptible(); + transformerConditionVariable.close(); return progressState; } @@ -208,15 +203,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MSG_END, END_REASON_CANCELLED, /* unused */ 0, /* transformationException */ null) .sendToTarget(); clock.onThreadBlocked(); - cancellingConditionVariable.blockUninterruptible(); + transformerConditionVariable.blockUninterruptible(); + transformerConditionVariable.close(); if (cancelException != null) { throw cancelException; } } private boolean handleMessage(Message msg) { - // Handle end messages even if resources have been released to report release timeouts. - if (released && msg.what != MSG_END) { + // Some messages cannot be ignored when resources have been released. End messages must be + // handled to report release timeouts and to unblock the transformer condition variable in case + // of cancellation. Progress update messages must be handled to unblock the transformer + // condition variable. + if (released && msg.what != MSG_END && msg.what != MSG_UPDATE_PROGRESS) { return true; } try { @@ -241,6 +240,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* endReason= */ msg.arg1, /* transformationException= */ (TransformationException) msg.obj); break; + case MSG_UPDATE_PROGRESS: + updateProgressInternal(/* progressHolder= */ (ProgressHolder) msg.obj); + break; default: return false; } @@ -343,60 +345,53 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; internalHandler.post(internalHandlerThread::quitSafely); } - if (!forCancellation) { - TransformationException exception = transformationException; - if (exception == null) { - // We only report the exception caused by releasing the resources if there is no other - // exception. It is more intuitive to call the error callback only once and reporting the - // exception caused by releasing the resources can be confusing if it is a consequence of - // the first exception. - exception = releaseTransformationException; - } - - if (exception != null) { - listener.onTransformationError(exception); - } else { - listener.onTransformationCompleted(checkNotNull(transformationResult)); - } + if (forCancellation) { + transformerConditionVariable.open(); + return; } - cancellingConditionVariable.open(); + TransformationException exception = transformationException; + if (exception == null) { + // We only report the exception caused by releasing the resources if there is no other + // exception. It is more intuitive to call the error callback only once and reporting the + // exception caused by releasing the resources can be confusing if it is a consequence of the + // first exception. + exception = releaseTransformationException; + } + + if (exception != null) { + listener.onTransformationError(exception); + } else { + listener.onTransformationCompleted(checkNotNull(transformationResult)); + } } - private class ComponentListener - implements ExoPlayerAssetLoader.Listener, SamplePipeline.Listener { + private void updateProgressInternal(ProgressHolder progressHolder) { + progressState = exoPlayerAssetLoader.getProgress(progressHolder); + transformerConditionVariable.open(); + } - private static final long MIN_DURATION_BETWEEN_PROGRESS_UPDATES_MS = 100; + private class ComponentListener implements ExoPlayerAssetLoader.Listener { private final MediaItem mediaItem; private final FallbackListener fallbackListener; + private long durationUs; private int tracksAddedCount; - private long lastProgressUpdateMs; - private long lastProgressPositionMs; private volatile boolean trackRegistered; public ComponentListener(MediaItem mediaItem, FallbackListener fallbackListener) { this.mediaItem = mediaItem; this.fallbackListener = fallbackListener; + durationUs = C.TIME_UNSET; } // ExoPlayerAssetLoader.Listener implementation. @Override public void onDurationUs(long durationUs) { - applicationHandler.post( - () -> { - // Make progress permanently unavailable if the duration is unknown, so that it doesn't - // jump to a high value at the end of the transformation if the duration is set once the - // media is entirely loaded. - progressState = - durationUs <= 0 || durationUs == C.TIME_UNSET - ? PROGRESS_STATE_UNAVAILABLE - : PROGRESS_STATE_AVAILABLE; - TransformerInternal.this.durationUs = durationUs; - }); + this.durationUs = durationUs; } @Override @@ -462,22 +457,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; onTransformationError(transformationException); } - // SamplePipeline.Listener implementation. - - @Override - public void onInputBufferQueued(long positionUs) { - long positionMs = Util.usToMs(positionUs); - long elapsedTimeMs = clock.elapsedRealtime(); - if (elapsedTimeMs > lastProgressUpdateMs + MIN_DURATION_BETWEEN_PROGRESS_UPDATES_MS - && positionMs > lastProgressPositionMs) { - lastProgressUpdateMs = elapsedTimeMs; - // Store positionMs in a variable to make sure the thread reads the latest value. - lastProgressPositionMs = positionMs; - applicationHandler.post(() -> progressPositionMs = positionMs); - } - } - - @Override public void onTransformationError(TransformationException transformationException) { internalHandler .obtainMessage(MSG_END, END_REASON_ERROR, /* unused */ 0, transformationException) @@ -497,7 +476,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; forceSilentAudio ? durationUs : C.TIME_UNSET, encoderFactory, muxerWrapper, - /* listener= */ this, fallbackListener); } else if (MimeTypes.isVideo(inputFormat.sampleMimeType) && shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs)) { @@ -512,7 +490,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; decoderFactory, encoderFactory, muxerWrapper, - /* listener= */ this, + /* errorConsumer= */ this::onTransformationError, fallbackListener, debugViewProvider); } else { @@ -522,7 +500,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; streamOffsetUs, transformationRequest, muxerWrapper, - /* listener= */ this, fallbackListener); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 4b7efd1c02..90b3ed58c2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -36,6 +36,7 @@ import androidx.media3.common.FrameProcessingException; import androidx.media3.common.FrameProcessor; import androidx.media3.common.MimeTypes; import androidx.media3.common.SurfaceInfo; +import androidx.media3.common.util.Consumer; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; @@ -82,7 +83,7 @@ import org.checkerframework.dataflow.qual.Pure; Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper, - Listener listener, + Consumer errorConsumer, FallbackListener fallbackListener, DebugViewProvider debugViewProvider) throws TransformationException { @@ -91,8 +92,7 @@ import org.checkerframework.dataflow.qual.Pure; streamStartPositionUs, streamOffsetUs, transformationRequest.flattenForSlowMotion, - muxerWrapper, - listener); + muxerWrapper); if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) { if (transformationRequest.hdrMode @@ -179,7 +179,7 @@ import org.checkerframework.dataflow.qual.Pure; checkNotNull(frameProcessor) .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); } catch (TransformationException exception) { - listener.onTransformationError(exception); + errorConsumer.accept(exception); } } @@ -191,7 +191,7 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void onFrameProcessingError(FrameProcessingException exception) { - listener.onTransformationError( + errorConsumer.accept( TransformationException.createForFrameProcessingException( exception, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED)); } @@ -203,7 +203,7 @@ import org.checkerframework.dataflow.qual.Pure; try { encoderWrapper.signalEndOfInputStream(); } catch (TransformationException exception) { - listener.onTransformationError(exception); + errorConsumer.accept(exception); } } });