diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java new file mode 100644 index 0000000000..4f4029a38b --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.Assert.assertThrows; + +import android.content.Context; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.NullableType; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.transformer.ExperimentalFrameExtractor.Frame; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** End-to-end instrumentation test for {@link ExperimentalFrameExtractor}. */ +@RunWith(AndroidJUnit4.class) +public class FrameExtractorTest { + private static final String FILE_PATH = + "asset:///media/mp4/sample_with_increasing_timestamps_360p.mp4"; + private static final long TIMEOUT_SECONDS = 10; + + private final Context context = ApplicationProvider.getApplicationContext(); + + private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor; + + @After + public void tearDown() { + if (frameExtractor != null) { + frameExtractor.release(); + } + } + + @Test + public void extractFrame_oneFrame_returnsNearest() throws Exception { + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + + ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500); + + assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_531); + } + + @Test + public void extractFrame_pastDuration_returnsLastFrame() throws Exception { + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + + ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 200_000); + int lastVideoFramePresentationTimeMs = 17_029; + + assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs) + .isEqualTo(lastVideoFramePresentationTimeMs); + } + + @Test + public void extractFrame_repeatedPositionMs_returnsTheSameFrame() throws Exception { + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + + ListenableFuture frame0 = frameExtractor.getFrame(/* positionMs= */ 0); + ListenableFuture frame0Again = frameExtractor.getFrame(/* positionMs= */ 0); + ListenableFuture frame33 = frameExtractor.getFrame(/* positionMs= */ 33); + ListenableFuture frame34 = frameExtractor.getFrame(/* positionMs= */ 34); + ListenableFuture frame34Again = frameExtractor.getFrame(/* positionMs= */ 34); + + assertThat(frame0.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0); + assertThat(frame0Again.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0); + assertThat(frame33.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(33); + assertThat(frame34.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(66); + assertThat(frame34Again.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(66); + } + + @Test + public void extractFrame_randomAccess_returnsCorrectFrames() throws Exception { + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + + ListenableFuture frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000); + ListenableFuture frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000); + ListenableFuture frame7 = frameExtractor.getFrame(/* positionMs= */ 7_000); + ListenableFuture frame2 = frameExtractor.getFrame(/* positionMs= */ 2_000); + ListenableFuture frame8 = frameExtractor.getFrame(/* positionMs= */ 8_000); + + assertThat(frame5.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(5_032); + assertThat(frame3.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(3_032); + assertThat(frame7.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(7_031); + assertThat(frame2.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(2_032); + assertThat(frame8.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_031); + } + + @Test + public void extractFrame_invalidInput_reportsErrorViaFuture() { + String filePath = "asset:///nonexistent"; + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(filePath)); + + ListenableFuture frame0 = frameExtractor.getFrame(/* positionMs= */ 0); + + ExecutionException thrown = + assertThrows(ExecutionException.class, () -> frame0.get(TIMEOUT_SECONDS, SECONDS)); + assertThat(thrown).hasCauseThat().isInstanceOf(ExoPlaybackException.class); + assertThat(((ExoPlaybackException) thrown.getCause()).errorCode) + .isEqualTo(ERROR_CODE_IO_FILE_NOT_FOUND); + } + + @Test + public void extractFrame_oneFrame_completesViaCallback() throws Exception { + frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH)); + AtomicReference<@NullableType Frame> frameAtomicReference = new AtomicReference<>(); + AtomicReference<@NullableType Throwable> throwableAtomicReference = new AtomicReference<>(); + ConditionVariable frameReady = new ConditionVariable(); + + ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 0); + Futures.addCallback( + frameFuture, + new FutureCallback() { + @Override + public void onSuccess(Frame result) { + frameAtomicReference.set(result); + frameReady.open(); + } + + @Override + public void onFailure(Throwable t) { + throwableAtomicReference.set(t); + frameReady.open(); + } + }, + directExecutor()); + frameReady.block(/* timeoutMs= */ TIMEOUT_SECONDS * 1000); + + assertThat(throwableAtomicReference.get()).isNull(); + assertThat(frameAtomicReference.get().presentationTimeMs).isEqualTo(0); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java new file mode 100644 index 0000000000..5653141d45 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -0,0 +1,239 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.PlaybackException.ERROR_CODE_FAILED_RUNTIME_CHECK; +import static androidx.media3.common.Player.DISCONTINUITY_REASON_SEEK; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Util.usToMs; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import android.content.Context; +import android.os.Handler; +import androidx.annotation.Nullable; +import androidx.media3.common.Effect; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.MediaItem; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.Player; +import androidx.media3.common.util.NullableType; +import androidx.media3.effect.GlEffect; +import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.PassthroughShaderProgram; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.analytics.AnalyticsListener; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.util.concurrent.atomic.AtomicReference; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Extracts decoded frames from {@link MediaItem}. + * + *

This class is experimental and will be renamed or removed in a future release. + * + *

Frame extractor instances must be accessed from a single application thread. + */ +/* package */ final class ExperimentalFrameExtractor implements AnalyticsListener { + + /** Stores an extracted and decoded video frame. */ + // TODO: b/350498258 - Add a Bitmap field to Frame. + public static final class Frame { + + /** The presentation timestamp of the extracted frame, in milliseconds. */ + public final long presentationTimeMs; + + private Frame(long presentationTimeMs) { + this.presentationTimeMs = presentationTimeMs; + } + } + + private final ExoPlayer player; + private final Handler playerApplicationThreadHandler; + + /** + * A {@link SettableFuture} representing the frame currently being extracted. Accessed on both the + * {@linkplain ExoPlayer#getApplicationLooper() ExoPlayer application thread}, and the video + * effects GL thread. + */ + private final AtomicReference<@NullableType SettableFuture> + frameBeingExtractedFutureAtomicReference; + + /** + * The last {@link SettableFuture} returned by {@link #getFrame(long)}. Accessed on the frame + * extractor application thread. + */ + private SettableFuture lastRequestedFrameFuture; + + /** + * The last {@link Frame} that was extracted successfully. Accessed on the {@linkplain + * ExoPlayer#getApplicationLooper() ExoPlayer application thread}. + */ + private @MonotonicNonNull Frame lastExtractedFrame; + + /** + * Creates an instance. + * + * @param context {@link Context}. + * @param mediaItem The {@link MediaItem} from which frames are extracted. + */ + // TODO: b/350498258 - Support changing the MediaItem. + // TODO: b/350498258 - Add configuration options such as SeekParameters. + // TODO: b/350498258 - Support video effects. + public ExperimentalFrameExtractor(Context context, MediaItem mediaItem) { + player = new ExoPlayer.Builder(context).build(); + playerApplicationThreadHandler = new Handler(player.getApplicationLooper()); + lastRequestedFrameFuture = SettableFuture.create(); + // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects + // returning incorrect timestamps if we seek the player before rendering starts from zero. + frameBeingExtractedFutureAtomicReference = new AtomicReference<>(lastRequestedFrameFuture); + // TODO: b/350498258 - Refactor this and remove declaring this reference as initialized + // to satisfy the nullness checker. + @SuppressWarnings("nullness:assignment") + @Initialized + ExperimentalFrameExtractor thisRef = this; + playerApplicationThreadHandler.post( + () -> { + player.addAnalyticsListener(thisRef); + player.setVideoEffects(buildVideoEffects()); + player.setMediaItem(mediaItem); + player.setPlayWhenReady(false); + player.prepare(); + }); + } + + /** + * Extracts a representative {@link Frame} for the specified video position. + * + * @param positionMs The time position in the {@link MediaItem} for which a frame is extracted. + * @return A {@link ListenableFuture} of the result. + */ + public ListenableFuture getFrame(long positionMs) { + SettableFuture frameSettableFuture = SettableFuture.create(); + // Process frameSettableFuture after lastRequestedFrameFuture completes. + // If lastRequestedFrameFuture is done, the callbacks are invoked immediately. + Futures.addCallback( + lastRequestedFrameFuture, + new FutureCallback() { + @Override + public void onSuccess(Frame result) { + playerApplicationThreadHandler.post( + () -> { + lastExtractedFrame = result; + @Nullable PlaybackException playerError; + if (player.isReleased()) { + playerError = + new PlaybackException( + "The player is already released", + null, + ERROR_CODE_FAILED_RUNTIME_CHECK); + } else { + playerError = player.getPlayerError(); + } + if (playerError != null) { + frameSettableFuture.setException(playerError); + } else { + checkState( + frameBeingExtractedFutureAtomicReference.compareAndSet( + null, frameSettableFuture)); + player.seekTo(positionMs); + } + }); + } + + @Override + public void onFailure(Throwable t) { + frameSettableFuture.setException(t); + } + }, + directExecutor()); + lastRequestedFrameFuture = frameSettableFuture; + return lastRequestedFrameFuture; + } + + /** + * Releases the underlying resources. This method must be called when the frame extractor is no + * longer required. The frame extractor must not be used after calling this method. + */ + public void release() { + // TODO: b/350498258 - Block the caller until exoPlayer.release() returns. + playerApplicationThreadHandler.removeCallbacksAndMessages(null); + playerApplicationThreadHandler.post(player::release); + } + + // AnalyticsListener + + @Override + public void onPlayerError(EventTime eventTime, PlaybackException error) { + // Fail the next frame to be extracted. Errors will propagate to later pending requests via + // Future callbacks. + @Nullable + SettableFuture frameBeingExtractedFuture = + frameBeingExtractedFutureAtomicReference.getAndSet(null); + if (frameBeingExtractedFuture != null) { + frameBeingExtractedFuture.setException(error); + } + } + + @Override + public void onPositionDiscontinuity( + EventTime eventTime, + Player.PositionInfo oldPosition, + Player.PositionInfo newPosition, + @Player.DiscontinuityReason int reason) { + if (oldPosition.equals(newPosition) && reason == DISCONTINUITY_REASON_SEEK) { + // When the new seeking position resolves to the old position, no frames are rendered. + // Repeat the previously returned frame. + SettableFuture frameBeingExtractedFuture = + checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null)); + frameBeingExtractedFuture.set(checkNotNull(lastExtractedFrame)); + } + } + + private ImmutableList buildVideoEffects() { + return ImmutableList.of(new FrameReader()); + } + + private final class FrameReader implements GlEffect { + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + // TODO: b/350498258 - Support HDR. + return new FrameReadingGlShaderProgram(); + } + } + + private final class FrameReadingGlShaderProgram extends PassthroughShaderProgram { + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + SettableFuture frameBeingExtractedFuture = + checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null)); + // TODO: b/350498258 - Read the input texture contents into a Bitmap. + frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs))); + // Drop frame: do not call outputListener.onOutputFrameAvailable(). + // Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame(). + // The effects pipeline will unblock and receive new frames when flushed after a seek. + getInputListener().onInputFrameProcessed(inputTexture); + } + } +}