mirror of
https://github.com/samsonjs/media.git
synced 2026-03-25 09:25:53 +00:00
Experimental frame extraction based on ExoPlayer
A skeleton implementation that doesn't actually return decoded frames. In the current state, we use ExoPlayer to seek to a position, and ExoPlayer.setVideoEffects to record the presentation time selected. Seeking and processing frames are synchronized via ListenableFuture callbacks. PiperOrigin-RevId: 695691183
This commit is contained in:
dancho
Copybara-Service
parent
4acd1b970c
commit
ed288fca46
2 changed files with 396 additions and 0 deletions
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
* Copyright 2024 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.PlaybackException.ERROR_CODE_IO_FILE_NOT_FOUND;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
|
||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import android.content.Context;
|
||||
import androidx.media3.common.MediaItem;
|
||||
import androidx.media3.common.util.ConditionVariable;
|
||||
import androidx.media3.common.util.NullableType;
|
||||
import androidx.media3.exoplayer.ExoPlaybackException;
|
||||
import androidx.media3.transformer.ExperimentalFrameExtractor.Frame;
|
||||
import androidx.test.core.app.ApplicationProvider;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.util.concurrent.FutureCallback;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/** End-to-end instrumentation test for {@link ExperimentalFrameExtractor}. */
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public class FrameExtractorTest {
|
||||
private static final String FILE_PATH =
|
||||
"asset:///media/mp4/sample_with_increasing_timestamps_360p.mp4";
|
||||
private static final long TIMEOUT_SECONDS = 10;
|
||||
|
||||
private final Context context = ApplicationProvider.getApplicationContext();
|
||||
|
||||
private @MonotonicNonNull ExperimentalFrameExtractor frameExtractor;
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
if (frameExtractor != null) {
|
||||
frameExtractor.release();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_oneFrame_returnsNearest() throws Exception {
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
|
||||
|
||||
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
|
||||
|
||||
assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_531);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_pastDuration_returnsLastFrame() throws Exception {
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
|
||||
|
||||
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 200_000);
|
||||
int lastVideoFramePresentationTimeMs = 17_029;
|
||||
|
||||
assertThat(frameFuture.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs)
|
||||
.isEqualTo(lastVideoFramePresentationTimeMs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_repeatedPositionMs_returnsTheSameFrame() throws Exception {
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
|
||||
|
||||
ListenableFuture<Frame> frame0 = frameExtractor.getFrame(/* positionMs= */ 0);
|
||||
ListenableFuture<Frame> frame0Again = frameExtractor.getFrame(/* positionMs= */ 0);
|
||||
ListenableFuture<Frame> frame33 = frameExtractor.getFrame(/* positionMs= */ 33);
|
||||
ListenableFuture<Frame> frame34 = frameExtractor.getFrame(/* positionMs= */ 34);
|
||||
ListenableFuture<Frame> frame34Again = frameExtractor.getFrame(/* positionMs= */ 34);
|
||||
|
||||
assertThat(frame0.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0);
|
||||
assertThat(frame0Again.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(0);
|
||||
assertThat(frame33.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(33);
|
||||
assertThat(frame34.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(66);
|
||||
assertThat(frame34Again.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(66);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_randomAccess_returnsCorrectFrames() throws Exception {
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
|
||||
|
||||
ListenableFuture<Frame> frame5 = frameExtractor.getFrame(/* positionMs= */ 5_000);
|
||||
ListenableFuture<Frame> frame3 = frameExtractor.getFrame(/* positionMs= */ 3_000);
|
||||
ListenableFuture<Frame> frame7 = frameExtractor.getFrame(/* positionMs= */ 7_000);
|
||||
ListenableFuture<Frame> frame2 = frameExtractor.getFrame(/* positionMs= */ 2_000);
|
||||
ListenableFuture<Frame> frame8 = frameExtractor.getFrame(/* positionMs= */ 8_000);
|
||||
|
||||
assertThat(frame5.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(5_032);
|
||||
assertThat(frame3.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(3_032);
|
||||
assertThat(frame7.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(7_031);
|
||||
assertThat(frame2.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(2_032);
|
||||
assertThat(frame8.get(TIMEOUT_SECONDS, SECONDS).presentationTimeMs).isEqualTo(8_031);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_invalidInput_reportsErrorViaFuture() {
|
||||
String filePath = "asset:///nonexistent";
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(filePath));
|
||||
|
||||
ListenableFuture<Frame> frame0 = frameExtractor.getFrame(/* positionMs= */ 0);
|
||||
|
||||
ExecutionException thrown =
|
||||
assertThrows(ExecutionException.class, () -> frame0.get(TIMEOUT_SECONDS, SECONDS));
|
||||
assertThat(thrown).hasCauseThat().isInstanceOf(ExoPlaybackException.class);
|
||||
assertThat(((ExoPlaybackException) thrown.getCause()).errorCode)
|
||||
.isEqualTo(ERROR_CODE_IO_FILE_NOT_FOUND);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void extractFrame_oneFrame_completesViaCallback() throws Exception {
|
||||
frameExtractor = new ExperimentalFrameExtractor(context, MediaItem.fromUri(FILE_PATH));
|
||||
AtomicReference<@NullableType Frame> frameAtomicReference = new AtomicReference<>();
|
||||
AtomicReference<@NullableType Throwable> throwableAtomicReference = new AtomicReference<>();
|
||||
ConditionVariable frameReady = new ConditionVariable();
|
||||
|
||||
ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 0);
|
||||
Futures.addCallback(
|
||||
frameFuture,
|
||||
new FutureCallback<Frame>() {
|
||||
@Override
|
||||
public void onSuccess(Frame result) {
|
||||
frameAtomicReference.set(result);
|
||||
frameReady.open();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable t) {
|
||||
throwableAtomicReference.set(t);
|
||||
frameReady.open();
|
||||
}
|
||||
},
|
||||
directExecutor());
|
||||
frameReady.block(/* timeoutMs= */ TIMEOUT_SECONDS * 1000);
|
||||
|
||||
assertThat(throwableAtomicReference.get()).isNull();
|
||||
assertThat(frameAtomicReference.get().presentationTimeMs).isEqualTo(0);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
/*
|
||||
* Copyright 2024 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.PlaybackException.ERROR_CODE_FAILED_RUNTIME_CHECK;
|
||||
import static androidx.media3.common.Player.DISCONTINUITY_REASON_SEEK;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Util.usToMs;
|
||||
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
|
||||
|
||||
import android.content.Context;
|
||||
import android.os.Handler;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.MediaItem;
|
||||
import androidx.media3.common.PlaybackException;
|
||||
import androidx.media3.common.Player;
|
||||
import androidx.media3.common.util.NullableType;
|
||||
import androidx.media3.effect.GlEffect;
|
||||
import androidx.media3.effect.GlShaderProgram;
|
||||
import androidx.media3.effect.PassthroughShaderProgram;
|
||||
import androidx.media3.exoplayer.ExoPlayer;
|
||||
import androidx.media3.exoplayer.analytics.AnalyticsListener;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.util.concurrent.FutureCallback;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.SettableFuture;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.checkerframework.checker.initialization.qual.Initialized;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* Extracts decoded frames from {@link MediaItem}.
|
||||
*
|
||||
* <p>This class is experimental and will be renamed or removed in a future release.
|
||||
*
|
||||
* <p>Frame extractor instances must be accessed from a single application thread.
|
||||
*/
|
||||
/* package */ final class ExperimentalFrameExtractor implements AnalyticsListener {
|
||||
|
||||
/** Stores an extracted and decoded video frame. */
|
||||
// TODO: b/350498258 - Add a Bitmap field to Frame.
|
||||
public static final class Frame {
|
||||
|
||||
/** The presentation timestamp of the extracted frame, in milliseconds. */
|
||||
public final long presentationTimeMs;
|
||||
|
||||
private Frame(long presentationTimeMs) {
|
||||
this.presentationTimeMs = presentationTimeMs;
|
||||
}
|
||||
}
|
||||
|
||||
private final ExoPlayer player;
|
||||
private final Handler playerApplicationThreadHandler;
|
||||
|
||||
/**
|
||||
* A {@link SettableFuture} representing the frame currently being extracted. Accessed on both the
|
||||
* {@linkplain ExoPlayer#getApplicationLooper() ExoPlayer application thread}, and the video
|
||||
* effects GL thread.
|
||||
*/
|
||||
private final AtomicReference<@NullableType SettableFuture<Frame>>
|
||||
frameBeingExtractedFutureAtomicReference;
|
||||
|
||||
/**
|
||||
* The last {@link SettableFuture} returned by {@link #getFrame(long)}. Accessed on the frame
|
||||
* extractor application thread.
|
||||
*/
|
||||
private SettableFuture<Frame> lastRequestedFrameFuture;
|
||||
|
||||
/**
|
||||
* The last {@link Frame} that was extracted successfully. Accessed on the {@linkplain
|
||||
* ExoPlayer#getApplicationLooper() ExoPlayer application thread}.
|
||||
*/
|
||||
private @MonotonicNonNull Frame lastExtractedFrame;
|
||||
|
||||
/**
|
||||
* Creates an instance.
|
||||
*
|
||||
* @param context {@link Context}.
|
||||
* @param mediaItem The {@link MediaItem} from which frames are extracted.
|
||||
*/
|
||||
// TODO: b/350498258 - Support changing the MediaItem.
|
||||
// TODO: b/350498258 - Add configuration options such as SeekParameters.
|
||||
// TODO: b/350498258 - Support video effects.
|
||||
public ExperimentalFrameExtractor(Context context, MediaItem mediaItem) {
|
||||
player = new ExoPlayer.Builder(context).build();
|
||||
playerApplicationThreadHandler = new Handler(player.getApplicationLooper());
|
||||
lastRequestedFrameFuture = SettableFuture.create();
|
||||
// TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects
|
||||
// returning incorrect timestamps if we seek the player before rendering starts from zero.
|
||||
frameBeingExtractedFutureAtomicReference = new AtomicReference<>(lastRequestedFrameFuture);
|
||||
// TODO: b/350498258 - Refactor this and remove declaring this reference as initialized
|
||||
// to satisfy the nullness checker.
|
||||
@SuppressWarnings("nullness:assignment")
|
||||
@Initialized
|
||||
ExperimentalFrameExtractor thisRef = this;
|
||||
playerApplicationThreadHandler.post(
|
||||
() -> {
|
||||
player.addAnalyticsListener(thisRef);
|
||||
player.setVideoEffects(buildVideoEffects());
|
||||
player.setMediaItem(mediaItem);
|
||||
player.setPlayWhenReady(false);
|
||||
player.prepare();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a representative {@link Frame} for the specified video position.
|
||||
*
|
||||
* @param positionMs The time position in the {@link MediaItem} for which a frame is extracted.
|
||||
* @return A {@link ListenableFuture} of the result.
|
||||
*/
|
||||
public ListenableFuture<Frame> getFrame(long positionMs) {
|
||||
SettableFuture<Frame> frameSettableFuture = SettableFuture.create();
|
||||
// Process frameSettableFuture after lastRequestedFrameFuture completes.
|
||||
// If lastRequestedFrameFuture is done, the callbacks are invoked immediately.
|
||||
Futures.addCallback(
|
||||
lastRequestedFrameFuture,
|
||||
new FutureCallback<Frame>() {
|
||||
@Override
|
||||
public void onSuccess(Frame result) {
|
||||
playerApplicationThreadHandler.post(
|
||||
() -> {
|
||||
lastExtractedFrame = result;
|
||||
@Nullable PlaybackException playerError;
|
||||
if (player.isReleased()) {
|
||||
playerError =
|
||||
new PlaybackException(
|
||||
"The player is already released",
|
||||
null,
|
||||
ERROR_CODE_FAILED_RUNTIME_CHECK);
|
||||
} else {
|
||||
playerError = player.getPlayerError();
|
||||
}
|
||||
if (playerError != null) {
|
||||
frameSettableFuture.setException(playerError);
|
||||
} else {
|
||||
checkState(
|
||||
frameBeingExtractedFutureAtomicReference.compareAndSet(
|
||||
null, frameSettableFuture));
|
||||
player.seekTo(positionMs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable t) {
|
||||
frameSettableFuture.setException(t);
|
||||
}
|
||||
},
|
||||
directExecutor());
|
||||
lastRequestedFrameFuture = frameSettableFuture;
|
||||
return lastRequestedFrameFuture;
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the underlying resources. This method must be called when the frame extractor is no
|
||||
* longer required. The frame extractor must not be used after calling this method.
|
||||
*/
|
||||
public void release() {
|
||||
// TODO: b/350498258 - Block the caller until exoPlayer.release() returns.
|
||||
playerApplicationThreadHandler.removeCallbacksAndMessages(null);
|
||||
playerApplicationThreadHandler.post(player::release);
|
||||
}
|
||||
|
||||
// AnalyticsListener
|
||||
|
||||
@Override
|
||||
public void onPlayerError(EventTime eventTime, PlaybackException error) {
|
||||
// Fail the next frame to be extracted. Errors will propagate to later pending requests via
|
||||
// Future callbacks.
|
||||
@Nullable
|
||||
SettableFuture<Frame> frameBeingExtractedFuture =
|
||||
frameBeingExtractedFutureAtomicReference.getAndSet(null);
|
||||
if (frameBeingExtractedFuture != null) {
|
||||
frameBeingExtractedFuture.setException(error);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPositionDiscontinuity(
|
||||
EventTime eventTime,
|
||||
Player.PositionInfo oldPosition,
|
||||
Player.PositionInfo newPosition,
|
||||
@Player.DiscontinuityReason int reason) {
|
||||
if (oldPosition.equals(newPosition) && reason == DISCONTINUITY_REASON_SEEK) {
|
||||
// When the new seeking position resolves to the old position, no frames are rendered.
|
||||
// Repeat the previously returned frame.
|
||||
SettableFuture<Frame> frameBeingExtractedFuture =
|
||||
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null));
|
||||
frameBeingExtractedFuture.set(checkNotNull(lastExtractedFrame));
|
||||
}
|
||||
}
|
||||
|
||||
private ImmutableList<Effect> buildVideoEffects() {
|
||||
return ImmutableList.of(new FrameReader());
|
||||
}
|
||||
|
||||
private final class FrameReader implements GlEffect {
|
||||
@Override
|
||||
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
|
||||
// TODO: b/350498258 - Support HDR.
|
||||
return new FrameReadingGlShaderProgram();
|
||||
}
|
||||
}
|
||||
|
||||
private final class FrameReadingGlShaderProgram extends PassthroughShaderProgram {
|
||||
@Override
|
||||
public void queueInputFrame(
|
||||
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
SettableFuture<Frame> frameBeingExtractedFuture =
|
||||
checkNotNull(frameBeingExtractedFutureAtomicReference.getAndSet(null));
|
||||
// TODO: b/350498258 - Read the input texture contents into a Bitmap.
|
||||
frameBeingExtractedFuture.set(new Frame(usToMs(presentationTimeUs)));
|
||||
// Drop frame: do not call outputListener.onOutputFrameAvailable().
|
||||
// Block effects pipeline: do not call inputListener.onReadyToAcceptInputFrame().
|
||||
// The effects pipeline will unblock and receive new frames when flushed after a seek.
|
||||
getInputListener().onInputFrameProcessed(inputTexture);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue