sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-04 11:05:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Internal changes only

Browse source 
Internal changes

PiperOrigin-RevId: 613155879
This commit is contained in:
Googler 2024-03-06 03:41:56 -08:00 committed by Copybara-Service
parent b1b8b2e6f9
commit 4d0af794df
1 changed files with 128 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View file

@ -53,6 +53,7 @@ import androidx.media3.exoplayer.ExoPlaybackException;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@ -62,6 +63,7 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.Executor;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
@ -73,6 +75,47 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
public final class CompositingVideoSinkProvider
implements VideoSinkProvider, VideoGraph.Listener, VideoFrameRenderControl.FrameRenderer {
/** Listener for {@link CompositingVideoSinkProvider} events. */
public interface Listener {
/**
* Called when the video frame processor renders the first frame.
*
* @param compositingVideoSinkProvider The compositing video sink provider which triggered this
* event.
*/
void onFirstFrameRendered(CompositingVideoSinkProvider compositingVideoSinkProvider);
/**
* Called when the video frame processor dropped a frame.
*
* @param compositingVideoSinkProvider The compositing video sink provider which triggered this
* event.
*/
void onFrameDropped(CompositingVideoSinkProvider compositingVideoSinkProvider);
/**
* Called before a frame is rendered for the first time since setting the surface, and each time
* there's a change in the size, rotation or pixel aspect ratio of the video being rendered.
*
* @param compositingVideoSinkProvider The compositing video sink provider which triggered this
* event.
* @param videoSize The video size.
*/
void onVideoSizeChanged(
CompositingVideoSinkProvider compositingVideoSinkProvider, VideoSize videoSize);
/**
* Called when the video frame processor encountered an error.
*
* @param compositingVideoSinkProvider The compositing video sink provider which triggered this
* event.
* @param videoFrameProcessingException The error.
*/
void onError(
CompositingVideoSinkProvider compositingVideoSinkProvider,
VideoFrameProcessingException videoFrameProcessingException);
}
/** A builder for {@link CompositingVideoSinkProvider} instances. */
public static final class Builder {
private final Context context;
@ -96,6 +139,7 @@ public final class CompositingVideoSinkProvider
* @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory}.
* @return This builder, for convenience.
*/
@CanIgnoreReturnValue
public Builder setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
@ -111,6 +155,7 @@ public final class CompositingVideoSinkProvider
* @param previewingVideoGraphFactory The {@link PreviewingVideoGraph.Factory}.
* @return This builder, for convenience.
*/
@CanIgnoreReturnValue
public Builder setPreviewingVideoGraphFactory(
PreviewingVideoGraph.Factory previewingVideoGraphFactory) {
this.previewingVideoGraphFactory = previewingVideoGraphFactory;
@ -154,6 +199,7 @@ public final class CompositingVideoSinkProvider
private final Context context;
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
private final CopyOnWriteArraySet<CompositingVideoSinkProvider.Listener> listeners;
private Clock clock;
private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl;
@ -165,20 +211,35 @@ public final class CompositingVideoSinkProvider
private @MonotonicNonNull VideoSinkImpl videoSinkImpl;
private @MonotonicNonNull List<Effect> videoEffects;
@Nullable private Pair<Surface, Size> currentSurfaceAndSize;
private VideoSink.Listener listener;
private Executor listenerExecutor;
private int pendingFlushCount;
private @State int state;
private CompositingVideoSinkProvider(Builder builder) {
this.context = builder.context;
this.previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory);
this.listeners = new CopyOnWriteArraySet<>();
clock = Clock.DEFAULT;
listener = VideoSink.Listener.NO_OP;
listenerExecutor = NO_OP_EXECUTOR;
state = STATE_CREATED;
}
/**
* Adds a {@link CompositingVideoSinkProvider.Listener}.
*
* @param listener The listener to be added.
*/
public void addListener(CompositingVideoSinkProvider.Listener listener) {
listeners.add(listener);
}
/**
* Removes a {@link CompositingVideoSinkProvider.Listener}.
*
* @param listener The listener to be removed.
*/
public void removeListener(CompositingVideoSinkProvider.Listener listener) {
listeners.remove(listener);
}
// VideoSinkProvider methods
@Override
@ -220,6 +281,7 @@ public final class CompositingVideoSinkProvider
throw new VideoSink.VideoSinkException(e, sourceFormat);
}
videoSinkImpl.setVideoEffects(checkNotNull(videoEffects));
addListener(videoSinkImpl);
state = STATE_INITIALIZED;
}
@ -342,15 +404,9 @@ public final class CompositingVideoSinkProvider
@Override
public void onError(VideoFrameProcessingException exception) {
VideoSink.Listener currentListener = this.listener;
listenerExecutor.execute(
() -> {
VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
currentListener.onError(
videoSink,
new VideoSink.VideoSinkException(
exception, checkStateNotNull(videoSink.inputFormat)));
});
for (CompositingVideoSinkProvider.Listener listener : listeners) {
listener.onError(/* compositingVideoSinkProvider= */ this, exception);
}
}
// FrameRenderer methods
@ -363,18 +419,18 @@ public final class CompositingVideoSinkProvider
.setHeight(videoSize.height)
.setSampleMimeType(MimeTypes.VIDEO_RAW)
.build();
VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
VideoSink.Listener currentListener = this.listener;
listenerExecutor.execute(() -> currentListener.onVideoSizeChanged(videoSink, videoSize));
for (CompositingVideoSinkProvider.Listener listener : listeners) {
listener.onVideoSizeChanged(/* compositingVideoSinkProvider= */ this, videoSize);
}
}
@Override
public void renderFrame(
long renderTimeNs, long presentationTimeUs, long streamOffsetUs, boolean isFirstFrame) {
if (isFirstFrame && listenerExecutor != NO_OP_EXECUTOR) {
VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
VideoSink.Listener currentListener = this.listener;
listenerExecutor.execute(() -> currentListener.onFirstFrameRendered(videoSink));
if (isFirstFrame && currentSurfaceAndSize != null) {
for (CompositingVideoSinkProvider.Listener listener : listeners) {
listener.onFirstFrameRendered(/* compositingVideoSinkProvider= */ this);
}
}
if (videoFrameMetadataListener != null) {
// TODO b/292111083 - outputFormat is initialized after the first frame is rendered because
@ -391,9 +447,9 @@ public final class CompositingVideoSinkProvider
@Override
public void dropFrame() {
VideoSink.Listener currentListener = this.listener;
listenerExecutor.execute(
() -> currentListener.onFrameDropped(checkStateNotNull(videoSinkImpl)));
for (CompositingVideoSinkProvider.Listener listener : listeners) {
listener.onFrameDropped(/* compositingVideoSinkProvider= */ this);
}
checkStateNotNull(videoGraph).renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME);
}
@ -424,16 +480,6 @@ public final class CompositingVideoSinkProvider
// Internal methods
private void setListener(VideoSink.Listener listener, Executor executor) {
if (Objects.equals(listener, this.listener)) {
checkState(Objects.equals(executor, listenerExecutor));
return;
}
this.listener = listener;
this.listenerExecutor = executor;
}
private void maybeSetOutputSurfaceInfo(@Nullable Surface surface, int width, int height) {
if (videoGraph != null) {
// Update the surface on the video graph and the video frame release control together.
@ -490,7 +536,8 @@ public final class CompositingVideoSinkProvider
}
/** Receives input from an ExoPlayer renderer and forwards it to the video graph. */
private static final class VideoSinkImpl implements VideoSink {
private static final class VideoSinkImpl
implements VideoSink, CompositingVideoSinkProvider.Listener {
private final Context context;
private final CompositingVideoSinkProvider compositingVideoSinkProvider;
private final VideoFrameProcessor videoFrameProcessor;
@ -513,6 +560,8 @@ public final class CompositingVideoSinkProvider
private boolean hasRegisteredFirstInputStream;
private long pendingInputStreamBufferPresentationTimeUs;
private VideoSink.Listener listener;
private Executor listenerExecutor;
/** Creates a new instance. */
public VideoSinkImpl(
@ -534,6 +583,8 @@ public final class CompositingVideoSinkProvider
videoEffects = new ArrayList<>();
finalBufferPresentationTimeUs = C.TIME_UNSET;
lastBufferPresentationTimeUs = C.TIME_UNSET;
listener = VideoSink.Listener.NO_OP;
listenerExecutor = NO_OP_EXECUTOR;
}
// VideoSink impl
@ -605,7 +656,13 @@ public final class CompositingVideoSinkProvider
@Override
public void setListener(Listener listener, Executor executor) {
compositingVideoSinkProvider.setListener(listener, executor);
if (Objects.equals(listener, this.listener)) {
checkState(Objects.equals(executor, listenerExecutor));
return;
}
this.listener = listener;
listenerExecutor = executor;
}
@Override
@ -727,6 +784,42 @@ public final class CompositingVideoSinkProvider
.build());
}
// CompositingVideoSinkProvider.Listener implementation
@Override
public void onFirstFrameRendered(CompositingVideoSinkProvider compositingVideoSinkProvider) {
VideoSink.Listener currentListener = listener;
listenerExecutor.execute(() -> currentListener.onFirstFrameRendered(/* videoSink= */ this));
}
@Override
public void onFrameDropped(CompositingVideoSinkProvider compositingVideoSinkProvider) {
VideoSink.Listener currentListener = listener;
listenerExecutor.execute(
() -> currentListener.onFrameDropped(checkStateNotNull(/* reference= */ this)));
}
@Override
public void onVideoSizeChanged(
CompositingVideoSinkProvider compositingVideoSinkProvider, VideoSize videoSize) {
VideoSink.Listener currentListener = listener;
listenerExecutor.execute(
() -> currentListener.onVideoSizeChanged(/* videoSink= */ this, videoSize));
}
@Override
public void onError(
CompositingVideoSinkProvider compositingVideoSinkProvider,
VideoFrameProcessingException videoFrameProcessingException) {
VideoSink.Listener currentListener = listener;
listenerExecutor.execute(
() ->
currentListener.onError(
/* videoSink= */ this,
new VideoSinkException(
videoFrameProcessingException, checkStateNotNull(this.inputFormat))));
}
private static final class ScaleAndRotateAccessor {
private static @MonotonicNonNull Constructor<?>
scaleAndRotateTransformationBuilderConstructor;