From 438ae0ed6ac1059e8aaa9380f476c1403f24a986 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Mon, 22 May 2023 14:24:57 +0100 Subject: [PATCH] ExoPlayer: Add setVideoFrameProcessorFactory(). This allows apps to use a custom VideoFrameProcessor implementation for video playback. This may be useful, for example, when outputting to a texture. PiperOrigin-RevId: 534044831 --- RELEASENOTES.md | 3 + .../androidx/media3/exoplayer/ExoPlayer.java | 14 ++ .../media3/exoplayer/ExoPlayerImpl.java | 10 ++ .../androidx/media3/exoplayer/Renderer.java | 15 ++- .../media3/exoplayer/SimpleExoPlayer.java | 8 ++ .../video/MediaCodecVideoRenderer.java | 127 ++++++++++-------- .../media3/test/utils/StubExoPlayer.java | 7 + 7 files changed, 124 insertions(+), 60 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1a8830f5c3..30ceb87a22 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -90,6 +90,9 @@ * Remove deprecated `ExoPlayer.retry()`, use `prepare()` instead. * Remove deprecated zero-arg `DefaultTrackSelector` constructor, use `DefaultTrackSelector(Context)` instead. +* Core library: + * Add `ExoPlayer.setVideoFrameProcessorFactory()` for using `Effect` with + a custom `VideoFrameProcessor.Factory` during video playback. ### 1.0.2 (2023-05-18) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java index 77cccb41ab..0c3dc7fc63 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java @@ -44,6 +44,7 @@ import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; import androidx.media3.common.Tracks; +import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.CueGroup; import androidx.media3.common.util.Clock; @@ -1562,6 +1563,19 @@ public interface ExoPlayer extends Player { @UnstableApi void setVideoEffects(List videoEffects); + /** + * Sets a {@link VideoFrameProcessor.Factory} to create the {@link VideoFrameProcessor} that + * applies video effects set in {@link #setVideoEffects}. + * + *

See {@link #setVideoEffects} for limitations. + * + * @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} to use to apply the + * video effects. + */ + @RequiresApi(18) + @UnstableApi + void setVideoFrameProcessorFactory(VideoFrameProcessor.Factory videoFrameProcessorFactory); + /** * Sets the {@link C.VideoScalingMode}. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index 8301ab99e1..3deac14e16 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -32,6 +32,7 @@ import static androidx.media3.exoplayer.Renderer.MSG_SET_SCALING_MODE; import static androidx.media3.exoplayer.Renderer.MSG_SET_SKIP_SILENCE_ENABLED; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_EFFECTS; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER; +import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT; import static androidx.media3.exoplayer.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION; import static androidx.media3.exoplayer.Renderer.MSG_SET_VOLUME; @@ -77,6 +78,7 @@ import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; +import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.Cue; import androidx.media3.common.text.CueGroup; @@ -1238,6 +1240,14 @@ import java.util.concurrent.TimeoutException; sendRendererMessage(TRACK_TYPE_VIDEO, MSG_SET_VIDEO_EFFECTS, videoEffects); } + @Override + public void setVideoFrameProcessorFactory( + VideoFrameProcessor.Factory videoFrameProcessorFactory) { + verifyApplicationThread(); + sendRendererMessage( + TRACK_TYPE_VIDEO, MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, videoFrameProcessorFactory); + } + @Override public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) { verifyApplicationThread(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java index 739337df6d..28e58f282b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/Renderer.java @@ -27,6 +27,7 @@ import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.Player; +import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -90,9 +91,9 @@ public interface Renderer extends PlayerMessage.Target { * #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link * #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link * #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link - * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS} or - * {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May also be an app-defined value (see {@link - * #MSG_CUSTOM_BASE}). + * #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS}, + * {@link #MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY} or {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May + * also be an app-defined value (see {@link #MSG_CUSTOM_BASE}). */ @Documented @Retention(RetentionPolicy.SOURCE) @@ -112,6 +113,7 @@ public interface Renderer extends PlayerMessage.Target { MSG_SET_AUDIO_SESSION_ID, MSG_SET_WAKEUP_LISTENER, MSG_SET_VIDEO_EFFECTS, + MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, MSG_SET_VIDEO_OUTPUT_RESOLUTION }) public @interface MessageType {} @@ -219,12 +221,17 @@ public interface Renderer extends PlayerMessage.Target { * {@link List} containing {@linkplain Effect video effects}. */ int MSG_SET_VIDEO_EFFECTS = 13; + /** + * The type of a message that can be passed to a video renderer. The message payload should be a + * {@link VideoFrameProcessor.Factory}. + */ + int MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY = 14; /** * The type of a message that can be passed to a video renderer to set the desired output * resolution. The message payload should be a {@link Size} of the desired output width and * height. Use this method only when playing with video {@linkplain Effect effects}. */ - int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 14; + int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 15; /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * renderers. These custom constants must be greater than or equal to this value. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java index edda6b71cb..6e2ff0c857 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java @@ -43,6 +43,7 @@ import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; +import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; import androidx.media3.common.text.CueGroup; import androidx.media3.common.util.Clock; @@ -671,6 +672,13 @@ public class SimpleExoPlayer extends BasePlayer player.setVideoEffects(videoEffects); } + @Override + public void setVideoFrameProcessorFactory( + VideoFrameProcessor.Factory videoFrameProcessorFactory) { + blockUntilConstructorFinished(); + player.setVideoFrameProcessorFactory(videoFrameProcessorFactory); + } + @Override public void setSkipSilenceEnabled(boolean skipSilenceEnabled) { blockUntilConstructorFinished(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 3d848899ce..7f72e8cf11 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -687,6 +687,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { List videoEffects = (List) checkNotNull(message); videoFrameProcessorManager.setVideoEffects(videoEffects); break; + case MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY: + VideoFrameProcessor.Factory videoFrameProcessorFactory = + (VideoFrameProcessor.Factory) checkNotNull(message); + videoFrameProcessorManager.setVideoFrameProcessorFactory(videoFrameProcessorFactory); + break; case MSG_SET_VIDEO_OUTPUT_RESOLUTION: Size outputResolution = (Size) checkNotNull(message); if (outputResolution.getWidth() != 0 @@ -1876,6 +1881,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private final ArrayDeque> pendingFrameFormats; private @MonotonicNonNull Handler handler; + private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory; @Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private CopyOnWriteArrayList videoEffects; @Nullable private Format inputFormat; @@ -1937,6 +1943,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { this.videoEffects.addAll(videoEffects); } + /** Sets the {@link VideoFrameProcessor.Factory}. */ + public void setVideoFrameProcessorFactory( + VideoFrameProcessor.Factory videoFrameProcessorFactory) { + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + } + /** Returns whether video frame processing is enabled. */ public boolean isEnabled() { return videoFrameProcessor != null; @@ -2009,66 +2021,69 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees)); } + videoFrameProcessorFactory = + videoFrameProcessorFactory == null + ? VideoFrameProcessorAccessor.getFrameProcessorFactory() + : videoFrameProcessorFactory; videoFrameProcessor = - VideoFrameProcessorAccessor.getFrameProcessorFactory() - .create( - renderer.context, - checkNotNull(videoEffects), - DebugViewProvider.NONE, - inputAndOutputColorInfos.first, - inputAndOutputColorInfos.second, - /* renderFramesAutomatically= */ false, - /* listenerExecutor= */ handler::post, - new VideoFrameProcessor.Listener() { - @Override - public void onOutputSizeChanged(int width, int height) { - @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; - checkStateNotNull(inputFormat); - // TODO(b/264889146): Handle Effect that changes output size based on pts. - processedFrameSize = - new VideoSize( - width, - height, - // VideoFrameProcessor is configured to produce rotation free - // frames. - /* unappliedRotationDegrees= */ 0, - // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. - /* pixelWidthHeightRatio= */ 1.f); - pendingOutputSizeChange = true; - } + videoFrameProcessorFactory.create( + renderer.context, + checkNotNull(videoEffects), + DebugViewProvider.NONE, + inputAndOutputColorInfos.first, + inputAndOutputColorInfos.second, + /* renderFramesAutomatically= */ false, + /* listenerExecutor= */ handler::post, + new VideoFrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; + checkStateNotNull(inputFormat); + // TODO(b/264889146): Handle Effect that changes output size based on pts. + processedFrameSize = + new VideoSize( + width, + height, + // VideoFrameProcessor is configured to produce rotation free + // frames. + /* unappliedRotationDegrees= */ 0, + // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. + /* pixelWidthHeightRatio= */ 1.f); + pendingOutputSizeChange = true; + } - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (registeredLastFrame) { - checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); - } - processedFramesTimestampsUs.add(presentationTimeUs); - // TODO(b/257464707) Support extensively modified media. - if (registeredLastFrame - && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { - processedLastFrame = true; - } - if (pendingOutputSizeChange) { - // Report the size change on releasing this frame. - pendingOutputSizeChange = false; - pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; - } - } + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (registeredLastFrame) { + checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); + } + processedFramesTimestampsUs.add(presentationTimeUs); + // TODO(b/257464707) Support extensively modified media. + if (registeredLastFrame + && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { + processedLastFrame = true; + } + if (pendingOutputSizeChange) { + // Report the size change on releasing this frame. + pendingOutputSizeChange = false; + pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; + } + } - @Override - public void onError(VideoFrameProcessingException exception) { - renderer.setPendingPlaybackException( - renderer.createRendererException( - exception, - inputFormat, - PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } + @Override + public void onError(VideoFrameProcessingException exception) { + renderer.setPendingPlaybackException( + renderer.createRendererException( + exception, + inputFormat, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } - @Override - public void onEnded() { - throw new IllegalStateException(); - } - }); + @Override + public void onEnded() { + throw new IllegalStateException(); + } + }); videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE); this.initialStreamOffsetUs = initialStreamOffsetUs; } catch (Exception e) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java index 730fefd7e1..b2e20db5e9 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java @@ -24,6 +24,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; +import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Clock; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.DecoderCounters; @@ -249,6 +250,12 @@ public class StubExoPlayer extends StubPlayer implements ExoPlayer { throw new UnsupportedOperationException(); } + @Override + public void setVideoFrameProcessorFactory( + VideoFrameProcessor.Factory videoFrameProcessorFactory) { + throw new UnsupportedOperationException(); + } + @Override public void setVideoScalingMode(int videoScalingMode) { throw new UnsupportedOperationException();