diff --git a/extensions/av1/README.md b/extensions/av1/README.md index 5f049df4f5..e20d6b258e 100644 --- a/extensions/av1/README.md +++ b/extensions/av1/README.md @@ -79,23 +79,26 @@ a custom track selector the choice of `Renderer` is up to your implementation. You need to make sure you are passing a `Libgav1VideoRenderer` to the player and then you need to implement your own logic to use the renderer for a given track. +## Rendering options ## + There are two possibilities for rendering the output `Libgav1VideoRenderer` gets from the libgav1 decoder: -* Native rendering with `ANativeWindow` -* OpenGL rendering. +* GL rendering using GL shader for color space conversion + * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by + setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message + of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of + `VideoDecoderOutputBufferRenderer` as its object. -`SimpleExoPlayer` uses `ANativeWindow` rendering. To enable this mode send the -renderer a message of type `C.MSG_SET_SURFACE` with a `Surface` as its object. -`Libgav1VideoRenderer` can also output to a `VideoDecoderSurfaceView` when -not being used via `SimpleExoPlayer`, in which case color space conversion will -be performed using a GL shader. To enable this mode, send the renderer a message -of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as -its object. +* Native rendering using `ANativeWindow` + * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled + by default. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of + type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object. Note: Although the default option uses `ANativeWindow`, based on our testing the -GL rendering mode has better performance, so should be preferred by apps that -can use `VideoDecoderSurfaceView`. +GL rendering mode has better performance, so should be preferred ## Links ## diff --git a/extensions/vp9/README.md b/extensions/vp9/README.md index b7ea254a6c..a2e8736f82 100644 --- a/extensions/vp9/README.md +++ b/extensions/vp9/README.md @@ -107,11 +107,26 @@ a custom track selector the choice of `Renderer` is up to your implementation, so you need to make sure you are passing an `LibvpxVideoRenderer` to the player, then implement your own logic to use the renderer for a given track. -`LibvpxVideoRenderer` can optionally output to a `VideoDecoderSurfaceView` when -not being used via `SimpleExoPlayer`, in which case color space conversion will -be performed using a GL shader. To enable this mode, send the renderer a message -of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as -its object, instead of sending `MSG_SET_SURFACE` with a `Surface`. +## Rendering options ## + +There are two possibilities for rendering the output `LibvpxVideoRenderer` +gets from the libvpx decoder: + +* GL rendering using GL shader for color space conversion + * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by + setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`. + * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of + type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of + `VideoDecoderOutputBufferRenderer` as its object. + +* Native rendering using `ANativeWindow` + * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled + by default. + * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of + type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object. + +Note: Although the default option uses `ANativeWindow`, based on our testing the +GL rendering mode has better performance, so should be preferred. ## Links ## diff --git a/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java b/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java index 3dd039118c..85ceabc909 100644 --- a/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java +++ b/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java @@ -124,7 +124,7 @@ public class VpxPlaybackTest { player .createMessage(videoRenderer) .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER) - .setPayload(new VideoDecoderSurfaceView(context)) + .setPayload(new VideoDecoderSurfaceView(context).getOutputBufferRenderer()) .send(); player.prepare(mediaSource); player.setPlayWhenReady(true); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/Player.java b/library/core/src/main/java/com/google/android/exoplayer2/Player.java index 5f00916892..5380f6a1e5 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/Player.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/Player.java @@ -31,6 +31,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray; import com.google.android.exoplayer2.text.TextOutput; import com.google.android.exoplayer2.trackselection.TrackSelectionArray; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoListener; import com.google.android.exoplayer2.video.spherical.CameraMotionListener; @@ -280,6 +281,13 @@ public interface Player { * @param textureView The texture view to clear. */ void clearVideoTextureView(TextureView textureView); + + /** + * Sets the output buffer renderer. + * + * @param outputBufferRenderer The output buffer renderer. + */ + void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer); } /** The text component of a {@link Player}. */ diff --git a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java index 4e779c2a78..911890d529 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java @@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.Clock; import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.PriorityTaskManager; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoRendererEventListener; import com.google.android.exoplayer2.video.spherical.CameraMotionListener; @@ -584,8 +585,8 @@ public class SimpleExoPlayer extends BasePlayer Log.w(TAG, "Replacing existing SurfaceTextureListener."); } textureView.setSurfaceTextureListener(componentListener); - SurfaceTexture surfaceTexture = textureView.isAvailable() ? textureView.getSurfaceTexture() - : null; + SurfaceTexture surfaceTexture = + textureView.isAvailable() ? textureView.getSurfaceTexture() : null; if (surfaceTexture == null) { setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true); maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0); @@ -604,6 +605,23 @@ public class SimpleExoPlayer extends BasePlayer } } + @Override + public void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer) { + verifyApplicationThread(); + removeSurfaceCallbacks(); + List messages = new ArrayList<>(); + for (Renderer renderer : renderers) { + if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) { + messages.add( + player + .createMessage(renderer) + .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER) + .setPayload(outputBufferRenderer) + .send()); + } + } + } + @Override public void addAudioListener(AudioListener listener) { audioListeners.add(listener); @@ -695,12 +713,12 @@ public class SimpleExoPlayer extends BasePlayer /** * Sets the stream type for audio playback, used by the underlying audio track. - *

- * Setting the stream type during playback may introduce a short gap in audio output as the audio - * track is recreated. A new audio session id will also be generated. - *

- * Calling this method overwrites any attributes set previously by calling - * {@link #setAudioAttributes(AudioAttributes)}. + * + *

Setting the stream type during playback may introduce a short gap in audio output as the + * audio track is recreated. A new audio session id will also be generated. + * + *

Calling this method overwrites any attributes set previously by calling {@link + * #setAudioAttributes(AudioAttributes)}. * * @deprecated Use {@link #setAudioAttributes(AudioAttributes)}. * @param streamType The stream type for audio playback. @@ -1473,11 +1491,11 @@ public class SimpleExoPlayer extends BasePlayer } @Override - public void onVideoDecoderInitialized(String decoderName, long initializedTimestampMs, - long initializationDurationMs) { + public void onVideoDecoderInitialized( + String decoderName, long initializedTimestampMs, long initializationDurationMs) { for (VideoRendererEventListener videoDebugListener : videoDebugListeners) { - videoDebugListener.onVideoDecoderInitialized(decoderName, initializedTimestampMs, - initializationDurationMs); + videoDebugListener.onVideoDecoderInitialized( + decoderName, initializedTimestampMs, initializationDurationMs); } } @@ -1497,8 +1515,8 @@ public class SimpleExoPlayer extends BasePlayer } @Override - public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees, - float pixelWidthHeightRatio) { + public void onVideoSizeChanged( + int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) { for (com.google.android.exoplayer2.video.VideoListener videoListener : videoListeners) { // Prevent duplicate notification if a listener is both a VideoRendererEventListener and // a VideoListener, as they have the same method signature. @@ -1508,8 +1526,8 @@ public class SimpleExoPlayer extends BasePlayer } } for (VideoRendererEventListener videoDebugListener : videoDebugListeners) { - videoDebugListener.onVideoSizeChanged(width, height, unappliedRotationDegrees, - pixelWidthHeightRatio); + videoDebugListener.onVideoSizeChanged( + width, height, unappliedRotationDegrees, pixelWidthHeightRatio); } } @@ -1563,11 +1581,11 @@ public class SimpleExoPlayer extends BasePlayer } @Override - public void onAudioDecoderInitialized(String decoderName, long initializedTimestampMs, - long initializationDurationMs) { + public void onAudioDecoderInitialized( + String decoderName, long initializedTimestampMs, long initializationDurationMs) { for (AudioRendererEventListener audioDebugListener : audioDebugListeners) { - audioDebugListener.onAudioDecoderInitialized(decoderName, initializedTimestampMs, - initializationDurationMs); + audioDebugListener.onAudioDecoderInitialized( + decoderName, initializedTimestampMs, initializationDurationMs); } } @@ -1580,8 +1598,8 @@ public class SimpleExoPlayer extends BasePlayer } @Override - public void onAudioSinkUnderrun(int bufferSize, long bufferSizeMs, - long elapsedSinceLastFeedMs) { + public void onAudioSinkUnderrun( + int bufferSize, long bufferSizeMs, long elapsedSinceLastFeedMs) { for (AudioRendererEventListener audioDebugListener : audioDebugListeners) { audioDebugListener.onAudioSinkUnderrun(bufferSize, bufferSizeMs, elapsedSinceLastFeedMs); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java index c85dff0c26..183b1c063c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java @@ -28,7 +28,8 @@ import javax.microedition.khronos.opengles.GL10; * GLSurfaceView.Renderer implementation that can render YUV Frames returned by a video decoder * after decoding. It does the YUV to RGB color conversion in the Fragment Shader. */ -/* package */ class VideoDecoderRenderer implements GLSurfaceView.Renderer { +/* package */ class VideoDecoderRenderer + implements GLSurfaceView.Renderer, VideoDecoderOutputBufferRenderer { private static final float[] kColorConversion601 = { 1.164f, 1.164f, 1.164f, @@ -82,6 +83,7 @@ import javax.microedition.khronos.opengles.GL10; private static final FloatBuffer TEXTURE_VERTICES = GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f}); + private final GLSurfaceView surfaceView; private final int[] yuvTextures = new int[3]; private final AtomicReference pendingOutputBufferReference; @@ -98,7 +100,8 @@ import javax.microedition.khronos.opengles.GL10; private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread. - public VideoDecoderRenderer() { + public VideoDecoderRenderer(GLSurfaceView surfaceView) { + this.surfaceView = surfaceView; pendingOutputBufferReference = new AtomicReference<>(); textureCoords = new FloatBuffer[3]; texLocations = new int[3]; @@ -109,21 +112,6 @@ import javax.microedition.khronos.opengles.GL10; } } - /** - * Set a frame to be rendered. This should be followed by a call to - * VideoDecoderSurfaceView.requestRender() to actually render the frame. - * - * @param outputBuffer OutputBuffer containing the YUV Frame to be rendered - */ - public void setFrame(VideoDecoderOutputBuffer outputBuffer) { - VideoDecoderOutputBuffer oldPendingOutputBuffer = - pendingOutputBufferReference.getAndSet(outputBuffer); - if (oldPendingOutputBuffer != null) { - // The old pending output buffer will never be used for rendering, so release it now. - oldPendingOutputBuffer.release(); - } - } - @Override public void onSurfaceCreated(GL10 unused, EGLConfig config) { program = GlUtil.compileProgram(VERTEX_SHADER, FRAGMENT_SHADER); @@ -223,6 +211,17 @@ import javax.microedition.khronos.opengles.GL10; GlUtil.checkGlError(); } + @Override + public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) { + VideoDecoderOutputBuffer oldPendingOutputBuffer = + pendingOutputBufferReference.getAndSet(outputBuffer); + if (oldPendingOutputBuffer != null) { + // The old pending output buffer will never be used for rendering, so release it now. + oldPendingOutputBuffer.release(); + } + surfaceView.requestRender(); + } + private void setupTextures() { GLES20.glGenTextures(3, yuvTextures, 0); for (int i = 0; i < 3; i++) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java index f2a4c2d002..0436c919a0 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java @@ -21,28 +21,40 @@ import android.util.AttributeSet; import androidx.annotation.Nullable; /** A GLSurfaceView extension that scales itself to the given aspect ratio. */ -public class VideoDecoderSurfaceView extends GLSurfaceView - implements VideoDecoderOutputBufferRenderer { +public class VideoDecoderSurfaceView extends GLSurfaceView { private final VideoDecoderRenderer renderer; + /** + * Creates VideoDecoderSurfaceView. + * + * @param context A {@link Context}. + */ public VideoDecoderSurfaceView(Context context) { this(context, /* attrs= */ null); } + /** + * Creates VideoDecoderSurfaceView. + * + * @param context A {@link Context}. + * @param attrs Custom attributes. + */ public VideoDecoderSurfaceView(Context context, @Nullable AttributeSet attrs) { super(context, attrs); - renderer = new VideoDecoderRenderer(); + renderer = new VideoDecoderRenderer(this); setPreserveEGLContextOnPause(true); setEGLContextClientVersion(2); setRenderer(renderer); setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); } - @Override - public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) { - renderer.setFrame(outputBuffer); - requestRender(); + /** + * Returns the output buffer renderer used. + * + * @return {@link VideoDecoderOutputBuffer}. + */ + public VideoDecoderOutputBufferRenderer getOutputBufferRenderer() { + return renderer; } - } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java index 67701cbb93..20775ceb8f 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java @@ -64,6 +64,7 @@ import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.ErrorMessageProvider; import com.google.android.exoplayer2.util.RepeatModeUtil; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoDecoderSurfaceView; import com.google.android.exoplayer2.video.VideoListener; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -276,6 +277,7 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider private static final int SURFACE_TYPE_SURFACE_VIEW = 1; private static final int SURFACE_TYPE_TEXTURE_VIEW = 2; private static final int SURFACE_TYPE_MONO360_VIEW = 3; + private static final int SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW = 4; // LINT.ThenChange(../../../../../../res/values/attrs.xml) @Nullable private final AspectRatioFrameLayout contentFrame; @@ -411,6 +413,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider sphericalSurfaceView.setSingleTapListener(componentListener); surfaceView = sphericalSurfaceView; break; + case SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW: + surfaceView = new VideoDecoderSurfaceView(context); + break; default: surfaceView = new SurfaceView(context); break; @@ -539,6 +544,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider oldVideoComponent.clearVideoTextureView((TextureView) surfaceView); } else if (surfaceView instanceof SphericalSurfaceView) { ((SphericalSurfaceView) surfaceView).setVideoComponent(null); + } else if (surfaceView instanceof VideoDecoderSurfaceView) { + oldVideoComponent.setOutputBufferRenderer(null); } else if (surfaceView instanceof SurfaceView) { oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView); } @@ -565,6 +572,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider newVideoComponent.setVideoTextureView((TextureView) surfaceView); } else if (surfaceView instanceof SphericalSurfaceView) { ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent); + } else if (surfaceView instanceof VideoDecoderSurfaceView) { + newVideoComponent.setOutputBufferRenderer( + ((VideoDecoderSurfaceView) surfaceView).getOutputBufferRenderer()); } else if (surfaceView instanceof SurfaceView) { newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView); } @@ -736,8 +746,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider * buffering spinner is not displayed by default. * * @param showBuffering The mode that defines when the buffering spinner is displayed. One of - * {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and - * {@link #SHOW_BUFFERING_ALWAYS}. + * {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and {@link + * #SHOW_BUFFERING_ALWAYS}. */ public void setShowBuffering(@ShowBuffering int showBuffering) { if (this.showBuffering != showBuffering) { diff --git a/library/ui/src/main/res/values/attrs.xml b/library/ui/src/main/res/values/attrs.xml index 706fba0e0b..b342d5d888 100644 --- a/library/ui/src/main/res/values/attrs.xml +++ b/library/ui/src/main/res/values/attrs.xml @@ -30,6 +30,7 @@ +