sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-13 12:35:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support GL rendering with SimpleExoPlayer and PlayerView

Browse source 
PiperOrigin-RevId: 273760294
This commit is contained in:
sofijajvc 2019-10-09 17:24:08 +01:00 committed by Ian Baker
parent a35a0925ed
commit 5cf82a5079
9 changed files with 132 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
extensions/av1/README.md
View file

@ -79,23 +79,26 @@ a custom track selector the choice of `Renderer` is up to your implementation.
You need to make sure you are passing a `Libgav1VideoRenderer` to the player and
then you need to implement your own logic to use the renderer for a given track.
## Rendering options ##
There are two possibilities for rendering the output `Libgav1VideoRenderer`
gets from the libgav1 decoder:
* Native rendering with `ANativeWindow`
* OpenGL rendering.
* GL rendering using GL shader for color space conversion
* If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
* Otherwise, enable this option by sending `Libgav1VideoRenderer` a message
of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
`VideoDecoderOutputBufferRenderer` as its object.
`SimpleExoPlayer` uses `ANativeWindow` rendering. To enable this mode send the
renderer a message of type `C.MSG_SET_SURFACE` with a `Surface` as its object.
`Libgav1VideoRenderer` can also output to a `VideoDecoderSurfaceView` when
not being used via `SimpleExoPlayer`, in which case color space conversion will
be performed using a GL shader. To enable this mode, send the renderer a message
of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
its object.
* Native rendering using `ANativeWindow`
* If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
by default.
* Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of
type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
Note: Although the default option uses `ANativeWindow`, based on our testing the
GL rendering mode has better performance, so should be preferred by apps that
can use `VideoDecoderSurfaceView`.
GL rendering mode has better performance, so should be preferred
## Links ##

25
extensions/vp9/README.md
View file

@ -107,11 +107,26 @@ a custom track selector the choice of `Renderer` is up to your implementation,
so you need to make sure you are passing an `LibvpxVideoRenderer` to the
player, then implement your own logic to use the renderer for a given track.
`LibvpxVideoRenderer` can optionally output to a `VideoDecoderSurfaceView` when
not being used via `SimpleExoPlayer`, in which case color space conversion will
be performed using a GL shader. To enable this mode, send the renderer a message
of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
its object, instead of sending `MSG_SET_SURFACE` with a `Surface`.
## Rendering options ##
There are two possibilities for rendering the output `LibvpxVideoRenderer`
gets from the libvpx decoder:
* GL rendering using GL shader for color space conversion
* If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
* Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
`VideoDecoderOutputBufferRenderer` as its object.
* Native rendering using `ANativeWindow`
* If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
by default.
* Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
Note: Although the default option uses `ANativeWindow`, based on our testing the
GL rendering mode has better performance, so should be preferred.
## Links ##

2
extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java
View file

@ -124,7 +124,7 @@ public class VpxPlaybackTest {
player
.createMessage(videoRenderer)
.setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
.setPayload(new VideoDecoderSurfaceView(context))
.setPayload(new VideoDecoderSurfaceView(context).getOutputBufferRenderer())
.send();
player.prepare(mediaSource);
player.setPlayWhenReady(true);

8
library/core/src/main/java/com/google/android/exoplayer2/Player.java
View file

@ -31,6 +31,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray;
import com.google.android.exoplayer2.text.TextOutput;
import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@ -280,6 +281,13 @@ public interface Player {
* @param textureView The texture view to clear.
*/
void clearVideoTextureView(TextureView textureView);
/**
* Sets the output buffer renderer.
*
* @param outputBufferRenderer The output buffer renderer.
*/
void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer);
}
/** The text component of a {@link Player}. */

62
library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
View file

@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoRendererEventListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@ -584,8 +585,8 @@ public class SimpleExoPlayer extends BasePlayer
Log.w(TAG, "Replacing existing SurfaceTextureListener.");
}
textureView.setSurfaceTextureListener(componentListener);
SurfaceTexture surfaceTexture = textureView.isAvailable() ? textureView.getSurfaceTexture()
: null;
SurfaceTexture surfaceTexture =
textureView.isAvailable() ? textureView.getSurfaceTexture() : null;
if (surfaceTexture == null) {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
@ -604,6 +605,23 @@ public class SimpleExoPlayer extends BasePlayer
}
}
@Override
public void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer) {
verifyApplicationThread();
removeSurfaceCallbacks();
List<PlayerMessage> messages = new ArrayList<>();
for (Renderer renderer : renderers) {
if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
messages.add(
player
.createMessage(renderer)
.setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
.setPayload(outputBufferRenderer)
.send());
}
}
}
@Override
public void addAudioListener(AudioListener listener) {
audioListeners.add(listener);
@ -695,12 +713,12 @@ public class SimpleExoPlayer extends BasePlayer
/**
* Sets the stream type for audio playback, used by the underlying audio track.
* <p>
* Setting the stream type during playback may introduce a short gap in audio output as the audio
* track is recreated. A new audio session id will also be generated.
* <p>
* Calling this method overwrites any attributes set previously by calling
* {@link #setAudioAttributes(AudioAttributes)}.
*
* <p>Setting the stream type during playback may introduce a short gap in audio output as the
* audio track is recreated. A new audio session id will also be generated.
*
* <p>Calling this method overwrites any attributes set previously by calling {@link
* #setAudioAttributes(AudioAttributes)}.
*
* @deprecated Use {@link #setAudioAttributes(AudioAttributes)}.
* @param streamType The stream type for audio playback.
@ -1473,11 +1491,11 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void onVideoDecoderInitialized(String decoderName, long initializedTimestampMs,
long initializationDurationMs) {
public void onVideoDecoderInitialized(
String decoderName, long initializedTimestampMs, long initializationDurationMs) {
for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
videoDebugListener.onVideoDecoderInitialized(decoderName, initializedTimestampMs,
initializationDurationMs);
videoDebugListener.onVideoDecoderInitialized(
decoderName, initializedTimestampMs, initializationDurationMs);
}
}
@ -1497,8 +1515,8 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees,
float pixelWidthHeightRatio) {
public void onVideoSizeChanged(
int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {
for (com.google.android.exoplayer2.video.VideoListener videoListener : videoListeners) {
// Prevent duplicate notification if a listener is both a VideoRendererEventListener and
// a VideoListener, as they have the same method signature.
@ -1508,8 +1526,8 @@ public class SimpleExoPlayer extends BasePlayer
}
}
for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
videoDebugListener.onVideoSizeChanged(width, height, unappliedRotationDegrees,
pixelWidthHeightRatio);
videoDebugListener.onVideoSizeChanged(
width, height, unappliedRotationDegrees, pixelWidthHeightRatio);
}
}
@ -1563,11 +1581,11 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void onAudioDecoderInitialized(String decoderName, long initializedTimestampMs,
long initializationDurationMs) {
public void onAudioDecoderInitialized(
String decoderName, long initializedTimestampMs, long initializationDurationMs) {
for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
audioDebugListener.onAudioDecoderInitialized(decoderName, initializedTimestampMs,
initializationDurationMs);
audioDebugListener.onAudioDecoderInitialized(
decoderName, initializedTimestampMs, initializationDurationMs);
}
}
@ -1580,8 +1598,8 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void onAudioSinkUnderrun(int bufferSize, long bufferSizeMs,
long elapsedSinceLastFeedMs) {
public void onAudioSinkUnderrun(
int bufferSize, long bufferSizeMs, long elapsedSinceLastFeedMs) {
for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
audioDebugListener.onAudioSinkUnderrun(bufferSize, bufferSizeMs, elapsedSinceLastFeedMs);
}

33
library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java
View file

@ -28,7 +28,8 @@ import javax.microedition.khronos.opengles.GL10;
* GLSurfaceView.Renderer implementation that can render YUV Frames returned by a video decoder
* after decoding. It does the YUV to RGB color conversion in the Fragment Shader.
*/
/* package */ class VideoDecoderRenderer implements GLSurfaceView.Renderer {
/* package */ class VideoDecoderRenderer
implements GLSurfaceView.Renderer, VideoDecoderOutputBufferRenderer {
private static final float[] kColorConversion601 = {
1.164f, 1.164f, 1.164f,
@ -82,6 +83,7 @@ import javax.microedition.khronos.opengles.GL10;
private static final FloatBuffer TEXTURE_VERTICES =
GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
private final GLSurfaceView surfaceView;
private final int[] yuvTextures = new int[3];
private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
@ -98,7 +100,8 @@ import javax.microedition.khronos.opengles.GL10;
private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
public VideoDecoderRenderer() {
public VideoDecoderRenderer(GLSurfaceView surfaceView) {
this.surfaceView = surfaceView;
pendingOutputBufferReference = new AtomicReference<>();
textureCoords = new FloatBuffer[3];
texLocations = new int[3];
@ -109,21 +112,6 @@ import javax.microedition.khronos.opengles.GL10;
}
}
/**
* Set a frame to be rendered. This should be followed by a call to
* VideoDecoderSurfaceView.requestRender() to actually render the frame.
*
* @param outputBuffer OutputBuffer containing the YUV Frame to be rendered
*/
public void setFrame(VideoDecoderOutputBuffer outputBuffer) {
VideoDecoderOutputBuffer oldPendingOutputBuffer =
pendingOutputBufferReference.getAndSet(outputBuffer);
if (oldPendingOutputBuffer != null) {
// The old pending output buffer will never be used for rendering, so release it now.
oldPendingOutputBuffer.release();
}
}
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
program = GlUtil.compileProgram(VERTEX_SHADER, FRAGMENT_SHADER);
@ -223,6 +211,17 @@ import javax.microedition.khronos.opengles.GL10;
GlUtil.checkGlError();
}
@Override
public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
VideoDecoderOutputBuffer oldPendingOutputBuffer =
pendingOutputBufferReference.getAndSet(outputBuffer);
if (oldPendingOutputBuffer != null) {
// The old pending output buffer will never be used for rendering, so release it now.
oldPendingOutputBuffer.release();
}
surfaceView.requestRender();
}
private void setupTextures() {
GLES20.glGenTextures(3, yuvTextures, 0);
for (int i = 0; i < 3; i++) {

28
library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java
View file

@ -21,28 +21,40 @@ import android.util.AttributeSet;
import androidx.annotation.Nullable;
/** A GLSurfaceView extension that scales itself to the given aspect ratio. */
public class VideoDecoderSurfaceView extends GLSurfaceView
implements VideoDecoderOutputBufferRenderer {
public class VideoDecoderSurfaceView extends GLSurfaceView {
private final VideoDecoderRenderer renderer;
/**
* Creates VideoDecoderSurfaceView.
*
* @param context A {@link Context}.
*/
public VideoDecoderSurfaceView(Context context) {
this(context, /* attrs= */ null);
}
/**
* Creates VideoDecoderSurfaceView.
*
* @param context A {@link Context}.
* @param attrs Custom attributes.
*/
public VideoDecoderSurfaceView(Context context, @Nullable AttributeSet attrs) {
super(context, attrs);
renderer = new VideoDecoderRenderer();
renderer = new VideoDecoderRenderer(this);
setPreserveEGLContextOnPause(true);
setEGLContextClientVersion(2);
setRenderer(renderer);
setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
}
@Override
public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
renderer.setFrame(outputBuffer);
requestRender();
/**
* Returns the output buffer renderer used.
*
* @return {@link VideoDecoderOutputBuffer}.
*/
public VideoDecoderOutputBufferRenderer getOutputBufferRenderer() {
return renderer;
}
}

14
library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
View file

@ -64,6 +64,7 @@ import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.ErrorMessageProvider;
import com.google.android.exoplayer2.util.RepeatModeUtil;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoDecoderSurfaceView;
import com.google.android.exoplayer2.video.VideoListener;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
@ -276,6 +277,7 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
private static final int SURFACE_TYPE_SURFACE_VIEW = 1;
private static final int SURFACE_TYPE_TEXTURE_VIEW = 2;
private static final int SURFACE_TYPE_MONO360_VIEW = 3;
private static final int SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW = 4;
// LINT.ThenChange(../../../../../../res/values/attrs.xml)
@Nullable private final AspectRatioFrameLayout contentFrame;
@ -411,6 +413,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
sphericalSurfaceView.setSingleTapListener(componentListener);
surfaceView = sphericalSurfaceView;
break;
case SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW:
surfaceView = new VideoDecoderSurfaceView(context);
break;
default:
surfaceView = new SurfaceView(context);
break;
@ -539,6 +544,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) {
((SphericalSurfaceView) surfaceView).setVideoComponent(null);
} else if (surfaceView instanceof VideoDecoderSurfaceView) {
oldVideoComponent.setOutputBufferRenderer(null);
} else if (surfaceView instanceof SurfaceView) {
oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
}
@ -565,6 +572,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
newVideoComponent.setVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) {
((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
} else if (surfaceView instanceof VideoDecoderSurfaceView) {
newVideoComponent.setOutputBufferRenderer(
((VideoDecoderSurfaceView) surfaceView).getOutputBufferRenderer());
} else if (surfaceView instanceof SurfaceView) {
newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
}
@ -736,8 +746,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider
* buffering spinner is not displayed by default.
*
* @param showBuffering The mode that defines when the buffering spinner is displayed. One of
* {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and
* {@link #SHOW_BUFFERING_ALWAYS}.
* {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and {@link
* #SHOW_BUFFERING_ALWAYS}.
*/
public void setShowBuffering(@ShowBuffering int showBuffering) {
if (this.showBuffering != showBuffering) {

1
library/ui/src/main/res/values/attrs.xml
View file

@ -30,6 +30,7 @@
<enum name="surface_view" value="1"/>
<enum name="texture_view" value="2"/>
<enum name="spherical_view" value="3"/>
<enum name="video_decoder_surface_view" value="4"/>
</attr>
<!-- Must be kept in sync with RepeatModeUtil -->