Add OpenGL support to av1 extension: Libgav1VideoRenderer

Move reusable code from LibvpxVideoRenderer to SimpleDecoderVideoRenderer.
Pass outputBuffer to renderOutputBuffer method instead of keeping the reference in the renderer.

PiperOrigin-RevId: 272899549
This commit is contained in:
sofijajvc 2019-10-04 18:05:52 +01:00 committed by Oliver Woodman
parent 27b90fba0e
commit 8f3a363dd2
2 changed files with 171 additions and 165 deletions

View file

@ -28,7 +28,6 @@ import com.google.android.exoplayer2.PlayerMessage.Target;
import com.google.android.exoplayer2.decoder.SimpleDecoder;
import com.google.android.exoplayer2.drm.DrmSessionManager;
import com.google.android.exoplayer2.drm.ExoMediaCrypto;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer;
@ -72,13 +71,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
private final boolean disableLoopFilter;
private final int threads;
private Surface surface;
private VideoDecoderOutputBufferRenderer outputBufferRenderer;
@C.VideoOutputMode private int outputMode;
private VpxDecoder decoder;
private VideoDecoderOutputBuffer outputBuffer;
private VideoFrameMetadataListener frameMetadataListener;
/**
@ -197,7 +190,6 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
this.threads = threads;
this.numInputBuffers = numInputBuffers;
this.numOutputBuffers = numOutputBuffers;
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
}
@Override
@ -236,53 +228,37 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
disableLoopFilter,
enableRowMultiThreadMode,
threads);
decoder.setOutputMode(outputMode);
TraceUtil.endSection();
return decoder;
}
@Override
@Nullable
protected VideoDecoderOutputBuffer dequeueOutputBuffer() throws VpxDecoderException {
outputBuffer = decoder.dequeueOutputBuffer();
return outputBuffer;
}
@Override
protected void renderOutputBuffer(long presentationTimeUs, Format outputFormat)
throws VpxDecoderException {
protected void renderOutputBuffer(
VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
throws VideoDecoderException {
if (frameMetadataListener != null) {
frameMetadataListener.onVideoFrameAboutToBeRendered(
presentationTimeUs, System.nanoTime(), outputFormat);
}
super.renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
}
int bufferMode = outputBuffer.mode;
boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;
boolean renderYuv = bufferMode == C.VIDEO_OUTPUT_MODE_YUV && outputBufferRenderer != null;
if (!renderYuv && !renderSurface) {
dropOutputBuffer(outputBuffer);
} else {
maybeNotifyVideoSizeChanged(outputBuffer.width, outputBuffer.height);
if (renderYuv) {
outputBufferRenderer.setOutputBuffer(outputBuffer);
// The renderer will release the buffer.
} else { // renderSurface
decoder.renderToSurface(outputBuffer, surface);
outputBuffer.release();
}
onFrameRendered(surface);
@Override
protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
throws VpxDecoderException {
if (decoder == null) {
throw new VpxDecoderException(
"Failed to render output buffer to surface: decoder is not initialized.");
}
decoder.renderToSurface(outputBuffer, surface);
outputBuffer.release();
}
@Override
protected void clearOutputBuffer() {
super.clearOutputBuffer();
outputBuffer = null;
}
@Override
protected boolean hasOutputSurface() {
return outputMode != C.VIDEO_OUTPUT_MODE_NONE;
protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) {
if (decoder != null) {
decoder.setOutputMode(outputMode);
}
}
// PlayerMessage.Target implementation.
@ -290,44 +266,13 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
@Override
public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
if (messageType == C.MSG_SET_SURFACE) {
setOutput((Surface) message, null);
setOutputSurface((Surface) message);
} else if (messageType == C.MSG_SET_OUTPUT_BUFFER_RENDERER) {
setOutput(null, (VideoDecoderOutputBufferRenderer) message);
setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
} else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
frameMetadataListener = (VideoFrameMetadataListener) message;
} else {
super.handleMessage(messageType, message);
}
}
// Internal methods.
private void setOutput(
@Nullable Surface surface, @Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
// At most one output may be non-null. Both may be null if the output is being cleared.
Assertions.checkState(surface == null || outputBufferRenderer == null);
if (this.surface != surface || this.outputBufferRenderer != outputBufferRenderer) {
// The output has changed.
this.surface = surface;
this.outputBufferRenderer = outputBufferRenderer;
if (surface != null) {
outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
} else {
outputMode =
outputBufferRenderer != null ? C.VIDEO_OUTPUT_MODE_YUV : C.VIDEO_OUTPUT_MODE_NONE;
}
if (hasOutputSurface()) {
if (decoder != null) {
decoder.setOutputMode(outputMode);
}
onOutputSurfaceChanged();
} else {
// The output has been removed. We leave the outputMode of the underlying decoder unchanged
// in anticipation that a subsequent output will likely be of the same type.
onOutputSurfaceRemoved();
}
} else if (hasOutputSurface()) {
onOutputSurfaceReset(surface);
}
}
}

View file

@ -86,6 +86,10 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
decoder;
private VideoDecoderInputBuffer inputBuffer;
private VideoDecoderOutputBuffer outputBuffer;
@Nullable private Surface surface;
@Nullable private VideoDecoderOutputBufferRenderer outputBufferRenderer;
@C.VideoOutputMode private int outputMode;
@Nullable private DrmSession<ExoMediaCrypto> decoderDrmSession;
@Nullable private DrmSession<ExoMediaCrypto> sourceDrmSession;
@ -147,6 +151,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance();
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
}
// BaseRenderer implementation.
@ -210,7 +215,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
}
if (inputFormat != null
&& (isSourceReady() || outputBuffer != null)
&& (renderedFirstFrame || !hasOutputSurface())) {
&& (renderedFirstFrame || !hasOutput())) {
// Ready. If we were joining then we've now joined, so clear the joining deadline.
joiningDeadlineMs = C.TIME_UNSET;
return true;
@ -227,6 +232,8 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
}
}
// Protected methods.
@Override
protected void onEnabled(boolean joining) throws ExoPlaybackException {
decoderCounters = new DecoderCounters();
@ -316,7 +323,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
inputBuffer = null;
if (outputBuffer != null) {
outputBuffer.release();
clearOutputBuffer();
outputBuffer = null;
}
decoder.flush();
decoderReceivedBuffers = false;
@ -327,7 +334,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
@CallSuper
protected void releaseDecoder() {
inputBuffer = null;
clearOutputBuffer();
outputBuffer = null;
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
decoderReceivedBuffers = false;
buffersInCodecCount = 0;
@ -339,16 +346,6 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
setDecoderDrmSession(null);
}
private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
DrmSession.replaceSessionReferences(sourceDrmSession, session);
sourceDrmSession = session;
}
private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
DrmSession.replaceSessionReferences(decoderDrmSession, session);
decoderDrmSession = session;
}
/**
* Called when a new format is read from the upstream source.
*
@ -527,96 +524,126 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto)
throws VideoDecoderException;
/**
* Dequeues output buffer.
*
* @return Dequeued video decoder output buffer, or null if an output buffer isn't available.
* @throws VideoDecoderException If an error occurs while dequeuing the output buffer.
*/
@Nullable
protected abstract VideoDecoderOutputBuffer dequeueOutputBuffer() throws VideoDecoderException;
/** Clears output buffer. */
protected void clearOutputBuffer() {
outputBuffer = null;
}
/**
* Renders the specified output buffer.
*
* <p>The implementation of this method takes ownership of the output buffer and is responsible
* for calling {@link VideoDecoderOutputBuffer#release()} either immediately or in the future.
*
* @param outputBuffer {@link VideoDecoderOutputBuffer} to render.
* @param presentationTimeUs Presentation time in microseconds.
* @param outputFormat Output format.
* @param outputFormat Output {@link Format}.
* @throws VideoDecoderException If an error occurs when rendering the output buffer.
*/
// TODO: The output buffer is not being passed to this method currently. Due to the need of
// decoder-specific output buffer type, the reference to the output buffer is being kept in the
// subclass. Once the common output buffer is established, this method can be updated to receive
// the output buffer as an argument. See [Internal: b/139174707].
protected abstract void renderOutputBuffer(long presentationTimeUs, Format outputFormat)
throws VideoDecoderException;
/**
* Returns whether the renderer has output surface.
*
* @return Whether the renderer has output surface.
*/
protected abstract boolean hasOutputSurface();
/** Called when the output surface is changed. */
protected final void onOutputSurfaceChanged() {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
// We haven't rendered to the new output yet.
clearRenderedFirstFrame();
if (getState() == STATE_STARTED) {
setJoiningDeadlineMs();
protected void renderOutputBuffer(
VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
throws VideoDecoderException {
lastRenderTimeUs = C.msToUs(SystemClock.elapsedRealtime() * 1000);
int bufferMode = outputBuffer.mode;
boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;
boolean renderYuv = bufferMode == C.VIDEO_OUTPUT_MODE_YUV && outputBufferRenderer != null;
if (!renderYuv && !renderSurface) {
dropOutputBuffer(outputBuffer);
} else {
maybeNotifyVideoSizeChanged(outputBuffer.width, outputBuffer.height);
if (renderYuv) {
outputBufferRenderer.setOutputBuffer(outputBuffer);
} else {
renderOutputBufferToSurface(outputBuffer, surface);
}
consecutiveDroppedFrameCount = 0;
decoderCounters.renderedOutputBufferCount++;
maybeNotifyRenderedFirstFrame();
}
}
/** Called when the output surface is removed. */
protected final void onOutputSurfaceRemoved() {
clearReportedVideoSize();
clearRenderedFirstFrame();
}
/**
* Renders the specified output buffer to the passed surface.
*
* <p>The implementation of this method takes ownership of the output buffer and is responsible
* for calling {@link VideoDecoderOutputBuffer#release()} either immediately or in the future.
*
* @param outputBuffer {@link VideoDecoderOutputBuffer} to render.
* @param surface Output {@link Surface}.
* @throws VideoDecoderException If an error occurs when rendering the output buffer.
*/
protected abstract void renderOutputBufferToSurface(
VideoDecoderOutputBuffer outputBuffer, Surface surface) throws VideoDecoderException;
/**
* Called when the output surface is set again to the same non-null value.
* Sets output surface.
*
* @param surface Output surface.
* @param surface Surface.
*/
protected final void onOutputSurfaceReset(Surface surface) {
// The output is unchanged and non-null. If we know the video size and/or have already
// rendered to the output, report these again immediately.
maybeRenotifyVideoSizeChanged();
maybeRenotifyRenderedFirstFrame(surface);
}
/**
* Notifies event dispatcher if video size changed.
*
* @param width New video width.
* @param height New video height.
*/
protected final void maybeNotifyVideoSizeChanged(int width, int height) {
if (reportedWidth != width || reportedHeight != height) {
reportedWidth = width;
reportedHeight = height;
eventDispatcher.videoSizeChanged(
width, height, /* unappliedRotationDegrees= */ 0, /* pixelWidthHeightRatio= */ 1);
protected final void setOutputSurface(@Nullable Surface surface) {
if (this.surface != surface) {
// The output has changed.
this.surface = surface;
if (surface != null) {
outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
if (decoder != null) {
setDecoderOutputMode(outputMode);
}
onOutputChanged();
} else {
// The output has been removed. We leave the outputMode of the underlying decoder unchanged
// in anticipation that a subsequent output will likely be of the same type.
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
onOutputRemoved();
}
} else if (surface != null) {
// The output is unchanged and non-null.
onOutputReset();
}
}
/** Called after rendering a frame. */
protected final void onFrameRendered(Surface surface) {
consecutiveDroppedFrameCount = 0;
decoderCounters.renderedOutputBufferCount++;
maybeNotifyRenderedFirstFrame(surface);
/**
* Sets output buffer renderer.
*
* @param outputBufferRenderer Output buffer renderer.
*/
protected final void setOutputBufferRenderer(
@Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
if (this.outputBufferRenderer != outputBufferRenderer) {
// The output has changed.
this.outputBufferRenderer = outputBufferRenderer;
if (outputBufferRenderer != null) {
outputMode = C.VIDEO_OUTPUT_MODE_YUV;
if (decoder != null) {
setDecoderOutputMode(outputMode);
}
onOutputChanged();
} else {
// The output has been removed. We leave the outputMode of the underlying decoder unchanged
// in anticipation that a subsequent output will likely be of the same type.
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
onOutputRemoved();
}
} else if (outputBufferRenderer != null) {
// The output is unchanged and non-null.
onOutputReset();
}
}
/**
* Sets output mode of the decoder.
*
* @param outputMode Output mode.
*/
protected abstract void setDecoderOutputMode(@C.VideoOutputMode int outputMode);
// Internal methods.
private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
DrmSession.replaceSessionReferences(sourceDrmSession, session);
sourceDrmSession = session;
}
private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
DrmSession.replaceSessionReferences(decoderDrmSession, session);
decoderDrmSession = session;
}
private void maybeInitDecoder() throws ExoPlaybackException {
if (decoder != null) {
return;
@ -642,6 +669,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
try {
long decoderInitializingTimestamp = SystemClock.elapsedRealtime();
decoder = createDecoder(inputFormat, mediaCrypto);
setDecoderOutputMode(outputMode);
long decoderInitializedTimestamp = SystemClock.elapsedRealtime();
onDecoderInitialized(
decoder.getName(),
@ -731,7 +759,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
throws ExoPlaybackException, VideoDecoderException {
if (outputBuffer == null) {
outputBuffer = dequeueOutputBuffer();
outputBuffer = decoder.dequeueOutputBuffer();
if (outputBuffer == null) {
return false;
}
@ -746,7 +774,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
maybeInitDecoder();
} else {
outputBuffer.release();
clearOutputBuffer();
outputBuffer = null;
outputStreamEnded = true;
}
return false;
@ -755,7 +783,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
boolean processedOutputBuffer = processOutputBuffer(positionUs, elapsedRealtimeUs);
if (processedOutputBuffer) {
onProcessedOutputBuffer(outputBuffer.timeUs);
clearOutputBuffer();
outputBuffer = null;
}
return processedOutputBuffer;
}
@ -777,7 +805,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
}
long earlyUs = outputBuffer.timeUs - positionUs;
if (!hasOutputSurface()) {
if (!hasOutput()) {
// Skip frames in sync with playback, so we'll be at the right frame if the mode changes.
if (isBufferLate(earlyUs)) {
skipOutputBuffer(outputBuffer);
@ -797,8 +825,7 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
if (!renderedFirstFrame
|| (isStarted
&& shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) {
lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
renderOutputBuffer(presentationTimeUs, outputFormat);
renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
return true;
}
@ -815,14 +842,39 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
}
if (earlyUs < 30000) {
lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
renderOutputBuffer(presentationTimeUs, outputFormat);
renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
return true;
}
return false;
}
private boolean hasOutput() {
return outputMode != C.VIDEO_OUTPUT_MODE_NONE;
}
private void onOutputChanged() {
// If we know the video size, report it again immediately.
maybeRenotifyVideoSizeChanged();
// We haven't rendered to the new output yet.
clearRenderedFirstFrame();
if (getState() == STATE_STARTED) {
setJoiningDeadlineMs();
}
}
private void onOutputRemoved() {
clearReportedVideoSize();
clearRenderedFirstFrame();
}
private void onOutputReset() {
// The output is unchanged and non-null. If we know the video size and/or have already
// rendered to the output, report these again immediately.
maybeRenotifyVideoSizeChanged();
maybeRenotifyRenderedFirstFrame();
}
private boolean shouldWaitForKeys(boolean bufferEncrypted) throws ExoPlaybackException {
if (decoderDrmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
return false;
@ -845,14 +897,14 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
renderedFirstFrame = false;
}
private void maybeNotifyRenderedFirstFrame(Surface surface) {
private void maybeNotifyRenderedFirstFrame() {
if (!renderedFirstFrame) {
renderedFirstFrame = true;
eventDispatcher.renderedFirstFrame(surface);
}
}
private void maybeRenotifyRenderedFirstFrame(Surface surface) {
private void maybeRenotifyRenderedFirstFrame() {
if (renderedFirstFrame) {
eventDispatcher.renderedFirstFrame(surface);
}
@ -863,6 +915,15 @@ public abstract class SimpleDecoderVideoRenderer extends BaseRenderer {
reportedHeight = Format.NO_VALUE;
}
private void maybeNotifyVideoSizeChanged(int width, int height) {
if (reportedWidth != width || reportedHeight != height) {
reportedWidth = width;
reportedHeight = height;
eventDispatcher.videoSizeChanged(
width, height, /* unappliedRotationDegrees= */ 0, /* pixelWidthHeightRatio= */ 1);
}
}
private void maybeRenotifyVideoSizeChanged() {
if (reportedWidth != Format.NO_VALUE || reportedHeight != Format.NO_VALUE) {
eventDispatcher.videoSizeChanged(