sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-25 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use a single message for setting video renderer outputs

Browse source 
Previously, we had separate MSG_SET_SURFACE and
MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER messages for
setting different types of supported output. Use of these
constants to switch between outputs during use of a player
was confusing because not all video renderers support both
message types.

To switch from VideoDecoderOutputBufferRenderer to a Surface,
it was sufficient just to send MSG_SET_SURFACE, since all
video renderers support this and clear any other output that
might be set. Conversely, to switch in the opposite direction,
just sending a MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER was
not sufficient, because not all video renderers handle this
message to clear any previous output. Hence it was necessary to
explicitly clear a previously set surface using a separate
MSG_SET_SURFACE message. Passing two messages to switch the
output may prevent renderers from implementing the output switch
efficiently.

This change passes all outputs using a single message type, and
requires that all renderers treat unsupported outputs as though
null were passed (i.e., they clear any existing output). There
are some other miscellaneous improvements:

1. Non-surface outputs are now passed to onRenderedFirstFrame.
   This fixes a bug in SimpleExoPlayer's onRenderedFirstFrame,
   where previously it could not correctly equality check the
   output corresponding to the event to its current output in
   the VideoDecoderOutputBufferRenderer case.
2. Fix SimpleExoPlayer to report surface size changes for the
   VideoDecoderOutputBufferRenderer case. Even though the
   surface is rendered to indirectly in this case, we can still
   query (and listen to changes to) the surface's size.

PiperOrigin-RevId: 368215850
This commit is contained in:
olly 2021-04-13 16:06:04 +01:00 committed by marcbaechinger
parent 84282d7c32
commit 3032252fde
15 changed files with 157 additions and 209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
library/common/src/main/java/com/google/android/exoplayer2/C.java
View file

@ -782,7 +782,7 @@ public final class C {
*/
public static final UUID PLAYREADY_UUID = new UUID(0x9A04F07998404286L, 0xAB92E65BE0885F95L);
/** @deprecated Use {@code Renderer.MSG_SET_SURFACE}. */
/** @deprecated Use {@code Renderer.MSG_SET_VIDEO_OUTPUT}. */
@Deprecated public static final int MSG_SET_SURFACE = 1;
/** @deprecated Use {@code Renderer.MSG_SET_VOLUME}. */
@ -803,9 +803,6 @@ public final class C {
/** @deprecated Use {@code Renderer.MSG_SET_CAMERA_MOTION_LISTENER}. */
@Deprecated public static final int MSG_SET_CAMERA_MOTION_LISTENER = 7;
/** @deprecated Use {@code Renderer.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER}. */
@Deprecated public static final int MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER = 8;
/** @deprecated Use {@code Renderer.MSG_CUSTOM_BASE}. */
@Deprecated public static final int MSG_CUSTOM_BASE = 10000;

2
library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java
View file

@ -196,7 +196,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
// PlayerMessage.Target implementation.
@Override
public void handleMessage(int what, @Nullable Object object) throws ExoPlaybackException {
public void handleMessage(int messageType, @Nullable Object payload) throws ExoPlaybackException {
// Do nothing.
}

21
library/core/src/main/java/com/google/android/exoplayer2/Renderer.java
View file

@ -24,7 +24,6 @@ import com.google.android.exoplayer2.audio.AuxEffectInfo;
import com.google.android.exoplayer2.source.SampleStream;
import com.google.android.exoplayer2.util.MediaClock;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.DecoderVideoRenderer;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@ -76,11 +75,14 @@ public interface Renderer extends PlayerMessage.Target {
/**
* The type of a message that can be passed to a video renderer via {@link
* ExoPlayer#createMessage(Target)}. The message payload should be the target {@link Surface}, or
* null.
* ExoPlayer#createMessage(Target)}. The message payload is normally a {@link Surface}, however
* some video renderers may accept other outputs (e.g., {@link VideoDecoderOutputBufferRenderer}).
*
* <p>If the receiving renderer does not support the payload type as an output, then it will clear
* any existing output that it has.
*/
@SuppressWarnings("deprecation")
int MSG_SET_SURFACE = C.MSG_SET_SURFACE;
int MSG_SET_VIDEO_OUTPUT = C.MSG_SET_SURFACE;
/**
* A type of a message that can be passed to an audio renderer via {@link
* ExoPlayer#createMessage(Target)}. The message payload should be a {@link Float} with 0 being
@ -142,17 +144,6 @@ public interface Renderer extends PlayerMessage.Target {
*/
@SuppressWarnings("deprecation")
int MSG_SET_CAMERA_MOTION_LISTENER = C.MSG_SET_CAMERA_MOTION_LISTENER;
/**
* The type of a message that can be passed to a {@link DecoderVideoRenderer} via {@link
* ExoPlayer#createMessage(Target)}. The message payload should be the target {@link
* VideoDecoderOutputBufferRenderer}, or null.
*
* <p>This message is intended only for use with extension renderers that expect a {@link
* VideoDecoderOutputBufferRenderer}. For other use cases, an output surface should be passed via
* {@link #MSG_SET_SURFACE} instead.
*/
@SuppressWarnings("deprecation")
int MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER = C.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER;
/**
* The type of a message that can be passed to an audio renderer via {@link
* ExoPlayer#createMessage(Target)}. The message payload should be a {@link Boolean} instance

143
library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
View file

@ -61,7 +61,6 @@ import com.google.android.exoplayer2.util.ConditionVariable;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoDecoderGLSurfaceView;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoListener;
@ -587,11 +586,12 @@ public class SimpleExoPlayer extends BasePlayer
@Nullable private Format videoFormat;
@Nullable private Format audioFormat;
@Nullable private AudioTrack keepSessionIdAudioTrack;
@Nullable private Surface surface;
private boolean ownsSurface;
@C.VideoScalingMode private int videoScalingMode;
@Nullable private Object videoOutput;
@Nullable private Surface ownedSurface;
@Nullable private SurfaceHolder surfaceHolder;
private boolean surfaceHolderSurfaceIsVideoOutput;
@Nullable private TextureView textureView;
@C.VideoScalingMode private int videoScalingMode;
private int surfaceWidth;
private int surfaceHeight;
@Nullable private DecoderCounters videoDecoderCounters;
@ -797,14 +797,14 @@ public class SimpleExoPlayer extends BasePlayer
public void clearVideoSurface() {
verifyApplicationThread();
removeSurfaceCallbacks();
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ false);
setVideoOutputInternal(/* videoOutput= */ null);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
}
@Override
public void clearVideoSurface(@Nullable Surface surface) {
verifyApplicationThread();
if (surface != null && surface == this.surface) {
if (surface != null && surface == videoOutput) {
clearVideoSurface();
}
}
@ -813,10 +813,7 @@ public class SimpleExoPlayer extends BasePlayer
public void setVideoSurface(@Nullable Surface surface) {
verifyApplicationThread();
removeSurfaceCallbacks();
if (surface != null) {
setVideoDecoderOutputBufferRenderer(/* videoDecoderOutputBufferRenderer= */ null);
}
setVideoSurfaceInternal(surface, /* ownsSurface= */ false);
setVideoOutputInternal(surface);
int newSurfaceSize = surface == null ? 0 : C.LENGTH_UNSET;
maybeNotifySurfaceSizeChanged(/* width= */ newSurfaceSize, /* height= */ newSurfaceSize);
}
@ -824,23 +821,20 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public void setVideoSurfaceHolder(@Nullable SurfaceHolder surfaceHolder) {
verifyApplicationThread();
removeSurfaceCallbacks();
if (surfaceHolder != null) {
setVideoDecoderOutputBufferRenderer(/* videoDecoderOutputBufferRenderer= */ null);
}
this.surfaceHolder = surfaceHolder;
if (surfaceHolder == null) {
setVideoSurfaceInternal(null, /* ownsSurface= */ false);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
clearVideoSurface();
} else {
removeSurfaceCallbacks();
this.surfaceHolderSurfaceIsVideoOutput = true;
this.surfaceHolder = surfaceHolder;
surfaceHolder.addCallback(componentListener);
Surface surface = surfaceHolder.getSurface();
if (surface != null && surface.isValid()) {
setVideoSurfaceInternal(surface, /* ownsSurface= */ false);
setVideoOutputInternal(surface);
Rect surfaceSize = surfaceHolder.getSurfaceFrame();
maybeNotifySurfaceSizeChanged(surfaceSize.width(), surfaceSize.height());
} else {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ false);
setVideoOutputInternal(/* videoOutput= */ null);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
}
}
@ -850,7 +844,7 @@ public class SimpleExoPlayer extends BasePlayer
public void clearVideoSurfaceHolder(@Nullable SurfaceHolder surfaceHolder) {
verifyApplicationThread();
if (surfaceHolder != null && surfaceHolder == this.surfaceHolder) {
setVideoSurfaceHolder(null);
clearVideoSurface();
}
}
@ -858,11 +852,21 @@ public class SimpleExoPlayer extends BasePlayer
public void setVideoSurfaceView(@Nullable SurfaceView surfaceView) {
verifyApplicationThread();
if (surfaceView instanceof VideoDecoderOutputBufferRenderer) {
VideoDecoderOutputBufferRenderer videoDecoderOutputBufferRenderer =
(VideoDecoderOutputBufferRenderer) surfaceView;
clearVideoSurface();
removeSurfaceCallbacks();
setVideoOutputInternal(surfaceView);
// Although we won't use the surface directly as the video output, still use the holder to
// query the surface size, to be informed in changes to the size via componentListener, and
// for equality checking in clearVideoSurfaceHolder.
surfaceHolderSurfaceIsVideoOutput = false;
surfaceHolder = surfaceView.getHolder();
setVideoDecoderOutputBufferRenderer(videoDecoderOutputBufferRenderer);
surfaceHolder.addCallback(componentListener);
Surface surface = surfaceHolder.getSurface();
if (surface != null && surface.isValid()) {
Rect surfaceSize = surfaceHolder.getSurfaceFrame();
maybeNotifySurfaceSizeChanged(surfaceSize.width(), surfaceSize.height());
} else {
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
}
} else {
setVideoSurfaceHolder(surfaceView == null ? null : surfaceView.getHolder());
}
@ -871,39 +875,29 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public void clearVideoSurfaceView(@Nullable SurfaceView surfaceView) {
verifyApplicationThread();
if (surfaceView instanceof VideoDecoderGLSurfaceView) {
if (surfaceView.getHolder() == surfaceHolder) {
setVideoDecoderOutputBufferRenderer(null);
surfaceHolder = null;
}
} else {
clearVideoSurfaceHolder(surfaceView == null ? null : surfaceView.getHolder());
}
clearVideoSurfaceHolder(surfaceView == null ? null : surfaceView.getHolder());
}
@Override
public void setVideoTextureView(@Nullable TextureView textureView) {
verifyApplicationThread();
removeSurfaceCallbacks();
if (textureView != null) {
setVideoDecoderOutputBufferRenderer(/* videoDecoderOutputBufferRenderer= */ null);
}
this.textureView = textureView;
if (textureView == null) {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
clearVideoSurface();
} else {
removeSurfaceCallbacks();
this.textureView = textureView;
if (textureView.getSurfaceTextureListener() != null) {
Log.w(TAG, "Replacing existing SurfaceTextureListener.");
}
textureView.setSurfaceTextureListener(componentListener);
@Nullable
SurfaceTexture surfaceTexture =
textureView.isAvailable() ? textureView.getSurfaceTexture() : null;
if (surfaceTexture == null) {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
setVideoOutputInternal(/* videoOutput= */ null);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
} else {
setVideoSurfaceInternal(new Surface(surfaceTexture), /* ownsSurface= */ true);
setSurfaceTextureInternal(surfaceTexture);
maybeNotifySurfaceSizeChanged(textureView.getWidth(), textureView.getHeight());
}
}
@ -913,7 +907,7 @@ public class SimpleExoPlayer extends BasePlayer
public void clearVideoTextureView(@Nullable TextureView textureView) {
verifyApplicationThread();
if (textureView != null && textureView == this.textureView) {
setVideoTextureView(null);
clearVideoSurface();
}
}
@ -1563,11 +1557,9 @@ public class SimpleExoPlayer extends BasePlayer
player.release();
analyticsCollector.release();
removeSurfaceCallbacks();
if (surface != null) {
if (ownsSurface) {
surface.release();
}
surface = null;
if (ownedSurface != null) {
ownedSurface.release();
ownedSurface = null;
}
if (isPriorityTaskManagerRegistered) {
Assertions.checkNotNull(priorityTaskManager).remove(C.PRIORITY_PLAYBACK);
@ -1834,22 +1826,29 @@ public class SimpleExoPlayer extends BasePlayer
}
}
private void setVideoSurfaceInternal(@Nullable Surface surface, boolean ownsSurface) {
// Note: We don't turn this method into a no-op if the surface is being replaced with itself
// so as to ensure onRenderedFirstFrame callbacks are still called in this case.
private void setSurfaceTextureInternal(SurfaceTexture surfaceTexture) {
Surface surface = new Surface(surfaceTexture);
setVideoOutputInternal(surface);
ownedSurface = surface;
}
private void setVideoOutputInternal(@Nullable Object videoOutput) {
// Note: We don't turn this method into a no-op if the output is being replaced with itself so
// as to ensure onRenderedFirstFrame callbacks are still called in this case.
List<PlayerMessage> messages = new ArrayList<>();
for (Renderer renderer : renderers) {
if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
messages.add(
player
.createMessage(renderer)
.setType(Renderer.MSG_SET_SURFACE)
.setPayload(surface)
.setType(Renderer.MSG_SET_VIDEO_OUTPUT)
.setPayload(videoOutput)
.send());
}
}
if (this.surface != null && this.surface != surface) {
// We're replacing a surface. Block to ensure that it's not accessed after the method returns.
if (this.videoOutput != null && this.videoOutput != videoOutput) {
// We're replacing an output. Block to ensure that this output will not be accessed by the
// renderers after this method returns.
try {
for (PlayerMessage message : messages) {
message.blockUntilDelivered(detachSurfaceTimeoutMs);
@ -1863,21 +1862,13 @@ public class SimpleExoPlayer extends BasePlayer
ExoPlaybackException.createForRenderer(
new ExoTimeoutException(ExoTimeoutException.TIMEOUT_OPERATION_DETACH_SURFACE)));
}
// If we created the previous surface, we are responsible for releasing it.
if (this.ownsSurface) {
this.surface.release();
if (this.videoOutput == ownedSurface) {
// We're replacing a surface that we are responsible for releasing.
ownedSurface.release();
ownedSurface = null;
}
}
this.surface = surface;
this.ownsSurface = ownsSurface;
}
private void setVideoDecoderOutputBufferRenderer(
@Nullable VideoDecoderOutputBufferRenderer videoDecoderOutputBufferRenderer) {
sendRendererMessage(
C.TRACK_TYPE_VIDEO,
Renderer.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER,
videoDecoderOutputBufferRenderer);
this.videoOutput = videoOutput;
}
private void maybeNotifySurfaceSizeChanged(int width, int height) {
@ -2060,9 +2051,9 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void onRenderedFirstFrame(@Nullable Surface surface, long renderTimeMs) {
analyticsCollector.onRenderedFirstFrame(surface, renderTimeMs);
if (SimpleExoPlayer.this.surface == surface) {
public void onRenderedFirstFrame(Object output, long renderTimeMs) {
analyticsCollector.onRenderedFirstFrame(output, renderTimeMs);
if (videoOutput == output) {
for (VideoListener videoListener : videoListeners) {
videoListener.onRenderedFirstFrame();
}
@ -2178,7 +2169,9 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public void surfaceCreated(SurfaceHolder holder) {
setVideoSurfaceInternal(holder.getSurface(), false);
if (surfaceHolderSurfaceIsVideoOutput) {
setVideoOutputInternal(holder.getSurface());
}
}
@Override
@ -2188,7 +2181,9 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ false);
if (surfaceHolderSurfaceIsVideoOutput) {
setVideoOutputInternal(/* videoOutput= */ null);
}
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
}
@ -2196,7 +2191,7 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
setVideoSurfaceInternal(new Surface(surfaceTexture), /* ownsSurface= */ true);
setSurfaceTextureInternal(surfaceTexture);
maybeNotifySurfaceSizeChanged(width, height);
}
@ -2207,7 +2202,7 @@ public class SimpleExoPlayer extends BasePlayer
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
setVideoOutputInternal(/* videoOutput= */ null);
maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
return true;
}

9
library/core/src/main/java/com/google/android/exoplayer2/analytics/AnalyticsCollector.java
View file

@ -19,7 +19,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.os.Looper;
import android.util.SparseArray;
import android.view.Surface;
import androidx.annotation.CallSuper;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
@ -442,17 +441,13 @@ public class AnalyticsCollector
eventTime, width, height, unappliedRotationDegrees, pixelWidthHeightRatio));
}
@SuppressWarnings("deprecation") // Calling deprecated listener method.
@Override
public final void onRenderedFirstFrame(@Nullable Surface surface, long renderTimeMs) {
public final void onRenderedFirstFrame(Object output, long renderTimeMs) {
EventTime eventTime = generateReadingMediaPeriodEventTime();
sendEvent(
eventTime,
AnalyticsListener.EVENT_RENDERED_FIRST_FRAME,
listener -> {
listener.onRenderedFirstFrame(eventTime, surface);
listener.onRenderedFirstFrame(eventTime, surface, renderTimeMs);
});
listener -> listener.onRenderedFirstFrame(eventTime, output, renderTimeMs));
}
@Override

12
library/core/src/main/java/com/google/android/exoplayer2/analytics/AnalyticsListener.java
View file

@ -48,6 +48,7 @@ import com.google.android.exoplayer2.source.MediaSource.MediaPeriodId;
import com.google.android.exoplayer2.source.TrackGroupArray;
import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
import com.google.android.exoplayer2.util.ExoFlags;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.common.base.Objects;
import java.io.IOException;
import java.lang.annotation.Documented;
@ -1020,16 +1021,11 @@ public interface AnalyticsListener {
* renderer was reset, or since the stream being rendered was changed.
*
* @param eventTime The event time.
* @param surface The {@link Surface} to which a frame has been rendered, or {@code null} if the
* renderer renders to something that isn't a {@link Surface}.
* @param output The output to which a frame has been rendered. Normally a {@link Surface},
* however may also be other output types (e.g., a {@link VideoDecoderOutputBufferRenderer}).
* @param renderTimeMs {@link SystemClock#elapsedRealtime()} when the first frame was rendered.
*/
default void onRenderedFirstFrame(
EventTime eventTime, @Nullable Surface surface, long renderTimeMs) {}
/** @deprecated Use {@link #onRenderedFirstFrame(EventTime, Surface, long)} instead. */
@Deprecated
default void onRenderedFirstFrame(EventTime eventTime, @Nullable Surface surface) {}
default void onRenderedFirstFrame(EventTime eventTime, Object output, long renderTimeMs) {}
/**
* Called before a frame is rendered for the first time since setting the surface, and each time

5
library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java
View file

@ -19,7 +19,6 @@ import static java.lang.Math.min;
import android.os.SystemClock;
import android.text.TextUtils;
import android.view.Surface;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException;
@ -453,8 +452,8 @@ public class EventLogger implements AnalyticsListener {
}
@Override
public void onRenderedFirstFrame(EventTime eventTime, @Nullable Surface surface) {
logd(eventTime, "renderedFirstFrame", String.valueOf(surface));
public void onRenderedFirstFrame(EventTime eventTime, Object output, long renderTimeMs) {
logd(eventTime, "renderedFirstFrame", String.valueOf(output));
}
@Override

92
library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java
View file

@ -29,6 +29,7 @@ import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.BaseRenderer;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.C.VideoOutputMode;
import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.ExoPlayer;
import com.google.android.exoplayer2.Format;
@ -59,11 +60,9 @@ import java.lang.annotation.RetentionPolicy;
* on the playback thread:
*
* <ul>
* <li>Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload
* should be the target {@link Surface}, or null.
* <li>Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output
* buffer renderer. The message payload should be the target {@link
* VideoDecoderOutputBufferRenderer}, or null.
* <li>Message with type {@link #MSG_SET_VIDEO_OUTPUT} to set the output surface. The message
* payload should be the target {@link Surface} or {@link VideoDecoderOutputBufferRenderer},
* or null. Other non-null payloads have the effect of clearing the output.
* <li>Message with type {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER} to set a listener for
* metadata associated with frames being rendered. The message payload should be the {@link
* VideoFrameMetadataListener}, or null.
@ -113,10 +112,11 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
private VideoDecoderInputBuffer inputBuffer;
private VideoDecoderOutputBuffer outputBuffer;
@Nullable private Surface surface;
@VideoOutputMode private int outputMode;
@Nullable private Object output;
@Nullable private Surface outputSurface;
@Nullable private VideoDecoderOutputBufferRenderer outputBufferRenderer;
@Nullable private VideoFrameMetadataListener frameMetadataListener;
@C.VideoOutputMode private int outputMode;
@Nullable private DrmSession decoderDrmSession;
@Nullable private DrmSession sourceDrmSession;
@ -248,10 +248,8 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
@Override
public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
if (messageType == MSG_SET_SURFACE) {
setOutputSurface((Surface) message);
} else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) {
setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
if (messageType == MSG_SET_VIDEO_OUTPUT) {
setOutput(message);
} else if (messageType == MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
frameMetadataListener = (VideoFrameMetadataListener) message;
} else {
@ -560,7 +558,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
}
lastRenderTimeUs = C.msToUs(SystemClock.elapsedRealtime() * 1000);
int bufferMode = outputBuffer.mode;
boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;
boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && outputSurface != null;
boolean renderYuv = bufferMode == C.VIDEO_OUTPUT_MODE_YUV && outputBufferRenderer != null;
if (!renderYuv && !renderSurface) {
dropOutputBuffer(outputBuffer);
@ -569,7 +567,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
if (renderYuv) {
outputBufferRenderer.setOutputBuffer(outputBuffer);
} else {
renderOutputBufferToSurface(outputBuffer, surface);
renderOutputBufferToSurface(outputBuffer, outputSurface);
}
consecutiveDroppedFrameCount = 0;
decoderCounters.renderedOutputBufferCount++;
@ -590,47 +588,26 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
protected abstract void renderOutputBufferToSurface(
VideoDecoderOutputBuffer outputBuffer, Surface surface) throws DecoderException;
/**
* Sets output surface.
*
* @param surface Surface.
*/
protected final void setOutputSurface(@Nullable Surface surface) {
if (this.surface != surface) {
// The output has changed.
this.surface = surface;
if (surface != null) {
outputBufferRenderer = null;
outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
if (decoder != null) {
setDecoderOutputMode(outputMode);
}
onOutputChanged();
} else {
// The output has been removed. We leave the outputMode of the underlying decoder unchanged
// in anticipation that a subsequent output will likely be of the same type.
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
onOutputRemoved();
}
} else if (surface != null) {
// The output is unchanged and non-null.
onOutputReset();
/** Sets the video output. */
protected final void setOutput(@Nullable Object output) {
if (output instanceof Surface) {
outputSurface = (Surface) output;
outputBufferRenderer = null;
outputMode = C.VIDEO_OUTPUT_MODE_SURFACE_YUV;
} else if (output instanceof VideoDecoderOutputBufferRenderer) {
outputSurface = null;
outputBufferRenderer = (VideoDecoderOutputBufferRenderer) output;
outputMode = C.VIDEO_OUTPUT_MODE_YUV;
} else {
// Handle unsupported outputs by clearing the output.
output = null;
outputSurface = null;
outputBufferRenderer = null;
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
}
}
/**
* Sets output buffer renderer.
*
* @param outputBufferRenderer Output buffer renderer.
*/
protected final void setOutputBufferRenderer(
@Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
if (this.outputBufferRenderer != outputBufferRenderer) {
// The output has changed.
this.outputBufferRenderer = outputBufferRenderer;
if (outputBufferRenderer != null) {
surface = null;
outputMode = C.VIDEO_OUTPUT_MODE_YUV;
if (this.output != output) {
this.output = output;
if (output != null) {
if (decoder != null) {
setDecoderOutputMode(outputMode);
}
@ -638,10 +615,9 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
} else {
// The output has been removed. We leave the outputMode of the underlying decoder unchanged
// in anticipation that a subsequent output will likely be of the same type.
outputMode = C.VIDEO_OUTPUT_MODE_NONE;
onOutputRemoved();
}
} else if (outputBufferRenderer != null) {
} else if (output != null) {
// The output is unchanged and non-null.
onOutputReset();
}
@ -652,7 +628,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
*
* @param outputMode Output mode.
*/
protected abstract void setDecoderOutputMode(@C.VideoOutputMode int outputMode);
protected abstract void setDecoderOutputMode(@VideoOutputMode int outputMode);
/**
* Evaluates whether the existing decoder can be reused for a new {@link Format}.
@ -927,13 +903,13 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
renderedFirstFrameAfterEnable = true;
if (!renderedFirstFrameAfterReset) {
renderedFirstFrameAfterReset = true;
eventDispatcher.renderedFirstFrame(surface);
eventDispatcher.renderedFirstFrame(output);
}
}
private void maybeRenotifyRenderedFirstFrame() {
if (renderedFirstFrameAfterReset) {
eventDispatcher.renderedFirstFrame(surface);
eventDispatcher.renderedFirstFrame(output);
}
}

15
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View file

@ -76,8 +76,9 @@ import java.util.List;
* on the playback thread:
*
* <ul>
* <li>Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload
* should be the target {@link Surface}, or null.
* <li>Message with type {@link #MSG_SET_VIDEO_OUTPUT} to set the output. The message payload
* should be the target {@link Surface}, or null to clear the output. Other non-null payloads
* have the effect of clearing the output.
* <li>Message with type {@link #MSG_SET_SCALING_MODE} to set the video scaling mode. The message
* payload should be one of the integer scaling modes in {@link C.VideoScalingMode}. Note that
* the scaling mode only applies if the {@link Surface} targeted by this renderer is owned by
@ -506,8 +507,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
switch (messageType) {
case MSG_SET_SURFACE:
setSurface((Surface) message);
case MSG_SET_VIDEO_OUTPUT:
setOutput(message);
break;
case MSG_SET_SCALING_MODE:
scalingMode = (Integer) message;
@ -533,7 +534,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
}
private void setSurface(Surface surface) throws ExoPlaybackException {
private void setOutput(@Nullable Object output) throws ExoPlaybackException {
// Handle unsupported (i.e., non-Surface) outputs by clearing the surface.
@Nullable Surface surface = output instanceof Surface ? (Surface) output : null;
if (surface == null) {
// Use a dummy surface if possible.
if (dummySurface != null) {
@ -546,6 +550,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
}
}
// We only need to update the codec if the surface has changed.
if (this.surface != surface) {
this.surface = surface;

22
library/core/src/main/java/com/google/android/exoplayer2/video/VideoRendererEventListener.java
View file

@ -125,18 +125,14 @@ public interface VideoRendererEventListener {
int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {}
/**
* Called when a frame is rendered for the first time since setting the surface, or since the
* Called when a frame is rendered for the first time since setting the output, or since the
* renderer was reset, or since the stream being rendered was changed.
*
* @param surface The {@link Surface} to which a first frame has been rendered, or {@code null} if
* the renderer renders to something that isn't a {@link Surface}.
* @param output The output of the video renderer. Normally a {@link Surface}, however some video
* renderers may have other output types (e.g., a {@link VideoDecoderOutputBufferRenderer}).
* @param renderTimeMs The {@link SystemClock#elapsedRealtime()} when the frame was rendered.
*/
default void onRenderedFirstFrame(@Nullable Surface surface, long renderTimeMs) {}
/** @deprecated Use {@link #onRenderedFirstFrame(Surface, long)}. */
@Deprecated
default void onRenderedFirstFrame(@Nullable Surface surface) {}
default void onRenderedFirstFrame(Object output, long renderTimeMs) {}
/**
* Called when a decoder is released.
@ -251,16 +247,12 @@ public interface VideoRendererEventListener {
}
}
/** Invokes {@link VideoRendererEventListener#onRenderedFirstFrame(Surface, long)}. */
public void renderedFirstFrame(@Nullable Surface surface) {
/** Invokes {@link VideoRendererEventListener#onRenderedFirstFrame(Object, long)}. */
public void renderedFirstFrame(Object output) {
if (handler != null) {
// TODO: Replace this timestamp with the actual frame release time.
long renderTimeMs = SystemClock.elapsedRealtime();
handler.post(
() -> {
castNonNull(listener).onRenderedFirstFrame(surface);
castNonNull(listener).onRenderedFirstFrame(surface, renderTimeMs);
});
handler.post(() -> castNonNull(listener).onRenderedFirstFrame(output, renderTimeMs));
}
}

2
library/core/src/test/java/com/google/android/exoplayer2/ExoPlayerTest.java
View file

@ -2560,7 +2560,7 @@ public final class ExoPlayerTest {
.start()
.blockUntilActionScheduleFinished(TIMEOUT_MS)
.blockUntilEnded(TIMEOUT_MS);
assertThat(Collections.frequency(rendererMessages, Renderer.MSG_SET_SURFACE)).isEqualTo(2);
assertThat(Collections.frequency(rendererMessages, Renderer.MSG_SET_VIDEO_OUTPUT)).isEqualTo(2);
}
@Test

3
library/core/src/test/java/com/google/android/exoplayer2/analytics/AnalyticsCollectorTest.java
View file

@ -2309,8 +2309,7 @@ public final class AnalyticsCollectorTest {
}
@Override
public void onRenderedFirstFrame(
EventTime eventTime, @Nullable Surface surface, long renderTimeMs) {
public void onRenderedFirstFrame(EventTime eventTime, Object output, long renderTimeMs) {
reportedEvents.add(new ReportedEvent(EVENT_RENDERED_FIRST_FRAME, eventTime));
}

19
library/core/src/test/java/com/google/android/exoplayer2/video/DecoderVideoRendererTest.java
View file

@ -17,7 +17,8 @@ package com.google.android.exoplayer2.video;
import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.oneByteSample;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -170,7 +171,7 @@ public final class DecoderVideoRendererTest {
};
}
};
renderer.setOutputSurface(surface);
renderer.setOutput(surface);
}
@After
@ -211,7 +212,7 @@ public final class DecoderVideoRendererTest {
ShadowLooper.idleMainLooper();
}
verify(eventListener).onRenderedFirstFrame(any());
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
@Test
@ -242,7 +243,7 @@ public final class DecoderVideoRendererTest {
ShadowLooper.idleMainLooper();
}
verify(eventListener, never()).onRenderedFirstFrame(any());
verify(eventListener, never()).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
@Test
@ -273,7 +274,7 @@ public final class DecoderVideoRendererTest {
ShadowLooper.idleMainLooper();
}
verify(eventListener).onRenderedFirstFrame(any());
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
// TODO: Fix rendering of first frame at stream transition.
@ -325,7 +326,8 @@ public final class DecoderVideoRendererTest {
ShadowLooper.idleMainLooper();
}
verify(eventListener, times(2)).onRenderedFirstFrame(any());
verify(eventListener, times(2))
.onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
// TODO: Fix rendering of first frame at stream transition.
@ -376,11 +378,12 @@ public final class DecoderVideoRendererTest {
ShadowLooper.idleMainLooper();
}
verify(eventListener).onRenderedFirstFrame(any());
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
// Render to streamOffsetUs and verify the new first frame gets rendered.
renderer.render(/* positionUs= */ 100, SystemClock.elapsedRealtime() * 1000);
verify(eventListener, times(2)).onRenderedFirstFrame(any());
verify(eventListener, times(2))
.onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
}

2
library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java
View file

@ -120,7 +120,7 @@ public class MediaCodecVideoRendererTest {
};
surface = new Surface(new SurfaceTexture(/* texName= */ 0));
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_SURFACE, surface);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
}
@After

14
testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeVideoRenderer.java
View file

@ -18,7 +18,6 @@ package com.google.android.exoplayer2.testutil;
import android.os.Handler;
import android.os.SystemClock;
import android.view.Surface;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException;
@ -35,7 +34,7 @@ public class FakeVideoRenderer extends FakeRenderer {
private final VideoRendererEventListener.EventDispatcher eventDispatcher;
private final DecoderCounters decoderCounters;
private @MonotonicNonNull Format format;
@Nullable private Surface surface;
@Nullable private Object output;
private long streamOffsetUs;
private boolean renderedFirstFrameAfterReset;
private boolean mayRenderFirstFrameAfterEnableIfNotStarted;
@ -97,8 +96,8 @@ public class FakeVideoRenderer extends FakeRenderer {
@Override
public void handleMessage(int messageType, @Nullable Object payload) throws ExoPlaybackException {
switch (messageType) {
case MSG_SET_SURFACE:
surface = (Surface) payload;
case MSG_SET_VIDEO_OUTPUT:
output = payload;
renderedFirstFrameAfterReset = false;
break;
default:
@ -110,17 +109,18 @@ public class FakeVideoRenderer extends FakeRenderer {
protected boolean shouldProcessBuffer(long bufferTimeUs, long playbackPositionUs) {
boolean shouldProcess = super.shouldProcessBuffer(bufferTimeUs, playbackPositionUs);
boolean shouldRenderFirstFrame =
surface != null
output != null
&& (!renderedFirstFrameAfterEnable
? (getState() == Renderer.STATE_STARTED
|| mayRenderFirstFrameAfterEnableIfNotStarted)
: !renderedFirstFrameAfterReset);
shouldProcess |= shouldRenderFirstFrame && playbackPositionUs >= streamOffsetUs;
if (shouldProcess && !renderedFirstFrameAfterReset && surface != null) {
@Nullable Object output = this.output;
if (shouldProcess && !renderedFirstFrameAfterReset && output != null) {
@MonotonicNonNull Format format = Assertions.checkNotNull(this.format);
eventDispatcher.videoSizeChanged(
format.width, format.height, format.rotationDegrees, format.pixelWidthHeightRatio);
eventDispatcher.renderedFirstFrame(surface);
eventDispatcher.renderedFirstFrame(output);
renderedFirstFrameAfterReset = true;
renderedFirstFrameAfterEnable = true;
}