mirror of
https://github.com/samsonjs/media.git
synced 2026-04-12 12:25:47 +00:00
9a124120ff
- Make MediaCodecTrackRenderer.isReady more permissive. This largely fixes #21 - Bring WebmExtractor closer to FragmentedMp4Extractor. The two will probably be placed under a common interface fairly soon, which will allow significant code deduplication.
451 lines
18 KiB
Java
451 lines
18 KiB
Java
/*
|
|
* Copyright (C) 2014 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
package com.google.android.exoplayer;
|
|
|
|
import com.google.android.exoplayer.drm.DrmSessionManager;
|
|
import com.google.android.exoplayer.util.MimeTypes;
|
|
import com.google.android.exoplayer.util.TraceUtil;
|
|
|
|
import android.annotation.TargetApi;
|
|
import android.media.MediaCodec;
|
|
import android.media.MediaCrypto;
|
|
import android.os.Handler;
|
|
import android.os.SystemClock;
|
|
import android.view.Surface;
|
|
|
|
import java.nio.ByteBuffer;
|
|
|
|
/**
|
|
* Decodes and renders video using {@link MediaCodec}.
|
|
*/
|
|
@TargetApi(16)
|
|
public class MediaCodecVideoTrackRenderer extends MediaCodecTrackRenderer {
|
|
|
|
/**
|
|
* Interface definition for a callback to be notified of {@link MediaCodecVideoTrackRenderer}
|
|
* events.
|
|
*/
|
|
public interface EventListener extends MediaCodecTrackRenderer.EventListener {
|
|
|
|
/**
|
|
* Invoked to report the number of frames dropped by the renderer. Dropped frames are reported
|
|
* whenever the renderer is stopped having dropped frames, and optionally, whenever the count
|
|
* reaches a specified threshold whilst the renderer is started.
|
|
*
|
|
* @param count The number of dropped frames.
|
|
* @param elapsed The duration in milliseconds over which the frames were dropped. This
|
|
* duration is timed from when the renderer was started or from when dropped frames were
|
|
* last reported (whichever was more recent), and not from when the first of the reported
|
|
* drops occurred.
|
|
*/
|
|
void onDroppedFrames(int count, long elapsed);
|
|
|
|
/**
|
|
* Invoked each time there's a change in the size of the video being rendered.
|
|
*
|
|
* @param width The video width in pixels.
|
|
* @param height The video height in pixels.
|
|
*/
|
|
void onVideoSizeChanged(int width, int height);
|
|
|
|
/**
|
|
* Invoked when a frame is rendered to a surface for the first time following that surface
|
|
* having been set as the target for the renderer.
|
|
*
|
|
* @param surface The surface to which a first frame has been rendered.
|
|
*/
|
|
void onDrawnToSurface(Surface surface);
|
|
|
|
}
|
|
|
|
// TODO: Use MediaFormat constants if these get exposed through the API. See [redacted].
|
|
private static final String KEY_CROP_LEFT = "crop-left";
|
|
private static final String KEY_CROP_RIGHT = "crop-right";
|
|
private static final String KEY_CROP_BOTTOM = "crop-bottom";
|
|
private static final String KEY_CROP_TOP = "crop-top";
|
|
|
|
/**
|
|
* The type of a message that can be passed to an instance of this class via
|
|
* {@link ExoPlayer#sendMessage} or {@link ExoPlayer#blockingSendMessage}. The message object
|
|
* should be the target {@link Surface}, or null.
|
|
*/
|
|
public static final int MSG_SET_SURFACE = 1;
|
|
|
|
private final EventListener eventListener;
|
|
private final long allowedJoiningTimeUs;
|
|
private final int videoScalingMode;
|
|
private final int maxDroppedFrameCountToNotify;
|
|
|
|
private Surface surface;
|
|
private boolean drawnToSurface;
|
|
private boolean renderedFirstFrame;
|
|
private long joiningDeadlineUs;
|
|
private long droppedFrameAccumulationStartTimeMs;
|
|
private int droppedFrameCount;
|
|
|
|
private int currentWidth;
|
|
private int currentHeight;
|
|
private int lastReportedWidth;
|
|
private int lastReportedHeight;
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, int videoScalingMode) {
|
|
this(source, null, true, videoScalingMode);
|
|
}
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param drmSessionManager For use with encrypted content. May be null if support for encrypted
|
|
* content is not required.
|
|
* @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions.
|
|
* For example a media file may start with a short clear region so as to allow playback to
|
|
* begin in parallel with key acquisision. This parameter specifies whether the renderer is
|
|
* permitted to play clear regions of encrypted media files before {@code drmSessionManager}
|
|
* has obtained the keys necessary to decrypt encrypted regions of the media.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, DrmSessionManager drmSessionManager,
|
|
boolean playClearSamplesWithoutKeys, int videoScalingMode) {
|
|
this(source, drmSessionManager, playClearSamplesWithoutKeys, videoScalingMode, 0);
|
|
}
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
|
|
* can attempt to seamlessly join an ongoing playback.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, int videoScalingMode,
|
|
long allowedJoiningTimeMs) {
|
|
this(source, null, true, videoScalingMode, allowedJoiningTimeMs);
|
|
}
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param drmSessionManager For use with encrypted content. May be null if support for encrypted
|
|
* content is not required.
|
|
* @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions.
|
|
* For example a media file may start with a short clear region so as to allow playback to
|
|
* begin in parallel with key acquisision. This parameter specifies whether the renderer is
|
|
* permitted to play clear regions of encrypted media files before {@code drmSessionManager}
|
|
* has obtained the keys necessary to decrypt encrypted regions of the media.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
|
|
* can attempt to seamlessly join an ongoing playback.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, DrmSessionManager drmSessionManager,
|
|
boolean playClearSamplesWithoutKeys, int videoScalingMode, long allowedJoiningTimeMs) {
|
|
this(source, drmSessionManager, playClearSamplesWithoutKeys, videoScalingMode,
|
|
allowedJoiningTimeMs, null, null, -1);
|
|
}
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
|
|
* can attempt to seamlessly join an ongoing playback.
|
|
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
|
|
* null if delivery of events is not required.
|
|
* @param eventListener A listener of events. May be null if delivery of events is not required.
|
|
* @param maxDroppedFrameCountToNotify The maximum number of frames that can be dropped between
|
|
* invocations of {@link EventListener#onDroppedFrames(int, long)}.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, int videoScalingMode,
|
|
long allowedJoiningTimeMs, Handler eventHandler, EventListener eventListener,
|
|
int maxDroppedFrameCountToNotify) {
|
|
this(source, null, true, videoScalingMode, allowedJoiningTimeMs, eventHandler, eventListener,
|
|
maxDroppedFrameCountToNotify);
|
|
}
|
|
|
|
/**
|
|
* @param source The upstream source from which the renderer obtains samples.
|
|
* @param drmSessionManager For use with encrypted content. May be null if support for encrypted
|
|
* content is not required.
|
|
* @param playClearSamplesWithoutKeys Encrypted media may contain clear (un-encrypted) regions.
|
|
* For example a media file may start with a short clear region so as to allow playback to
|
|
* begin in parallel with key acquisision. This parameter specifies whether the renderer is
|
|
* permitted to play clear regions of encrypted media files before {@code drmSessionManager}
|
|
* has obtained the keys necessary to decrypt encrypted regions of the media.
|
|
* @param videoScalingMode The scaling mode to pass to
|
|
* {@link MediaCodec#setVideoScalingMode(int)}.
|
|
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
|
|
* can attempt to seamlessly join an ongoing playback.
|
|
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
|
|
* null if delivery of events is not required.
|
|
* @param eventListener A listener of events. May be null if delivery of events is not required.
|
|
* @param maxDroppedFrameCountToNotify The maximum number of frames that can be dropped between
|
|
* invocations of {@link EventListener#onDroppedFrames(int, long)}.
|
|
*/
|
|
public MediaCodecVideoTrackRenderer(SampleSource source, DrmSessionManager drmSessionManager,
|
|
boolean playClearSamplesWithoutKeys, int videoScalingMode, long allowedJoiningTimeMs,
|
|
Handler eventHandler, EventListener eventListener, int maxDroppedFrameCountToNotify) {
|
|
super(source, drmSessionManager, playClearSamplesWithoutKeys, eventHandler, eventListener);
|
|
this.videoScalingMode = videoScalingMode;
|
|
this.allowedJoiningTimeUs = allowedJoiningTimeMs * 1000;
|
|
this.eventListener = eventListener;
|
|
this.maxDroppedFrameCountToNotify = maxDroppedFrameCountToNotify;
|
|
joiningDeadlineUs = -1;
|
|
currentWidth = -1;
|
|
currentHeight = -1;
|
|
lastReportedWidth = -1;
|
|
lastReportedHeight = -1;
|
|
}
|
|
|
|
@Override
|
|
protected boolean handlesMimeType(String mimeType) {
|
|
return MimeTypes.isVideo(mimeType) && super.handlesMimeType(mimeType);
|
|
}
|
|
|
|
@Override
|
|
protected void onEnabled(long startTimeUs, boolean joining) {
|
|
super.onEnabled(startTimeUs, joining);
|
|
renderedFirstFrame = false;
|
|
if (joining && allowedJoiningTimeUs > 0) {
|
|
joiningDeadlineUs = SystemClock.elapsedRealtime() * 1000L + allowedJoiningTimeUs;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
protected void seekTo(long timeUs) throws ExoPlaybackException {
|
|
super.seekTo(timeUs);
|
|
renderedFirstFrame = false;
|
|
joiningDeadlineUs = -1;
|
|
}
|
|
|
|
@Override
|
|
protected boolean isReady() {
|
|
if (super.isReady()) {
|
|
// Ready. If we were joining then we've now joined, so clear the joining deadline.
|
|
joiningDeadlineUs = -1;
|
|
return true;
|
|
} else if (joiningDeadlineUs == -1) {
|
|
// Not joining.
|
|
return false;
|
|
} else if (SystemClock.elapsedRealtime() * 1000 < joiningDeadlineUs) {
|
|
// Joining and still within the joining deadline.
|
|
return true;
|
|
} else {
|
|
// The joining deadline has been exceeded. Give up and clear the deadline.
|
|
joiningDeadlineUs = -1;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
@Override
|
|
protected void onStarted() {
|
|
super.onStarted();
|
|
droppedFrameCount = 0;
|
|
droppedFrameAccumulationStartTimeMs = SystemClock.elapsedRealtime();
|
|
}
|
|
|
|
@Override
|
|
protected void onStopped() {
|
|
super.onStopped();
|
|
joiningDeadlineUs = -1;
|
|
notifyAndResetDroppedFrameCount();
|
|
}
|
|
|
|
@Override
|
|
public void onDisabled() {
|
|
super.onDisabled();
|
|
currentWidth = -1;
|
|
currentHeight = -1;
|
|
lastReportedWidth = -1;
|
|
lastReportedHeight = -1;
|
|
}
|
|
|
|
@Override
|
|
public void handleMessage(int messageType, Object message) throws ExoPlaybackException {
|
|
if (messageType == MSG_SET_SURFACE) {
|
|
setSurface((Surface) message);
|
|
} else {
|
|
super.handleMessage(messageType, message);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param surface The surface to set.
|
|
* @throws ExoPlaybackException
|
|
*/
|
|
private void setSurface(Surface surface) throws ExoPlaybackException {
|
|
if (this.surface == surface) {
|
|
return;
|
|
}
|
|
this.surface = surface;
|
|
this.drawnToSurface = false;
|
|
int state = getState();
|
|
if (state == TrackRenderer.STATE_ENABLED || state == TrackRenderer.STATE_STARTED) {
|
|
releaseCodec();
|
|
maybeInitCodec();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
protected boolean shouldInitCodec() {
|
|
return super.shouldInitCodec() && surface != null;
|
|
}
|
|
|
|
// Override configureCodec to provide the surface.
|
|
@Override
|
|
protected void configureCodec(MediaCodec codec, android.media.MediaFormat format,
|
|
MediaCrypto crypto) {
|
|
codec.configure(format, surface, crypto, 0);
|
|
codec.setVideoScalingMode(videoScalingMode);
|
|
}
|
|
|
|
@Override
|
|
protected void onOutputFormatChanged(android.media.MediaFormat format) {
|
|
boolean hasCrop = format.containsKey(KEY_CROP_RIGHT) && format.containsKey(KEY_CROP_LEFT)
|
|
&& format.containsKey(KEY_CROP_BOTTOM) && format.containsKey(KEY_CROP_TOP);
|
|
currentWidth = hasCrop
|
|
? format.getInteger(KEY_CROP_RIGHT) - format.getInteger(KEY_CROP_LEFT) + 1
|
|
: format.getInteger(android.media.MediaFormat.KEY_WIDTH);
|
|
currentHeight = hasCrop
|
|
? format.getInteger(KEY_CROP_BOTTOM) - format.getInteger(KEY_CROP_TOP) + 1
|
|
: format.getInteger(android.media.MediaFormat.KEY_HEIGHT);
|
|
}
|
|
|
|
@Override
|
|
protected boolean canReconfigureCodec(MediaCodec codec, boolean codecIsAdaptive,
|
|
MediaFormat oldFormat, MediaFormat newFormat) {
|
|
// TODO: Relax this check to also allow non-H264 adaptive decoders.
|
|
return newFormat.mimeType.equals(MimeTypes.VIDEO_H264)
|
|
&& oldFormat.mimeType.equals(MimeTypes.VIDEO_H264)
|
|
&& codecIsAdaptive
|
|
|| (oldFormat.width == newFormat.width && oldFormat.height == newFormat.height);
|
|
}
|
|
|
|
@Override
|
|
protected boolean processOutputBuffer(long timeUs, MediaCodec codec, ByteBuffer buffer,
|
|
MediaCodec.BufferInfo bufferInfo, int bufferIndex, boolean shouldSkip) {
|
|
if (shouldSkip) {
|
|
skipOutputBuffer(codec, bufferIndex);
|
|
return true;
|
|
}
|
|
|
|
long earlyUs = bufferInfo.presentationTimeUs - timeUs;
|
|
if (earlyUs < -30000) {
|
|
// We're more than 30ms late rendering the frame.
|
|
dropOutputBuffer(codec, bufferIndex);
|
|
return true;
|
|
}
|
|
|
|
if (!renderedFirstFrame) {
|
|
renderOutputBuffer(codec, bufferIndex);
|
|
renderedFirstFrame = true;
|
|
return true;
|
|
}
|
|
|
|
if (getState() == TrackRenderer.STATE_STARTED && earlyUs < 30000) {
|
|
if (earlyUs > 11000) {
|
|
// We're a little too early to render the frame. Sleep until the frame can be rendered.
|
|
// Note: The 11ms threshold was chosen fairly arbitrarily.
|
|
try {
|
|
// Subtracting 10000 rather than 11000 ensures that the sleep time will be at least 1ms.
|
|
Thread.sleep((earlyUs - 10000) / 1000);
|
|
} catch (InterruptedException e) {
|
|
Thread.currentThread().interrupt();
|
|
}
|
|
}
|
|
renderOutputBuffer(codec, bufferIndex);
|
|
return true;
|
|
}
|
|
|
|
// We're either not playing, or it's not time to render the frame yet.
|
|
return false;
|
|
}
|
|
|
|
private void skipOutputBuffer(MediaCodec codec, int bufferIndex) {
|
|
TraceUtil.beginSection("skipVideoBuffer");
|
|
codec.releaseOutputBuffer(bufferIndex, false);
|
|
TraceUtil.endSection();
|
|
codecCounters.skippedOutputBufferCount++;
|
|
}
|
|
|
|
private void dropOutputBuffer(MediaCodec codec, int bufferIndex) {
|
|
TraceUtil.beginSection("dropVideoBuffer");
|
|
codec.releaseOutputBuffer(bufferIndex, false);
|
|
TraceUtil.endSection();
|
|
codecCounters.droppedOutputBufferCount++;
|
|
droppedFrameCount++;
|
|
if (droppedFrameCount == maxDroppedFrameCountToNotify) {
|
|
notifyAndResetDroppedFrameCount();
|
|
}
|
|
}
|
|
|
|
private void renderOutputBuffer(MediaCodec codec, int bufferIndex) {
|
|
if (lastReportedWidth != currentWidth || lastReportedHeight != currentHeight) {
|
|
lastReportedWidth = currentWidth;
|
|
lastReportedHeight = currentHeight;
|
|
notifyVideoSizeChanged(currentWidth, currentHeight);
|
|
}
|
|
TraceUtil.beginSection("renderVideoBuffer");
|
|
codec.releaseOutputBuffer(bufferIndex, true);
|
|
TraceUtil.endSection();
|
|
codecCounters.renderedOutputBufferCount++;
|
|
if (!drawnToSurface) {
|
|
drawnToSurface = true;
|
|
notifyDrawnToSurface(surface);
|
|
}
|
|
}
|
|
|
|
private void notifyVideoSizeChanged(final int width, final int height) {
|
|
if (eventHandler != null && eventListener != null) {
|
|
eventHandler.post(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
eventListener.onVideoSizeChanged(width, height);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
private void notifyDrawnToSurface(final Surface surface) {
|
|
if (eventHandler != null && eventListener != null) {
|
|
eventHandler.post(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
eventListener.onDrawnToSurface(surface);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
private void notifyAndResetDroppedFrameCount() {
|
|
if (eventHandler != null && eventListener != null && droppedFrameCount > 0) {
|
|
long now = SystemClock.elapsedRealtime();
|
|
final int countToNotify = droppedFrameCount;
|
|
final long elapsedToNotify = now - droppedFrameAccumulationStartTimeMs;
|
|
droppedFrameCount = 0;
|
|
droppedFrameAccumulationStartTimeMs = now;
|
|
eventHandler.post(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
eventListener.onDroppedFrames(countToNotify, elapsedToNotify);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
}
|