diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java index a73be489d1..c9bc1f58cf 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java @@ -55,6 +55,7 @@ public final class MimeTypes { public static final String VIDEO_DOLBY_VISION = BASE_TYPE_VIDEO + "/dolby-vision"; public static final String VIDEO_OGG = BASE_TYPE_VIDEO + "/ogg"; public static final String VIDEO_AVI = BASE_TYPE_VIDEO + "/x-msvideo"; + public static final String VIDEO_JPEG = BASE_TYPE_VIDEO + "/JPEG"; //RFC 3555 public static final String VIDEO_UNKNOWN = BASE_TYPE_VIDEO + "/x-unknown"; // audio/ MIME types diff --git a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java index 0d1c126dc5..4b0fff8ffc 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java @@ -27,6 +27,7 @@ import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.AudioSink; import com.google.android.exoplayer2.audio.DefaultAudioSink; import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; +import com.google.android.exoplayer2.video.BitmapFactoryVideoRenderer; import com.google.android.exoplayer2.mediacodec.DefaultMediaCodecAdapterFactory; import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter; import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; @@ -395,6 +396,7 @@ public class DefaultRenderersFactory implements RenderersFactory { eventListener, MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); out.add(videoRenderer); + out.add(new BitmapFactoryVideoRenderer(eventHandler, eventListener)); if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { return; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java new file mode 100644 index 0000000000..36b67c12b7 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java @@ -0,0 +1,230 @@ +package com.google.android.exoplayer2.video; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Point; +import android.graphics.Rect; +import android.os.Handler; +import android.view.Surface; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.BaseRenderer; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.FormatHolder; +import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.decoder.DecoderCounters; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.MimeTypes; +import java.nio.ByteBuffer; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class BitmapFactoryVideoRenderer extends BaseRenderer { + private static final String TAG = "BitmapFactoryRenderer"; + final VideoRendererEventListener.EventDispatcher eventDispatcher; + @Nullable + Surface surface; + private boolean firstFrameRendered; + private final Rect rect = new Rect(); + private final Point lastSurface = new Point(); + private VideoSize lastVideoSize = VideoSize.UNKNOWN; + @Nullable + private ThreadPoolExecutor renderExecutor; + @Nullable + private Thread thread; + private long currentTimeUs; + private long nextFrameUs; + private long frameUs; + private boolean ended; + private DecoderCounters decoderCounters; + + public BitmapFactoryVideoRenderer(@Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener) { + super(C.TRACK_TYPE_VIDEO); + eventDispatcher = new VideoRendererEventListener.EventDispatcher(eventHandler, eventListener); + } + + @NonNull + @Override + public String getName() { + return TAG; + } + + @Override + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) + throws ExoPlaybackException { + firstFrameRendered = ended = false; + renderExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<>(3)); + decoderCounters = new DecoderCounters(); + eventDispatcher.enabled(decoderCounters); + } + + @Override + protected void onDisabled() { + renderExecutor.shutdownNow(); + eventDispatcher.disabled(decoderCounters); + } + + @Override + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) + throws ExoPlaybackException { + nextFrameUs = startPositionUs; + for (final Format format : formats) { + @NonNull final FormatHolder formatHolder = getFormatHolder(); + @Nullable final Format currentFormat = formatHolder.format; + if (formatHolder.format == null || !currentFormat.equals(format)) { + getFormatHolder().format = format; + eventDispatcher.inputFormatChanged(format, null); + frameUs = (long)(1_000_000L / format.frameRate); + } + } + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + synchronized (eventDispatcher) { + currentTimeUs = positionUs; + eventDispatcher.notify(); + } + if (renderExecutor.getActiveCount() > 0) { + if (positionUs > nextFrameUs) { + long us = (positionUs - nextFrameUs) + frameUs; + long dropped = us / frameUs; + eventDispatcher.droppedFrames((int)dropped, us); + nextFrameUs += frameUs * dropped; + } + return; + } + final FormatHolder formatHolder = getFormatHolder(); + final DecoderInputBuffer decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + int result = readSource(formatHolder, decoderInputBuffer, 0); + if (result == C.RESULT_BUFFER_READ) { + renderExecutor.execute(new RenderRunnable(decoderInputBuffer, nextFrameUs)); + nextFrameUs += frameUs; + } else if (result == C.RESULT_END_OF_INPUT) { + ended = true; + } + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + nextFrameUs = positionUs; + @Nullable + final Thread thread = this.thread; + if (thread != null) { + thread.interrupt(); + } + } + + @Override + public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { + if (messageType == MSG_SET_VIDEO_OUTPUT) { + if (message instanceof Surface) { + surface = (Surface) message; + } else { + surface = null; + } + } + super.handleMessage(messageType, message); + } + + @Override + public boolean isReady() { + return surface != null; + } + + @Override + public boolean isEnded() { + return ended && renderExecutor.getActiveCount() == 0; + } + + @Override + public int supportsFormat(Format format) throws ExoPlaybackException { + //Technically could support any format BitmapFactory supports + if (MimeTypes.VIDEO_JPEG.equals(format.sampleMimeType)) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + return RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE); + } + + class RenderRunnable implements Runnable { + final DecoderInputBuffer decoderInputBuffer; + final long renderUs; + + RenderRunnable(final DecoderInputBuffer decoderInputBuffer, long renderUs) { + this.decoderInputBuffer = decoderInputBuffer; + this.renderUs = renderUs; + } + + public void run() { + synchronized (eventDispatcher) { + while (currentTimeUs < renderUs) { + try { + thread = Thread.currentThread(); + eventDispatcher.wait(); + } catch (InterruptedException e) { + //If we are interrupted, treat as a cancel + return; + } finally { + thread = null; + } + } + } + @Nullable + final ByteBuffer byteBuffer = decoderInputBuffer.data; + @Nullable + final Surface surface = BitmapFactoryVideoRenderer.this.surface; + if (byteBuffer != null && surface != null) { + final Bitmap bitmap; + try { + bitmap = BitmapFactory.decodeByteArray(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.arrayOffset() + byteBuffer.position()); + } catch (Exception e) { + eventDispatcher.videoCodecError(e); + return; + } + if (bitmap == null) { + eventDispatcher.videoCodecError(new NullPointerException("Decode bytes failed")); + return; + } + //Log.d(TAG, "Drawing: " + bitmap.getWidth() + "x" + bitmap.getHeight()); + final Canvas canvas = surface.lockCanvas(null); + + final Rect clipBounds = canvas.getClipBounds(); + final VideoSize videoSize = new VideoSize(bitmap.getWidth(), bitmap.getHeight()); + final boolean videoSizeChanged; + if (videoSize.equals(lastVideoSize)) { + videoSizeChanged = false; + } else { + lastVideoSize = videoSize; + eventDispatcher.videoSizeChanged(videoSize); + videoSizeChanged = true; + } + if (lastSurface.x != clipBounds.width() || lastSurface.y != clipBounds.height() || + videoSizeChanged) { + lastSurface.x = clipBounds.width(); + lastSurface.y = clipBounds.height(); + final float scaleX = lastSurface.x / (float)videoSize.width; + final float scaleY = lastSurface.y / (float)videoSize.height; + final float scale = Math.min(scaleX, scaleY); + final float width = videoSize.width * scale; + final float height = videoSize.height * scale; + final int x = (int)(lastSurface.x - width) / 2; + final int y = (int)(lastSurface.y - height) / 2; + rect.set(x, y, x + (int)width, y + (int) height); + } + canvas.drawBitmap(bitmap, null, rect, null); + + surface.unlockCanvasAndPost(canvas); + decoderCounters.renderedOutputBufferCount++; + if (!firstFrameRendered) { + firstFrameRendered = true; + eventDispatcher.renderedFirstFrame(surface); + } + } + } + } +} diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java index e0c6cea720..8eea67db55 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java @@ -24,7 +24,7 @@ public class StreamHeaderBox extends ResidentBox { //final String mimeType = MimeTypes.VIDEO_H263; //Doesn't seem to be supported on Android - //STREAM_MAP.put('M' | ('P' << 8) | ('4' << 16) | ('2' << 24), MimeTypes.VIDEO_MP4); + STREAM_MAP.put('M' | ('P' << 8) | ('4' << 16) | ('2' << 24), MimeTypes.VIDEO_AVI); STREAM_MAP.put('H' | ('2' << 8) | ('6' << 16) | ('4' << 24), MimeTypes.VIDEO_H264); STREAM_MAP.put('a' | ('v' << 8) | ('c' << 16) | ('1' << 24), MimeTypes.VIDEO_H264); STREAM_MAP.put('A' | ('V' << 8) | ('C' << 16) | ('1' << 24), MimeTypes.VIDEO_H264); @@ -32,7 +32,7 @@ public class StreamHeaderBox extends ResidentBox { STREAM_MAP.put('x' | ('v' << 8) | ('i' << 16) | ('d' << 24), mimeType); STREAM_MAP.put('X' | ('V' << 8) | ('I' << 16) | ('D' << 24), mimeType); - STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.IMAGE_JPEG); + STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.VIDEO_JPEG); } StreamHeaderBox(int type, int size, ByteBuffer byteBuffer) { @@ -52,8 +52,7 @@ public class StreamHeaderBox extends ResidentBox { } /** - * How long each sample covers - * @return + * @return sample duration in us */ public long getUsPerSample() { return getScale() * 1_000_000L / getRate();