diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1b8fb18a67..18a4b2141a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -55,9 +55,6 @@ * Add `DataSpec.Builder` and deprecate most `DataSpec` constructors. * Add `DataSpec.customData` to allow applications to pass custom data through `DataSource` chains. - * Add a sample count parameter to `MediaCodecRenderer.processOutputBuffer` - and `AudioSink.handleBuffer` to allow batching multiple encoded frames - in one buffer. * Add a `Format.Builder` and deprecate all `Format.create*` methods and most `Format.copyWith*` methods. * Split `Format.bitrate` into `Format.averageBitrate` and @@ -133,6 +130,11 @@ directly instead. * Update `CachedContentIndex` to use `SecureRandom` for generating the initialization vector used to encrypt the cache contents. +* Audio: + * Add a sample count parameter to `MediaCodecRenderer.processOutputBuffer` + and `AudioSink.handleBuffer` to allow batching multiple encoded frames + in one buffer. + * No longer use a `MediaCodec` in audio passthrough mode. * DASH: * Merge trick play adaptation sets (i.e., adaptation sets marked with `http://dashif.org/guidelines/trickmode`) into the same `TrackGroup` as diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 90fb1f8110..578a51b685 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -217,7 +217,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @TunnelingSupport int tunnelingSupport = Util.SDK_INT >= 21 ? TUNNELING_SUPPORTED : TUNNELING_NOT_SUPPORTED; boolean supportsFormatDrm = supportsFormatDrm(format); - if (supportsFormatDrm && usePassthrough(format.channelCount, mimeType)) { + if (supportsFormatDrm + && usePassthrough(format.channelCount, mimeType) + && MediaCodecUtil.getPassthroughDecoderInfo() != null) { return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_NOT_SEAMLESS, tunnelingSupport); } if ((MimeTypes.AUDIO_RAW.equals(mimeType) @@ -256,7 +258,10 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media return Collections.emptyList(); } if (usePassthrough(format.channelCount, mimeType)) { - return Collections.singletonList(MediaCodecUtil.getPassthroughDecoderInfo()); + @Nullable MediaCodecInfo codecInfo = MediaCodecUtil.getPassthroughDecoderInfo(); + if (codecInfo != null) { + return Collections.singletonList(codecInfo); + } } List decoderInfos = mediaCodecSelector.getDecoderInfos( @@ -273,19 +278,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media return Collections.unmodifiableList(decoderInfos); } - /** - * Returns whether encoded audio passthrough should be used for playing back the input format. - * - * @param channelCount The number of channels in the input media, or {@link Format#NO_VALUE} if - * not known. - * @param mimeType The type of input media. - * @return Whether passthrough playback is supported. - * @throws DecoderQueryException If there was an error querying the available passthrough - * decoders. - */ - protected boolean usePassthrough(int channelCount, String mimeType) throws DecoderQueryException { - return getPassthroughEncoding(channelCount, mimeType) != C.ENCODING_INVALID - && MediaCodecUtil.getPassthroughDecoderInfo() != null; + @Override + protected boolean usePassthrough(int channelCount, String mimeType) { + return getPassthroughEncoding(channelCount, mimeType) != C.ENCODING_INVALID; } @Override @@ -423,20 +418,15 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } else { channelMap = null; } + configureAudioSink(encoding, channelCount, sampleRate, channelMap); + } - try { - audioSink.configure( - encoding, - channelCount, - sampleRate, - 0, - channelMap, - inputFormat.encoderDelay, - inputFormat.encoderPadding); - } catch (AudioSink.ConfigurationException e) { - // TODO(internal: b/145658993) Use outputFormat instead. - throw createRendererException(e, inputFormat); - } + @Override + protected void onOutputPassthroughFormatChanged(Format outputFormat) throws ExoPlaybackException { + @C.Encoding + int encoding = getPassthroughEncoding(outputFormat.channelCount, outputFormat.sampleMimeType); + configureAudioSink( + encoding, outputFormat.channelCount, outputFormat.sampleRate, /* channelMap= */ null); } /** @@ -602,7 +592,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media protected boolean processOutputBuffer( long positionUs, long elapsedRealtimeUs, - MediaCodec codec, + @Nullable MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, @@ -612,7 +602,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media boolean isLastBuffer, Format format) throws ExoPlaybackException { - if (codecNeedsEosBufferTimestampWorkaround + if (codec != null + && codecNeedsEosBufferTimestampWorkaround && bufferPresentationTimeUs == 0 && (bufferFlags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 && getLargestQueuedPresentationTimeUs() != C.TIME_UNSET) { @@ -626,7 +617,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } if (isDecodeOnlyBuffer) { - codec.releaseOutputBuffer(bufferIndex, false); + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, false); + } decoderCounters.skippedOutputBufferCount++; audioSink.handleDiscontinuity(); return true; @@ -641,7 +634,9 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } if (fullyConsumed) { - codec.releaseOutputBuffer(bufferIndex, false); + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, false); + } decoderCounters.renderedOutputBufferCount++; return true; } @@ -769,6 +764,24 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media return mediaFormat; } + private void configureAudioSink( + int encoding, int channelCount, int sampleRate, @Nullable int[] channelMap) + throws ExoPlaybackException { + try { + audioSink.configure( + encoding, + channelCount, + sampleRate, + /* specifiedBufferSize= */ 0, + channelMap, + inputFormat.encoderDelay, + inputFormat.encoderPadding); + } catch (AudioSink.ConfigurationException e) { + // TODO(internal: b/145658993) Use outputFormat instead. + throw createRendererException(e, inputFormat); + } + } + private void updateCurrentPosition() { long newCurrentPositionUs = audioSink.getCurrentPositionUs(isEnded()); if (newCurrentPositionUs != AudioSink.CURRENT_POSITION_NOT_SET) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/BatchBuffer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/BatchBuffer.java new file mode 100644 index 0000000000..3c40fe02d4 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/BatchBuffer.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.mediacodec; + +import androidx.annotation.IntRange; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.Assertions; +import java.nio.ByteBuffer; + +/** Buffer that stores multiple encoded access units to allow batch processing. */ +/* package */ final class BatchBuffer extends DecoderInputBuffer { + /** Arbitrary limit to the number of access unit in a full batch buffer. */ + public static final int DEFAULT_BATCH_SIZE_ACCESS_UNITS = 32; + /** + * Arbitrary limit to the memory used by a full batch buffer to avoid using too much memory for + * very high bitrate. Equivalent of 75s of mp3 at highest bitrate (320kb/s) and 30s of AAC LC at + * highest bitrate (800kb/s). That limit is ignored for the first access unit to avoid stalling + * stream with huge access units. + */ + private static final int BATCH_SIZE_BYTES = 3 * 1000 * 1024; + + private final DecoderInputBuffer nextAccessUnitBuffer; + private boolean hasPendingAccessUnit; + + private long firstAccessUnitTimeUs; + private int accessUnitCount; + private int maxAccessUnitCount; + + public BatchBuffer() { + super(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); + nextAccessUnitBuffer = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); + clear(); + } + + /** Sets the maximum number of access units the buffer can contain before being full. */ + public void setMaxAccessUnitCount(@IntRange(from = 1) int maxAccessUnitCount) { + Assertions.checkArgument(maxAccessUnitCount > 0); + this.maxAccessUnitCount = maxAccessUnitCount; + } + + /** Gets the maximum number of access units the buffer can contain before being full. */ + public int getMaxAccessUnitCount() { + return maxAccessUnitCount; + } + + /** Resets the state of this object to what it was after construction. */ + @Override + public void clear() { + flush(); + maxAccessUnitCount = DEFAULT_BATCH_SIZE_ACCESS_UNITS; + } + + /** Clear all access units from the BatchBuffer to empty it. */ + public void flush() { + clearMainBuffer(); + nextAccessUnitBuffer.clear(); + hasPendingAccessUnit = false; + } + + /** Clears the state of the batch buffer to be ready to receive a new sequence of access units. */ + public void batchWasConsumed() { + clearMainBuffer(); + if (hasPendingAccessUnit) { + putAccessUnit(nextAccessUnitBuffer); + hasPendingAccessUnit = false; + } + } + + /** + * Gets the buffer to fill-out that will then be append to the batch buffer with {@link + * #commitNextAccessUnit()}. + */ + public DecoderInputBuffer getNextAccessUnitBuffer() { + return nextAccessUnitBuffer; + } + + /** Gets the timestamp of the first access unit in the buffer. */ + public long getFirstAccessUnitTimeUs() { + return firstAccessUnitTimeUs; + } + + /** Gets the timestamp of the last access unit in the buffer. */ + public long getLastAccessUnitTimeUs() { + return timeUs; + } + + /** Gets the number of access units contained in this batch buffer. */ + public int getAccessUnitCount() { + return accessUnitCount; + } + + /** If the buffer contains no access units. */ + public boolean isEmpty() { + return accessUnitCount == 0; + } + + /** If more access units should be added to the batch buffer. */ + public boolean isFull() { + return accessUnitCount >= maxAccessUnitCount + || (data != null && data.position() >= BATCH_SIZE_BYTES) + || hasPendingAccessUnit; + } + + /** + * Appends the staged access unit in this batch buffer. + * + * @throws IllegalStateException If calling this method on a full or end of stream batch buffer. + * @throws IllegalArgumentException If the {@code accessUnit} is encrypted or has + * supplementalData, as batching of those state has not been implemented. + */ + public void commitNextAccessUnit() { + DecoderInputBuffer accessUnit = nextAccessUnitBuffer; + Assertions.checkState(!isFull() && !isEndOfStream()); + Assertions.checkArgument(!accessUnit.isEncrypted() && !accessUnit.hasSupplementalData()); + if (!canBatch(accessUnit)) { + hasPendingAccessUnit = true; // Delay the putAccessUnit until the batch buffer is empty. + return; + } + putAccessUnit(accessUnit); + } + + private boolean canBatch(DecoderInputBuffer accessUnit) { + if (isEmpty()) { + return true; // Batching with an empty batch must always succeed or the stream will stall. + } + if (accessUnit.isDecodeOnly() != isDecodeOnly()) { + return false; // Decode only and non decode only access units can not be batched together. + } + + @Nullable ByteBuffer accessUnitData = accessUnit.data; + if (accessUnitData != null + && this.data != null + && this.data.position() + accessUnitData.limit() >= BATCH_SIZE_BYTES) { + return false; // The batch buffer does not have the capacity to add this access unit. + } + return true; + } + + private void putAccessUnit(DecoderInputBuffer accessUnit) { + @Nullable ByteBuffer accessUnitData = accessUnit.data; + if (accessUnitData != null) { + accessUnit.flip(); + ensureSpaceForWrite(accessUnitData.remaining()); + this.data.put(accessUnitData); + } + + if (accessUnit.isEndOfStream()) { + setFlags(C.BUFFER_FLAG_END_OF_STREAM); + } + if (accessUnit.isDecodeOnly()) { + setFlags(C.BUFFER_FLAG_DECODE_ONLY); + } + if (accessUnit.isKeyFrame()) { + setFlags(C.BUFFER_FLAG_KEY_FRAME); + } + accessUnitCount++; + timeUs = accessUnit.timeUs; + if (accessUnitCount == 1) { // First read of the buffer + firstAccessUnitTimeUs = timeUs; + } + accessUnit.clear(); + } + + private void clearMainBuffer() { + super.clear(); + accessUnitCount = 0; + firstAccessUnitTimeUs = C.TIME_UNSET; + timeUs = C.TIME_UNSET; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index b4b0d6ecd9..611631038a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -46,6 +46,7 @@ import com.google.android.exoplayer2.source.MediaPeriod; import com.google.android.exoplayer2.source.SampleStream; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.NalUnitUtil; import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.TraceUtil; @@ -361,6 +362,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private final float assumedMinimumCodecOperatingRate; private final DecoderInputBuffer buffer; private final DecoderInputBuffer flagsOnlyBuffer; + private final BatchBuffer passthroughBatchBuffer; private final TimedValueQueue formatQueue; private final ArrayList decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; @@ -401,6 +403,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private ByteBuffer outputBuffer; private boolean isDecodeOnlyOutputBuffer; private boolean isLastOutputBuffer; + private boolean passthroughEnabled; + private boolean passthroughDrainAndReinitialize; private boolean codecReconfigured; @ReconfigurationState private int codecReconfigurationState; @DrainState private int codecDrainState; @@ -453,6 +457,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; outputStreamOffsetUs = C.TIME_UNSET; + passthroughBatchBuffer = new BatchBuffer(); resetCodecStateForRelease(); } @@ -567,9 +572,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @Nullable MediaCrypto crypto, float codecOperatingRate); - protected final void maybeInitCodec() throws ExoPlaybackException { - if (codec != null || inputFormat == null) { - // We have a codec already, or we don't have a format with which to instantiate one. + protected final void maybeInitCodecOrPassthrough() throws ExoPlaybackException { + if (codec != null || passthroughEnabled || inputFormat == null) { + // We have a codec or using passthrough, or don't have a format to decide how to render. + return; + } + + if (inputFormat.drmInitData == null + && usePassthrough(inputFormat.channelCount, inputFormat.sampleMimeType)) { + initPassthrough(inputFormat); return; } @@ -618,6 +629,18 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } } + /** + * Returns whether encoded passthrough should be used for playing back the input format. + * + * @param channelCount The number of channels in the input media, or {@link Format#NO_VALUE} if + * not known. + * @param mimeType The type of input media. + * @return Whether passthrough playback is supported. + */ + protected boolean usePassthrough(int channelCount, String mimeType) { + return false; + } + protected boolean shouldInitCodec(MediaCodecInfo codecInfo) { return true; } @@ -695,7 +718,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { inputStreamEnded = false; outputStreamEnded = false; pendingOutputEndOfStream = false; - flushOrReinitializeCodec(); + if (passthroughEnabled) { + passthroughBatchBuffer.flush(); + } else { + flushOrReinitializeCodec(); + } // If there is a format change on the input side still pending propagation to the output, we // need to queue a format next time a buffer is read. This is because we may not read a new // input format after the position reset. @@ -735,12 +762,19 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @Override protected void onReset() { try { + disablePassthrough(); releaseCodec(); } finally { setSourceDrmSession(null); } } + private void disablePassthrough() { + passthroughDrainAndReinitialize = false; + passthroughBatchBuffer.clear(); + passthroughEnabled = false; + } + protected void releaseCodec() { try { if (codecAdapter != null) { @@ -791,8 +825,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return; } // We have a format. - maybeInitCodec(); - if (codec != null) { + maybeInitCodecOrPassthrough(); + if (passthroughEnabled) { + TraceUtil.beginSection("renderPassthrough"); + while (renderPassthrough(positionUs, elapsedRealtimeUs)) {} + TraceUtil.endSection(); + } else if (codec != null) { long renderStartTimeMs = SystemClock.elapsedRealtime(); TraceUtil.beginSection("drainAndFeed"); while (drainOutputBuffer(positionUs, elapsedRealtimeUs) @@ -821,7 +859,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { * This method is a no-op if the codec is {@code null}. * *

The implementation of this method calls {@link #flushOrReleaseCodec()}, and {@link - * #maybeInitCodec()} if the codec needs to be re-instantiated. + * #maybeInitCodecOrPassthrough()} if the codec needs to be re-instantiated. * * @return Whether the codec was released and reinitialized, rather than being flushed. * @throws ExoPlaybackException If an error occurs re-instantiating the codec. @@ -829,7 +867,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected final boolean flushOrReinitializeCodec() throws ExoPlaybackException { boolean released = flushOrReleaseCodec(); if (released) { - maybeInitCodec(); + maybeInitCodecOrPassthrough(); } return released; } @@ -1022,6 +1060,26 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return codecInfos; } + /** + * Configures passthrough where no codec is used. Called instead of {@link + * #configureCodec(MediaCodecInfo, MediaCodec, Format, MediaCrypto, float)} when no codec is used + * in passthrough. + */ + private void initPassthrough(Format format) { + disablePassthrough(); // In case of transition between 2 passthrough formats. + + String mimeType = format.sampleMimeType; + if (!MimeTypes.AUDIO_AAC.equals(mimeType) + && !MimeTypes.AUDIO_MPEG.equals(mimeType) + && !MimeTypes.AUDIO_OPUS.equals(mimeType)) { + // TODO(b/154746451): Batching provokes frame drops in non offload passthrough. + passthroughBatchBuffer.setMaxAccessUnitCount(1); + } else { + passthroughBatchBuffer.setMaxAccessUnitCount(BatchBuffer.DEFAULT_BATCH_SIZE_ACCESS_UNITS); + } + passthroughEnabled = true; + } + private void initCodec(MediaCodecInfo codecInfo, MediaCrypto crypto) throws Exception { long codecInitializingTimestamp; long codecInitializedTimestamp; @@ -1373,13 +1431,19 @@ public abstract class MediaCodecRenderer extends BaseRenderer { setSourceDrmSession(formatHolder.drmSession); inputFormat = newFormat; + if (passthroughEnabled) { + passthroughDrainAndReinitialize = true; + return; // Need to drain passthrough first. + } + if (codec == null) { - maybeInitCodec(); + maybeInitCodecOrPassthrough(); return; } - // We have an existing codec that we may need to reconfigure or re-initialize. If the existing - // codec instance is being kept then its operating rate may need to be updated. + // We have an existing codec that we may need to reconfigure or re-initialize or release it to + // switch to passthrough. If the existing codec instance is being kept then its operating rate + // may need to be updated. if ((sourceDrmSession == null && codecDrmSession != null) || (sourceDrmSession != null && codecDrmSession == null) @@ -1473,6 +1537,19 @@ public abstract class MediaCodecRenderer extends BaseRenderer { // Do nothing. } + /** + * Called when the output {@link Format} changes in passthrough. + * + *

The default implementation is a no-op. + * + * @param outputFormat The new output {@link MediaFormat}. + * @throws ExoPlaybackException Thrown if an error occurs handling the new output media format. + */ + // TODO(b/154849417): merge with {@link #onOutputFormatChanged(Format)}. + protected void onOutputPassthroughFormatChanged(Format outputFormat) throws ExoPlaybackException { + // Do nothing. + } + /** * Handles supplemental data associated with an input buffer. * @@ -1821,7 +1898,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { * iteration of the rendering loop. * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the * start of the current iteration of the rendering loop. - * @param codec The {@link MediaCodec} instance. + * @param codec The {@link MediaCodec} instance, or null in passthrough mode. * @param buffer The output buffer to process. * @param bufferIndex The index of the output buffer. * @param bufferFlags The flags attached to the output buffer. @@ -1838,7 +1915,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected abstract boolean processOutputBuffer( long positionUs, long elapsedRealtimeUs, - MediaCodec codec, + @Nullable MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, @@ -1950,7 +2027,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private void reinitializeCodec() throws ExoPlaybackException { releaseCodec(); - maybeInitCodec(); + maybeInitCodecOrPassthrough(); } private boolean isDecodeOnlyBuffer(long presentationTimeUs) { @@ -2016,6 +2093,116 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return (FrameworkMediaCrypto) mediaCrypto; } + /** + * Processes any pending batch of buffers without using a decoder, and drains a new batch of + * buffers from the source. + * + * @param positionUs The current media time in microseconds, measured at the start of the current + * iteration of the rendering loop. + * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the + * start of the current iteration of the rendering loop. + * @return If more buffers are ready to be rendered. + * @throws ExoPlaybackException If an error occurred while processing a buffer or handling a + * format change. + */ + private boolean renderPassthrough(long positionUs, long elapsedRealtimeUs) + throws ExoPlaybackException { + BatchBuffer batchBuffer = passthroughBatchBuffer; + + // Let's process the pending buffer if any. + Assertions.checkState(!outputStreamEnded); + if (!batchBuffer.isEmpty()) { // Optimisation: Do not process buffer if empty. + if (processOutputBuffer( + positionUs, + elapsedRealtimeUs, + /* codec= */ null, + batchBuffer.data, + outputIndex, + /* bufferFlags= */ 0, + batchBuffer.getAccessUnitCount(), + batchBuffer.getFirstAccessUnitTimeUs(), + batchBuffer.isDecodeOnly(), + batchBuffer.isEndOfStream(), + outputFormat)) { + // Buffer completely processed + onProcessedOutputBuffer(batchBuffer.getLastAccessUnitTimeUs()); + } else { + return false; // Could not process buffer, let's try later. + } + } + if (batchBuffer.isEndOfStream()) { + outputStreamEnded = true; + return false; + } + batchBuffer.batchWasConsumed(); + + if (passthroughDrainAndReinitialize) { + if (!batchBuffer.isEmpty()) { + return true; // Drain the batch buffer before propagating the format change. + } + disablePassthrough(); // The new format might not be supported in passthrough. + passthroughDrainAndReinitialize = false; + maybeInitCodecOrPassthrough(); + if (!passthroughEnabled) { + return false; // The new format is not supported in passthrough. + } + } + + // Now refill the empty buffer for the next iteration. + Assertions.checkState(!inputStreamEnded); + FormatHolder formatHolder = getFormatHolder(); + boolean formatChange = readBatchFromSource(formatHolder, batchBuffer); + + if (!batchBuffer.isEmpty() && waitingForFirstSampleInFormat) { + // This is the first buffer in a new format, the output format must be updated. + outputFormat = Assertions.checkNotNull(inputFormat); + onOutputPassthroughFormatChanged(outputFormat); + waitingForFirstSampleInFormat = false; + } + + if (formatChange) { + onInputFormatChanged(formatHolder); + } + + if (batchBuffer.isEndOfStream()) { + inputStreamEnded = true; + } + + if (batchBuffer.isEmpty()) { + return false; // The buffer could not be filled, there is nothing more to do. + } + batchBuffer.flip(); // Buffer at least partially full, it can now be processed. + return true; + } + + /** + * Fills the buffer with multiple access unit from the source. Has otherwise the same semantic as + * {@link #readSource(FormatHolder, DecoderInputBuffer, boolean)}. Will stop early on format + * change, EOS or source starvation. + * + * @return If the format has changed. + */ + private boolean readBatchFromSource(FormatHolder formatHolder, BatchBuffer batchBuffer) { + while (!batchBuffer.isFull() && !batchBuffer.isEndOfStream()) { + @SampleStream.ReadDataResult + int result = + readSource( + formatHolder, batchBuffer.getNextAccessUnitBuffer(), /* formatRequired= */ false); + switch (result) { + case C.RESULT_FORMAT_READ: + return true; + case C.RESULT_NOTHING_READ: + return false; + case C.RESULT_BUFFER_READ: + batchBuffer.commitNextAccessUnit(); + break; + default: + throw new IllegalStateException(); // Unsupported result + } + } + return false; + } + private static boolean isMediaCodecException(IllegalStateException error) { if (Util.SDK_INT >= 21 && isMediaCodecExceptionV21(error)) { return true; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 847dac3a79..9dc0c7230d 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -514,7 +514,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { setOutputSurfaceV23(codec, surface); } else { releaseCodec(); - maybeInitCodec(); + maybeInitCodecOrPassthrough(); } } if (surface != null && surface != dummySurface) { @@ -753,7 +753,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { protected boolean processOutputBuffer( long positionUs, long elapsedRealtimeUs, - MediaCodec codec, + @Nullable MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, @@ -763,6 +763,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { boolean isLastBuffer, Format format) throws ExoPlaybackException { + Assertions.checkNotNull(codec); // Can not render video without codec + if (initialPositionUs == C.TIME_UNSET) { initialPositionUs = positionUs; } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/BatchBufferTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/BatchBufferTest.java new file mode 100644 index 0000000000..ac40b4b39a --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/BatchBufferTest.java @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.mediacodec; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.testutil.TestUtil; +import java.nio.ByteBuffer; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link BatchBuffer}. */ +@RunWith(AndroidJUnit4.class) +public final class BatchBufferTest { + + /** Bigger than {@link BatchBuffer#BATCH_SIZE_BYTES} */ + private static final int BUFFER_SIZE_LARGER_THAN_BATCH_SIZE_BYTES = 100 * 1000 * 1000; + /** Smaller than {@link BatchBuffer#BATCH_SIZE_BYTES} */ + private static final int BUFFER_SIZE_MUCH_SMALLER_THAN_BATCH_SIZE_BYTES = 100; + + private static final byte[] TEST_ACCESS_UNIT = + TestUtil.buildTestData(BUFFER_SIZE_MUCH_SMALLER_THAN_BATCH_SIZE_BYTES); + private static final byte[] TEST_HUGE_ACCESS_UNIT = + TestUtil.buildTestData(BUFFER_SIZE_LARGER_THAN_BATCH_SIZE_BYTES); + + private final BatchBuffer batchBuffer = new BatchBuffer(); + + @Test + public void newBatchBuffer_isEmpty() { + assertIsCleared(batchBuffer); + } + + @Test + public void clear_empty_isEmpty() { + batchBuffer.clear(); + + assertIsCleared(batchBuffer); + } + + @Test + public void clear_afterInsertingAccessUnit_isEmpty() { + batchBuffer.commitNextAccessUnit(); + + batchBuffer.clear(); + + assertIsCleared(batchBuffer); + } + + @Test + public void commitNextAccessUnit_addsAccessUnit() { + batchBuffer.commitNextAccessUnit(); + + assertThat(batchBuffer.getAccessUnitCount()).isEqualTo(1); + } + + @Test + public void commitNextAccessUnit_untilFull_isFullAndNotEmpty() { + fillBatchBuffer(batchBuffer); + + assertThat(batchBuffer.isEmpty()).isFalse(); + assertThat(batchBuffer.isFull()).isTrue(); + } + + @Test + public void commitNextAccessUnit_whenFull_throws() { + batchBuffer.setMaxAccessUnitCount(1); + batchBuffer.commitNextAccessUnit(); + + assertThrows(IllegalStateException.class, batchBuffer::commitNextAccessUnit); + } + + @Test + public void commitNextAccessUnit_whenAccessUnitIsDecodeOnly_isDecodeOnly() { + batchBuffer.getNextAccessUnitBuffer().setFlags(C.BUFFER_FLAG_DECODE_ONLY); + + batchBuffer.commitNextAccessUnit(); + + assertThat(batchBuffer.isDecodeOnly()).isTrue(); + } + + @Test + public void commitNextAccessUnit_whenAccessUnitIsEndOfStream_isEndOfSteam() { + batchBuffer.getNextAccessUnitBuffer().setFlags(C.BUFFER_FLAG_END_OF_STREAM); + + batchBuffer.commitNextAccessUnit(); + + assertThat(batchBuffer.isEndOfStream()).isTrue(); + } + + @Test + public void commitNextAccessUnit_whenAccessUnitIsKeyFrame_isKeyFrame() { + batchBuffer.getNextAccessUnitBuffer().setFlags(C.BUFFER_FLAG_KEY_FRAME); + + batchBuffer.commitNextAccessUnit(); + + assertThat(batchBuffer.isKeyFrame()).isTrue(); + } + + @Test + public void commitNextAccessUnit_withData_dataIsCopiedInTheBatch() { + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + + batchBuffer.commitNextAccessUnit(); + batchBuffer.flip(); + + assertThat(batchBuffer.getAccessUnitCount()).isEqualTo(1); + assertThat(batchBuffer.data).isEqualTo(ByteBuffer.wrap(TEST_ACCESS_UNIT)); + } + + @Test + public void commitNextAccessUnit_nextAccessUnit_isClear() { + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + batchBuffer.getNextAccessUnitBuffer().setFlags(C.BUFFER_FLAG_KEY_FRAME); + + batchBuffer.commitNextAccessUnit(); + + DecoderInputBuffer nextAccessUnit = batchBuffer.getNextAccessUnitBuffer(); + assertThat(nextAccessUnit.data).isNotNull(); + assertThat(nextAccessUnit.data.position()).isEqualTo(0); + assertThat(nextAccessUnit.isKeyFrame()).isFalse(); + } + + @Test + public void commitNextAccessUnit_twice_bothAccessUnitAreConcatenated() { + // Commit TEST_ACCESS_UNIT + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + batchBuffer.commitNextAccessUnit(); + // Commit TEST_ACCESS_UNIT again + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + + batchBuffer.commitNextAccessUnit(); + batchBuffer.flip(); + + byte[] expected = TestUtil.joinByteArrays(TEST_ACCESS_UNIT, TEST_ACCESS_UNIT); + assertThat(batchBuffer.data).isEqualTo(ByteBuffer.wrap(expected)); + } + + @Test + public void commitNextAccessUnit_whenAccessUnitIsHugeAndBatchBufferNotEmpty_isMarkedPending() { + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + batchBuffer.commitNextAccessUnit(); + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_HUGE_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_HUGE_ACCESS_UNIT); + batchBuffer.commitNextAccessUnit(); + + batchBuffer.batchWasConsumed(); + batchBuffer.flip(); + + assertThat(batchBuffer.getAccessUnitCount()).isEqualTo(1); + assertThat(batchBuffer.data).isEqualTo(ByteBuffer.wrap(TEST_HUGE_ACCESS_UNIT)); + } + + @Test + public void batchWasConsumed_whenNotEmpty_isEmpty() { + fillBatchBuffer(batchBuffer); + + batchBuffer.batchWasConsumed(); + + assertIsCleared(batchBuffer); + } + + @Test + public void batchWasConsumed_whenFull_isEmpty() { + fillBatchBuffer(batchBuffer); + + batchBuffer.batchWasConsumed(); + + assertIsCleared(batchBuffer); + } + + @Test + public void getMaxAccessUnitCount_whenSetToAPositiveValue_returnsIt() { + batchBuffer.setMaxAccessUnitCount(20); + + assertThat(batchBuffer.getMaxAccessUnitCount()).isEqualTo(20); + } + + @Test + public void setMaxAccessUnitCount_whenSetToNegative_throws() { + assertThrows(IllegalArgumentException.class, () -> batchBuffer.setMaxAccessUnitCount(-19)); + } + + @Test + public void setMaxAccessUnitCount_whenSetToZero_throws() { + assertThrows(IllegalArgumentException.class, () -> batchBuffer.setMaxAccessUnitCount(0)); + } + + @Test + public void setMaxAccessUnitCount_whenSetToTheNumberOfAccessUnitInTheBatch_isFull() { + batchBuffer.commitNextAccessUnit(); + + batchBuffer.setMaxAccessUnitCount(1); + + assertThat(batchBuffer.isFull()).isTrue(); + } + + @Test + public void batchWasConsumed_whenAccessUnitIsPending_pendingAccessUnitIsInTheBatch() { + batchBuffer.commitNextAccessUnit(); + batchBuffer.getNextAccessUnitBuffer().setFlags(C.BUFFER_FLAG_DECODE_ONLY); + batchBuffer.getNextAccessUnitBuffer().ensureSpaceForWrite(TEST_ACCESS_UNIT.length); + batchBuffer.getNextAccessUnitBuffer().data.put(TEST_ACCESS_UNIT); + batchBuffer.commitNextAccessUnit(); + + batchBuffer.batchWasConsumed(); + batchBuffer.flip(); + + assertThat(batchBuffer.getAccessUnitCount()).isEqualTo(1); + assertThat(batchBuffer.isDecodeOnly()).isTrue(); + assertThat(batchBuffer.data).isEqualTo(ByteBuffer.wrap(TEST_ACCESS_UNIT)); + } + + private static void fillBatchBuffer(BatchBuffer batchBuffer) { + int maxAccessUnit = batchBuffer.getMaxAccessUnitCount(); + while (!batchBuffer.isFull()) { + assertThat(maxAccessUnit--).isNotEqualTo(0); + batchBuffer.commitNextAccessUnit(); + } + } + + private static void assertIsCleared(BatchBuffer batchBuffer) { + assertThat(batchBuffer.getFirstAccessUnitTimeUs()).isEqualTo(C.TIME_UNSET); + assertThat(batchBuffer.getLastAccessUnitTimeUs()).isEqualTo(C.TIME_UNSET); + assertThat(batchBuffer.getAccessUnitCount()).isEqualTo(0); + assertThat(batchBuffer.isEmpty()).isTrue(); + assertThat(batchBuffer.isFull()).isFalse(); + } +}