From aeb17e6a8830081794a40be1963d134fcdb5fdcb Mon Sep 17 00:00:00 2001 From: Oliver Woodman Date: Thu, 6 Nov 2014 19:22:14 +0000 Subject: [PATCH] HLS improvements + steps towards ABR. --- .../android/exoplayer/hls/HlsChunkSource.java | 17 +- .../exoplayer/hls/HlsSampleSource.java | 97 ++++++++---- .../google/android/exoplayer/hls/TsChunk.java | 36 ++--- .../exoplayer/parser/ts/TsExtractor.java | 145 ++++++++++++------ 4 files changed, 182 insertions(+), 113 deletions(-) diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsChunkSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsChunkSource.java index 8ea6b27e29..2dc1158cd5 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/HlsChunkSource.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsChunkSource.java @@ -45,7 +45,6 @@ public class HlsChunkSource { private final HlsMasterPlaylist masterPlaylist; private final HlsMediaPlaylistParser mediaPlaylistParser; - private long liveStartTimeUs; /* package */ HlsMediaPlaylist mediaPlaylist; /* package */ boolean mediaPlaylistWasLive; /* package */ long lastMediaPlaylistLoadTimeMs; @@ -168,25 +167,22 @@ public class HlsChunkSource { DataSpec dataSpec = new DataSpec(chunkUri, 0, C.LENGTH_UNBOUNDED, null); - long startTimeUs = segment.startTimeUs; - long endTimeUs = startTimeUs + (long) (segment.durationSecs * 1000000); + long startTimeUs; int nextChunkMediaSequence = chunkMediaSequence + 1; - if (mediaPlaylistWasLive) { if (queue.isEmpty()) { - liveStartTimeUs = startTimeUs; startTimeUs = 0; - endTimeUs -= liveStartTimeUs; } else { - startTimeUs -= liveStartTimeUs; - endTimeUs -= liveStartTimeUs; + startTimeUs = queue.get(queue.size() - 1).endTimeUs; } } else { // Not live. + startTimeUs = segment.startTimeUs; if (chunkIndex == mediaPlaylist.segments.size() - 1) { nextChunkMediaSequence = -1; } } + long endTimeUs = startTimeUs + (long) (segment.durationSecs * 1000000); DataSource dataSource; if (encryptedDataSource != null) { @@ -194,9 +190,8 @@ public class HlsChunkSource { } else { dataSource = upstreamDataSource; } - - out.chunk = new TsChunk(dataSource, dataSpec, 0, startTimeUs, endTimeUs, - nextChunkMediaSequence, segment.discontinuity); + out.chunk = new TsChunk(dataSource, dataSpec, 0, 0, startTimeUs, endTimeUs, + nextChunkMediaSequence, segment.discontinuity, false); } private boolean shouldRerequestMediaPlaylist() { diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java index f05d9cb8ec..7482904986 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java @@ -48,10 +48,11 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { private static final long MAX_SAMPLE_INTERLEAVING_OFFSET_US = 5000000; private static final int NO_RESET_PENDING = -1; - private final TsExtractor extractor; + private final TsExtractor.SamplePool samplePool; private final LoadControl loadControl; private final HlsChunkSource chunkSource; private final HlsChunkOperationHolder currentLoadableHolder; + private final LinkedList extractors; private final LinkedList mediaChunks; private final List readOnlyHlsChunks; private final int bufferSizeContribution; @@ -84,7 +85,8 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { this.bufferSizeContribution = bufferSizeContribution; this.frameAccurateSeeking = frameAccurateSeeking; this.remainingReleaseCount = downstreamRendererCount; - extractor = new TsExtractor(); + samplePool = new TsExtractor.SamplePool(); + extractors = new LinkedList(); currentLoadableHolder = new HlsChunkOperationHolder(); mediaChunks = new LinkedList(); readOnlyHlsChunks = Collections.unmodifiableList(mediaChunks); @@ -100,6 +102,10 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { loadControl.register(this, bufferSizeContribution); } continueBufferingInternal(); + if (extractors.isEmpty()) { + return false; + } + TsExtractor extractor = extractors.get(0); if (extractor.isPrepared()) { trackCount = extractor.getTrackCount(); trackEnabledStates = new boolean[trackCount]; @@ -171,39 +177,38 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { } TsChunk mediaChunk = mediaChunks.getFirst(); + int currentVariant = mediaChunk.variantIndex; + + TsExtractor extractor; + if (extractors.isEmpty()) { + extractor = new TsExtractor(mediaChunk.startTimeUs, samplePool); + extractors.addLast(extractor); + if (mediaChunk.discardFromFirstKeyframes) { + extractor.discardFromNextKeyframes(); + } + } else { + extractor = extractors.getLast(); + } + if (mediaChunk.isReadFinished() && mediaChunks.size() > 1) { discardDownstreamHlsChunk(); mediaChunk = mediaChunks.getFirst(); - } - - boolean haveSufficientSamples = false; - if (mediaChunk.hasPendingDiscontinuity()) { - if (extractor.hasSamples()) { - // There are samples from before the discontinuity yet to be read from the extractor, so - // we don't want to reset the extractor yet. - haveSufficientSamples = true; - } else { - extractor.reset(mediaChunk.startTimeUs); - mediaChunk.clearPendingDiscontinuity(); - if (pendingDiscontinuities == null) { - // We're not prepared yet. - } else { - for (int i = 0; i < pendingDiscontinuities.length; i++) { - pendingDiscontinuities[i] = true; - } - } + if (mediaChunk.discontinuity || mediaChunk.variantIndex != currentVariant) { + extractor = new TsExtractor(mediaChunk.startTimeUs, samplePool); + extractors.addLast(extractor); + } + if (mediaChunk.discardFromFirstKeyframes) { + extractor.discardFromNextKeyframes(); } } - if (!mediaChunk.hasPendingDiscontinuity()) { - // Allow the extractor to consume from the current chunk. - NonBlockingInputStream inputStream = mediaChunk.getNonBlockingInputStream(); - haveSufficientSamples = extractor.consumeUntil(inputStream, - downstreamPositionUs + MAX_SAMPLE_INTERLEAVING_OFFSET_US); + // Allow the extractor to consume from the current chunk. + NonBlockingInputStream inputStream = mediaChunk.getNonBlockingInputStream(); + boolean haveSufficientSamples = extractor.consumeUntil(inputStream, + downstreamPositionUs + MAX_SAMPLE_INTERLEAVING_OFFSET_US); + if (!haveSufficientSamples) { // If we can't read any more, then we always say we have sufficient samples. - if (!haveSufficientSamples) { - haveSufficientSamples = mediaChunk.isLastChunk() && mediaChunk.isReadFinished(); - } + haveSufficientSamples = mediaChunk.isLastChunk() && mediaChunk.isReadFinished(); } if (!haveSufficientSamples && currentLoadableException != null) { @@ -223,7 +228,28 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { return DISCONTINUITY_READ; } - if (onlyReadDiscontinuity || isPendingReset() || !extractor.isPrepared()) { + if (onlyReadDiscontinuity || isPendingReset()) { + return NOTHING_READ; + } + + if (extractors.isEmpty()) { + return NOTHING_READ; + } + + TsExtractor extractor = extractors.getFirst(); + while (extractors.size() > 1 && !extractor.hasSamples()) { + // We're finished reading from the extractor for all tracks, and so can discard it. + extractors.removeFirst().clear(); + extractor = extractors.getFirst(); + } + int extractorIndex = 0; + while (extractors.size() > extractorIndex + 1 && !extractor.hasSamples(track)) { + // We're finished reading from the extractor for this particular track, so advance to the + // next one for the current read. + extractor = extractors.get(++extractorIndex); + } + + if (!extractor.isPrepared()) { return NOTHING_READ; } @@ -265,9 +291,9 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { if (mediaChunk == null) { restartFrom(positionUs); } else { + discardExtractors(); discardDownstreamHlsChunks(mediaChunk); - mediaChunk.reset(); - extractor.reset(mediaChunk.startTimeUs); + mediaChunk.resetReadPosition(); updateLoadControl(); } } @@ -494,13 +520,20 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { TsChunk mediaChunk = (TsChunk) currentLoadable; mediaChunks.add(mediaChunk); if (isPendingReset()) { - extractor.reset(mediaChunk.startTimeUs); + discardExtractors(); pendingResetPositionUs = NO_RESET_PENDING; } } loader.startLoading(currentLoadable, this); } + private void discardExtractors() { + for (int i = 0; i < extractors.size(); i++) { + extractors.get(i).clear(); + } + extractors.clear(); + } + /** * Discards downstream media chunks until {@code untilChunk} if found. {@code untilChunk} is not * itself discarded. Null can be passed to discard all media chunks. diff --git a/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java b/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java index ee872bef83..e69b8c88e0 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java @@ -23,6 +23,10 @@ import com.google.android.exoplayer.upstream.DataSpec; */ public final class TsChunk extends HlsChunk { + /** + * The index of the variant in the master playlist. + */ + public final int variantIndex; /** * The start time of the media contained by the chunk. */ @@ -38,44 +42,38 @@ public final class TsChunk extends HlsChunk { /** * The encoding discontinuity indicator. */ - private final boolean discontinuity; - - private boolean pendingDiscontinuity; + public final boolean discontinuity; + /** + * For each track, whether samples from the first keyframe (inclusive) should be discarded. + */ + public final boolean discardFromFirstKeyframes; /** * @param dataSource A {@link DataSource} for loading the data. * @param dataSpec Defines the data to be loaded. * @param trigger The reason for this chunk being selected. + * @param variantIndex The index of the variant in the master playlist. * @param startTimeUs The start time of the media contained by the chunk, in microseconds. * @param endTimeUs The end time of the media contained by the chunk, in microseconds. * @param nextChunkIndex The index of the next chunk, or -1 if this is the last chunk. * @param discontinuity The encoding discontinuity indicator. + * @param discardFromFirstKeyframes For each contained media stream, whether samples from the + * first keyframe (inclusive) should be discarded. */ - public TsChunk(DataSource dataSource, DataSpec dataSpec, int trigger, long startTimeUs, - long endTimeUs, int nextChunkIndex, boolean discontinuity) { + public TsChunk(DataSource dataSource, DataSpec dataSpec, int trigger, int variantIndex, + long startTimeUs, long endTimeUs, int nextChunkIndex, boolean discontinuity, + boolean discardFromFirstKeyframes) { super(dataSource, dataSpec, trigger); + this.variantIndex = variantIndex; this.startTimeUs = startTimeUs; this.endTimeUs = endTimeUs; this.nextChunkIndex = nextChunkIndex; this.discontinuity = discontinuity; - this.pendingDiscontinuity = discontinuity; + this.discardFromFirstKeyframes = discardFromFirstKeyframes; } public boolean isLastChunk() { return nextChunkIndex == -1; } - public void reset() { - resetReadPosition(); - pendingDiscontinuity = discontinuity; - } - - public boolean hasPendingDiscontinuity() { - return pendingDiscontinuity; - } - - public void clearPendingDiscontinuity() { - pendingDiscontinuity = false; - } - } diff --git a/library/src/main/java/com/google/android/exoplayer/parser/ts/TsExtractor.java b/library/src/main/java/com/google/android/exoplayer/parser/ts/TsExtractor.java index 3a95798903..d305aa4f31 100644 --- a/library/src/main/java/com/google/android/exoplayer/parser/ts/TsExtractor.java +++ b/library/src/main/java/com/google/android/exoplayer/parser/ts/TsExtractor.java @@ -23,11 +23,13 @@ import com.google.android.exoplayer.util.CodecSpecificDataUtil; import com.google.android.exoplayer.util.MimeTypes; import android.annotation.SuppressLint; +import android.media.MediaCodec; import android.media.MediaExtractor; import android.util.Log; import android.util.Pair; import android.util.SparseArray; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.Queue; @@ -47,26 +49,27 @@ public final class TsExtractor { private static final int TS_STREAM_TYPE_H264 = 0x1B; private static final int TS_STREAM_TYPE_ID3 = 0x15; - private static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024; - private final BitsArray tsPacketBuffer; private final SparseArray pesPayloadReaders; // Indexed by streamType private final SparseArray tsPayloadReaders; // Indexed by pid - private final Queue samplesPool; + private final SamplePool samplePool; private boolean prepared; - /* package */ boolean pendingTimestampOffsetUpdate; - /* package */ long pendingTimestampOffsetUs; + /* package */ boolean pendingFirstSampleTimestampAdjustment; + /* package */ long firstSampleTimestamp; /* package */ long sampleTimestampOffsetUs; /* package */ long largestParsedTimestampUs; + /* package */ boolean discardFromNextKeyframes; - public TsExtractor() { + public TsExtractor(long firstSampleTimestamp, SamplePool samplePool) { + this.firstSampleTimestamp = firstSampleTimestamp; + this.samplePool = samplePool; + pendingFirstSampleTimestampAdjustment = true; tsPacketBuffer = new BitsArray(); pesPayloadReaders = new SparseArray(); tsPayloadReaders = new SparseArray(); tsPayloadReaders.put(TS_PAT_PID, new PatReader()); - samplesPool = new LinkedList(); largestParsedTimestampUs = Long.MIN_VALUE; } @@ -105,22 +108,19 @@ public final class TsExtractor { } /** - * Resets the extractor's internal state. + * Flushes any pending or incomplete samples, returning them to the sample pool. */ - public void reset(long nextSampleTimestampUs) { - prepared = false; - tsPacketBuffer.reset(); - tsPayloadReaders.clear(); - tsPayloadReaders.put(TS_PAT_PID, new PatReader()); - // Clear each reader before discarding it, so as to recycle any queued Sample objects. + public void clear() { for (int i = 0; i < pesPayloadReaders.size(); i++) { pesPayloadReaders.valueAt(i).clear(); } - pesPayloadReaders.clear(); - // Configure for subsequent read operations. - pendingTimestampOffsetUpdate = true; - pendingTimestampOffsetUs = nextSampleTimestampUs; - largestParsedTimestampUs = Long.MIN_VALUE; + } + + /** + * For each track, whether to discard samples from the next keyframe (inclusive). + */ + public void discardFromNextKeyframes() { + discardFromNextKeyframes = true; } /** @@ -153,31 +153,43 @@ public final class TsExtractor { */ public boolean getSample(int track, SampleHolder out) { Assertions.checkState(prepared); - Queue queue = pesPayloadReaders.valueAt(track).samplesQueue; + Queue queue = pesPayloadReaders.valueAt(track).sampleQueue; if (queue.isEmpty()) { return false; } Sample sample = queue.remove(); convert(sample, out); - recycleSample(sample); + samplePool.recycle(sample); return true; } /** - * Whether samples are available for reading from {@link #getSample(int, SampleHolder)}. + * Whether samples are available for reading from {@link #getSample(int, SampleHolder)} for any + * track. * - * @return True if samples are available for reading from {@link #getSample(int, SampleHolder)}. - * False otherwise. + * @return True if samples are available for reading from {@link #getSample(int, SampleHolder)} + * for any track. False otherwise. */ public boolean hasSamples() { for (int i = 0; i < pesPayloadReaders.size(); i++) { - if (!pesPayloadReaders.valueAt(i).samplesQueue.isEmpty()) { + if (hasSamples(i)) { return true; } } return false; } + /** + * Whether samples are available for reading from {@link #getSample(int, SampleHolder)} for the + * specified track. + * + * @return True if samples are available for reading from {@link #getSample(int, SampleHolder)} + * for the specified track. False otherwise. + */ + public boolean hasSamples(int track) { + return !pesPayloadReaders.valueAt(track).sampleQueue.isEmpty(); + } + private boolean checkPrepared() { int pesPayloadReaderCount = pesPayloadReaders.size(); if (pesPayloadReaderCount == 0) { @@ -251,18 +263,6 @@ public final class TsExtractor { out.timeUs = in.timeUs; } - /* package */ Sample getSample() { - if (samplesPool.isEmpty()) { - return new Sample(DEFAULT_BUFFER_SEGMENT_SIZE); - } - return samplesPool.remove(); - } - - /* package */ void recycleSample(Sample sample) { - sample.reset(); - samplesPool.add(sample); - } - /** * Parses payload data. */ @@ -484,12 +484,14 @@ public final class TsExtractor { */ private abstract class PesPayloadReader { - public final Queue samplesQueue; + public final Queue sampleQueue; private MediaFormat mediaFormat; + private boolean foundFirstKeyframe; + private boolean foundLastKeyframe; protected PesPayloadReader() { - this.samplesQueue = new LinkedList(); + this.sampleQueue = new LinkedList(); } public boolean hasMediaFormat() { @@ -507,8 +509,8 @@ public final class TsExtractor { public abstract void read(BitsArray pesBuffer, int pesPayloadSize, long pesTimeUs); public void clear() { - while (!samplesQueue.isEmpty()) { - recycleSample(samplesQueue.remove()); + while (!sampleQueue.isEmpty()) { + samplePool.recycle(sampleQueue.remove()); } } @@ -520,17 +522,31 @@ public final class TsExtractor { * @param sampleTimeUs The sample time stamp. */ protected void addSample(BitsArray buffer, int sampleSize, long sampleTimeUs, int flags) { - Sample sample = getSample(); + Sample sample = samplePool.get(); addToSample(sample, buffer, sampleSize); sample.flags = flags; sample.timeUs = sampleTimeUs; addSample(sample); } + @SuppressLint("InlinedApi") protected void addSample(Sample sample) { + boolean isKeyframe = (sample.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0; + if (isKeyframe) { + if (!foundFirstKeyframe) { + foundFirstKeyframe = true; + } + if (discardFromNextKeyframes) { + foundLastKeyframe = true; + } + } adjustTimestamp(sample); - largestParsedTimestampUs = Math.max(largestParsedTimestampUs, sample.timeUs); - samplesQueue.add(sample); + if (foundFirstKeyframe && !foundLastKeyframe) { + largestParsedTimestampUs = Math.max(largestParsedTimestampUs, sample.timeUs); + sampleQueue.add(sample); + } else { + samplePool.recycle(sample); + } } protected void addToSample(Sample sample, BitsArray buffer, int size) { @@ -542,9 +558,9 @@ public final class TsExtractor { } private void adjustTimestamp(Sample sample) { - if (pendingTimestampOffsetUpdate) { - sampleTimestampOffsetUs = pendingTimestampOffsetUs - sample.timeUs; - pendingTimestampOffsetUpdate = false; + if (pendingFirstSampleTimestampAdjustment) { + sampleTimestampOffsetUs = firstSampleTimestamp - sample.timeUs; + pendingFirstSampleTimestampAdjustment = false; } sample.timeUs += sampleTimestampOffsetUs; } @@ -583,7 +599,7 @@ public final class TsExtractor { if (currentSample != null) { addSample(currentSample); } - currentSample = getSample(); + currentSample = samplePool.get(); pesPayloadSize -= readOneH264Frame(pesBuffer, false); currentSample.timeUs = pesTimeUs; @@ -615,7 +631,7 @@ public final class TsExtractor { public void clear() { super.clear(); if (currentSample != null) { - recycleSample(currentSample); + samplePool.recycle(currentSample); currentSample = null; } } @@ -742,8 +758,35 @@ public final class TsExtractor { } /** - * Simplified version of SampleHolder for internal buffering. - */ + * A pool from which the extractor can obtain sample objects for internal use. + */ + public static class SamplePool { + + private static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024; + + private final ArrayList samples; + + public SamplePool() { + samples = new ArrayList(); + } + + /* package */ Sample get() { + if (samples.isEmpty()) { + return new Sample(DEFAULT_BUFFER_SEGMENT_SIZE); + } + return samples.remove(samples.size() - 1); + } + + /* package */ void recycle(Sample sample) { + sample.reset(); + samples.add(sample); + } + + } + + /** + * Simplified version of SampleHolder for internal buffering. + */ private static class Sample { public byte[] data;