From 8e2801ce9b4ab0091ec1681832f9de4c1acdbdc8 Mon Sep 17 00:00:00 2001 From: Oliver Woodman Date: Wed, 26 Nov 2014 12:08:46 +0000 Subject: [PATCH] Improve HLS ABR. - Add options to switch abruptly at segment boundaries. Third parties who guarantee keyframes at the start of segments will want this, because it makes switching more efficient and hence rebuffering less likely. - Switch quality faster when performing a splicing switch (when we detect that we need to switch variant, we now immediately request the same segment as we did last time for the new variant, rather than requesting one more segment for the old variant before doing this. --- .../demo/full/player/HlsRendererBuilder.java | 3 +- .../demo/simple/HlsRendererBuilder.java | 3 +- .../android/exoplayer/hls/HlsChunkSource.java | 197 +++++++++++------- .../exoplayer/hls/HlsSampleSource.java | 45 ++-- .../google/android/exoplayer/hls/TsChunk.java | 24 +-- .../android/exoplayer/hls/TsExtractor.java | 10 +- 6 files changed, 170 insertions(+), 112 deletions(-) diff --git a/demo/src/main/java/com/google/android/exoplayer/demo/full/player/HlsRendererBuilder.java b/demo/src/main/java/com/google/android/exoplayer/demo/full/player/HlsRendererBuilder.java index 79db5d6566..5a3a3535e9 100644 --- a/demo/src/main/java/com/google/android/exoplayer/demo/full/player/HlsRendererBuilder.java +++ b/demo/src/main/java/com/google/android/exoplayer/demo/full/player/HlsRendererBuilder.java @@ -80,8 +80,9 @@ public class HlsRendererBuilder implements RendererBuilder, ManifestCallback + * The initially selected variant will be used throughout playback. + */ + public static final int ADAPTIVE_MODE_NONE = 0; + + /** + * Adaptive switches splice overlapping segments of the old and new variants. + *

+ * When performing a switch from one variant to another, overlapping segments will be requested + * from both the old and new variants. These segments will then be spliced together, allowing + * a seamless switch from one variant to another even if keyframes are misaligned or if keyframes + * are not positioned at the start of each segment. + *

+ * Note that where it can be guaranteed that the source content has keyframes positioned at the + * start of each segment, {@link #ADAPTIVE_MODE_ABRUPT} should always be used in preference to + * this mode. + */ + public static final int ADAPTIVE_MODE_SPLICE = 1; + + /** + * Adaptive switches are performed at segment boundaries. + *

+ * For this mode to perform seamless switches, the source content is required to have keyframes + * positioned at the start of each segment. If this is not the case a visual discontinuity may + * be experienced when switching from one variant to another. + *

+ * Note that where it can be guaranteed that the source content does have keyframes positioned at + * the start of each segment, this mode should always be used in preference to + * {@link #ADAPTIVE_MODE_SPLICE} because it requires fetching less data. + */ + public static final int ADAPTIVE_MODE_ABRUPT = 3; + private static final float BANDWIDTH_FRACTION = 0.8f; private static final long MIN_BUFFER_TO_SWITCH_UP_US = 5000000; private static final long MAX_BUFFER_TO_SWITCH_DOWN_US = 15000000; @@ -56,7 +90,7 @@ public class HlsChunkSource { private final Variant[] enabledVariants; private final BandwidthMeter bandwidthMeter; private final BitArray bitArray; - private final boolean enableAdaptive; + private final int adaptiveMode; private final Uri baseUri; private final int maxWidth; private final int maxHeight; @@ -79,27 +113,31 @@ public class HlsChunkSource { * @param bandwidthMeter provides an estimate of the currently available bandwidth. * @param variantIndices A subset of variant indices to consider, or null to consider all of the * variants in the master playlist. + * @param adaptiveMode The mode for switching from one variant to another. One of + * {@link #ADAPTIVE_MODE_NONE}, {@link #ADAPTIVE_MODE_ABRUPT} and + * {@link #ADAPTIVE_MODE_SPLICE}. */ public HlsChunkSource(DataSource dataSource, String playlistUrl, HlsPlaylist playlist, - BandwidthMeter bandwidthMeter, int[] variantIndices, boolean enableAdaptive) { + BandwidthMeter bandwidthMeter, int[] variantIndices, int adaptiveMode) { this.upstreamDataSource = dataSource; this.bandwidthMeter = bandwidthMeter; - this.enableAdaptive = enableAdaptive; + this.adaptiveMode = adaptiveMode; baseUri = playlist.baseUri; bitArray = new BitArray(); playlistParser = new HlsPlaylistParser(); if (playlist.type == HlsPlaylist.TYPE_MEDIA) { enabledVariants = new Variant[] {new Variant(0, playlistUrl, 0, null, -1, -1)}; - mediaPlaylists = new HlsMediaPlaylist[] {(HlsMediaPlaylist) playlist}; + mediaPlaylists = new HlsMediaPlaylist[1]; + lastMediaPlaylistLoadTimesMs = new long[1]; + setMediaPlaylist(0, (HlsMediaPlaylist) playlist); } else { Assertions.checkState(playlist.type == HlsPlaylist.TYPE_MASTER); enabledVariants = filterVariants((HlsMasterPlaylist) playlist, variantIndices); mediaPlaylists = new HlsMediaPlaylist[enabledVariants.length]; + lastMediaPlaylistLoadTimesMs = new long[enabledVariants.length]; } - lastMediaPlaylistLoadTimesMs = new long[enabledVariants.length]; - int maxWidth = -1; int maxHeight = -1; // Select the first variant from the master playlist that's enabled. @@ -144,24 +182,41 @@ public class HlsChunkSource { */ public HlsChunk getChunkOperation(TsChunk previousTsChunk, long seekPositionUs, long playbackPositionUs) { - - HlsMediaPlaylist mediaPlaylist = mediaPlaylists[variantIndex]; - if (mediaPlaylist == null) { - return newMediaPlaylistChunk(); + if (previousTsChunk != null && previousTsChunk.isLastChunk) { + // We're already finished. + return null; } + int nextVariantIndex = variantIndex; + boolean switchingVariant = false; + boolean switchingVariantSpliced = false; + if (adaptiveMode == ADAPTIVE_MODE_NONE) { + // Do nothing. + } else { + nextVariantIndex = getNextVariantIndex(previousTsChunk, playbackPositionUs); + switchingVariant = nextVariantIndex != variantIndex; + switchingVariantSpliced = switchingVariant && adaptiveMode == ADAPTIVE_MODE_SPLICE; + } + + HlsMediaPlaylist mediaPlaylist = mediaPlaylists[nextVariantIndex]; + if (mediaPlaylist == null) { + // We don't have the media playlist for the next variant. Request it now. + return newMediaPlaylistChunk(nextVariantIndex); + } + + variantIndex = nextVariantIndex; int chunkMediaSequence = 0; + boolean liveDiscontinuity = false; if (live) { if (previousTsChunk == null) { - chunkMediaSequence = getLiveStartChunkMediaSequence(); + chunkMediaSequence = getLiveStartChunkMediaSequence(variantIndex); } else { - // For live nextChunkIndex contains chunk media sequence number. - chunkMediaSequence = previousTsChunk.nextChunkIndex; - // If the updated playlist is far ahead and doesn't even have the last chunk from the - // queue, then try to catch up, skip a few chunks and start as if it was a new playlist. + chunkMediaSequence = switchingVariantSpliced + ? previousTsChunk.chunkIndex : previousTsChunk.chunkIndex + 1; if (chunkMediaSequence < mediaPlaylist.mediaSequence) { - // TODO: Trigger discontinuity in this case. - chunkMediaSequence = getLiveStartChunkMediaSequence(); + // If the chunk is no longer in the playlist. Skip ahead and start again. + chunkMediaSequence = getLiveStartChunkMediaSequence(variantIndex); + liveDiscontinuity = true; } } } else { @@ -170,19 +225,15 @@ public class HlsChunkSource { chunkMediaSequence = Util.binarySearchFloor(mediaPlaylist.segments, seekPositionUs, true, true) + mediaPlaylist.mediaSequence; } else { - chunkMediaSequence = previousTsChunk.nextChunkIndex; + chunkMediaSequence = switchingVariantSpliced + ? previousTsChunk.chunkIndex : previousTsChunk.chunkIndex + 1; } } - if (chunkMediaSequence == -1) { - // We've reached the end of the stream. - return null; - } - int chunkIndex = chunkMediaSequence - mediaPlaylist.mediaSequence; if (chunkIndex >= mediaPlaylist.segments.size()) { - if (mediaPlaylist.live && shouldRerequestMediaPlaylist()) { - return newMediaPlaylistChunk(); + if (mediaPlaylist.live && shouldRerequestMediaPlaylist(variantIndex)) { + return newMediaPlaylistChunk(variantIndex); } else { return null; } @@ -206,67 +257,59 @@ public class HlsChunkSource { clearEncryptedDataSource(); } + // Configure the data source and spec for the chunk. + DataSource dataSource = encryptedDataSource != null ? encryptedDataSource : upstreamDataSource; + DataSpec dataSpec = new DataSpec(chunkUri, segment.byterangeOffset, segment.byterangeLength, + null); + + // Compute start and end times, and the sequence number of the next chunk. long startTimeUs; - boolean splicingIn = previousTsChunk != null && previousTsChunk.splicingOut; - int nextChunkMediaSequence = chunkMediaSequence + 1; if (live) { if (previousTsChunk == null) { startTimeUs = 0; - } else if (splicingIn) { + } else if (switchingVariantSpliced) { startTimeUs = previousTsChunk.startTimeUs; } else { startTimeUs = previousTsChunk.endTimeUs; } - } else { - // Not live. + } else /* Not live */ { startTimeUs = segment.startTimeUs; } - if (!mediaPlaylist.live && chunkIndex == mediaPlaylist.segments.size() - 1) { - nextChunkMediaSequence = -1; - } - long endTimeUs = startTimeUs + (long) (segment.durationSecs * 1000000); - - int currentVariantIndex = variantIndex; - boolean splicingOut = false; - if (splicingIn) { - // Do nothing. - } else if (enableAdaptive && nextChunkMediaSequence != -1) { - int idealVariantIndex = getVariantIndexForBandwdith( - (int) (bandwidthMeter.getBitrateEstimate() * BANDWIDTH_FRACTION)); - long bufferedUs = startTimeUs - playbackPositionUs; - if ((idealVariantIndex > currentVariantIndex && bufferedUs < MAX_BUFFER_TO_SWITCH_DOWN_US) - || (idealVariantIndex < currentVariantIndex && bufferedUs > MIN_BUFFER_TO_SWITCH_UP_US)) { - variantIndex = idealVariantIndex; - } - splicingOut = variantIndex != currentVariantIndex; - if (splicingOut) { - // If we're splicing out, we want to load the same chunk again next time, but for a - // different variant. - nextChunkMediaSequence = chunkMediaSequence; - } - } - - // Configure the datasource for loading the chunk. - DataSource dataSource; - if (encryptedDataSource != null) { - dataSource = encryptedDataSource; - } else { - dataSource = upstreamDataSource; - } - DataSpec dataSpec = new DataSpec(chunkUri, segment.byterangeOffset, segment.byterangeLength, - null); + boolean isLastChunk = !mediaPlaylist.live && chunkIndex == mediaPlaylist.segments.size() - 1; // Configure the extractor that will read the chunk. TsExtractor extractor; - if (previousTsChunk == null || splicingIn || segment.discontinuity) { - extractor = new TsExtractor(startTimeUs, samplePool); + if (previousTsChunk == null || segment.discontinuity || switchingVariant || liveDiscontinuity) { + extractor = new TsExtractor(startTimeUs, samplePool, switchingVariantSpliced); } else { extractor = previousTsChunk.extractor; } - return new TsChunk(dataSource, dataSpec, extractor, enabledVariants[currentVariantIndex].index, - startTimeUs, endTimeUs, nextChunkMediaSequence, splicingOut); + return new TsChunk(dataSource, dataSpec, extractor, enabledVariants[variantIndex].index, + startTimeUs, endTimeUs, chunkMediaSequence, isLastChunk); + } + + private int getNextVariantIndex(TsChunk previousTsChunk, long playbackPositionUs) { + int idealVariantIndex = getVariantIndexForBandwdith( + (int) (bandwidthMeter.getBitrateEstimate() * BANDWIDTH_FRACTION)); + if (idealVariantIndex == variantIndex) { + // We're already using the ideal variant. + return variantIndex; + } + // We're not using the ideal variant for the available bandwidth, but only switch if the + // conditions are appropriate. + long bufferedPositionUs = previousTsChunk == null ? playbackPositionUs + : adaptiveMode == ADAPTIVE_MODE_SPLICE ? previousTsChunk.startTimeUs + : previousTsChunk.endTimeUs; + long bufferedUs = bufferedPositionUs - playbackPositionUs; + if ((idealVariantIndex > variantIndex && bufferedUs < MAX_BUFFER_TO_SWITCH_DOWN_US) + || (idealVariantIndex < variantIndex && bufferedUs > MIN_BUFFER_TO_SWITCH_UP_US)) { + // Switch variant. + return idealVariantIndex; + } + // Stick with the current variant for now. + return variantIndex; } private int getVariantIndexForBandwdith(int bandwidth) { @@ -278,7 +321,7 @@ public class HlsChunkSource { return enabledVariants.length - 1; } - private boolean shouldRerequestMediaPlaylist() { + private boolean shouldRerequestMediaPlaylist(int variantIndex) { // Don't re-request media playlist more often than one-half of the target duration. HlsMediaPlaylist mediaPlaylist = mediaPlaylists[variantIndex]; long timeSinceLastMediaPlaylistLoadMs = @@ -286,14 +329,14 @@ public class HlsChunkSource { return timeSinceLastMediaPlaylistLoadMs >= (mediaPlaylist.targetDurationSecs * 1000) / 2; } - private int getLiveStartChunkMediaSequence() { + private int getLiveStartChunkMediaSequence(int variantIndex) { // For live start playback from the third chunk from the end. HlsMediaPlaylist mediaPlaylist = mediaPlaylists[variantIndex]; int chunkIndex = mediaPlaylist.segments.size() > 3 ? mediaPlaylist.segments.size() - 3 : 0; return chunkIndex + mediaPlaylist.mediaSequence; } - private MediaPlaylistChunk newMediaPlaylistChunk() { + private MediaPlaylistChunk newMediaPlaylistChunk(int variantIndex) { Uri mediaPlaylistUri = Util.getMergedUri(baseUri, enabledVariants[variantIndex].url); DataSpec dataSpec = new DataSpec(mediaPlaylistUri, 0, C.LENGTH_UNBOUNDED, null); Uri baseUri = Util.parseBaseUri(mediaPlaylistUri.toString()); @@ -332,6 +375,13 @@ public class HlsChunkSource { encryptedDataSourceSecretKey = null; } + /* package */ void setMediaPlaylist(int variantIndex, HlsMediaPlaylist mediaPlaylist) { + lastMediaPlaylistLoadTimesMs[variantIndex] = SystemClock.elapsedRealtime(); + mediaPlaylists[variantIndex] = mediaPlaylist; + live |= mediaPlaylist.live; + durationUs = mediaPlaylist.durationUs; + } + private static Variant[] filterVariants(HlsMasterPlaylist masterPlaylist, int[] variantIndices) { List masterVariants = masterPlaylist.variants; ArrayList enabledVariants = new ArrayList(); @@ -408,10 +458,7 @@ public class HlsChunkSource { playlistBaseUri); Assertions.checkState(playlist.type == HlsPlaylist.TYPE_MEDIA); HlsMediaPlaylist mediaPlaylist = (HlsMediaPlaylist) playlist; - mediaPlaylists[variantIndex] = mediaPlaylist; - lastMediaPlaylistLoadTimesMs[variantIndex] = SystemClock.elapsedRealtime(); - live |= mediaPlaylist.live; - durationUs = mediaPlaylist.durationUs; + setMediaPlaylist(variantIndex, mediaPlaylist); } } diff --git a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java index 4982ca894c..e2ef450970 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/HlsSampleSource.java @@ -35,12 +35,18 @@ import java.util.LinkedList; */ public class HlsSampleSource implements SampleSource, Loader.Callback { + /** + * The default minimum number of times to retry loading data prior to failing. + */ + public static final int DEFAULT_MIN_LOADABLE_RETRY_COUNT = 1; + private static final long BUFFER_DURATION_US = 20000000; private static final int NO_RESET_PENDING = -1; private final HlsChunkSource chunkSource; private final LinkedList extractors; private final boolean frameAccurateSeeking; + private final int minLoadableRetryCount; private int remainingReleaseCount; private boolean prepared; @@ -65,11 +71,18 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { private int currentLoadableExceptionCount; private long currentLoadableExceptionTimestamp; - public HlsSampleSource(HlsChunkSource chunkSource, - boolean frameAccurateSeeking, int downstreamRendererCount) { + public HlsSampleSource(HlsChunkSource chunkSource, boolean frameAccurateSeeking, + int downstreamRendererCount) { + this(chunkSource, frameAccurateSeeking, downstreamRendererCount, + DEFAULT_MIN_LOADABLE_RETRY_COUNT); + } + + public HlsSampleSource(HlsChunkSource chunkSource, boolean frameAccurateSeeking, + int downstreamRendererCount, int minLoadableRetryCount) { this.chunkSource = chunkSource; this.frameAccurateSeeking = frameAccurateSeeking; this.remainingReleaseCount = downstreamRendererCount; + this.minLoadableRetryCount = minLoadableRetryCount; extractors = new LinkedList(); } @@ -97,8 +110,8 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { prepared = true; } } - if (!prepared && currentLoadableException != null) { - throw currentLoadableException; + if (!prepared) { + maybeThrowLoadableException(); } return prepared; } @@ -157,8 +170,8 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { return false; } boolean haveSamples = extractors.getFirst().hasSamples(); - if (!haveSamples && currentLoadableException != null) { - throw currentLoadableException; + if (!haveSamples) { + maybeThrowLoadableException(); } return haveSamples; } @@ -175,9 +188,7 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { } if (onlyReadDiscontinuity || isPendingReset() || extractors.isEmpty()) { - if (currentLoadableException != null) { - throw currentLoadableException; - } + maybeThrowLoadableException(); return NOTHING_READ; } @@ -202,9 +213,7 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { } if (!extractor.isPrepared()) { - if (currentLoadableException != null) { - throw currentLoadableException; - } + maybeThrowLoadableException(); return NOTHING_READ; } @@ -225,9 +234,7 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { return END_OF_STREAM; } - if (currentLoadableException != null) { - throw currentLoadableException; - } + maybeThrowLoadableException(); return NOTHING_READ; } @@ -283,7 +290,7 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { } finally { if (isTsChunk(currentLoadable)) { TsChunk tsChunk = (TsChunk) loadable; - loadingFinished = tsChunk.isLastChunk(); + loadingFinished = tsChunk.isLastChunk; } if (!currentLoadableExceptionFatal) { clearCurrentLoadable(); @@ -309,6 +316,12 @@ public class HlsSampleSource implements SampleSource, Loader.Callback { maybeStartLoading(); } + private void maybeThrowLoadableException() throws IOException { + if (currentLoadableException != null && currentLoadableExceptionCount > minLoadableRetryCount) { + throw currentLoadableException; + } + } + private void restartFrom(long positionUs) { pendingResetPositionUs = positionUs; loadingFinished = false; diff --git a/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java b/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java index 9222317839..4261dbd4ca 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/TsChunk.java @@ -39,14 +39,13 @@ public final class TsChunk extends HlsChunk { */ public final long endTimeUs; /** - * The index of the next media chunk, or -1 if this is the last media chunk in the stream. + * The chunk index. */ - public final int nextChunkIndex; + public final int chunkIndex; /** - * True if this is the final chunk being loaded for the current variant, as we splice to another - * one. False otherwise. + * True if this is the last chunk in the media. False otherwise. */ - public final boolean splicingOut; + public final boolean isLastChunk; /** * The extractor into which this chunk is being consumed. */ @@ -62,19 +61,18 @@ public final class TsChunk extends HlsChunk { * @param variantIndex The index of the variant in the master playlist. * @param startTimeUs The start time of the media contained by the chunk, in microseconds. * @param endTimeUs The end time of the media contained by the chunk, in microseconds. - * @param nextChunkIndex The index of the next chunk, or -1 if this is the last chunk. - * @param splicingOut True if this is the final chunk being loaded for the current variant, as we - * splice to another one. False otherwise. + * @param chunkIndex The index of the chunk. + * @param isLastChunk True if this is the last chunk in the media. False otherwise. */ public TsChunk(DataSource dataSource, DataSpec dataSpec, TsExtractor tsExtractor, - int variantIndex, long startTimeUs, long endTimeUs, int nextChunkIndex, boolean splicingOut) { + int variantIndex, long startTimeUs, long endTimeUs, int chunkIndex, boolean isLastChunk) { super(dataSource, dataSpec); this.extractor = tsExtractor; this.variantIndex = variantIndex; this.startTimeUs = startTimeUs; this.endTimeUs = endTimeUs; - this.nextChunkIndex = nextChunkIndex; - this.splicingOut = splicingOut; + this.chunkIndex = chunkIndex; + this.isLastChunk = isLastChunk; } @Override @@ -82,10 +80,6 @@ public final class TsChunk extends HlsChunk { // Do nothing. } - public boolean isLastChunk() { - return nextChunkIndex == -1; - } - @Override public boolean isLoadFinished() { return loadFinished; diff --git a/library/src/main/java/com/google/android/exoplayer/hls/TsExtractor.java b/library/src/main/java/com/google/android/exoplayer/hls/TsExtractor.java index d574078162..64541a1492 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/TsExtractor.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/TsExtractor.java @@ -56,6 +56,7 @@ public final class TsExtractor { private final SparseArray sampleQueues; // Indexed by streamType private final SparseArray tsPayloadReaders; // Indexed by pid private final SamplePool samplePool; + private final boolean shouldSpliceIn; /* package */ final long firstSampleTimestamp; // Accessed only by the consuming thread. @@ -69,9 +70,10 @@ public final class TsExtractor { private volatile boolean prepared; /* package */ volatile long largestParsedTimestampUs; - public TsExtractor(long firstSampleTimestamp, SamplePool samplePool) { + public TsExtractor(long firstSampleTimestamp, SamplePool samplePool, boolean shouldSpliceIn) { this.firstSampleTimestamp = firstSampleTimestamp; this.samplePool = samplePool; + this.shouldSpliceIn = shouldSpliceIn; pendingFirstSampleTimestampAdjustment = true; tsPacketBuffer = new BitArray(); sampleQueues = new SparseArray(); @@ -141,9 +143,9 @@ public final class TsExtractor { */ public void configureSpliceTo(TsExtractor nextExtractor) { Assertions.checkState(prepared); - if (spliceConfigured || !nextExtractor.isPrepared()) { - // The splice is already configured or the next extractor isn't ready to be spliced in. - // Already configured, or too early to splice. + if (spliceConfigured || !nextExtractor.shouldSpliceIn || !nextExtractor.isPrepared()) { + // The splice is already configured, or the next extractor doesn't want to be spliced in, or + // the next extractor isn't ready to be spliced in. return; } boolean spliceConfigured = true;