Discard HLS preload chunks when an HTTP 410 or 404 occurs

This change avoids an early fatal exception for replaced hinted parts.

Issue: #5011
PiperOrigin-RevId: 344828076

library/core/src/main/java/com/google/android/exoplayer2/util/StableApiCandidate.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.util;
+
+import static java.lang.annotation.ElementType.CONSTRUCTOR;
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PACKAGE;
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.CLASS;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to mark an API as a candidate for being part of a stable subset of the API.
+ *
+ * <p>Note: this is experimental, and no guarantees are made about the stability of APIs even if
+ * they are marked with this annotation.
+ */
+@Retention(CLASS)
+@Target({TYPE, METHOD, CONSTRUCTOR, FIELD, PACKAGE})
+public @interface StableApiCandidate {}

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsChunkSource.java

@@ -21,6 +21,7 @@ import static java.lang.Math.max;
 import android.net.Uri;
 import android.os.SystemClock;
 import android.util.Pair;
+import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import com.google.android.exoplayer2.C;
@@ -47,6 +48,8 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.primitives.Ints;
 import java.io.IOException;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -86,6 +89,29 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  /**
+   * Chunk publication state. One of {@link #CHUNK_PUBLICATION_STATE_PRELOAD}, {@link
+   * #CHUNK_PUBLICATION_STATE_PUBLISHED}, {@link #CHUNK_PUBLICATION_STATE_REMOVED}.
+   */
+  @IntDef({
+    CHUNK_PUBLICATION_STATE_PRELOAD,
+    CHUNK_PUBLICATION_STATE_PUBLISHED,
+    CHUNK_PUBLICATION_STATE_REMOVED
+  })
+  @Retention(RetentionPolicy.SOURCE)
+  @interface ChunkPublicationState {}
+
+  /** Indicates that the chunk is based on a preload hint. */
+  public static final int CHUNK_PUBLICATION_STATE_PRELOAD = 0;
+  /** Indicates that the chunk is definitely published. */
+  public static final int CHUNK_PUBLICATION_STATE_PUBLISHED = 1;
+  /**
+   * Indicates that the chunk has been removed from the playlist.
+   *
+   * <p>See RFC 8216, Section 6.2.6 also.
+   */
+  public static final int CHUNK_PUBLICATION_STATE_REMOVED = 2;
+
   /**
    * The maximum number of keys that the key cache can hold. This value must be 2 or greater in
    * order to hold initialization segment and media segment keys simultaneously.
@@ -222,29 +248,32 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   }
 
   /**
-   * Checks whether the previous media chunk is a preload chunk that has been removed in the current
+   * Returns the publication state of the given chunk.
-   * playlist.
    *
-   * @param previous The previous media chunk.
+   * @param mediaChunk The media chunk for which to evaluate the publication state.
-   * @return True if the previous media chunk has been removed in the current playlist.
+   * @return Whether the media chunk is {@link #CHUNK_PUBLICATION_STATE_PRELOAD a preload chunk},
+   *     has been {@link #CHUNK_PUBLICATION_STATE_REMOVED removed} or is definitely {@link
+   *     #CHUNK_PUBLICATION_STATE_PUBLISHED published}.
    */
-  public boolean isMediaChunkRemoved(HlsMediaChunk previous) {
+  @ChunkPublicationState
-    if (!previous.isPreload) {
+  public int getChunkPublicationState(HlsMediaChunk mediaChunk) {
-      return false;
+    if (mediaChunk.partIndex == C.INDEX_UNSET) {
+      // Chunks based on full segments can't be removed and are always published.
+      return CHUNK_PUBLICATION_STATE_PUBLISHED;
     }
-    Uri playlistUrl = playlistUrls[trackGroup.indexOf(previous.trackFormat)];
+    Uri playlistUrl = playlistUrls[trackGroup.indexOf(mediaChunk.trackFormat)];
     HlsMediaPlaylist mediaPlaylist =
         checkNotNull(playlistTracker.getPlaylistSnapshot(playlistUrl, /* isForPlayback= */ false));
-    int segmentIndexInPlaylist = (int) (previous.chunkIndex - mediaPlaylist.mediaSequence);
+    int segmentIndexInPlaylist = (int) (mediaChunk.chunkIndex - mediaPlaylist.mediaSequence);
     if (segmentIndexInPlaylist < 0) {
-      // The segment of the previous chunk is not in the current playlist anymore.
+      // The parent segment of the previous chunk is not in the current playlist anymore.
-      return false;
+      return CHUNK_PUBLICATION_STATE_PUBLISHED;
     }
     List<HlsMediaPlaylist.Part> partsInCurrentPlaylist =
         segmentIndexInPlaylist < mediaPlaylist.segments.size()
             ? mediaPlaylist.segments.get(segmentIndexInPlaylist).parts
             : mediaPlaylist.trailingParts;
-    if (previous.partIndex >= partsInCurrentPlaylist.size()) {
+    if (mediaChunk.partIndex >= partsInCurrentPlaylist.size()) {
       // In case the part hinted in the previous playlist has been wrongly assigned to the then full
       // but not yet terminated segment, we discard it regardless whether the URI is different or
       // not. While this is theoretically possible and unspecified, it appears to be an edge case
@@ -252,11 +281,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       // here but, if the chunk is not discarded, it could create unpredictable problems later,
       // because the media sequence in previous.chunkIndex does not match to the actual media
       // sequence in the new playlist.
-      return true;
+      return CHUNK_PUBLICATION_STATE_REMOVED;
     }
-    HlsMediaPlaylist.Part publishedPart = partsInCurrentPlaylist.get(previous.partIndex);
+    HlsMediaPlaylist.Part newPart = partsInCurrentPlaylist.get(mediaChunk.partIndex);
-    Uri publishedUri = Uri.parse(UriUtil.resolve(mediaPlaylist.baseUri, publishedPart.url));
+    if (newPart.isPreload) {
-    return !Util.areEqual(publishedUri, previous.dataSpec.uri);
+      // The playlist did not change and the part in the new playlist is still a preload hint.
+      return CHUNK_PUBLICATION_STATE_PRELOAD;
+    }
+    Uri newUri = Uri.parse(UriUtil.resolve(mediaPlaylist.baseUri, newPart.url));
+    return Util.areEqual(newUri, mediaChunk.dataSpec.uri)
+        ? CHUNK_PUBLICATION_STATE_PUBLISHED
+        : CHUNK_PUBLICATION_STATE_REMOVED;
   }
 
   /**

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaChunk.java

@@ -169,7 +169,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         segmentEndTimeInPeriodUs,
         segmentBaseHolder.mediaSequence,
         segmentBaseHolder.partIndex,
-        segmentBaseHolder.isPreload,
+        /* isPublished= */ !segmentBaseHolder.isPreload,
         discontinuitySequenceNumber,
         mediaSegment.hasGapTag,
         isMasterTimestampSource,
@@ -205,9 +205,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   /** The part index or {@link C#INDEX_UNSET} if the chunk is a full segment */
   public final int partIndex;
 
-  /** Whether this chunk is a preload chunk. */
-  public final boolean isPreload;
-
   @Nullable private final DataSource initDataSource;
   @Nullable private final DataSpec initDataSpec;
   @Nullable private final HlsMediaChunkExtractor previousExtractor;
@@ -233,6 +230,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   private boolean loadCompleted;
   private ImmutableList<Integer> sampleQueueFirstSampleIndices;
   private boolean extractorInvalidated;
+  private boolean isPublished;
 
   private HlsMediaChunk(
       HlsExtractorFactory extractorFactory,
@@ -251,7 +249,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       long endTimeUs,
       long chunkMediaSequence,
       int partIndex,
-      boolean isPreload,
+      boolean isPublished,
       int discontinuitySequenceNumber,
       boolean hasGapTag,
       boolean isMasterTimestampSource,
@@ -272,7 +270,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
         chunkMediaSequence);
     this.mediaSegmentEncrypted = mediaSegmentEncrypted;
     this.partIndex = partIndex;
-    this.isPreload = isPreload;
+    this.isPublished = isPublished;
     this.discontinuitySequenceNumber = discontinuitySequenceNumber;
     this.initDataSpec = initDataSpec;
     this.initDataSource = initDataSource;
@@ -356,6 +354,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
   }
 
+  /**
+   * Whether the chunk is a published chunk as opposed to a preload hint that may change when the
+   * playlist updates.
+   */
+  public boolean isPublished() {
+    return isPublished;
+  }
+
+  /**
+   * Sets the publish flag of the media chunk to indicate that it is not based on a part that is a
+   * preload hint in the playlist.
+   */
+  public void publish() {
+    isPublished = true;
+  }
+
   // Internal methods.
 
   @RequiresNonNull("output")

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java

@@ -15,6 +15,8 @@
  */
 package com.google.android.exoplayer2.source.hls;
 
+import static com.google.android.exoplayer2.source.hls.HlsChunkSource.CHUNK_PUBLICATION_STATE_PUBLISHED;
+import static com.google.android.exoplayer2.source.hls.HlsChunkSource.CHUNK_PUBLICATION_STATE_REMOVED;
 import static java.lang.Math.max;
 
 import android.net.Uri;
@@ -54,6 +56,7 @@ import com.google.android.exoplayer2.source.chunk.MediaChunkIterator;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.upstream.Allocator;
 import com.google.android.exoplayer2.upstream.DataReader;
+import com.google.android.exoplayer2.upstream.HttpDataSource;
 import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy;
 import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy.LoadErrorInfo;
 import com.google.android.exoplayer2.upstream.Loader;
@@ -506,10 +509,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
   /** Called when the playlist is updated. */
   public void onPlaylistUpdated() {
-    if (!loadingFinished
+    if (mediaChunks.isEmpty()) {
-        && loader.isLoading()
+      return;
-        && !mediaChunks.isEmpty()
+    }
-        && chunkSource.isMediaChunkRemoved(Iterables.getLast(mediaChunks))) {
+    HlsMediaChunk lastMediaChunk = Iterables.getLast(mediaChunks);
+    @HlsChunkSource.ChunkPublicationState
+    int chunkState = chunkSource.getChunkPublicationState(lastMediaChunk);
+    if (chunkState == CHUNK_PUBLICATION_STATE_PUBLISHED) {
+      lastMediaChunk.publish();
+    } else if (chunkState == CHUNK_PUBLICATION_STATE_REMOVED
+        && !loadingFinished
+        && loader.isLoading()) {
       loader.cancelLoading();
     }
   }
@@ -738,7 +748,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
 
     if (!readOnlyMediaChunks.isEmpty()
-        && chunkSource.isMediaChunkRemoved(Iterables.getLast(readOnlyMediaChunks))) {
+        && chunkSource.getChunkPublicationState(Iterables.getLast(readOnlyMediaChunks))
+            == CHUNK_PUBLICATION_STATE_REMOVED) {
       discardUpstream(mediaChunks.size() - 1);
     }
 
@@ -820,8 +831,19 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       long loadDurationMs,
       IOException error,
       int errorCount) {
-    long bytesLoaded = loadable.bytesLoaded();
     boolean isMediaChunk = isMediaChunk(loadable);
+    if (isMediaChunk
+        && !((HlsMediaChunk) loadable).isPublished()
+        && error instanceof HttpDataSource.InvalidResponseCodeException) {
+      int responseCode = ((HttpDataSource.InvalidResponseCodeException) error).responseCode;
+      if (responseCode == 410 || responseCode == 404) {
+        // According to RFC 8216, Section 6.2.6 a server should respond with an HTTP 404 (Not found)
+        // for requests of hinted parts that are replaced and not available anymore. We've seen test
+        // streams with HTTP 410 (Gone) also.
+        return Loader.RETRY;
+      }
+    }
+    long bytesLoaded = loadable.bytesLoaded();
     boolean exclusionSucceeded = false;
     LoadEventInfo loadEventInfo =
         new LoadEventInfo(