Add container format sniffing in HLS

Issue:#2025

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=211977802

RELEASENOTES.md

@@ -32,6 +32,8 @@
     [VR180](https://github.com/google/spatial-media/blob/master/docs/vr180.md).
 * HLS:
   * Support PlayReady.
+  * Add container format sniffing
+    ([#2025](https://github.com/google/ExoPlayer/issues/2025)).
   * Support alternative `EXT-X-KEY` tags.
   * Support `EXT-X-INDEPENDENT-SEGMENTS` in the master playlist.
   * Support variable substitution

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/DefaultHlsExtractorFactory.java

@@ -21,14 +21,18 @@ import android.util.Pair;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.drm.DrmInitData;
 import com.google.android.exoplayer2.extractor.Extractor;
+import com.google.android.exoplayer2.extractor.ExtractorInput;
 import com.google.android.exoplayer2.extractor.mp3.Mp3Extractor;
 import com.google.android.exoplayer2.extractor.mp4.FragmentedMp4Extractor;
 import com.google.android.exoplayer2.extractor.ts.Ac3Extractor;
 import com.google.android.exoplayer2.extractor.ts.AdtsExtractor;
 import com.google.android.exoplayer2.extractor.ts.DefaultTsPayloadReaderFactory;
 import com.google.android.exoplayer2.extractor.ts.TsExtractor;
+import com.google.android.exoplayer2.source.UnrecognizedInputFormatException;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TimestampAdjuster;
+import java.io.EOFException;
+import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -56,74 +60,187 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
       List<Format> muxedCaptionFormats,
       DrmInitData drmInitData,
       TimestampAdjuster timestampAdjuster,
-      Map<String, List<String>> responseHeaders) {
+      Map<String, List<String>> responseHeaders,
-    String lastPathSegment = uri.getLastPathSegment();
+      ExtractorInput extractorInput)
-    if (lastPathSegment == null) {
+      throws InterruptedException, IOException {
-      lastPathSegment = "";
+
+    if (previousExtractor != null) {
+      // A extractor has already been successfully used. Return one of the same type.
+      if (previousExtractor instanceof TsExtractor
+          || previousExtractor instanceof FragmentedMp4Extractor) {
+        // TS and fMP4 extractors can be reused.
+        return buildResult(previousExtractor);
+      } else if (previousExtractor instanceof WebvttExtractor) {
+        return buildResult(new WebvttExtractor(format.language, timestampAdjuster));
+      } else if (previousExtractor instanceof AdtsExtractor) {
+        return buildResult(new AdtsExtractor());
+      } else if (previousExtractor instanceof Ac3Extractor) {
+        return buildResult(new Ac3Extractor());
+      } else if (previousExtractor instanceof Mp3Extractor) {
+        return buildResult(new Mp3Extractor());
+      } else {
+        throw new IllegalArgumentException(
+            "Unexpected previousExtractor type: " + previousExtractor.getClass().getSimpleName());
+      }
     }
-    boolean isPackedAudioExtractor = false;
+
-    Extractor extractor;
+    // Try selecting the extractor by the file extension.
-    if (MimeTypes.TEXT_VTT.equals(format.sampleMimeType)
+    Extractor extractorByFileExtension =
-        || lastPathSegment.endsWith(WEBVTT_FILE_EXTENSION)
+        createExtractorByFileExtension(
-        || lastPathSegment.endsWith(VTT_FILE_EXTENSION)) {
+            uri, format, muxedCaptionFormats, drmInitData, timestampAdjuster);
-      extractor = new WebvttExtractor(format.language, timestampAdjuster);
+    extractorInput.resetPeekPosition();
-    } else if (lastPathSegment.endsWith(AAC_FILE_EXTENSION)) {
+    if (sniffQuietly(extractorByFileExtension, extractorInput)) {
-      isPackedAudioExtractor = true;
+      return buildResult(extractorByFileExtension);
-      extractor = new AdtsExtractor();
+    }
-    } else if (lastPathSegment.endsWith(AC3_FILE_EXTENSION)
+
-        || lastPathSegment.endsWith(EC3_FILE_EXTENSION)) {
+    // We need to manually sniff each known type, without retrying the one selected by file
-      isPackedAudioExtractor = true;
+    // extension.
-      extractor = new Ac3Extractor();
+
-    } else if (lastPathSegment.endsWith(MP3_FILE_EXTENSION)) {
+    if (!(extractorByFileExtension instanceof WebvttExtractor)) {
-      isPackedAudioExtractor = true;
+      WebvttExtractor webvttExtractor = new WebvttExtractor(format.language, timestampAdjuster);
-      extractor = new Mp3Extractor(0, 0);
+      if (sniffQuietly(webvttExtractor, extractorInput)) {
-    } else if (previousExtractor != null) {
+        return buildResult(webvttExtractor);
-      // Only reuse TS and fMP4 extractors.
+      }
-      extractor = previousExtractor;
+    }
-    } else if (lastPathSegment.endsWith(MP4_FILE_EXTENSION)
+
-        || lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)
+    if (!(extractorByFileExtension instanceof AdtsExtractor)) {
-        || lastPathSegment.startsWith(MP4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)) {
+      AdtsExtractor adtsExtractor = new AdtsExtractor();
-      extractor =
+      if (sniffQuietly(adtsExtractor, extractorInput)) {
+        return buildResult(adtsExtractor);
+      }
+    }
+
+    if (!(extractorByFileExtension instanceof Ac3Extractor)) {
+      Ac3Extractor ac3Extractor = new Ac3Extractor();
+      if (sniffQuietly(ac3Extractor, extractorInput)) {
+        return buildResult(ac3Extractor);
+      }
+    }
+
+    if (!(extractorByFileExtension instanceof Mp3Extractor)) {
+      Mp3Extractor mp3Extractor =
+          new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0);
+      if (sniffQuietly(mp3Extractor, extractorInput)) {
+        return buildResult(mp3Extractor);
+      }
+    }
+
+    if (!(extractorByFileExtension instanceof FragmentedMp4Extractor)) {
+      FragmentedMp4Extractor fragmentedMp4Extractor =
           new FragmentedMp4Extractor(
               /* flags= */ 0,
               timestampAdjuster,
               /* sideloadedTrack= */ null,
               drmInitData,
               muxedCaptionFormats != null ? muxedCaptionFormats : Collections.emptyList());
+      if (sniffQuietly(fragmentedMp4Extractor, extractorInput)) {
+        return buildResult(fragmentedMp4Extractor);
+      }
+    }
+
+    if (!(extractorByFileExtension instanceof TsExtractor)) {
+      TsExtractor tsExtractor = createTsExtractor(format, muxedCaptionFormats, timestampAdjuster);
+      if (sniffQuietly(tsExtractor, extractorInput)) {
+        return buildResult(tsExtractor);
+      }
+    }
+
+    throw new UnrecognizedInputFormatException(
+        "The segment does not seem to conform to any of the known HLS segment formats", uri);
+  }
+
+  private Extractor createExtractorByFileExtension(
+      Uri uri,
+      Format format,
+      List<Format> muxedCaptionFormats,
+      DrmInitData drmInitData,
+      TimestampAdjuster timestampAdjuster) {
+    String lastPathSegment = uri.getLastPathSegment();
+    if (lastPathSegment == null) {
+      lastPathSegment = "";
+    }
+    if (MimeTypes.TEXT_VTT.equals(format.sampleMimeType)
+        || lastPathSegment.endsWith(WEBVTT_FILE_EXTENSION)
+        || lastPathSegment.endsWith(VTT_FILE_EXTENSION)) {
+      return new WebvttExtractor(format.language, timestampAdjuster);
+    } else if (lastPathSegment.endsWith(AAC_FILE_EXTENSION)) {
+      return new AdtsExtractor();
+    } else if (lastPathSegment.endsWith(AC3_FILE_EXTENSION)
+        || lastPathSegment.endsWith(EC3_FILE_EXTENSION)) {
+      return new Ac3Extractor();
+    } else if (lastPathSegment.endsWith(MP3_FILE_EXTENSION)) {
+      return new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0);
+    } else if (lastPathSegment.endsWith(MP4_FILE_EXTENSION)
+        || lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)
+        || lastPathSegment.startsWith(MP4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)) {
+      return new FragmentedMp4Extractor(
+          /* flags= */ 0,
+          timestampAdjuster,
+          /* sideloadedTrack= */ null,
+          drmInitData,
+          muxedCaptionFormats != null ? muxedCaptionFormats : Collections.emptyList());
     } else {
       // For any other file extension, we assume TS format.
-      @DefaultTsPayloadReaderFactory.Flags
+      return createTsExtractor(format, muxedCaptionFormats, timestampAdjuster);
-      int esReaderFactoryFlags = DefaultTsPayloadReaderFactory.FLAG_IGNORE_SPLICE_INFO_STREAM;
-      if (muxedCaptionFormats != null) {
-        // The playlist declares closed caption renditions, we should ignore descriptors.
-        esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_OVERRIDE_CAPTION_DESCRIPTORS;
-      } else {
-        // The playlist does not provide any closed caption information. We preemptively declare a
-        // closed caption track on channel 0.
-        muxedCaptionFormats =
-            Collections.singletonList(
-                Format.createTextSampleFormat(
-                    /* id= */ null,
-                    MimeTypes.APPLICATION_CEA608,
-                    /* selectionFlags= */ 0,
-                    /* language= */ null));
-      }
-      String codecs = format.codecs;
-      if (!TextUtils.isEmpty(codecs)) {
-        // Sometimes AAC and H264 streams are declared in TS chunks even though they don't really
-        // exist. If we know from the codec attribute that they don't exist, then we can
-        // explicitly ignore them even if they're declared.
-        if (!MimeTypes.AUDIO_AAC.equals(MimeTypes.getAudioMediaMimeType(codecs))) {
-          esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_IGNORE_AAC_STREAM;
-        }
-        if (!MimeTypes.VIDEO_H264.equals(MimeTypes.getVideoMediaMimeType(codecs))) {
-          esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_IGNORE_H264_STREAM;
-        }
-      }
-      extractor = new TsExtractor(TsExtractor.MODE_HLS, timestampAdjuster,
-          new DefaultTsPayloadReaderFactory(esReaderFactoryFlags, muxedCaptionFormats));
     }
-    return Pair.create(extractor, isPackedAudioExtractor);
+  }
+
+  private static TsExtractor createTsExtractor(
+      Format format, List<Format> muxedCaptionFormats, TimestampAdjuster timestampAdjuster) {
+    @DefaultTsPayloadReaderFactory.Flags
+    int esReaderFactoryFlags = DefaultTsPayloadReaderFactory.FLAG_IGNORE_SPLICE_INFO_STREAM;
+    if (muxedCaptionFormats != null) {
+      // The playlist declares closed caption renditions, we should ignore descriptors.
+      esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_OVERRIDE_CAPTION_DESCRIPTORS;
+    } else {
+      // The playlist does not provide any closed caption information. We preemptively declare a
+      // closed caption track on channel 0.
+      muxedCaptionFormats =
+          Collections.singletonList(
+              Format.createTextSampleFormat(
+                  /* id= */ null,
+                  MimeTypes.APPLICATION_CEA608,
+                  /* selectionFlags= */ 0,
+                  /* language= */ null));
+    }
+    String codecs = format.codecs;
+    if (!TextUtils.isEmpty(codecs)) {
+      // Sometimes AAC and H264 streams are declared in TS chunks even though they don't really
+      // exist. If we know from the codec attribute that they don't exist, then we can
+      // explicitly ignore them even if they're declared.
+      if (!MimeTypes.AUDIO_AAC.equals(MimeTypes.getAudioMediaMimeType(codecs))) {
+        esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_IGNORE_AAC_STREAM;
+      }
+      if (!MimeTypes.VIDEO_H264.equals(MimeTypes.getVideoMediaMimeType(codecs))) {
+        esReaderFactoryFlags |= DefaultTsPayloadReaderFactory.FLAG_IGNORE_H264_STREAM;
+      }
+    }
+
+    return new TsExtractor(
+        TsExtractor.MODE_HLS,
+        timestampAdjuster,
+        new DefaultTsPayloadReaderFactory(esReaderFactoryFlags, muxedCaptionFormats));
+  }
+
+  private static Pair<Extractor, Boolean> buildResult(Extractor extractor) {
+    return new Pair<>(
+        extractor,
+        extractor instanceof AdtsExtractor
+            || extractor instanceof Ac3Extractor
+            || extractor instanceof Mp3Extractor);
+  }
+
+  private static boolean sniffQuietly(Extractor extractor, ExtractorInput input)
+      throws InterruptedException, IOException {
+    boolean result = false;
+    try {
+      result = extractor.sniff(input);
+    } catch (EOFException e) {
+      // Do nothing.
+    } finally {
+      input.resetPeekPosition();
+    }
+    return result;
   }
 
 }

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsExtractorFactory.java

@@ -20,7 +20,10 @@ import android.util.Pair;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.drm.DrmInitData;
 import com.google.android.exoplayer2.extractor.Extractor;
+import com.google.android.exoplayer2.extractor.ExtractorInput;
+import com.google.android.exoplayer2.extractor.PositionHolder;
 import com.google.android.exoplayer2.util.TimestampAdjuster;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -45,9 +48,14 @@ public interface HlsExtractorFactory {
    * @param timestampAdjuster Adjuster corresponding to the provided discontinuity sequence number.
    * @param responseHeaders The HTTP response headers associated with the media segment or
    *     initialization section to extract.
+   * @param sniffingExtractorInput The first extractor input that will be passed to the returned
+   *     extractor's {@link Extractor#read(ExtractorInput, PositionHolder)}. Must only be used to
+   *     call {@link Extractor#sniff(ExtractorInput)}.
    * @return A pair containing the {@link Extractor} and a boolean that indicates whether it is a
    *     packed audio extractor. The first element may be {@code previousExtractor} if the factory
    *     has determined it can be re-used.
+   * @throws InterruptedException If the thread is interrupted while sniffing.
+   * @throws IOException If an I/O error is encountered while sniffing.
    */
   Pair<Extractor, Boolean> createExtractor(
       Extractor previousExtractor,
@@ -56,5 +64,7 @@ public interface HlsExtractorFactory {
       List<Format> muxedCaptionFormats,
       DrmInitData drmInitData,
       TimestampAdjuster timestampAdjuster,
-      Map<String, List<String>> responseHeaders);
+      Map<String, List<String>> responseHeaders,
+      ExtractorInput sniffingExtractorInput)
+      throws InterruptedException, IOException;
 }

library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaChunk.java

@@ -73,15 +73,13 @@ import java.util.concurrent.atomic.AtomicInteger;
   private final List<Format> muxedCaptionFormats;
   private final DrmInitData drmInitData;
   private final Extractor previousExtractor;
+  private final Id3Decoder id3Decoder;
+  private final ParsableByteArray id3Data;
 
   private Extractor extractor;
-  private boolean isPackedAudioExtractor;
-  private Id3Decoder id3Decoder;
-  private ParsableByteArray id3Data;
   private HlsSampleStreamWrapper output;
   private int initSegmentBytesLoaded;
   private int nextLoadPosition;
-  private boolean id3TimestampPeeked;
   private boolean initLoadCompleted;
   private volatile boolean loadCanceled;
   private boolean loadCompleted;
@@ -158,6 +156,8 @@ import java.util.concurrent.atomic.AtomicInteger;
       previousExtractor = previousChunk.discontinuitySequenceNumber != discontinuitySequenceNumber
           || shouldSpliceIn ? null : previousChunk.extractor;
     } else {
+      id3Decoder = new Id3Decoder();
+      id3Data = new ParsableByteArray(Id3Decoder.ID3_HEADER_LENGTH);
       shouldSpliceIn = false;
     }
     this.previousExtractor = previousExtractor;
@@ -244,12 +244,6 @@ import java.util.concurrent.atomic.AtomicInteger;
     }
     try {
       ExtractorInput input = prepareExtraction(dataSource, loadDataSpec);
-      if (isPackedAudioExtractor && !id3TimestampPeeked) {
-        long id3Timestamp = peekId3PrivTimestamp(input);
-        id3TimestampPeeked = true;
-        output.setSampleOffsetUs(id3Timestamp != C.TIME_UNSET
-            ? timestampAdjuster.adjustTsTimestamp(id3Timestamp) : startTimeUs);
-      }
       if (skipLoadedBytes) {
         input.skipFully(nextLoadPosition);
       }
@@ -267,10 +261,16 @@ import java.util.concurrent.atomic.AtomicInteger;
   }
 
   private DefaultExtractorInput prepareExtraction(DataSource dataSource, DataSpec dataSpec)
-      throws IOException {
+      throws IOException, InterruptedException {
     long bytesToRead = dataSource.open(dataSpec);
 
+    DefaultExtractorInput extractorInput =
+        new DefaultExtractorInput(dataSource, dataSpec.absoluteStreamPosition, bytesToRead);
+
     if (extractor == null) {
+      long id3Timestamp = peekId3PrivTimestamp(extractorInput);
+      extractorInput.resetPeekPosition();
+
       Pair<Extractor, Boolean> extractorData =
           extractorFactory.createExtractor(
               previousExtractor,
@@ -279,22 +279,26 @@ import java.util.concurrent.atomic.AtomicInteger;
               muxedCaptionFormats,
               drmInitData,
               timestampAdjuster,
-              dataSource.getResponseHeaders());
+              dataSource.getResponseHeaders(),
+              extractorInput);
       extractor = extractorData.first;
-      isPackedAudioExtractor = extractorData.second;
       boolean reusingExtractor = extractor == previousExtractor;
-      initLoadCompleted = reusingExtractor && initDataSpec != null;
+      boolean isPackedAudioExtractor = extractorData.second;
-      if (isPackedAudioExtractor && id3Data == null) {
+      if (isPackedAudioExtractor) {
-        id3Decoder = new Id3Decoder();
+        output.setSampleOffsetUs(
-        id3Data = new ParsableByteArray(Id3Decoder.ID3_HEADER_LENGTH);
+            id3Timestamp != C.TIME_UNSET
+                ? timestampAdjuster.adjustTsTimestamp(id3Timestamp)
+                : startTimeUs);
       }
+      initLoadCompleted = reusingExtractor && initDataSpec != null;
+
       output.init(uid, shouldSpliceIn, reusingExtractor);
       if (!reusingExtractor) {
         extractor.init(output);
       }
     }
 
-    return new DefaultExtractorInput(dataSource, dataSpec.absoluteStreamPosition, bytesToRead);
+    return extractorInput;
   }
 
   /**
@@ -309,7 +313,8 @@ import java.util.concurrent.atomic.AtomicInteger;
    */
   private long peekId3PrivTimestamp(ExtractorInput input) throws IOException, InterruptedException {
     input.resetPeekPosition();
-    if (!input.peekFully(id3Data.data, 0, Id3Decoder.ID3_HEADER_LENGTH, true)) {
+    if (input.getLength() < Id3Decoder.ID3_HEADER_LENGTH
+        || !input.peekFully(id3Data.data, 0, Id3Decoder.ID3_HEADER_LENGTH, true)) {
       return C.TIME_UNSET;
     }
     id3Data.reset(Id3Decoder.ID3_HEADER_LENGTH);