Add experimental opt-in to parse HLS subtitles during extraction

PiperOrigin-RevId: 578524318

RELEASENOTES.md

@@ -46,6 +46,9 @@
 *   HLS Extension:
     *   Reduce `HlsMediaPeriod` to package-private visibility. This type
         shouldn't be directly depended on from outside the HLS package.
+    *   Add experimental support for parsing subtitles during extraction. You
+        can enable this using
+        `HlsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`.
 *   DASH Extension:
     *   Extend experimental support for parsing subtitles during extraction to
         work with standalone text files (previously it only worked with

libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java

@@ -17,6 +17,7 @@ package androidx.media3.exoplayer.hls;
 
 import static androidx.media3.common.util.Assertions.checkState;
 
+import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.Format;
 import androidx.media3.common.util.TimestampAdjuster;
@@ -27,6 +28,8 @@ import androidx.media3.extractor.ExtractorOutput;
 import androidx.media3.extractor.PositionHolder;
 import androidx.media3.extractor.mp3.Mp3Extractor;
 import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
 import androidx.media3.extractor.ts.Ac3Extractor;
 import androidx.media3.extractor.ts.Ac4Extractor;
 import androidx.media3.extractor.ts.AdtsExtractor;
@@ -45,6 +48,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
   @VisibleForTesting /* package */ final Extractor extractor;
   private final Format multivariantPlaylistFormat;
   private final TimestampAdjuster timestampAdjuster;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   /**
    * Creates a new instance.
@@ -55,9 +59,35 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
    */
   public BundledHlsMediaChunkExtractor(
       Extractor extractor, Format multivariantPlaylistFormat, TimestampAdjuster timestampAdjuster) {
+    this(
+        extractor,
+        multivariantPlaylistFormat,
+        timestampAdjuster,
+        /* subtitleParserFactory= */ null);
+  }
+
+  /**
+   * Creates a new instance.
+   *
+   * @param extractor The underlying {@link Extractor}.
+   * @param multivariantPlaylistFormat The {@link Format} obtained from the multivariant playlist.
+   * @param timestampAdjuster A {@link TimestampAdjuster} to adjust sample timestamps.
+   * @param subtitleParserFactory A {@link SubtitleParser.Factory} to be used with WebVTT subtitles.
+   *     If the value is null, subtitles will be parsed during decoding, otherwise - during
+   *     extraction. Decoding will only work if this subtitleParserFactory supports the provided
+   *     multivariantPlaylistFormat.
+   */
+  // TODO(b/289983417): Once the subtitle-parsing-during-extraction is the only available flow, make
+  // this constructor public and remove @Nullable from subtitleParserFactory
+  /* package */ BundledHlsMediaChunkExtractor(
+      Extractor extractor,
+      Format multivariantPlaylistFormat,
+      TimestampAdjuster timestampAdjuster,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.extractor = extractor;
     this.multivariantPlaylistFormat = multivariantPlaylistFormat;
     this.timestampAdjuster = timestampAdjuster;
+    this.subtitleParserFactory = subtitleParserFactory;
   }
 
   @Override
@@ -97,6 +127,11 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
     if (extractor instanceof WebvttExtractor) {
       newExtractorInstance =
           new WebvttExtractor(multivariantPlaylistFormat.language, timestampAdjuster);
+      if (subtitleParserFactory != null
+          && subtitleParserFactory.supportsFormat(multivariantPlaylistFormat)) {
+        newExtractorInstance =
+            new SubtitleTranscodingExtractor(newExtractorInstance, subtitleParserFactory);
+      }
     } else if (extractor instanceof AdtsExtractor) {
       newExtractorInstance = new AdtsExtractor();
     } else if (extractor instanceof Ac3Extractor) {
@@ -110,7 +145,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
           "Unexpected extractor type for recreation: " + extractor.getClass().getSimpleName());
     }
     return new BundledHlsMediaChunkExtractor(
-        newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster);
+        newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster, subtitleParserFactory);
   }
 
   @Override

libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java

@@ -32,6 +32,8 @@ import androidx.media3.extractor.Extractor;
 import androidx.media3.extractor.ExtractorInput;
 import androidx.media3.extractor.mp3.Mp3Extractor;
 import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
+import androidx.media3.extractor.text.SubtitleParser;
+import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
 import androidx.media3.extractor.ts.Ac3Extractor;
 import androidx.media3.extractor.ts.Ac4Extractor;
 import androidx.media3.extractor.ts.AdtsExtractor;
@@ -63,6 +65,10 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
       };
 
   private final @DefaultTsPayloadReaderFactory.Flags int payloadReaderFactoryFlags;
+
+  /** Non-null if subtitles should be parsed during extraction, null otherwise. */
+  @Nullable private SubtitleParser.Factory subtitleParserFactory;
+
   private final boolean exposeCea608WhenMissingDeclarations;
 
   /**
@@ -127,7 +133,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
           checkNotNull(
               createExtractorByFileType(fileType, format, muxedCaptionFormats, timestampAdjuster));
       if (sniffQuietly(extractor, sniffingExtractorInput)) {
-        return new BundledHlsMediaChunkExtractor(extractor, format, timestampAdjuster);
+        return new BundledHlsMediaChunkExtractor(
+            extractor, format, timestampAdjuster, subtitleParserFactory);
       }
       if (fallBackExtractor == null
           && (fileType == formatInferredFileType
@@ -141,7 +148,24 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
     }
 
     return new BundledHlsMediaChunkExtractor(
-        checkNotNull(fallBackExtractor), format, timestampAdjuster);
+        checkNotNull(fallBackExtractor), format, timestampAdjuster, subtitleParserFactory);
+  }
+
+  /**
+   * Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or null
+   * to parse subtitles during decoding. The default is null (subtitles parsed after decoding).
+   *
+   * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+   * in a future release.
+   *
+   * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
+   *     extraction.
+   * @return This factory, for convenience.
+   */
+  public DefaultHlsExtractorFactory experimentalSetSubtitleParserFactory(
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
+    this.subtitleParserFactory = subtitleParserFactory;
+    return this;
   }
 
   private static void addFileTypeIfValidAndNotPresent(
@@ -162,7 +186,12 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
     // LINT.IfChange(extractor_instantiation)
     switch (fileType) {
       case FileTypes.WEBVTT:
-        return new WebvttExtractor(format.language, timestampAdjuster);
+        if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(format)) {
+          return new SubtitleTranscodingExtractor(
+              new WebvttExtractor(format.language, timestampAdjuster), subtitleParserFactory);
+        } else {
+          return new WebvttExtractor(format.language, timestampAdjuster);
+        }
       case FileTypes.ADTS:
         return new AdtsExtractor();
       case FileTypes.AC3:

libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java

@@ -51,6 +51,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
 import androidx.media3.exoplayer.upstream.CmcdConfiguration;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.extractor.Extractor;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.common.primitives.Ints;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -85,6 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final PlayerId playerId;
   private final HlsSampleStreamWrapper.Callback sampleStreamWrapperCallback;
   private final long timestampAdjusterInitializationTimeoutMs;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   @Nullable private MediaPeriod.Callback mediaPeriodCallback;
   private int pendingPrepareCount;
@@ -139,7 +141,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       @HlsMediaSource.MetadataType int metadataType,
       boolean useSessionKeys,
       PlayerId playerId,
-      long timestampAdjusterInitializationTimeoutMs) {
+      long timestampAdjusterInitializationTimeoutMs,
+      @Nullable SubtitleParser.Factory subtitleParserFactory) {
     this.extractorFactory = extractorFactory;
     this.playlistTracker = playlistTracker;
     this.dataSourceFactory = dataSourceFactory;
@@ -164,6 +167,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     sampleStreamWrappers = new HlsSampleStreamWrapper[0];
     enabledSampleStreamWrappers = new HlsSampleStreamWrapper[0];
     manifestUrlIndicesPerWrapper = new int[0][];
+    this.subtitleParserFactory = subtitleParserFactory;
   }
 
   public void release() {
@@ -517,21 +521,43 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     for (int i = 0; i < subtitleRenditions.size(); i++) {
       Rendition subtitleRendition = subtitleRenditions.get(i);
       String sampleStreamWrapperUid = "subtitle:" + i + ":" + subtitleRendition.name;
+      // Format for HlsChunkSource to createExtractor with
+      Format originalSubtitleFormat = subtitleRendition.format;
       HlsSampleStreamWrapper sampleStreamWrapper =
           buildSampleStreamWrapper(
               sampleStreamWrapperUid,
               C.TRACK_TYPE_TEXT,
               new Uri[] {subtitleRendition.url},
-              new Format[] {subtitleRendition.format},
+              new Format[] {originalSubtitleFormat},
               null,
               Collections.emptyList(),
               overridingDrmInitData,
               positionUs);
       manifestUrlIndicesPerWrapper.add(new int[] {i});
       sampleStreamWrappers.add(sampleStreamWrapper);
-      sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
+      if (subtitleParserFactory != null
-          new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, subtitleRendition.format)},
+          && subtitleParserFactory.supportsFormat(originalSubtitleFormat)) {
-          /* primaryTrackGroupIndex= */ 0);
+        Format updatedSubtitleFormat =
+            originalSubtitleFormat
+                .buildUpon()
+                .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
+                .setCueReplacementBehavior(
+                    subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat))
+                .setCodecs(
+                    originalSubtitleFormat.sampleMimeType
+                        + (originalSubtitleFormat.codecs != null
+                            ? " " + originalSubtitleFormat.codecs
+                            : ""))
+                .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
+                .build();
+        sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
+            new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, updatedSubtitleFormat)},
+            /* primaryTrackGroupIndex= */ 0);
+      } else {
+        sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
+            new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, originalSubtitleFormat)},
+            /* primaryTrackGroupIndex= */ 0);
+      }
     }
 
     this.sampleStreamWrappers = sampleStreamWrappers.toArray(new HlsSampleStreamWrapper[0]);

libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java

@@ -58,6 +58,8 @@ import androidx.media3.exoplayer.upstream.CmcdConfiguration;
 import androidx.media3.exoplayer.upstream.DefaultLoadErrorHandlingPolicy;
 import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
 import androidx.media3.extractor.Extractor;
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
+import androidx.media3.extractor.text.SubtitleParser;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.io.IOException;
 import java.lang.annotation.Documented;
@@ -111,6 +113,7 @@ public final class HlsMediaSource extends BaseMediaSource
     @Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory;
     private DrmSessionManagerProvider drmSessionManagerProvider;
     private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
+    @Nullable private SubtitleParser.Factory subtitleParserFactory;
     private boolean allowChunklessPreparation;
     private @MetadataType int metadataType;
     private boolean useSessionKeys;
@@ -196,6 +199,40 @@ public final class HlsMediaSource extends BaseMediaSource
       return this;
     }
 
+    /**
+     * Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
+     * part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
+     * as part of rendering).
+     *
+     * <p>This method is experimental. Its default value may change, or it may be renamed or removed
+     * in a future release.
+     *
+     * <p>This method may only be used with {@link DefaultHlsExtractorFactory}.
+     *
+     * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
+     *     rendering.
+     * @return This factory, for convenience.
+     */
+    // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
+    //  DefaultMediaSourceFactory via the MediaSource.Factory interface).
+    public Factory experimentalParseSubtitlesDuringExtraction(
+        boolean parseSubtitlesDuringExtraction) {
+      if (parseSubtitlesDuringExtraction) {
+        if (subtitleParserFactory == null) {
+          this.subtitleParserFactory = new DefaultSubtitleParserFactory();
+        }
+      } else {
+        this.subtitleParserFactory = null;
+      }
+      if (extractorFactory instanceof DefaultHlsExtractorFactory) {
+        ((DefaultHlsExtractorFactory) extractorFactory)
+            .experimentalSetSubtitleParserFactory(subtitleParserFactory);
+      } else {
+        throw new IllegalStateException();
+      }
+      return this;
+    }
+
     /**
      * Sets the factory from which playlist parsers will be obtained.
      *
@@ -381,6 +418,7 @@ public final class HlsMediaSource extends BaseMediaSource
           mediaItem,
           hlsDataSourceFactory,
           extractorFactory,
+          subtitleParserFactory,
           compositeSequenceableLoaderFactory,
           cmcdConfiguration,
           drmSessionManagerProvider.get(mediaItem),
@@ -412,6 +450,7 @@ public final class HlsMediaSource extends BaseMediaSource
   private final HlsPlaylistTracker playlistTracker;
   private final long elapsedRealTimeOffsetMs;
   private final long timestampAdjusterInitializationTimeoutMs;
+  @Nullable private final SubtitleParser.Factory subtitleParserFactory;
 
   private MediaItem.LiveConfiguration liveConfiguration;
   @Nullable private TransferListener mediaTransferListener;
@@ -423,6 +462,7 @@ public final class HlsMediaSource extends BaseMediaSource
       MediaItem mediaItem,
       HlsDataSourceFactory dataSourceFactory,
       HlsExtractorFactory extractorFactory,
+      @Nullable SubtitleParser.Factory subtitleParserFactory,
       CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
       @Nullable CmcdConfiguration cmcdConfiguration,
       DrmSessionManager drmSessionManager,
@@ -437,6 +477,7 @@ public final class HlsMediaSource extends BaseMediaSource
     this.liveConfiguration = mediaItem.liveConfiguration;
     this.dataSourceFactory = dataSourceFactory;
     this.extractorFactory = extractorFactory;
+    this.subtitleParserFactory = subtitleParserFactory;
     this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
     this.cmcdConfiguration = cmcdConfiguration;
     this.drmSessionManager = drmSessionManager;
@@ -511,7 +552,8 @@ public final class HlsMediaSource extends BaseMediaSource
         metadataType,
         useSessionKeys,
         getPlayerId(),
-        timestampAdjusterInitializationTimeoutMs);
+        timestampAdjusterInitializationTimeoutMs,
+        subtitleParserFactory);
   }
 
   @Override

libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java

@@ -97,7 +97,8 @@ public final class HlsMediaPeriodTest {
               HlsMediaSource.METADATA_TYPE_ID3,
               /* useSessionKeys= */ false,
               PlayerId.UNSET,
-              /* timestampAdjusterInitializationTimeoutMs= */ 0);
+              /* timestampAdjusterInitializationTimeoutMs= */ 0,
+              /* subtitleParserFactory= */ null);
         };
 
     MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration(

libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java

@@ -21,7 +21,9 @@ import android.graphics.SurfaceTexture;
 import android.view.Surface;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.Player;
+import androidx.media3.datasource.DefaultDataSource;
 import androidx.media3.exoplayer.ExoPlayer;
+import androidx.media3.exoplayer.hls.HlsMediaSource;
 import androidx.media3.test.utils.CapturingRenderersFactory;
 import androidx.media3.test.utils.DumpFileAsserts;
 import androidx.media3.test.utils.FakeClock;
@@ -30,7 +32,6 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
 import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -44,14 +45,15 @@ public final class HlsPlaybackTest {
       ShadowMediaCodecConfig.forAllSupportedMimeTypes();
 
   @Test
-  @Ignore(
-      "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
   public void webvttSubtitles() throws Exception {
     Context applicationContext = ApplicationProvider.getApplicationContext();
     CapturingRenderersFactory capturingRenderersFactory =
         new CapturingRenderersFactory(applicationContext);
     ExoPlayer player =
         new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setMediaSourceFactory(
+                new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
+                    .experimentalParseSubtitlesDuringExtraction(true))
             .setClock(new FakeClock(/* isAutoAdvancing= */ true))
             .build();
     player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));