From b23b1bb693ed6be4fff23278b4fcf8420c4a6784 Mon Sep 17 00:00:00 2001
From: jbibik <jbibik@google.com>
Date: Tue, 14 Nov 2023 10:07:57 -0800
Subject: [PATCH] Add HLS support for parsing TTML subs muxed into MP4 during
 extraction

This removes the flakiness of the HLS playback test as well.

Previously, this flow only worked for standalone WebVTT subtitles (https://github.com/androidx/media/commit/7b762642db96e0790002474e1da768aa7757e1d4)

PiperOrigin-RevId: 582359383
---
 RELEASENOTES.md                                      |  2 ++
 .../exoplayer/hls/DefaultHlsExtractorFactory.java    |  6 +++++-
 .../exoplayer/hls/e2etest/HlsPlaybackTest.java       |  6 +++---
 .../test/assets/playbackdumps/hls/ttml-in-mp4.dump   | 12 ++++++++----
 4 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index abfcc3a15b..f13f7a5361 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -69,6 +69,8 @@
     *   Add experimental support for parsing subtitles during extraction. You
         can enable this using
         `HlsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`.
+        This works for standalone WebVTT subtitles and IMSC1 subtitles muxed
+        into MP4.
 *   DASH Extension:
     *   Extend experimental support for parsing subtitles during extraction to
         work with standalone text files (previously it only worked with
diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java
index c607d15c45..fc88fd3021 100644
--- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java
+++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java
@@ -201,7 +201,11 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
       case FileTypes.MP3:
         return new Mp3Extractor(/* flags= */ 0, /* forcedFirstSampleTimestampUs= */ 0);
       case FileTypes.MP4:
-        return createFragmentedMp4Extractor(timestampAdjuster, format, muxedCaptionFormats);
+        Extractor mp4Extractor =
+            createFragmentedMp4Extractor(timestampAdjuster, format, muxedCaptionFormats);
+        return subtitleParserFactory != null
+            ? new SubtitleTranscodingExtractor(mp4Extractor, subtitleParserFactory)
+            : mp4Extractor;
       case FileTypes.TS:
         return createTsExtractor(
             payloadReaderFactoryFlags,
diff --git a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java
index 1ecb12e2ac..58f640c8b5 100644
--- a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java
+++ b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java
@@ -32,7 +32,6 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
 import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -72,14 +71,15 @@ public final class HlsPlaybackTest {
   }
 
   @Test
-  @Ignore(
-      "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
   public void ttmlInMp4() throws Exception {
     Context applicationContext = ApplicationProvider.getApplicationContext();
     CapturingRenderersFactory capturingRenderersFactory =
         new CapturingRenderersFactory(applicationContext);
     ExoPlayer player =
         new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setMediaSourceFactory(
+                new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
+                    .experimentalParseSubtitlesDuringExtraction(true))
             .setClock(new FakeClock(/* isAutoAdvancing= */ true))
             .build();
     player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
diff --git a/libraries/test_data/src/test/assets/playbackdumps/hls/ttml-in-mp4.dump b/libraries/test_data/src/test/assets/playbackdumps/hls/ttml-in-mp4.dump
index 55283c8d1f..98332c6cd4 100644
--- a/libraries/test_data/src/test/assets/playbackdumps/hls/ttml-in-mp4.dump
+++ b/libraries/test_data/src/test/assets/playbackdumps/hls/ttml-in-mp4.dump
@@ -222,22 +222,26 @@ TextOutput:
     presentationTimeUs = 0
     Cues = []
   Subtitle[1]:
+    presentationTimeUs = 20000
+    Cue[0]:
+      text = First Subtitle after 20ms of silence
+  Subtitle[2]:
     presentationTimeUs = 250000
     Cue[0]:
       text = First Subtitle after 20ms of silence
              Second Subtitle overlaps with first
-  Subtitle[2]:
+  Subtitle[3]:
     presentationTimeUs = 400000
     Cue[0]:
       text = First Subtitle after 20ms of silence
-  Subtitle[3]:
+  Subtitle[4]:
     presentationTimeUs = 500000
     Cues = []
-  Subtitle[4]:
+  Subtitle[5]:
     presentationTimeUs = 700000
     Cue[0]:
       text = Last subtitle
              And More
-  Subtitle[5]:
+  Subtitle[6]:
     presentationTimeUs = 950000
     Cues = []