From 97efa70852b3d2fbc6768a5c5a462bdd9048599e Mon Sep 17 00:00:00 2001 From: jbibik Date: Sat, 28 Oct 2023 06:57:13 -0700 Subject: [PATCH] Add DASH support for parsing standalone TTML files during extraction This change applies to standalone TTML files linked directly from the manifest. As a result, we no longer have the flakiness in the DashPlaybackTest which uses sidecar-loaded (standalone file) TTML subtitles. We experimentally opt into parsing subtitles during extraction and use SubtitleExtractor in hybrid mode. PiperOrigin-RevId: 577457256 --- RELEASENOTES.md | 3 +++ .../main/java/androidx/media3/common/MimeTypes.java | 2 ++ .../source/chunk/BundledChunkExtractor.java | 13 ++++++++++--- .../exoplayer/dash/e2etest/DashPlaybackTest.java | 6 +++--- .../media3/extractor/text/SubtitleExtractor.java | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 14fc676220..a5356c779e 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -38,6 +38,9 @@ * RTMP Extension: * HLS Extension: * DASH Extension: + * Extend experimental support for parsing subtitles during extraction to + work with standalone TTML files (previously it only worked with + subtitles muxed into MP4 segments). * Smooth Streaming Extension: * RTSP Extension: * Decoder Extensions (FFmpeg, VP9, AV1, MIDI, etc.): diff --git a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java index 20218f170f..fdfb10327f 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java @@ -25,6 +25,7 @@ import com.google.common.base.Ascii; import java.util.ArrayList; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.checkerframework.dataflow.qual.Pure; /** Defines common MIME types and helper methods. */ public final class MimeTypes { @@ -218,6 +219,7 @@ public final class MimeTypes { * "application" as their base type. */ @UnstableApi + @Pure public static boolean isText(@Nullable String mimeType) { return BASE_TYPE_TEXT.equals(getTopLevelType(mimeType)) || APPLICATION_MEDIA3_CUES.equals(mimeType) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java index f05a5d63e4..28ee84a819 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/BundledChunkExtractor.java @@ -37,6 +37,7 @@ import androidx.media3.extractor.SeekMap; import androidx.media3.extractor.TrackOutput; import androidx.media3.extractor.mkv.MatroskaExtractor; import androidx.media3.extractor.mp4.FragmentedMp4Extractor; +import androidx.media3.extractor.text.SubtitleExtractor; import androidx.media3.extractor.text.SubtitleParser; import androidx.media3.extractor.text.SubtitleTranscodingExtractor; import java.io.IOException; @@ -86,8 +87,14 @@ public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtrac @Nullable String containerMimeType = representationFormat.containerMimeType; Extractor extractor; if (MimeTypes.isText(containerMimeType)) { - // Text types do not need an extractor. - return null; + if (subtitleParserFactory == null) { + // Subtitles will be parsed after decoding + return null; + } else { + extractor = + new SubtitleExtractor( + subtitleParserFactory.create(representationFormat), representationFormat); + } } else if (MimeTypes.isMatroska(containerMimeType)) { extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES); } else { @@ -103,7 +110,7 @@ public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtrac closedCaptionFormats, playerEmsgTrackOutput); } - if (subtitleParserFactory != null) { + if (subtitleParserFactory != null && !MimeTypes.isText(containerMimeType)) { extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory); } return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat); diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java index 78930c7b5a..8b85b0e574 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java @@ -38,7 +38,6 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig; import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -52,8 +51,6 @@ public final class DashPlaybackTest { ShadowMediaCodecConfig.forAllSupportedMimeTypes(); @Test - @Ignore( - "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].") public void ttmlStandaloneXmlFile() throws Exception { Context applicationContext = ApplicationProvider.getApplicationContext(); CapturingRenderersFactory capturingRenderersFactory = @@ -61,6 +58,9 @@ public final class DashPlaybackTest { ExoPlayer player = new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .setMediaSourceFactory( + new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext)) + .experimentalParseSubtitlesDuringExtraction(true)) .build(); player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1))); PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java index 40ef8eac0c..575f4d44af 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleExtractor.java @@ -131,13 +131,13 @@ public class SubtitleExtractor implements Extractor { public void init(ExtractorOutput output) { checkState(state == STATE_CREATED); trackOutput = output.track(/* id= */ 0, C.TRACK_TYPE_TEXT); + trackOutput.format(format); output.endTracks(); output.seekMap( new IndexSeekMap( /* positions= */ new long[] {0}, /* timesUs= */ new long[] {0}, /* durationUs= */ C.TIME_UNSET)); - trackOutput.format(format); state = STATE_INITIALIZED; }