diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c6a6d2c1a2..29872f8bb7 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -46,6 +46,9 @@ * HLS Extension: * Reduce `HlsMediaPeriod` to package-private visibility. This type shouldn't be directly depended on from outside the HLS package. + * Add experimental support for parsing subtitles during extraction. You + can enable this using + `HlsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`. * DASH Extension: * Extend experimental support for parsing subtitles during extraction to work with standalone text files (previously it only worked with diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java index 5b699ce0a0..662de583d6 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java @@ -17,6 +17,7 @@ package androidx.media3.exoplayer.hls; import static androidx.media3.common.util.Assertions.checkState; +import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.Format; import androidx.media3.common.util.TimestampAdjuster; @@ -27,6 +28,8 @@ import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; import androidx.media3.extractor.mp3.Mp3Extractor; import androidx.media3.extractor.mp4.FragmentedMp4Extractor; +import androidx.media3.extractor.text.SubtitleParser; +import androidx.media3.extractor.text.SubtitleTranscodingExtractor; import androidx.media3.extractor.ts.Ac3Extractor; import androidx.media3.extractor.ts.Ac4Extractor; import androidx.media3.extractor.ts.AdtsExtractor; @@ -45,6 +48,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract @VisibleForTesting /* package */ final Extractor extractor; private final Format multivariantPlaylistFormat; private final TimestampAdjuster timestampAdjuster; + @Nullable private final SubtitleParser.Factory subtitleParserFactory; /** * Creates a new instance. @@ -55,9 +59,35 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract */ public BundledHlsMediaChunkExtractor( Extractor extractor, Format multivariantPlaylistFormat, TimestampAdjuster timestampAdjuster) { + this( + extractor, + multivariantPlaylistFormat, + timestampAdjuster, + /* subtitleParserFactory= */ null); + } + + /** + * Creates a new instance. + * + * @param extractor The underlying {@link Extractor}. + * @param multivariantPlaylistFormat The {@link Format} obtained from the multivariant playlist. + * @param timestampAdjuster A {@link TimestampAdjuster} to adjust sample timestamps. + * @param subtitleParserFactory A {@link SubtitleParser.Factory} to be used with WebVTT subtitles. + * If the value is null, subtitles will be parsed during decoding, otherwise - during + * extraction. Decoding will only work if this subtitleParserFactory supports the provided + * multivariantPlaylistFormat. + */ + // TODO(b/289983417): Once the subtitle-parsing-during-extraction is the only available flow, make + // this constructor public and remove @Nullable from subtitleParserFactory + /* package */ BundledHlsMediaChunkExtractor( + Extractor extractor, + Format multivariantPlaylistFormat, + TimestampAdjuster timestampAdjuster, + @Nullable SubtitleParser.Factory subtitleParserFactory) { this.extractor = extractor; this.multivariantPlaylistFormat = multivariantPlaylistFormat; this.timestampAdjuster = timestampAdjuster; + this.subtitleParserFactory = subtitleParserFactory; } @Override @@ -97,6 +127,11 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract if (extractor instanceof WebvttExtractor) { newExtractorInstance = new WebvttExtractor(multivariantPlaylistFormat.language, timestampAdjuster); + if (subtitleParserFactory != null + && subtitleParserFactory.supportsFormat(multivariantPlaylistFormat)) { + newExtractorInstance = + new SubtitleTranscodingExtractor(newExtractorInstance, subtitleParserFactory); + } } else if (extractor instanceof AdtsExtractor) { newExtractorInstance = new AdtsExtractor(); } else if (extractor instanceof Ac3Extractor) { @@ -110,7 +145,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract "Unexpected extractor type for recreation: " + extractor.getClass().getSimpleName()); } return new BundledHlsMediaChunkExtractor( - newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster); + newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster, subtitleParserFactory); } @Override diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java index 2ca8d08dbf..c607d15c45 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java @@ -32,6 +32,8 @@ import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.mp3.Mp3Extractor; import androidx.media3.extractor.mp4.FragmentedMp4Extractor; +import androidx.media3.extractor.text.SubtitleParser; +import androidx.media3.extractor.text.SubtitleTranscodingExtractor; import androidx.media3.extractor.ts.Ac3Extractor; import androidx.media3.extractor.ts.Ac4Extractor; import androidx.media3.extractor.ts.AdtsExtractor; @@ -63,6 +65,10 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { }; private final @DefaultTsPayloadReaderFactory.Flags int payloadReaderFactoryFlags; + + /** Non-null if subtitles should be parsed during extraction, null otherwise. */ + @Nullable private SubtitleParser.Factory subtitleParserFactory; + private final boolean exposeCea608WhenMissingDeclarations; /** @@ -127,7 +133,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { checkNotNull( createExtractorByFileType(fileType, format, muxedCaptionFormats, timestampAdjuster)); if (sniffQuietly(extractor, sniffingExtractorInput)) { - return new BundledHlsMediaChunkExtractor(extractor, format, timestampAdjuster); + return new BundledHlsMediaChunkExtractor( + extractor, format, timestampAdjuster, subtitleParserFactory); } if (fallBackExtractor == null && (fileType == formatInferredFileType @@ -141,7 +148,24 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { } return new BundledHlsMediaChunkExtractor( - checkNotNull(fallBackExtractor), format, timestampAdjuster); + checkNotNull(fallBackExtractor), format, timestampAdjuster, subtitleParserFactory); + } + + /** + * Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or null + * to parse subtitles during decoding. The default is null (subtitles parsed after decoding). + * + *

This method is experimental. Its default value may change, or it may be renamed or removed + * in a future release. + * + * @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during + * extraction. + * @return This factory, for convenience. + */ + public DefaultHlsExtractorFactory experimentalSetSubtitleParserFactory( + @Nullable SubtitleParser.Factory subtitleParserFactory) { + this.subtitleParserFactory = subtitleParserFactory; + return this; } private static void addFileTypeIfValidAndNotPresent( @@ -162,7 +186,12 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory { // LINT.IfChange(extractor_instantiation) switch (fileType) { case FileTypes.WEBVTT: - return new WebvttExtractor(format.language, timestampAdjuster); + if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(format)) { + return new SubtitleTranscodingExtractor( + new WebvttExtractor(format.language, timestampAdjuster), subtitleParserFactory); + } else { + return new WebvttExtractor(format.language, timestampAdjuster); + } case FileTypes.ADTS: return new AdtsExtractor(); case FileTypes.AC3: diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java index b5ac8fa9da..9a3b7a8b70 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java @@ -51,6 +51,7 @@ import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.extractor.Extractor; +import androidx.media3.extractor.text.SubtitleParser; import com.google.common.primitives.Ints; import java.io.IOException; import java.util.ArrayList; @@ -85,6 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final PlayerId playerId; private final HlsSampleStreamWrapper.Callback sampleStreamWrapperCallback; private final long timestampAdjusterInitializationTimeoutMs; + @Nullable private final SubtitleParser.Factory subtitleParserFactory; @Nullable private MediaPeriod.Callback mediaPeriodCallback; private int pendingPrepareCount; @@ -139,7 +141,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @HlsMediaSource.MetadataType int metadataType, boolean useSessionKeys, PlayerId playerId, - long timestampAdjusterInitializationTimeoutMs) { + long timestampAdjusterInitializationTimeoutMs, + @Nullable SubtitleParser.Factory subtitleParserFactory) { this.extractorFactory = extractorFactory; this.playlistTracker = playlistTracker; this.dataSourceFactory = dataSourceFactory; @@ -164,6 +167,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sampleStreamWrappers = new HlsSampleStreamWrapper[0]; enabledSampleStreamWrappers = new HlsSampleStreamWrapper[0]; manifestUrlIndicesPerWrapper = new int[0][]; + this.subtitleParserFactory = subtitleParserFactory; } public void release() { @@ -517,21 +521,43 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; for (int i = 0; i < subtitleRenditions.size(); i++) { Rendition subtitleRendition = subtitleRenditions.get(i); String sampleStreamWrapperUid = "subtitle:" + i + ":" + subtitleRendition.name; + // Format for HlsChunkSource to createExtractor with + Format originalSubtitleFormat = subtitleRendition.format; HlsSampleStreamWrapper sampleStreamWrapper = buildSampleStreamWrapper( sampleStreamWrapperUid, C.TRACK_TYPE_TEXT, new Uri[] {subtitleRendition.url}, - new Format[] {subtitleRendition.format}, + new Format[] {originalSubtitleFormat}, null, Collections.emptyList(), overridingDrmInitData, positionUs); manifestUrlIndicesPerWrapper.add(new int[] {i}); sampleStreamWrappers.add(sampleStreamWrapper); - sampleStreamWrapper.prepareWithMultivariantPlaylistInfo( - new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, subtitleRendition.format)}, - /* primaryTrackGroupIndex= */ 0); + if (subtitleParserFactory != null + && subtitleParserFactory.supportsFormat(originalSubtitleFormat)) { + Format updatedSubtitleFormat = + originalSubtitleFormat + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior( + subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat)) + .setCodecs( + originalSubtitleFormat.sampleMimeType + + (originalSubtitleFormat.codecs != null + ? " " + originalSubtitleFormat.codecs + : "")) + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .build(); + sampleStreamWrapper.prepareWithMultivariantPlaylistInfo( + new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, updatedSubtitleFormat)}, + /* primaryTrackGroupIndex= */ 0); + } else { + sampleStreamWrapper.prepareWithMultivariantPlaylistInfo( + new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, originalSubtitleFormat)}, + /* primaryTrackGroupIndex= */ 0); + } } this.sampleStreamWrappers = sampleStreamWrappers.toArray(new HlsSampleStreamWrapper[0]); diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java index 3c49efbae0..e2cf2e7b9d 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java @@ -58,6 +58,8 @@ import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.DefaultLoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.extractor.Extractor; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.extractor.text.SubtitleParser; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.lang.annotation.Documented; @@ -111,6 +113,7 @@ public final class HlsMediaSource extends BaseMediaSource @Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory; private DrmSessionManagerProvider drmSessionManagerProvider; private LoadErrorHandlingPolicy loadErrorHandlingPolicy; + @Nullable private SubtitleParser.Factory subtitleParserFactory; private boolean allowChunklessPreparation; private @MetadataType int metadataType; private boolean useSessionKeys; @@ -196,6 +199,40 @@ public final class HlsMediaSource extends BaseMediaSource return this; } + /** + * Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as + * part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed + * as part of rendering). + * + *

This method is experimental. Its default value may change, or it may be renamed or removed + * in a future release. + * + *

This method may only be used with {@link DefaultHlsExtractorFactory}. + * + * @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or + * rendering. + * @return This factory, for convenience. + */ + // TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on + // DefaultMediaSourceFactory via the MediaSource.Factory interface). + public Factory experimentalParseSubtitlesDuringExtraction( + boolean parseSubtitlesDuringExtraction) { + if (parseSubtitlesDuringExtraction) { + if (subtitleParserFactory == null) { + this.subtitleParserFactory = new DefaultSubtitleParserFactory(); + } + } else { + this.subtitleParserFactory = null; + } + if (extractorFactory instanceof DefaultHlsExtractorFactory) { + ((DefaultHlsExtractorFactory) extractorFactory) + .experimentalSetSubtitleParserFactory(subtitleParserFactory); + } else { + throw new IllegalStateException(); + } + return this; + } + /** * Sets the factory from which playlist parsers will be obtained. * @@ -381,6 +418,7 @@ public final class HlsMediaSource extends BaseMediaSource mediaItem, hlsDataSourceFactory, extractorFactory, + subtitleParserFactory, compositeSequenceableLoaderFactory, cmcdConfiguration, drmSessionManagerProvider.get(mediaItem), @@ -412,6 +450,7 @@ public final class HlsMediaSource extends BaseMediaSource private final HlsPlaylistTracker playlistTracker; private final long elapsedRealTimeOffsetMs; private final long timestampAdjusterInitializationTimeoutMs; + @Nullable private final SubtitleParser.Factory subtitleParserFactory; private MediaItem.LiveConfiguration liveConfiguration; @Nullable private TransferListener mediaTransferListener; @@ -423,6 +462,7 @@ public final class HlsMediaSource extends BaseMediaSource MediaItem mediaItem, HlsDataSourceFactory dataSourceFactory, HlsExtractorFactory extractorFactory, + @Nullable SubtitleParser.Factory subtitleParserFactory, CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory, @Nullable CmcdConfiguration cmcdConfiguration, DrmSessionManager drmSessionManager, @@ -437,6 +477,7 @@ public final class HlsMediaSource extends BaseMediaSource this.liveConfiguration = mediaItem.liveConfiguration; this.dataSourceFactory = dataSourceFactory; this.extractorFactory = extractorFactory; + this.subtitleParserFactory = subtitleParserFactory; this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory; this.cmcdConfiguration = cmcdConfiguration; this.drmSessionManager = drmSessionManager; @@ -511,7 +552,8 @@ public final class HlsMediaSource extends BaseMediaSource metadataType, useSessionKeys, getPlayerId(), - timestampAdjusterInitializationTimeoutMs); + timestampAdjusterInitializationTimeoutMs, + subtitleParserFactory); } @Override diff --git a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java index 9acace701f..bdebc756be 100644 --- a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java +++ b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java @@ -97,7 +97,8 @@ public final class HlsMediaPeriodTest { HlsMediaSource.METADATA_TYPE_ID3, /* useSessionKeys= */ false, PlayerId.UNSET, - /* timestampAdjusterInitializationTimeoutMs= */ 0); + /* timestampAdjusterInitializationTimeoutMs= */ 0, + /* subtitleParserFactory= */ null); }; MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( diff --git a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java index 7fd7b5a231..0cd6728217 100644 --- a/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java +++ b/libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java @@ -21,7 +21,9 @@ import android.graphics.SurfaceTexture; import android.view.Surface; import androidx.media3.common.MediaItem; import androidx.media3.common.Player; +import androidx.media3.datasource.DefaultDataSource; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.hls.HlsMediaSource; import androidx.media3.test.utils.CapturingRenderersFactory; import androidx.media3.test.utils.DumpFileAsserts; import androidx.media3.test.utils.FakeClock; @@ -30,7 +32,6 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig; import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -44,14 +45,15 @@ public final class HlsPlaybackTest { ShadowMediaCodecConfig.forAllSupportedMimeTypes(); @Test - @Ignore( - "Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].") public void webvttSubtitles() throws Exception { Context applicationContext = ApplicationProvider.getApplicationContext(); CapturingRenderersFactory capturingRenderersFactory = new CapturingRenderersFactory(applicationContext); ExoPlayer player = new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setMediaSourceFactory( + new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext)) + .experimentalParseSubtitlesDuringExtraction(true)) .setClock(new FakeClock(/* isAutoAdvancing= */ true)) .build(); player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));