sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-06 11:25:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add experimental opt-in to parse HLS subtitles during extraction

Browse source 
PiperOrigin-RevId: 578524318
This commit is contained in:
jbibik 2023-11-01 08:23:33 -07:00 committed by Copybara-Service
parent 72b7019578
commit 7b762642db
7 changed files with 152 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
RELEASENOTES.md
View file

@ -46,6 +46,9 @@
* HLS Extension:
* Reduce `HlsMediaPeriod` to package-private visibility. This type
shouldn't be directly depended on from outside the HLS package.
* Add experimental support for parsing subtitles during extraction. You
can enable this using
`HlsMediaSource.Factory.experimentalParseSubtitlesDuringExtraction()`.
* DASH Extension:
* Extend experimental support for parsing subtitles during extraction to
work with standalone text files (previously it only worked with

37
libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/BundledHlsMediaChunkExtractor.java
View file

@ -17,6 +17,7 @@ package androidx.media3.exoplayer.hls;
import static androidx.media3.common.util.Assertions.checkState;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.Format;
import androidx.media3.common.util.TimestampAdjuster;
@ -27,6 +28,8 @@ import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.mp3.Mp3Extractor;
import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
import androidx.media3.extractor.ts.Ac3Extractor;
import androidx.media3.extractor.ts.Ac4Extractor;
import androidx.media3.extractor.ts.AdtsExtractor;
@ -45,6 +48,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
@VisibleForTesting /* package */ final Extractor extractor;
private final Format multivariantPlaylistFormat;
private final TimestampAdjuster timestampAdjuster;
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
/**
* Creates a new instance.
@ -55,9 +59,35 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
*/
public BundledHlsMediaChunkExtractor(
Extractor extractor, Format multivariantPlaylistFormat, TimestampAdjuster timestampAdjuster) {
this(
extractor,
multivariantPlaylistFormat,
timestampAdjuster,
/* subtitleParserFactory= */ null);
}
/**
* Creates a new instance.
*
* @param extractor The underlying {@link Extractor}.
* @param multivariantPlaylistFormat The {@link Format} obtained from the multivariant playlist.
* @param timestampAdjuster A {@link TimestampAdjuster} to adjust sample timestamps.
* @param subtitleParserFactory A {@link SubtitleParser.Factory} to be used with WebVTT subtitles.
* If the value is null, subtitles will be parsed during decoding, otherwise - during
* extraction. Decoding will only work if this subtitleParserFactory supports the provided
* multivariantPlaylistFormat.
*/
// TODO(b/289983417): Once the subtitle-parsing-during-extraction is the only available flow, make
// this constructor public and remove @Nullable from subtitleParserFactory
/* package */ BundledHlsMediaChunkExtractor(
Extractor extractor,
Format multivariantPlaylistFormat,
TimestampAdjuster timestampAdjuster,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.extractor = extractor;
this.multivariantPlaylistFormat = multivariantPlaylistFormat;
this.timestampAdjuster = timestampAdjuster;
this.subtitleParserFactory = subtitleParserFactory;
}
@Override
@ -97,6 +127,11 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
if (extractor instanceof WebvttExtractor) {
newExtractorInstance =
new WebvttExtractor(multivariantPlaylistFormat.language, timestampAdjuster);
if (subtitleParserFactory != null
&& subtitleParserFactory.supportsFormat(multivariantPlaylistFormat)) {
newExtractorInstance =
new SubtitleTranscodingExtractor(newExtractorInstance, subtitleParserFactory);
}
} else if (extractor instanceof AdtsExtractor) {
newExtractorInstance = new AdtsExtractor();
} else if (extractor instanceof Ac3Extractor) {
@ -110,7 +145,7 @@ public final class BundledHlsMediaChunkExtractor implements HlsMediaChunkExtract
"Unexpected extractor type for recreation: " + extractor.getClass().getSimpleName());
}
return new BundledHlsMediaChunkExtractor(
newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster);
newExtractorInstance, multivariantPlaylistFormat, timestampAdjuster, subtitleParserFactory);
}
@Override

35
libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/DefaultHlsExtractorFactory.java
View file

@ -32,6 +32,8 @@ import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.mp3.Mp3Extractor;
import androidx.media3.extractor.mp4.FragmentedMp4Extractor;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.extractor.text.SubtitleTranscodingExtractor;
import androidx.media3.extractor.ts.Ac3Extractor;
import androidx.media3.extractor.ts.Ac4Extractor;
import androidx.media3.extractor.ts.AdtsExtractor;
@ -63,6 +65,10 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
};
private final @DefaultTsPayloadReaderFactory.Flags int payloadReaderFactoryFlags;
/** Non-null if subtitles should be parsed during extraction, null otherwise. */
@Nullable private SubtitleParser.Factory subtitleParserFactory;
private final boolean exposeCea608WhenMissingDeclarations;
/**
@ -127,7 +133,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
checkNotNull(
createExtractorByFileType(fileType, format, muxedCaptionFormats, timestampAdjuster));
if (sniffQuietly(extractor, sniffingExtractorInput)) {
return new BundledHlsMediaChunkExtractor(extractor, format, timestampAdjuster);
return new BundledHlsMediaChunkExtractor(
extractor, format, timestampAdjuster, subtitleParserFactory);
}
if (fallBackExtractor == null
&& (fileType == formatInferredFileType
@ -141,7 +148,24 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
}
return new BundledHlsMediaChunkExtractor(
checkNotNull(fallBackExtractor), format, timestampAdjuster);
checkNotNull(fallBackExtractor), format, timestampAdjuster, subtitleParserFactory);
}
/**
* Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or null
* to parse subtitles during decoding. The default is null (subtitles parsed after decoding).
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
* extraction.
* @return This factory, for convenience.
*/
public DefaultHlsExtractorFactory experimentalSetSubtitleParserFactory(
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.subtitleParserFactory = subtitleParserFactory;
return this;
}
private static void addFileTypeIfValidAndNotPresent(
@ -162,7 +186,12 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
// LINT.IfChange(extractor_instantiation)
switch (fileType) {
case FileTypes.WEBVTT:
return new WebvttExtractor(format.language, timestampAdjuster);
if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(format)) {
return new SubtitleTranscodingExtractor(
new WebvttExtractor(format.language, timestampAdjuster), subtitleParserFactory);
} else {
return new WebvttExtractor(format.language, timestampAdjuster);
}
case FileTypes.ADTS:
return new AdtsExtractor();
case FileTypes.AC3:

36
libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java
View file

@ -51,6 +51,7 @@ import androidx.media3.exoplayer.upstream.Allocator;
import androidx.media3.exoplayer.upstream.CmcdConfiguration;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.common.primitives.Ints;
import java.io.IOException;
import java.util.ArrayList;
@ -85,6 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final PlayerId playerId;
private final HlsSampleStreamWrapper.Callback sampleStreamWrapperCallback;
private final long timestampAdjusterInitializationTimeoutMs;
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
@Nullable private MediaPeriod.Callback mediaPeriodCallback;
private int pendingPrepareCount;
@ -139,7 +141,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@HlsMediaSource.MetadataType int metadataType,
boolean useSessionKeys,
PlayerId playerId,
long timestampAdjusterInitializationTimeoutMs) {
long timestampAdjusterInitializationTimeoutMs,
@Nullable SubtitleParser.Factory subtitleParserFactory) {
this.extractorFactory = extractorFactory;
this.playlistTracker = playlistTracker;
this.dataSourceFactory = dataSourceFactory;
@ -164,6 +167,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
sampleStreamWrappers = new HlsSampleStreamWrapper[0];
enabledSampleStreamWrappers = new HlsSampleStreamWrapper[0];
manifestUrlIndicesPerWrapper = new int[0][];
this.subtitleParserFactory = subtitleParserFactory;
}
public void release() {
@ -517,21 +521,43 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
for (int i = 0; i < subtitleRenditions.size(); i++) {
Rendition subtitleRendition = subtitleRenditions.get(i);
String sampleStreamWrapperUid = "subtitle:" + i + ":" + subtitleRendition.name;
// Format for HlsChunkSource to createExtractor with
Format originalSubtitleFormat = subtitleRendition.format;
HlsSampleStreamWrapper sampleStreamWrapper =
buildSampleStreamWrapper(
sampleStreamWrapperUid,
C.TRACK_TYPE_TEXT,
new Uri[] {subtitleRendition.url},
new Format[] {subtitleRendition.format},
new Format[] {originalSubtitleFormat},
null,
Collections.emptyList(),
overridingDrmInitData,
positionUs);
manifestUrlIndicesPerWrapper.add(new int[] {i});
sampleStreamWrappers.add(sampleStreamWrapper);
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, subtitleRendition.format)},
/* primaryTrackGroupIndex= */ 0);
if (subtitleParserFactory != null
&& subtitleParserFactory.supportsFormat(originalSubtitleFormat)) {
Format updatedSubtitleFormat =
originalSubtitleFormat
.buildUpon()
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
.setCueReplacementBehavior(
subtitleParserFactory.getCueReplacementBehavior(originalSubtitleFormat))
.setCodecs(
originalSubtitleFormat.sampleMimeType
+ (originalSubtitleFormat.codecs != null
? " " + originalSubtitleFormat.codecs
: ""))
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
.build();
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, updatedSubtitleFormat)},
/* primaryTrackGroupIndex= */ 0);
} else {
sampleStreamWrapper.prepareWithMultivariantPlaylistInfo(
new TrackGroup[] {new TrackGroup(sampleStreamWrapperUid, originalSubtitleFormat)},
/* primaryTrackGroupIndex= */ 0);
}
}
this.sampleStreamWrappers = sampleStreamWrappers.toArray(new HlsSampleStreamWrapper[0]);

44
libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaSource.java
View file

@ -58,6 +58,8 @@ import androidx.media3.exoplayer.upstream.CmcdConfiguration;
import androidx.media3.exoplayer.upstream.DefaultLoadErrorHandlingPolicy;
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy;
import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.extractor.text.SubtitleParser;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.IOException;
import java.lang.annotation.Documented;
@ -111,6 +113,7 @@ public final class HlsMediaSource extends BaseMediaSource
@Nullable private CmcdConfiguration.Factory cmcdConfigurationFactory;
private DrmSessionManagerProvider drmSessionManagerProvider;
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
@Nullable private SubtitleParser.Factory subtitleParserFactory;
private boolean allowChunklessPreparation;
private @MetadataType int metadataType;
private boolean useSessionKeys;
@ -196,6 +199,40 @@ public final class HlsMediaSource extends BaseMediaSource
return this;
}
/**
* Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
* part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
* as part of rendering).
*
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
* in a future release.
*
* <p>This method may only be used with {@link DefaultHlsExtractorFactory}.
*
* @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
* rendering.
* @return This factory, for convenience.
*/
// TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
// DefaultMediaSourceFactory via the MediaSource.Factory interface).
public Factory experimentalParseSubtitlesDuringExtraction(
boolean parseSubtitlesDuringExtraction) {
if (parseSubtitlesDuringExtraction) {
if (subtitleParserFactory == null) {
this.subtitleParserFactory = new DefaultSubtitleParserFactory();
}
} else {
this.subtitleParserFactory = null;
}
if (extractorFactory instanceof DefaultHlsExtractorFactory) {
((DefaultHlsExtractorFactory) extractorFactory)
.experimentalSetSubtitleParserFactory(subtitleParserFactory);
} else {
throw new IllegalStateException();
}
return this;
}
/**
* Sets the factory from which playlist parsers will be obtained.
*
@ -381,6 +418,7 @@ public final class HlsMediaSource extends BaseMediaSource
mediaItem,
hlsDataSourceFactory,
extractorFactory,
subtitleParserFactory,
compositeSequenceableLoaderFactory,
cmcdConfiguration,
drmSessionManagerProvider.get(mediaItem),
@ -412,6 +450,7 @@ public final class HlsMediaSource extends BaseMediaSource
private final HlsPlaylistTracker playlistTracker;
private final long elapsedRealTimeOffsetMs;
private final long timestampAdjusterInitializationTimeoutMs;
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
private MediaItem.LiveConfiguration liveConfiguration;
@Nullable private TransferListener mediaTransferListener;
@ -423,6 +462,7 @@ public final class HlsMediaSource extends BaseMediaSource
MediaItem mediaItem,
HlsDataSourceFactory dataSourceFactory,
HlsExtractorFactory extractorFactory,
@Nullable SubtitleParser.Factory subtitleParserFactory,
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
@Nullable CmcdConfiguration cmcdConfiguration,
DrmSessionManager drmSessionManager,
@ -437,6 +477,7 @@ public final class HlsMediaSource extends BaseMediaSource
this.liveConfiguration = mediaItem.liveConfiguration;
this.dataSourceFactory = dataSourceFactory;
this.extractorFactory = extractorFactory;
this.subtitleParserFactory = subtitleParserFactory;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
this.cmcdConfiguration = cmcdConfiguration;
this.drmSessionManager = drmSessionManager;
@ -511,7 +552,8 @@ public final class HlsMediaSource extends BaseMediaSource
metadataType,
useSessionKeys,
getPlayerId(),
timestampAdjusterInitializationTimeoutMs);
timestampAdjusterInitializationTimeoutMs,
subtitleParserFactory);
}
@Override

3
libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/HlsMediaPeriodTest.java
View file

@ -97,7 +97,8 @@ public final class HlsMediaPeriodTest {
HlsMediaSource.METADATA_TYPE_ID3,
/* useSessionKeys= */ false,
PlayerId.UNSET,
/* timestampAdjusterInitializationTimeoutMs= */ 0);
/* timestampAdjusterInitializationTimeoutMs= */ 0,
/* subtitleParserFactory= */ null);
};
MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration(

8
libraries/exoplayer_hls/src/test/java/androidx/media3/exoplayer/hls/e2etest/HlsPlaybackTest.java
View file

@ -21,7 +21,9 @@ import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.media3.common.MediaItem;
import androidx.media3.common.Player;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.hls.HlsMediaSource;
import androidx.media3.test.utils.CapturingRenderersFactory;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.media3.test.utils.FakeClock;
@ -30,7 +32,6 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -44,14 +45,15 @@ public final class HlsPlaybackTest {
ShadowMediaCodecConfig.forAllSupportedMimeTypes();
@Test
@Ignore(
"Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
public void webvttSubtitles() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setMediaSourceFactory(
new HlsMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
.experimentalParseSubtitlesDuringExtraction(true))
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));