From 592b5eafee3613646f0a8e6bfffd3eec6beef5a4 Mon Sep 17 00:00:00 2001 From: olly Date: Sun, 31 Mar 2019 23:40:58 +0100 Subject: [PATCH] Support multiple CC channels in DASH Issue: #5656 PiperOrigin-RevId: 241235377 --- RELEASENOTES.md | 5 +- .../source/dash/DashChunkSource.java | 10 +- .../source/dash/DashMediaPeriod.java | 185 ++++++++++++------ .../source/dash/DefaultDashChunkSource.java | 26 +-- 4 files changed, 148 insertions(+), 78 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 989d67a49a..2f49762ada 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -5,7 +5,10 @@ * Update to Mockito 2 * Add new `ExoPlaybackException` types for remote exceptions and out-of-memory errors. -* DASH: Parse role and accessibility descriptors into `Format.roleFlags`. +* DASH: + * Parse role and accessibility descriptors into `Format.roleFlags`. + * Support multiple CEA-608 channels muxed into FMP4 representations + ([#5656](https://github.com/google/ExoPlayer/issues/5656)). * HLS: * Work around lack of LA_URL attribute in PlayReady key request init data. * Prevent unnecessary reloads of initialization segments. diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java index f808d7c1b6..40d4e468bd 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java @@ -17,12 +17,14 @@ package com.google.android.exoplayer2.source.dash; import android.os.SystemClock; import androidx.annotation.Nullable; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.source.chunk.ChunkSource; import com.google.android.exoplayer2.source.dash.PlayerEmsgHandler.PlayerTrackEmsgHandler; import com.google.android.exoplayer2.source.dash.manifest.DashManifest; import com.google.android.exoplayer2.trackselection.TrackSelection; import com.google.android.exoplayer2.upstream.LoaderErrorThrower; import com.google.android.exoplayer2.upstream.TransferListener; +import java.util.List; /** * An {@link ChunkSource} for DASH streams. @@ -41,10 +43,8 @@ public interface DashChunkSource extends ChunkSource { * @param elapsedRealtimeOffsetMs If known, an estimate of the instantaneous difference between * server-side unix time and {@link SystemClock#elapsedRealtime()} in milliseconds, * specified as the server's unix time minus the local elapsed time. If unknown, set to 0. - * @param enableEventMessageTrack Whether the chunks generated by the source may output an event - * message track. - * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608 - * track. + * @param enableEventMessageTrack Whether to output an event message track. + * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output. * @param transferListener The transfer listener which should be informed of any data transfers. * May be null if no listener is available. * @return The created {@link DashChunkSource}. @@ -58,7 +58,7 @@ public interface DashChunkSource extends ChunkSource { int type, long elapsedRealtimeOffsetMs, boolean enableEventMessageTrack, - boolean enableCea608Track, + List closedCaptionFormats, @Nullable PlayerTrackEmsgHandler playerEmsgHandler, @Nullable TransferListener transferListener); } diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java index a2a7208d99..431a0a4bd9 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java @@ -56,6 +56,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.IdentityHashMap; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** A DASH {@link MediaPeriod}. */ /* package */ final class DashMediaPeriod @@ -63,6 +65,8 @@ import java.util.List; SequenceableLoader.Callback>, ChunkSampleStream.ReleaseCallback { + private static final Pattern CEA608_SERVICE_DESCRIPTOR_REGEX = Pattern.compile("CC([1-4])=(.+)"); + /* package */ final int id; private final DashChunkSource.Factory chunkSourceFactory; private final @Nullable TransferListener transferListener; @@ -457,18 +461,28 @@ import java.util.List; int primaryGroupCount = groupedAdaptationSetIndices.length; boolean[] primaryGroupHasEventMessageTrackFlags = new boolean[primaryGroupCount]; - boolean[] primaryGroupHasCea608TrackFlags = new boolean[primaryGroupCount]; - int totalEmbeddedTrackGroupCount = identifyEmbeddedTracks(primaryGroupCount, adaptationSets, - groupedAdaptationSetIndices, primaryGroupHasEventMessageTrackFlags, - primaryGroupHasCea608TrackFlags); + Format[][] primaryGroupCea608TrackFormats = new Format[primaryGroupCount][]; + int totalEmbeddedTrackGroupCount = + identifyEmbeddedTracks( + primaryGroupCount, + adaptationSets, + groupedAdaptationSetIndices, + primaryGroupHasEventMessageTrackFlags, + primaryGroupCea608TrackFormats); int totalGroupCount = primaryGroupCount + totalEmbeddedTrackGroupCount + eventStreams.size(); TrackGroup[] trackGroups = new TrackGroup[totalGroupCount]; TrackGroupInfo[] trackGroupInfos = new TrackGroupInfo[totalGroupCount]; - int trackGroupCount = buildPrimaryAndEmbeddedTrackGroupInfos(adaptationSets, - groupedAdaptationSetIndices, primaryGroupCount, primaryGroupHasEventMessageTrackFlags, - primaryGroupHasCea608TrackFlags, trackGroups, trackGroupInfos); + int trackGroupCount = + buildPrimaryAndEmbeddedTrackGroupInfos( + adaptationSets, + groupedAdaptationSetIndices, + primaryGroupCount, + primaryGroupHasEventMessageTrackFlags, + primaryGroupCea608TrackFormats, + trackGroups, + trackGroupInfos); buildManifestEventTrackGroupInfos(eventStreams, trackGroups, trackGroupInfos, trackGroupCount); @@ -524,39 +538,46 @@ import java.util.List; /** * Iterates through list of primary track groups and identifies embedded tracks. - *

+ * * @param primaryGroupCount The number of primary track groups. * @param adaptationSets The list of {@link AdaptationSet} of the current DASH period. - * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to - * the same primary group, grouped in primary track groups order. - * @param primaryGroupHasEventMessageTrackFlags An output array containing boolean flag, each - * indicates whether the corresponding primary track group contains an embedded event message - * track. - * @param primaryGroupHasCea608TrackFlags An output array containing boolean flag, each - * indicates whether the corresponding primary track group contains an embedded Cea608 track. - * @return Total number of embedded tracks. + * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to the + * same primary group, grouped in primary track groups order. + * @param primaryGroupHasEventMessageTrackFlags An output array to be filled with flags indicating + * whether each of the primary track groups contains an embedded event message track. + * @param primaryGroupCea608TrackFormats An output array to be filled with track formats for + * CEA-608 tracks embedded in each of the primary track groups. + * @return Total number of embedded track groups. */ - private static int identifyEmbeddedTracks(int primaryGroupCount, - List adaptationSets, int[][] groupedAdaptationSetIndices, - boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags) { - int numEmbeddedTrack = 0; + private static int identifyEmbeddedTracks( + int primaryGroupCount, + List adaptationSets, + int[][] groupedAdaptationSetIndices, + boolean[] primaryGroupHasEventMessageTrackFlags, + Format[][] primaryGroupCea608TrackFormats) { + int numEmbeddedTrackGroups = 0; for (int i = 0; i < primaryGroupCount; i++) { if (hasEventMessageTrack(adaptationSets, groupedAdaptationSetIndices[i])) { primaryGroupHasEventMessageTrackFlags[i] = true; - numEmbeddedTrack++; + numEmbeddedTrackGroups++; } - if (hasCea608Track(adaptationSets, groupedAdaptationSetIndices[i])) { - primaryGroupHasCea608TrackFlags[i] = true; - numEmbeddedTrack++; + primaryGroupCea608TrackFormats[i] = + getCea608TrackFormats(adaptationSets, groupedAdaptationSetIndices[i]); + if (primaryGroupCea608TrackFormats[i].length != 0) { + numEmbeddedTrackGroups++; } } - return numEmbeddedTrack; + return numEmbeddedTrackGroups; } - private static int buildPrimaryAndEmbeddedTrackGroupInfos(List adaptationSets, - int[][] groupedAdaptationSetIndices, int primaryGroupCount, - boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags, - TrackGroup[] trackGroups, TrackGroupInfo[] trackGroupInfos) { + private static int buildPrimaryAndEmbeddedTrackGroupInfos( + List adaptationSets, + int[][] groupedAdaptationSetIndices, + int primaryGroupCount, + boolean[] primaryGroupHasEventMessageTrackFlags, + Format[][] primaryGroupCea608TrackFormats, + TrackGroup[] trackGroups, + TrackGroupInfo[] trackGroupInfos) { int trackGroupCount = 0; for (int i = 0; i < primaryGroupCount; i++) { int[] adaptationSetIndices = groupedAdaptationSetIndices[i]; @@ -574,7 +595,7 @@ import java.util.List; int eventMessageTrackGroupIndex = primaryGroupHasEventMessageTrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET; int cea608TrackGroupIndex = - primaryGroupHasCea608TrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET; + primaryGroupCea608TrackFormats[i].length != 0 ? trackGroupCount++ : C.INDEX_UNSET; trackGroups[primaryTrackGroupIndex] = new TrackGroup(formats); trackGroupInfos[primaryTrackGroupIndex] = @@ -592,9 +613,7 @@ import java.util.List; TrackGroupInfo.embeddedEmsgTrack(adaptationSetIndices, primaryTrackGroupIndex); } if (cea608TrackGroupIndex != C.INDEX_UNSET) { - Format format = Format.createTextSampleFormat(firstAdaptationSet.id + ":cea608", - MimeTypes.APPLICATION_CEA608, 0, null); - trackGroups[cea608TrackGroupIndex] = new TrackGroup(format); + trackGroups[cea608TrackGroupIndex] = new TrackGroup(primaryGroupCea608TrackFormats[i]); trackGroupInfos[cea608TrackGroupIndex] = TrackGroupInfo.embeddedCea608Track(adaptationSetIndices, primaryTrackGroupIndex); } @@ -616,25 +635,39 @@ import java.util.List; private ChunkSampleStream buildSampleStream(TrackGroupInfo trackGroupInfo, TrackSelection selection, long positionUs) { int embeddedTrackCount = 0; - int[] embeddedTrackTypes = new int[2]; - Format[] embeddedTrackFormats = new Format[2]; boolean enableEventMessageTrack = trackGroupInfo.embeddedEventMessageTrackGroupIndex != C.INDEX_UNSET; + TrackGroup embeddedEventMessageTrackGroup = null; if (enableEventMessageTrack) { - embeddedTrackFormats[embeddedTrackCount] = - trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex).getFormat(0); - embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_METADATA; + embeddedEventMessageTrackGroup = + trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex); + embeddedTrackCount++; } - boolean enableCea608Track = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET; - if (enableCea608Track) { - embeddedTrackFormats[embeddedTrackCount] = - trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex).getFormat(0); - embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_TEXT; + boolean enableCea608Tracks = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET; + TrackGroup embeddedCea608TrackGroup = null; + if (enableCea608Tracks) { + embeddedCea608TrackGroup = trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex); + embeddedTrackCount += embeddedCea608TrackGroup.length; } - if (embeddedTrackCount < embeddedTrackTypes.length) { - embeddedTrackFormats = Arrays.copyOf(embeddedTrackFormats, embeddedTrackCount); - embeddedTrackTypes = Arrays.copyOf(embeddedTrackTypes, embeddedTrackCount); + + Format[] embeddedTrackFormats = new Format[embeddedTrackCount]; + int[] embeddedTrackTypes = new int[embeddedTrackCount]; + embeddedTrackCount = 0; + if (enableEventMessageTrack) { + embeddedTrackFormats[embeddedTrackCount] = embeddedEventMessageTrackGroup.getFormat(0); + embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_METADATA; + embeddedTrackCount++; } + List embeddedCea608TrackFormats = new ArrayList<>(); + if (enableCea608Tracks) { + for (int i = 0; i < embeddedCea608TrackGroup.length; i++) { + embeddedTrackFormats[embeddedTrackCount] = embeddedCea608TrackGroup.getFormat(i); + embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_TEXT; + embeddedCea608TrackFormats.add(embeddedTrackFormats[embeddedTrackCount]); + embeddedTrackCount++; + } + } + PlayerTrackEmsgHandler trackPlayerEmsgHandler = manifest.dynamic && enableEventMessageTrack ? playerEmsgHandler.newPlayerTrackEmsgHandler() @@ -649,7 +682,7 @@ import java.util.List; trackGroupInfo.trackType, elapsedRealtimeOffsetMs, enableEventMessageTrack, - enableCea608Track, + embeddedCea608TrackFormats, trackPlayerEmsgHandler, transferListener); ChunkSampleStream stream = @@ -694,18 +727,60 @@ import java.util.List; return false; } - private static boolean hasCea608Track(List adaptationSets, - int[] adaptationSetIndices) { + private static Format[] getCea608TrackFormats( + List adaptationSets, int[] adaptationSetIndices) { for (int i : adaptationSetIndices) { + AdaptationSet adaptationSet = adaptationSets.get(i); List descriptors = adaptationSets.get(i).accessibilityDescriptors; for (int j = 0; j < descriptors.size(); j++) { Descriptor descriptor = descriptors.get(j); if ("urn:scte:dash:cc:cea-608:2015".equals(descriptor.schemeIdUri)) { - return true; + String value = descriptor.value; + if (value == null) { + // There are embedded CEA-608 tracks, but service information is not declared. + return new Format[] {buildCea608TrackFormat(adaptationSet.id)}; + } + String[] services = Util.split(value, ";"); + Format[] formats = new Format[services.length]; + for (int k = 0; k < services.length; k++) { + Matcher matcher = CEA608_SERVICE_DESCRIPTOR_REGEX.matcher(services[k]); + if (!matcher.matches()) { + // If we can't parse service information for all services, assume a single track. + return new Format[] {buildCea608TrackFormat(adaptationSet.id)}; + } + formats[k] = + buildCea608TrackFormat( + adaptationSet.id, + /* language= */ matcher.group(2), + /* accessibilityChannel= */ Integer.parseInt(matcher.group(1))); + } + return formats; } } } - return false; + return new Format[0]; + } + + private static Format buildCea608TrackFormat(int adaptationSetId) { + return buildCea608TrackFormat( + adaptationSetId, /* language= */ null, /* accessibilityChannel= */ Format.NO_VALUE); + } + + private static Format buildCea608TrackFormat( + int adaptationSetId, String language, int accessibilityChannel) { + return Format.createTextSampleFormat( + adaptationSetId + + ":cea608" + + (accessibilityChannel != Format.NO_VALUE ? ":" + accessibilityChannel : ""), + MimeTypes.APPLICATION_CEA608, + /* codecs= */ null, + /* bitrate= */ Format.NO_VALUE, + /* selectionFlags= */ 0, + language, + accessibilityChannel, + /* drmInitData= */ null, + Format.OFFSET_SAMPLE_RELATIVE, + /* initializationData= */ null); } @SuppressWarnings("unchecked") @@ -761,7 +836,7 @@ import java.util.List; primaryTrackGroupIndex, embeddedEventMessageTrackGroupIndex, embeddedCea608TrackGroupIndex, - -1); + /* eventStreamGroupIndex= */ -1); } public static TrackGroupInfo embeddedEmsgTrack(int[] adaptationSetIndices, @@ -773,7 +848,7 @@ import java.util.List; primaryTrackGroupIndex, C.INDEX_UNSET, C.INDEX_UNSET, - -1); + /* eventStreamGroupIndex= */ -1); } public static TrackGroupInfo embeddedCea608Track(int[] adaptationSetIndices, @@ -785,7 +860,7 @@ import java.util.List; primaryTrackGroupIndex, C.INDEX_UNSET, C.INDEX_UNSET, - -1); + /* eventStreamGroupIndex= */ -1); } public static TrackGroupInfo mpdEventTrack(int eventStreamIndex) { @@ -793,7 +868,7 @@ import java.util.List; C.TRACK_TYPE_METADATA, CATEGORY_MANIFEST_EVENTS, new int[0], - -1, + /* primaryTrackGroupIndex= */ -1, C.INDEX_UNSET, C.INDEX_UNSET, eventStreamIndex); diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java index 6282195d67..057f0262d0 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java @@ -54,7 +54,6 @@ import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -86,7 +85,7 @@ public class DefaultDashChunkSource implements DashChunkSource { int trackType, long elapsedRealtimeOffsetMs, boolean enableEventMessageTrack, - boolean enableCea608Track, + List closedCaptionFormats, @Nullable PlayerTrackEmsgHandler playerEmsgHandler, @Nullable TransferListener transferListener) { DataSource dataSource = dataSourceFactory.createDataSource(); @@ -104,7 +103,7 @@ public class DefaultDashChunkSource implements DashChunkSource { elapsedRealtimeOffsetMs, maxSegmentsPerLoad, enableEventMessageTrack, - enableCea608Track, + closedCaptionFormats, playerEmsgHandler); } @@ -141,9 +140,8 @@ public class DefaultDashChunkSource implements DashChunkSource { * @param maxSegmentsPerLoad The maximum number of segments to combine into a single request. Note * that segments will only be combined if their {@link Uri}s are the same and if their data * ranges are adjacent. - * @param enableEventMessageTrack Whether the chunks generated by the source may output an event - * message track. - * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608 track. + * @param enableEventMessageTrack Whether to output an event message track. + * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output. * @param playerTrackEmsgHandler The {@link PlayerTrackEmsgHandler} instance to handle emsg * messages targeting the player. Maybe null if this is not necessary. */ @@ -158,7 +156,7 @@ public class DefaultDashChunkSource implements DashChunkSource { long elapsedRealtimeOffsetMs, int maxSegmentsPerLoad, boolean enableEventMessageTrack, - boolean enableCea608Track, + List closedCaptionFormats, @Nullable PlayerTrackEmsgHandler playerTrackEmsgHandler) { this.manifestLoaderErrorThrower = manifestLoaderErrorThrower; this.manifest = manifest; @@ -184,7 +182,7 @@ public class DefaultDashChunkSource implements DashChunkSource { trackType, representation, enableEventMessageTrack, - enableCea608Track, + closedCaptionFormats, playerTrackEmsgHandler); } } @@ -629,7 +627,7 @@ public class DefaultDashChunkSource implements DashChunkSource { int trackType, Representation representation, boolean enableEventMessageTrack, - boolean enableCea608Track, + List closedCaptionFormats, TrackOutput playerEmsgTrackOutput) { this( periodDurationUs, @@ -638,7 +636,7 @@ public class DefaultDashChunkSource implements DashChunkSource { trackType, representation, enableEventMessageTrack, - enableCea608Track, + closedCaptionFormats, playerEmsgTrackOutput), /* segmentNumShift= */ 0, representation.getIndex()); @@ -783,7 +781,7 @@ public class DefaultDashChunkSource implements DashChunkSource { int trackType, Representation representation, boolean enableEventMessageTrack, - boolean enableCea608Track, + List closedCaptionFormats, TrackOutput playerEmsgTrackOutput) { String containerMimeType = representation.format.containerMimeType; if (mimeTypeIsRawText(containerMimeType)) { @@ -799,12 +797,6 @@ public class DefaultDashChunkSource implements DashChunkSource { if (enableEventMessageTrack) { flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK; } - // TODO: Use caption format information from the manifest if available. - List closedCaptionFormats = - enableCea608Track - ? Collections.singletonList( - Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0, null)) - : Collections.emptyList(); extractor = new FragmentedMp4Extractor( flags, null, null, null, closedCaptionFormats, playerEmsgTrackOutput);