Support multiple CC channels in DASH

Issue: #5656
PiperOrigin-RevId: 241235377

RELEASENOTES.md

@@ -5,7 +5,10 @@
 * Update to Mockito 2
 * Add new `ExoPlaybackException` types for remote exceptions and out-of-memory
   errors.
-* DASH: Parse role and accessibility descriptors into `Format.roleFlags`.
+* DASH:
+  * Parse role and accessibility descriptors into `Format.roleFlags`.
+  * Support multiple CEA-608 channels muxed into FMP4 representations
+    ([#5656](https://github.com/google/ExoPlayer/issues/5656)).
 * HLS:
   * Work around lack of LA_URL attribute in PlayReady key request init data.
   * Prevent unnecessary reloads of initialization segments.

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashChunkSource.java

@@ -17,12 +17,14 @@ package com.google.android.exoplayer2.source.dash;
 
 import android.os.SystemClock;
 import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.source.chunk.ChunkSource;
 import com.google.android.exoplayer2.source.dash.PlayerEmsgHandler.PlayerTrackEmsgHandler;
 import com.google.android.exoplayer2.source.dash.manifest.DashManifest;
 import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.upstream.LoaderErrorThrower;
 import com.google.android.exoplayer2.upstream.TransferListener;
+import java.util.List;
 
 /**
  * An {@link ChunkSource} for DASH streams.
@@ -41,10 +43,8 @@ public interface DashChunkSource extends ChunkSource {
      * @param elapsedRealtimeOffsetMs If known, an estimate of the instantaneous difference between
      *     server-side unix time and {@link SystemClock#elapsedRealtime()} in milliseconds,
      *     specified as the server's unix time minus the local elapsed time. If unknown, set to 0.
-     * @param enableEventMessageTrack Whether the chunks generated by the source may output an event
+     * @param enableEventMessageTrack Whether to output an event message track.
-     *     message track.
+     * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output.
-     * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608
-     *     track.
      * @param transferListener The transfer listener which should be informed of any data transfers.
      *     May be null if no listener is available.
      * @return The created {@link DashChunkSource}.
@@ -58,7 +58,7 @@ public interface DashChunkSource extends ChunkSource {
         int type,
         long elapsedRealtimeOffsetMs,
         boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
         @Nullable PlayerTrackEmsgHandler playerEmsgHandler,
         @Nullable TransferListener transferListener);
   }

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java

@@ -56,6 +56,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /** A DASH {@link MediaPeriod}. */
 /* package */ final class DashMediaPeriod
@@ -63,6 +65,8 @@ import java.util.List;
         SequenceableLoader.Callback<ChunkSampleStream<DashChunkSource>>,
         ChunkSampleStream.ReleaseCallback<DashChunkSource> {
 
+  private static final Pattern CEA608_SERVICE_DESCRIPTOR_REGEX = Pattern.compile("CC([1-4])=(.+)");
+
   /* package */ final int id;
   private final DashChunkSource.Factory chunkSourceFactory;
   private final @Nullable TransferListener transferListener;
@@ -457,18 +461,28 @@ import java.util.List;
 
     int primaryGroupCount = groupedAdaptationSetIndices.length;
     boolean[] primaryGroupHasEventMessageTrackFlags = new boolean[primaryGroupCount];
-    boolean[] primaryGroupHasCea608TrackFlags = new boolean[primaryGroupCount];
+    Format[][] primaryGroupCea608TrackFormats = new Format[primaryGroupCount][];
-    int totalEmbeddedTrackGroupCount = identifyEmbeddedTracks(primaryGroupCount, adaptationSets,
+    int totalEmbeddedTrackGroupCount =
-        groupedAdaptationSetIndices, primaryGroupHasEventMessageTrackFlags,
+        identifyEmbeddedTracks(
-        primaryGroupHasCea608TrackFlags);
+            primaryGroupCount,
+            adaptationSets,
+            groupedAdaptationSetIndices,
+            primaryGroupHasEventMessageTrackFlags,
+            primaryGroupCea608TrackFormats);
 
     int totalGroupCount = primaryGroupCount + totalEmbeddedTrackGroupCount + eventStreams.size();
     TrackGroup[] trackGroups = new TrackGroup[totalGroupCount];
     TrackGroupInfo[] trackGroupInfos = new TrackGroupInfo[totalGroupCount];
 
-    int trackGroupCount = buildPrimaryAndEmbeddedTrackGroupInfos(adaptationSets,
+    int trackGroupCount =
-        groupedAdaptationSetIndices, primaryGroupCount, primaryGroupHasEventMessageTrackFlags,
+        buildPrimaryAndEmbeddedTrackGroupInfos(
-        primaryGroupHasCea608TrackFlags, trackGroups, trackGroupInfos);
+            adaptationSets,
+            groupedAdaptationSetIndices,
+            primaryGroupCount,
+            primaryGroupHasEventMessageTrackFlags,
+            primaryGroupCea608TrackFormats,
+            trackGroups,
+            trackGroupInfos);
 
     buildManifestEventTrackGroupInfos(eventStreams, trackGroups, trackGroupInfos, trackGroupCount);
 
@@ -524,39 +538,46 @@ import java.util.List;
 
   /**
    * Iterates through list of primary track groups and identifies embedded tracks.
-   * <p>
+   *
    * @param primaryGroupCount The number of primary track groups.
    * @param adaptationSets The list of {@link AdaptationSet} of the current DASH period.
-   * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to
+   * @param groupedAdaptationSetIndices The indices of {@link AdaptationSet} that belongs to the
-   *     the same primary group, grouped in primary track groups order.
+   *     same primary group, grouped in primary track groups order.
-   * @param primaryGroupHasEventMessageTrackFlags An output array containing boolean flag, each
+   * @param primaryGroupHasEventMessageTrackFlags An output array to be filled with flags indicating
-   *     indicates whether the corresponding primary track group contains an embedded event message
+   *     whether each of the primary track groups contains an embedded event message track.
-   *     track.
+   * @param primaryGroupCea608TrackFormats An output array to be filled with track formats for
-   * @param primaryGroupHasCea608TrackFlags An output array containing boolean flag, each
+   *     CEA-608 tracks embedded in each of the primary track groups.
-   *     indicates whether the corresponding primary track group contains an embedded Cea608 track.
+   * @return Total number of embedded track groups.
-   * @return Total number of embedded tracks.
    */
-  private static int identifyEmbeddedTracks(int primaryGroupCount,
+  private static int identifyEmbeddedTracks(
-      List<AdaptationSet> adaptationSets, int[][] groupedAdaptationSetIndices,
+      int primaryGroupCount,
-      boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags) {
+      List<AdaptationSet> adaptationSets,
-    int numEmbeddedTrack = 0;
+      int[][] groupedAdaptationSetIndices,
+      boolean[] primaryGroupHasEventMessageTrackFlags,
+      Format[][] primaryGroupCea608TrackFormats) {
+    int numEmbeddedTrackGroups = 0;
     for (int i = 0; i < primaryGroupCount; i++) {
       if (hasEventMessageTrack(adaptationSets, groupedAdaptationSetIndices[i])) {
         primaryGroupHasEventMessageTrackFlags[i] = true;
-        numEmbeddedTrack++;
+        numEmbeddedTrackGroups++;
       }
-      if (hasCea608Track(adaptationSets, groupedAdaptationSetIndices[i])) {
+      primaryGroupCea608TrackFormats[i] =
-        primaryGroupHasCea608TrackFlags[i] = true;
+          getCea608TrackFormats(adaptationSets, groupedAdaptationSetIndices[i]);
-        numEmbeddedTrack++;
+      if (primaryGroupCea608TrackFormats[i].length != 0) {
+        numEmbeddedTrackGroups++;
       }
     }
-    return numEmbeddedTrack;
+    return numEmbeddedTrackGroups;
   }
 
-  private static int buildPrimaryAndEmbeddedTrackGroupInfos(List<AdaptationSet> adaptationSets,
+  private static int buildPrimaryAndEmbeddedTrackGroupInfos(
-      int[][] groupedAdaptationSetIndices, int primaryGroupCount,
+      List<AdaptationSet> adaptationSets,
-      boolean[] primaryGroupHasEventMessageTrackFlags, boolean[] primaryGroupHasCea608TrackFlags,
+      int[][] groupedAdaptationSetIndices,
-      TrackGroup[] trackGroups, TrackGroupInfo[] trackGroupInfos) {
+      int primaryGroupCount,
+      boolean[] primaryGroupHasEventMessageTrackFlags,
+      Format[][] primaryGroupCea608TrackFormats,
+      TrackGroup[] trackGroups,
+      TrackGroupInfo[] trackGroupInfos) {
     int trackGroupCount = 0;
     for (int i = 0; i < primaryGroupCount; i++) {
       int[] adaptationSetIndices = groupedAdaptationSetIndices[i];
@@ -574,7 +595,7 @@ import java.util.List;
       int eventMessageTrackGroupIndex =
           primaryGroupHasEventMessageTrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET;
       int cea608TrackGroupIndex =
-          primaryGroupHasCea608TrackFlags[i] ? trackGroupCount++ : C.INDEX_UNSET;
+          primaryGroupCea608TrackFormats[i].length != 0 ? trackGroupCount++ : C.INDEX_UNSET;
 
       trackGroups[primaryTrackGroupIndex] = new TrackGroup(formats);
       trackGroupInfos[primaryTrackGroupIndex] =
@@ -592,9 +613,7 @@ import java.util.List;
             TrackGroupInfo.embeddedEmsgTrack(adaptationSetIndices, primaryTrackGroupIndex);
       }
       if (cea608TrackGroupIndex != C.INDEX_UNSET) {
-        Format format = Format.createTextSampleFormat(firstAdaptationSet.id + ":cea608",
+        trackGroups[cea608TrackGroupIndex] = new TrackGroup(primaryGroupCea608TrackFormats[i]);
-            MimeTypes.APPLICATION_CEA608, 0, null);
-        trackGroups[cea608TrackGroupIndex] = new TrackGroup(format);
         trackGroupInfos[cea608TrackGroupIndex] =
             TrackGroupInfo.embeddedCea608Track(adaptationSetIndices, primaryTrackGroupIndex);
       }
@@ -616,25 +635,39 @@ import java.util.List;
   private ChunkSampleStream<DashChunkSource> buildSampleStream(TrackGroupInfo trackGroupInfo,
       TrackSelection selection, long positionUs) {
     int embeddedTrackCount = 0;
-    int[] embeddedTrackTypes = new int[2];
-    Format[] embeddedTrackFormats = new Format[2];
     boolean enableEventMessageTrack =
         trackGroupInfo.embeddedEventMessageTrackGroupIndex != C.INDEX_UNSET;
+    TrackGroup embeddedEventMessageTrackGroup = null;
     if (enableEventMessageTrack) {
-      embeddedTrackFormats[embeddedTrackCount] =
+      embeddedEventMessageTrackGroup =
-          trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex).getFormat(0);
+          trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex);
-      embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_METADATA;
+      embeddedTrackCount++;
     }
-    boolean enableCea608Track = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET;
+    boolean enableCea608Tracks = trackGroupInfo.embeddedCea608TrackGroupIndex != C.INDEX_UNSET;
-    if (enableCea608Track) {
+    TrackGroup embeddedCea608TrackGroup = null;
-      embeddedTrackFormats[embeddedTrackCount] =
+    if (enableCea608Tracks) {
-          trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex).getFormat(0);
+      embeddedCea608TrackGroup = trackGroups.get(trackGroupInfo.embeddedCea608TrackGroupIndex);
-      embeddedTrackTypes[embeddedTrackCount++] = C.TRACK_TYPE_TEXT;
+      embeddedTrackCount += embeddedCea608TrackGroup.length;
     }
-    if (embeddedTrackCount < embeddedTrackTypes.length) {
+
-      embeddedTrackFormats = Arrays.copyOf(embeddedTrackFormats, embeddedTrackCount);
+    Format[] embeddedTrackFormats = new Format[embeddedTrackCount];
-      embeddedTrackTypes = Arrays.copyOf(embeddedTrackTypes, embeddedTrackCount);
+    int[] embeddedTrackTypes = new int[embeddedTrackCount];
+    embeddedTrackCount = 0;
+    if (enableEventMessageTrack) {
+      embeddedTrackFormats[embeddedTrackCount] = embeddedEventMessageTrackGroup.getFormat(0);
+      embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_METADATA;
+      embeddedTrackCount++;
     }
+    List<Format> embeddedCea608TrackFormats = new ArrayList<>();
+    if (enableCea608Tracks) {
+      for (int i = 0; i < embeddedCea608TrackGroup.length; i++) {
+        embeddedTrackFormats[embeddedTrackCount] = embeddedCea608TrackGroup.getFormat(i);
+        embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_TEXT;
+        embeddedCea608TrackFormats.add(embeddedTrackFormats[embeddedTrackCount]);
+        embeddedTrackCount++;
+      }
+    }
+
     PlayerTrackEmsgHandler trackPlayerEmsgHandler =
         manifest.dynamic && enableEventMessageTrack
             ? playerEmsgHandler.newPlayerTrackEmsgHandler()
@@ -649,7 +682,7 @@ import java.util.List;
             trackGroupInfo.trackType,
             elapsedRealtimeOffsetMs,
             enableEventMessageTrack,
-            enableCea608Track,
+            embeddedCea608TrackFormats,
             trackPlayerEmsgHandler,
             transferListener);
     ChunkSampleStream<DashChunkSource> stream =
@@ -694,18 +727,60 @@ import java.util.List;
     return false;
   }
 
-  private static boolean hasCea608Track(List<AdaptationSet> adaptationSets,
+  private static Format[] getCea608TrackFormats(
-      int[] adaptationSetIndices) {
+      List<AdaptationSet> adaptationSets, int[] adaptationSetIndices) {
     for (int i : adaptationSetIndices) {
+      AdaptationSet adaptationSet = adaptationSets.get(i);
       List<Descriptor> descriptors = adaptationSets.get(i).accessibilityDescriptors;
       for (int j = 0; j < descriptors.size(); j++) {
         Descriptor descriptor = descriptors.get(j);
         if ("urn:scte:dash:cc:cea-608:2015".equals(descriptor.schemeIdUri)) {
-          return true;
+          String value = descriptor.value;
+          if (value == null) {
+            // There are embedded CEA-608 tracks, but service information is not declared.
+            return new Format[] {buildCea608TrackFormat(adaptationSet.id)};
+          }
+          String[] services = Util.split(value, ";");
+          Format[] formats = new Format[services.length];
+          for (int k = 0; k < services.length; k++) {
+            Matcher matcher = CEA608_SERVICE_DESCRIPTOR_REGEX.matcher(services[k]);
+            if (!matcher.matches()) {
+              // If we can't parse service information for all services, assume a single track.
+              return new Format[] {buildCea608TrackFormat(adaptationSet.id)};
+            }
+            formats[k] =
+                buildCea608TrackFormat(
+                    adaptationSet.id,
+                    /* language= */ matcher.group(2),
+                    /* accessibilityChannel= */ Integer.parseInt(matcher.group(1)));
+          }
+          return formats;
         }
       }
     }
-    return false;
+    return new Format[0];
+  }
+
+  private static Format buildCea608TrackFormat(int adaptationSetId) {
+    return buildCea608TrackFormat(
+        adaptationSetId, /* language= */ null, /* accessibilityChannel= */ Format.NO_VALUE);
+  }
+
+  private static Format buildCea608TrackFormat(
+      int adaptationSetId, String language, int accessibilityChannel) {
+    return Format.createTextSampleFormat(
+        adaptationSetId
+            + ":cea608"
+            + (accessibilityChannel != Format.NO_VALUE ? ":" + accessibilityChannel : ""),
+        MimeTypes.APPLICATION_CEA608,
+        /* codecs= */ null,
+        /* bitrate= */ Format.NO_VALUE,
+        /* selectionFlags= */ 0,
+        language,
+        accessibilityChannel,
+        /* drmInitData= */ null,
+        Format.OFFSET_SAMPLE_RELATIVE,
+        /* initializationData= */ null);
   }
 
   @SuppressWarnings("unchecked")
@@ -761,7 +836,7 @@ import java.util.List;
           primaryTrackGroupIndex,
           embeddedEventMessageTrackGroupIndex,
           embeddedCea608TrackGroupIndex,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
     }
 
     public static TrackGroupInfo embeddedEmsgTrack(int[] adaptationSetIndices,
@@ -773,7 +848,7 @@ import java.util.List;
           primaryTrackGroupIndex,
           C.INDEX_UNSET,
           C.INDEX_UNSET,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
     }
 
     public static TrackGroupInfo embeddedCea608Track(int[] adaptationSetIndices,
@@ -785,7 +860,7 @@ import java.util.List;
           primaryTrackGroupIndex,
           C.INDEX_UNSET,
           C.INDEX_UNSET,
-          -1);
+          /* eventStreamGroupIndex= */ -1);
     }
 
     public static TrackGroupInfo mpdEventTrack(int eventStreamIndex) {
@@ -793,7 +868,7 @@ import java.util.List;
           C.TRACK_TYPE_METADATA,
           CATEGORY_MANIFEST_EVENTS,
           new int[0],
-          -1,
+          /* primaryTrackGroupIndex= */ -1,
           C.INDEX_UNSET,
           C.INDEX_UNSET,
           eventStreamIndex);

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java

@@ -54,7 +54,6 @@ import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -86,7 +85,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
         int trackType,
         long elapsedRealtimeOffsetMs,
         boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
         @Nullable PlayerTrackEmsgHandler playerEmsgHandler,
         @Nullable TransferListener transferListener) {
       DataSource dataSource = dataSourceFactory.createDataSource();
@@ -104,7 +103,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
           elapsedRealtimeOffsetMs,
           maxSegmentsPerLoad,
           enableEventMessageTrack,
-          enableCea608Track,
+          closedCaptionFormats,
           playerEmsgHandler);
     }
 
@@ -141,9 +140,8 @@ public class DefaultDashChunkSource implements DashChunkSource {
    * @param maxSegmentsPerLoad The maximum number of segments to combine into a single request. Note
    *     that segments will only be combined if their {@link Uri}s are the same and if their data
    *     ranges are adjacent.
-   * @param enableEventMessageTrack Whether the chunks generated by the source may output an event
+   * @param enableEventMessageTrack Whether to output an event message track.
-   *     message track.
+   * @param closedCaptionFormats The {@link Format Formats} of closed caption tracks to be output.
-   * @param enableCea608Track Whether the chunks generated by the source may output a CEA-608 track.
    * @param playerTrackEmsgHandler The {@link PlayerTrackEmsgHandler} instance to handle emsg
    *     messages targeting the player. Maybe null if this is not necessary.
    */
@@ -158,7 +156,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
       long elapsedRealtimeOffsetMs,
       int maxSegmentsPerLoad,
       boolean enableEventMessageTrack,
-      boolean enableCea608Track,
+      List<Format> closedCaptionFormats,
       @Nullable PlayerTrackEmsgHandler playerTrackEmsgHandler) {
     this.manifestLoaderErrorThrower = manifestLoaderErrorThrower;
     this.manifest = manifest;
@@ -184,7 +182,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
               trackType,
               representation,
               enableEventMessageTrack,
-              enableCea608Track,
+              closedCaptionFormats,
               playerTrackEmsgHandler);
     }
   }
@@ -629,7 +627,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
         int trackType,
         Representation representation,
         boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
         TrackOutput playerEmsgTrackOutput) {
       this(
           periodDurationUs,
@@ -638,7 +636,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
               trackType,
               representation,
               enableEventMessageTrack,
-              enableCea608Track,
+              closedCaptionFormats,
               playerEmsgTrackOutput),
           /* segmentNumShift= */ 0,
           representation.getIndex());
@@ -783,7 +781,7 @@ public class DefaultDashChunkSource implements DashChunkSource {
         int trackType,
         Representation representation,
         boolean enableEventMessageTrack,
-        boolean enableCea608Track,
+        List<Format> closedCaptionFormats,
         TrackOutput playerEmsgTrackOutput) {
       String containerMimeType = representation.format.containerMimeType;
       if (mimeTypeIsRawText(containerMimeType)) {
@@ -799,12 +797,6 @@ public class DefaultDashChunkSource implements DashChunkSource {
         if (enableEventMessageTrack) {
           flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
         }
-        // TODO: Use caption format information from the manifest if available.
-        List<Format> closedCaptionFormats =
-            enableCea608Track
-                ? Collections.singletonList(
-                    Format.createTextSampleFormat(null, MimeTypes.APPLICATION_CEA608, 0, null))
-                : Collections.emptyList();
         extractor =
             new FragmentedMp4Extractor(
                 flags, null, null, null, closedCaptionFormats, playerEmsgTrackOutput);