Extract audio type from TS descriptors

The audio type of the stream is defined by ISO/IEC 13818-1 in section 2.6.18.

libraries/common/src/main/java/androidx/media3/common/C.java

@@ -1547,6 +1547,31 @@ public final class C {
   /** The first frame was rendered. */
   @UnstableApi public static final int FIRST_FRAME_RENDERED = 3;
 
+  /**
+   * The audio type of the stream, as defined by ISO/IEC 13818-1, section 2.6.18.
+   *
+   * <p>One of {@link #AUDIO_TYPE_UNDEFINED}, {@link #AUDIO_TYPE_CLEAN_EFFECTS},
+   * {@link #AUDIO_TYPE_HEARING_IMPAIRED} or {@link #AUDIO_TYPE_VISUAL_IMPAIRED_COMMENTARY}.
+   */
+  @Documented
+  @Retention(RetentionPolicy.SOURCE)
+  @Target({FIELD, METHOD, PARAMETER, LOCAL_VARIABLE, TYPE_USE})
+  @IntDef({
+      AUDIO_TYPE_UNDEFINED,
+      AUDIO_TYPE_CLEAN_EFFECTS,
+      AUDIO_TYPE_HEARING_IMPAIRED,
+      AUDIO_TYPE_VISUAL_IMPAIRED_COMMENTARY
+  })
+  public @interface AudioType {}
+
+  public static final int AUDIO_TYPE_UNDEFINED = 0;
+  /** Indicates the track has no language. */
+  public static final int AUDIO_TYPE_CLEAN_EFFECTS = 1;
+  /** Indicates the track is prepared for the hearing impaired. */
+  public static final int AUDIO_TYPE_HEARING_IMPAIRED = 2;
+  /** Indicates the track is prepared for the visually impaired viewer. */
+  public static final int AUDIO_TYPE_VISUAL_IMPAIRED_COMMENTARY = 3;
+
   /**
    * @deprecated Use {@link Util#usToMs(long)}.
    */
@@ -1608,4 +1633,15 @@ public final class C {
       int mediaDrmErrorCode) {
     return Util.getErrorCodeForMediaDrmErrorCode(mediaDrmErrorCode);
   }
+
+  public static @C.RoleFlags int parseRoleFlagsFromAudioType(@AudioType int audioType) {
+    switch (audioType) {
+      case C.AUDIO_TYPE_HEARING_IMPAIRED:
+        return C.ROLE_FLAG_ENHANCED_DIALOG_INTELLIGIBILITY;
+      case C.AUDIO_TYPE_VISUAL_IMPAIRED_COMMENTARY:
+        return C.ROLE_FLAG_DESCRIBES_VIDEO;
+      default:
+        return 0;
+    }
+  }
 }

libraries/extractor/src/main/java/androidx/media3/extractor/DtsUtil.java

@@ -258,6 +258,7 @@ public final class DtsUtil {
    * @param frame The DTS Core frame to parse.
    * @param trackId The track identifier to set on the format.
    * @param language The language to set on the format.
+   * @param audioType The audio type to set on the format.
    * @param drmInitData {@link DrmInitData} to be included in the format.
    * @return The DTS format parsed from data in the header.
    */
@@ -265,6 +266,7 @@ public final class DtsUtil {
       byte[] frame,
       @Nullable String trackId,
       @Nullable String language,
+      @C.AudioType int audioType,
       @Nullable DrmInitData drmInitData) {
     ParsableBitArray frameBits = getNormalizedFrame(frame);
     frameBits.skipBits(32 + 1 + 5 + 1 + 7 + 14); // SYNC, FTYPE, SHORT, CPF, NBLKS, FSIZE
@@ -287,6 +289,7 @@ public final class DtsUtil {
         .setSampleRate(sampleRate)
         .setDrmInitData(drmInitData)
         .setLanguage(language)
+        .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
         .build();
   }
 

libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac3Reader.java

@@ -60,6 +60,7 @@ public final class Ac3Reader implements ElementaryStreamReader {
   private final ParsableBitArray headerScratchBits;
   private final ParsableByteArray headerScratchBytes;
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
 
   private @MonotonicNonNull String formatId;
   private @MonotonicNonNull TrackOutput output;
@@ -80,20 +81,22 @@ public final class Ac3Reader implements ElementaryStreamReader {
 
   /** Constructs a new reader for (E-)AC-3 elementary streams. */
   public Ac3Reader() {
-    this(null);
+    this(null, C.AUDIO_TYPE_UNDEFINED);
   }
 
   /**
    * Constructs a new reader for (E-)AC-3 elementary streams.
    *
    * @param language Track language.
+   * @param audioType Track audio type.
    */
-  public Ac3Reader(@Nullable String language) {
+  public Ac3Reader(@Nullable String language, @C.AudioType int audioType) {
     headerScratchBits = new ParsableBitArray(new byte[HEADER_SIZE]);
     headerScratchBytes = new ParsableByteArray(headerScratchBits.data);
     state = STATE_FINDING_SYNC;
     timeUs = C.TIME_UNSET;
     this.language = language;
+    this.audioType = audioType;
   }
 
   @Override
@@ -216,6 +219,7 @@ public final class Ac3Reader implements ElementaryStreamReader {
               .setChannelCount(frameInfo.channelCount)
               .setSampleRate(frameInfo.sampleRate)
               .setLanguage(language)
+              .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
               .setPeakBitrate(frameInfo.bitrate);
       // AC3 has constant bitrate, so averageBitrate = peakBitrate
       if (MimeTypes.AUDIO_AC3.equals(frameInfo.mimeType)) {

libraries/extractor/src/main/java/androidx/media3/extractor/ts/Ac4Reader.java

@@ -57,6 +57,7 @@ public final class Ac4Reader implements ElementaryStreamReader {
   private final ParsableBitArray headerScratchBits;
   private final ParsableByteArray headerScratchBytes;
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
 
   private @MonotonicNonNull String formatId;
   private @MonotonicNonNull TrackOutput output;
@@ -78,15 +79,16 @@ public final class Ac4Reader implements ElementaryStreamReader {
 
   /** Constructs a new reader for AC-4 elementary streams. */
   public Ac4Reader() {
-    this(null);
+    this(null, C.AUDIO_TYPE_UNDEFINED);
   }
 
   /**
    * Constructs a new reader for AC-4 elementary streams.
    *
    * @param language Track language.
+   * @param audioType Track audio type.
    */
-  public Ac4Reader(@Nullable String language) {
+  public Ac4Reader(@Nullable String language, @C.AudioType int audioType) {
     headerScratchBits = new ParsableBitArray(new byte[Ac4Util.HEADER_SIZE_FOR_PARSER]);
     headerScratchBytes = new ParsableByteArray(headerScratchBits.data);
     state = STATE_FINDING_SYNC;
@@ -95,6 +97,7 @@ public final class Ac4Reader implements ElementaryStreamReader {
     hasCRC = false;
     timeUs = C.TIME_UNSET;
     this.language = language;
+    this.audioType = audioType;
   }
 
   @Override
@@ -217,6 +220,7 @@ public final class Ac4Reader implements ElementaryStreamReader {
               .setChannelCount(frameInfo.channelCount)
               .setSampleRate(frameInfo.sampleRate)
               .setLanguage(language)
+              .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
               .build();
       output.format(format);
     }

libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java

@@ -71,6 +71,7 @@ public final class AdtsReader implements ElementaryStreamReader {
   private final ParsableBitArray adtsScratch;
   private final ParsableByteArray id3HeaderBuffer;
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
 
   private @MonotonicNonNull String formatId;
   private @MonotonicNonNull TrackOutput output;
@@ -105,14 +106,15 @@ public final class AdtsReader implements ElementaryStreamReader {
    * @param exposeId3 True if the reader should expose ID3 information.
    */
   public AdtsReader(boolean exposeId3) {
-    this(exposeId3, null);
+    this(exposeId3, null, C.AUDIO_TYPE_UNDEFINED);
   }
 
   /**
    * @param exposeId3 True if the reader should expose ID3 information.
    * @param language Track language.
+   * @param audioType Track audio type.
    */
-  public AdtsReader(boolean exposeId3, @Nullable String language) {
+  public AdtsReader(boolean exposeId3, @Nullable String language, @C.AudioType int audioType) {
     adtsScratch = new ParsableBitArray(new byte[HEADER_SIZE + CRC_SIZE]);
     id3HeaderBuffer = new ParsableByteArray(Arrays.copyOf(ID3_IDENTIFIER, ID3_HEADER_SIZE));
     setFindingSampleState();
@@ -122,6 +124,7 @@ public final class AdtsReader implements ElementaryStreamReader {
     timeUs = C.TIME_UNSET;
     this.exposeId3 = exposeId3;
     this.language = language;
+    this.audioType = audioType;
   }
 
   /** Returns whether an integer matches an ADTS SYNC word. */
@@ -510,6 +513,7 @@ public final class AdtsReader implements ElementaryStreamReader {
               .setSampleRate(aacConfig.sampleRateHz)
               .setInitializationData(Collections.singletonList(audioSpecificConfig))
               .setLanguage(language)
+              .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
               .build();
       // In this class a sample is an access unit, but the MediaFormat sample rate specifies the
       // number of PCM audio samples per second.

libraries/extractor/src/main/java/androidx/media3/extractor/ts/DefaultTsPayloadReaderFactory.java

@@ -151,20 +151,20 @@ public final class DefaultTsPayloadReaderFactory implements TsPayloadReader.Fact
     switch (streamType) {
       case TsExtractor.TS_STREAM_TYPE_MPA:
       case TsExtractor.TS_STREAM_TYPE_MPA_LSF:
-        return new PesReader(new MpegAudioReader(esInfo.language));
+        return new PesReader(new MpegAudioReader(esInfo.language, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_AAC_ADTS:
         return isSet(FLAG_IGNORE_AAC_STREAM)
             ? null
-            : new PesReader(new AdtsReader(false, esInfo.language));
+            : new PesReader(new AdtsReader(false, esInfo.language, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_AAC_LATM:
         return isSet(FLAG_IGNORE_AAC_STREAM)
             ? null
-            : new PesReader(new LatmReader(esInfo.language));
+            : new PesReader(new LatmReader(esInfo.language, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_AC3:
       case TsExtractor.TS_STREAM_TYPE_E_AC3:
-        return new PesReader(new Ac3Reader(esInfo.language));
+        return new PesReader(new Ac3Reader(esInfo.language, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_AC4:
-        return new PesReader(new Ac4Reader(esInfo.language));
+        return new PesReader(new Ac4Reader(esInfo.language, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_HDMV_DTS:
         if (!isSet(FLAG_ENABLE_HDMV_DTS_AUDIO_STREAMS)) {
           return null;
@@ -172,9 +172,9 @@ public final class DefaultTsPayloadReaderFactory implements TsPayloadReader.Fact
         // Fall through.
       case TsExtractor.TS_STREAM_TYPE_DTS:
       case TsExtractor.TS_STREAM_TYPE_DTS_HD:
-        return new PesReader(new DtsReader(esInfo.language, DtsReader.EXTSS_HEADER_SIZE_MAX));
+        return new PesReader(new DtsReader(esInfo.language, DtsReader.EXTSS_HEADER_SIZE_MAX, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_DTS_UHD:
-        return new PesReader(new DtsReader(esInfo.language, DtsReader.FTOC_MAX_HEADER_SIZE));
+        return new PesReader(new DtsReader(esInfo.language, DtsReader.FTOC_MAX_HEADER_SIZE, esInfo.audioType));
       case TsExtractor.TS_STREAM_TYPE_H262:
       case TsExtractor.TS_STREAM_TYPE_DC2_H262:
         return new PesReader(new H262Reader(buildUserDataReader(esInfo)));

libraries/extractor/src/main/java/androidx/media3/extractor/ts/DtsReader.java

@@ -68,6 +68,7 @@ public final class DtsReader implements ElementaryStreamReader {
   private final AtomicInteger uhdAudioChunkId;
 
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
 
   private @MonotonicNonNull String formatId;
   private @MonotonicNonNull TrackOutput output;
@@ -93,9 +94,10 @@ public final class DtsReader implements ElementaryStreamReader {
    * Constructs a new reader for DTS elementary streams.
    *
    * @param language Track language.
+   * @param audioType Track audio type.
    * @param maxHeaderSize Maximum size of the header in a frame.
    */
-  public DtsReader(@Nullable String language, int maxHeaderSize) {
+  public DtsReader(@Nullable String language, int maxHeaderSize, @C.AudioType int audioType) {
     headerScratchBytes = new ParsableByteArray(new byte[maxHeaderSize]);
     state = STATE_FINDING_SYNC;
     timeUs = C.TIME_UNSET;
@@ -103,6 +105,7 @@ public final class DtsReader implements ElementaryStreamReader {
     extensionSubstreamHeaderSize = C.LENGTH_UNSET;
     uhdHeaderSize = C.LENGTH_UNSET;
     this.language = language;
+    this.audioType = audioType;
   }
 
   @Override
@@ -263,7 +266,7 @@ public final class DtsReader implements ElementaryStreamReader {
   private void parseCoreHeader() {
     byte[] frameData = headerScratchBytes.getData();
     if (format == null) {
-      format = DtsUtil.parseDtsFormat(frameData, formatId, language, null);
+      format = DtsUtil.parseDtsFormat(frameData, formatId, language, audioType, null);
       output.format(format);
     }
     sampleSize = DtsUtil.getDtsFrameSize(frameData);

libraries/extractor/src/main/java/androidx/media3/extractor/ts/LatmReader.java

@@ -49,6 +49,7 @@ public final class LatmReader implements ElementaryStreamReader {
   private static final int SYNC_BYTE_SECOND = 0xE0;
 
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
   private final ParsableByteArray sampleDataBuffer;
   private final ParsableBitArray sampleBitArray;
 
@@ -78,9 +79,11 @@ public final class LatmReader implements ElementaryStreamReader {
 
   /**
    * @param language Track language.
+   * @param audioType Track audio type.
    */
-  public LatmReader(@Nullable String language) {
+  public LatmReader(@Nullable String language, @C.AudioType int audioType) {
     this.language = language;
+    this.audioType = audioType;
     sampleDataBuffer = new ParsableByteArray(INITIAL_BUFFER_SIZE);
     sampleBitArray = new ParsableBitArray(sampleDataBuffer.getData());
     timeUs = C.TIME_UNSET;
@@ -217,6 +220,7 @@ public final class LatmReader implements ElementaryStreamReader {
                 .setSampleRate(sampleRateHz)
                 .setInitializationData(Collections.singletonList(initData))
                 .setLanguage(language)
+                .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
                 .build();
         if (!format.equals(this.format)) {
           this.format = format;

libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpegAudioReader.java

@@ -44,6 +44,7 @@ public final class MpegAudioReader implements ElementaryStreamReader {
   private final ParsableByteArray headerScratch;
   private final MpegAudioUtil.Header header;
   @Nullable private final String language;
+  @C.AudioType private final int audioType;
 
   private @MonotonicNonNull TrackOutput output;
   private @MonotonicNonNull String formatId;
@@ -63,10 +64,10 @@ public final class MpegAudioReader implements ElementaryStreamReader {
   private long timeUs;
 
   public MpegAudioReader() {
-    this(null);
+    this(null, C.AUDIO_TYPE_UNDEFINED);
   }
 
-  public MpegAudioReader(@Nullable String language) {
+  public MpegAudioReader(@Nullable String language, @C.AudioType int audioType) {
     state = STATE_FINDING_HEADER;
     // The first byte of an MPEG Audio frame header is always 0xFF.
     headerScratch = new ParsableByteArray(4);
@@ -74,6 +75,7 @@ public final class MpegAudioReader implements ElementaryStreamReader {
     header = new MpegAudioUtil.Header();
     timeUs = C.TIME_UNSET;
     this.language = language;
+    this.audioType = audioType;
   }
 
   @Override
@@ -200,6 +202,7 @@ public final class MpegAudioReader implements ElementaryStreamReader {
               .setChannelCount(header.channels)
               .setSampleRate(header.sampleRate)
               .setLanguage(language)
+              .setRoleFlags(C.parseRoleFlagsFromAudioType(audioType))
               .build();
       output.format(format);
       hasOutputFormat = true;

libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsExtractor.java

@@ -752,7 +752,7 @@ public final class TsExtractor implements Extractor {
       if (mode == MODE_HLS && id3Reader == null) {
         // Setup an ID3 track regardless of whether there's a corresponding entry, in case one
         // appears intermittently during playback. See [Internal: b/20261500].
-        EsInfo id3EsInfo = new EsInfo(TS_STREAM_TYPE_ID3, null, null, Util.EMPTY_BYTE_ARRAY);
+        EsInfo id3EsInfo = new EsInfo(TS_STREAM_TYPE_ID3, null, C.AUDIO_TYPE_UNDEFINED, null, Util.EMPTY_BYTE_ARRAY);
         id3Reader = payloadReaderFactory.createPayloadReader(TS_STREAM_TYPE_ID3, id3EsInfo);
         if (id3Reader != null) {
           id3Reader.init(
@@ -842,6 +842,7 @@ public final class TsExtractor implements Extractor {
       int descriptorsEndPosition = descriptorsStartPosition + length;
       int streamType = -1;
       String language = null;
+      int audioType = C.AUDIO_TYPE_UNDEFINED;
       List<DvbSubtitleInfo> dvbSubtitleInfos = null;
       while (data.getPosition() < descriptorsEndPosition) {
         int descriptorTag = data.readUnsignedByte();
@@ -883,7 +884,7 @@ public final class TsExtractor implements Extractor {
           streamType = TS_STREAM_TYPE_DTS;
         } else if (descriptorTag == TS_PMT_DESC_ISO639_LANG) {
           language = data.readString(3).trim();
-          // Audio type is ignored.
+          audioType = data.readUnsignedByte();
         } else if (descriptorTag == TS_PMT_DESC_DVBSUBS) {
           streamType = TS_STREAM_TYPE_DVBSUBS;
           dvbSubtitleInfos = new ArrayList<>();
@@ -905,6 +906,7 @@ public final class TsExtractor implements Extractor {
       return new EsInfo(
           streamType,
           language,
+          audioType,
           dvbSubtitleInfos,
           Arrays.copyOfRange(data.getData(), descriptorsStartPosition, descriptorsEndPosition));
     }

libraries/extractor/src/main/java/androidx/media3/extractor/ts/TsPayloadReader.java

@@ -20,6 +20,7 @@ import static java.lang.annotation.ElementType.TYPE_USE;
 import android.util.SparseArray;
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
+import androidx.media3.common.C;
 import androidx.media3.common.ParserException;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.TimestampAdjuster;
@@ -67,6 +68,7 @@ public interface TsPayloadReader {
 
     public final int streamType;
     @Nullable public final String language;
+    @C.AudioType public final int audioType;
     public final List<DvbSubtitleInfo> dvbSubtitleInfos;
     public final byte[] descriptorBytes;
 
@@ -74,16 +76,19 @@ public interface TsPayloadReader {
      * @param streamType The type of the stream as defined by the {@link TsExtractor}{@code
      *     .TS_STREAM_TYPE_*}.
      * @param language The language of the stream, as defined by ISO/IEC 13818-1, section 2.6.18.
+     * @param audioType The audio type of the stream, as defined by ISO/IEC 13818-1, section 2.6.18.
      * @param dvbSubtitleInfos Information about DVB subtitles associated to the stream.
      * @param descriptorBytes The descriptor bytes associated to the stream.
      */
     public EsInfo(
         int streamType,
         @Nullable String language,
+        @C.AudioType int audioType,
         @Nullable List<DvbSubtitleInfo> dvbSubtitleInfos,
         byte[] descriptorBytes) {
       this.streamType = streamType;
       this.language = language;
+      this.audioType = audioType;
       this.dvbSubtitleInfos =
           dvbSubtitleInfos == null
               ? Collections.emptyList()