Add support for multiple table sections in PSI section

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=141073182

library/src/androidTest/java/com/google/android/exoplayer2/extractor/ts/SectionReaderTest.java

@@ -0,0 +1,195 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.extractor.ts;
+
+import com.google.android.exoplayer2.extractor.ExtractorOutput;
+import com.google.android.exoplayer2.extractor.TimestampAdjuster;
+import com.google.android.exoplayer2.testutil.FakeExtractorOutput;
+import com.google.android.exoplayer2.util.ParsableByteArray;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import junit.framework.TestCase;
+
+/**
+ * Test for {@link SectionReader}.
+ */
+public class SectionReaderTest extends TestCase {
+
+  private byte[] packetPayload;
+  private CustomSectionPayloadReader payloadReader;
+  private SectionReader reader;
+
+  @Override
+  public void setUp() {
+    packetPayload = new byte[512];
+    Arrays.fill(packetPayload, (byte) 0xFF);
+    payloadReader = new CustomSectionPayloadReader();
+    reader = new SectionReader(payloadReader);
+    reader.init(new TimestampAdjuster(0), new FakeExtractorOutput(),
+        new TsPayloadReader.TrackIdGenerator(0, 1));
+  }
+
+  public void testSingleOnePacketSection() {
+    packetPayload[0] = 3;
+    insertTableSection(4, (byte) 99, 3);
+    reader.consume(new ParsableByteArray(packetPayload), true);
+    assertEquals(Collections.singletonList(99), payloadReader.parsedTableIds);
+  }
+
+  public void testHeaderSplitAcrossPackets() {
+    packetPayload[0] = 3; // The first packet includes a pointer_field.
+    insertTableSection(4, (byte) 100, 3); // This section header spreads across both packets.
+
+    ParsableByteArray firstPacket = new ParsableByteArray(packetPayload, 5);
+    reader.consume(firstPacket, true);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray secondPacket = new ParsableByteArray(packetPayload);
+    secondPacket.setPosition(5);
+    reader.consume(secondPacket, false);
+    assertEquals(Collections.singletonList(100), payloadReader.parsedTableIds);
+  }
+
+  public void testFiveSectionsInTwoPackets() {
+    packetPayload[0] = 0; // The first packet includes a pointer_field.
+    insertTableSection(1, (byte) 101, 10);
+    insertTableSection(14, (byte) 102, 10);
+    insertTableSection(27, (byte) 103, 10);
+    packetPayload[40] = 0; // The second packet includes a pointer_field.
+    insertTableSection(41, (byte) 104, 10);
+    insertTableSection(54, (byte) 105, 10);
+
+    ParsableByteArray firstPacket = new ParsableByteArray(packetPayload, 40);
+    reader.consume(firstPacket, true);
+    assertEquals(Arrays.asList(101, 102, 103), payloadReader.parsedTableIds);
+
+    ParsableByteArray secondPacket = new ParsableByteArray(packetPayload);
+    secondPacket.setPosition(40);
+    reader.consume(secondPacket, true);
+    assertEquals(Arrays.asList(101, 102, 103, 104, 105), payloadReader.parsedTableIds);
+  }
+
+  public void testLongSectionAcrossFourPackets() {
+    packetPayload[0] = 13; // The first packet includes a pointer_field.
+    insertTableSection(1, (byte) 106, 10); // First section. Should be skipped.
+    // Second section spread across four packets. Should be consumed.
+    insertTableSection(14, (byte) 107, 300);
+    packetPayload[300] = 17; // The third packet includes a pointer_field.
+    // Third section, at the payload start of the fourth packet. Should be consumed.
+    insertTableSection(318, (byte) 108, 10);
+
+    ParsableByteArray firstPacket = new ParsableByteArray(packetPayload, 100);
+    reader.consume(firstPacket, true);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray secondPacket = new ParsableByteArray(packetPayload, 200);
+    secondPacket.setPosition(100);
+    reader.consume(secondPacket, false);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray thirdPacket = new ParsableByteArray(packetPayload, 300);
+    thirdPacket.setPosition(200);
+    reader.consume(thirdPacket, false);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray fourthPacket = new ParsableByteArray(packetPayload);
+    fourthPacket.setPosition(300);
+    reader.consume(fourthPacket, true);
+    assertEquals(Arrays.asList(107, 108), payloadReader.parsedTableIds);
+  }
+
+  public void testSeek() {
+    packetPayload[0] = 13; // The first packet includes a pointer_field.
+    insertTableSection(1, (byte) 109, 10); // First section. Should be skipped.
+    // Second section spread across four packets. Should be consumed.
+    insertTableSection(14, (byte) 110, 300);
+    packetPayload[300] = 17; // The third packet includes a pointer_field.
+    // Third section, at the payload start of the fourth packet. Should be consumed.
+    insertTableSection(318, (byte) 111, 10);
+
+    ParsableByteArray firstPacket = new ParsableByteArray(packetPayload, 100);
+    reader.consume(firstPacket, true);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray secondPacket = new ParsableByteArray(packetPayload, 200);
+    secondPacket.setPosition(100);
+    reader.consume(secondPacket, false);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    ParsableByteArray thirdPacket = new ParsableByteArray(packetPayload, 300);
+    thirdPacket.setPosition(200);
+    reader.consume(thirdPacket, false);
+    assertEquals(Collections.emptyList(), payloadReader.parsedTableIds);
+
+    reader.seek();
+
+    ParsableByteArray fourthPacket = new ParsableByteArray(packetPayload);
+    fourthPacket.setPosition(300);
+    reader.consume(fourthPacket, true);
+    assertEquals(Collections.singletonList(111), payloadReader.parsedTableIds);
+  }
+
+  public void testCrcChecks() {
+    byte[] correctCrcPat = new byte[] {
+        (byte) 0x0, (byte) 0x0, (byte) 0xb0, (byte) 0xd, (byte) 0x0, (byte) 0x1, (byte) 0xc1,
+        (byte) 0x0, (byte) 0x0, (byte) 0x0, (byte) 0x1, (byte) 0xe1, (byte) 0x0, (byte) 0xe8,
+        (byte) 0xf9, (byte) 0x5e, (byte) 0x7d};
+    byte[] incorrectCrcPat = Arrays.copyOf(correctCrcPat, correctCrcPat.length);
+    // Crc field is incorrect, and should not be passed to the payload reader.
+    incorrectCrcPat[16]--;
+    reader.consume(new ParsableByteArray(correctCrcPat), true);
+    assertEquals(Collections.singletonList(0), payloadReader.parsedTableIds);
+    reader.consume(new ParsableByteArray(incorrectCrcPat), true);
+    assertEquals(Collections.singletonList(0), payloadReader.parsedTableIds);
+  }
+
+  // Internal methods.
+
+  /**
+   * Inserts a private section header to {@link #packetPayload}.
+   *
+   * @param offset The position at which the header is inserted.
+   * @param tableId The table_id for the inserted section.
+   * @param sectionLength The value to use for private_section_length.
+   */
+  private void insertTableSection(int offset, byte tableId, int sectionLength) {
+    packetPayload[offset++] = tableId;
+    packetPayload[offset++] = (byte) ((sectionLength >> 8) & 0x0F);
+    packetPayload[offset] = (byte) (sectionLength & 0xFF);
+  }
+
+  // Internal classes.
+
+  private static final class CustomSectionPayloadReader implements SectionPayloadReader {
+
+    List<Integer> parsedTableIds;
+
+    @Override
+    public void init(TimestampAdjuster timestampAdjuster, ExtractorOutput extractorOutput,
+        TsPayloadReader.TrackIdGenerator idGenerator) {
+      parsedTableIds = new ArrayList<>();
+    }
+
+    @Override
+    public void consume(ParsableByteArray sectionData) {
+      parsedTableIds.add(sectionData.readUnsignedByte());
+    }
+
+  }
+
+}

library/src/main/java/com/google/android/exoplayer2/extractor/ts/SectionPayloadReader.java

@@ -40,8 +40,9 @@ public interface SectionPayloadReader {
   /**
    * Called by a {@link SectionReader} when a full section is received.
    *
-   * @param sectionData The data belonging to a section, including the section header but excluding
+   * @param sectionData The data belonging to a section starting from the table_id. If
-   *     the CRC_32 field.
+   *     section_syntax_indicator is set to '1', {@code sectionData} excludes the CRC_32 field.
+   *     Otherwise, all bytes belonging to the table section are included.
    */
   void consume(ParsableByteArray sectionData);
 

library/src/main/java/com/google/android/exoplayer2/extractor/ts/SectionReader.java

@@ -18,75 +18,116 @@ package com.google.android.exoplayer2.extractor.ts;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
 import com.google.android.exoplayer2.extractor.TimestampAdjuster;
-import com.google.android.exoplayer2.util.ParsableBitArray;
 import com.google.android.exoplayer2.util.ParsableByteArray;
 import com.google.android.exoplayer2.util.Util;
 
 /**
  * Reads section data packets and feeds the whole sections to a given {@link SectionPayloadReader}.
+ * Useful information on PSI sections can be found in ISO/IEC 13818-1, section 2.4.4.
  */
 public final class SectionReader implements TsPayloadReader {
 
   private static final int SECTION_HEADER_LENGTH = 3;
+  private static final int DEFAULT_SECTION_BUFFER_LENGTH = 32;
+  private static final int MAX_SECTION_LENGTH = 4098;
 
-  private final ParsableByteArray sectionData;
-  private final ParsableBitArray headerScratch;
   private final SectionPayloadReader reader;
-  private int sectionLength;
+  private final ParsableByteArray sectionData;
-  private int sectionBytesRead;
+
+  private int totalSectionLength;
+  private int bytesRead;
+  private boolean sectionSyntaxIndicator;
+  private boolean waitingForPayloadStart;
 
   public SectionReader(SectionPayloadReader reader) {
     this.reader = reader;
-    sectionData = new ParsableByteArray();
+    sectionData = new ParsableByteArray(DEFAULT_SECTION_BUFFER_LENGTH);
-    headerScratch = new ParsableBitArray(new byte[SECTION_HEADER_LENGTH]);
   }
 
   @Override
   public void init(TimestampAdjuster timestampAdjuster, ExtractorOutput extractorOutput,
       TrackIdGenerator idGenerator) {
     reader.init(timestampAdjuster, extractorOutput, idGenerator);
-    sectionLength = C.LENGTH_UNSET;
+    waitingForPayloadStart = true;
   }
 
   @Override
   public void seek() {
-    sectionLength = C.LENGTH_UNSET;
+    waitingForPayloadStart = true;
   }
 
   @Override
   public void consume(ParsableByteArray data, boolean payloadUnitStartIndicator) {
+    int payloadStartPosition = C.POSITION_UNSET;
     if (payloadUnitStartIndicator) {
-      int pointerField = data.readUnsignedByte();
+      int payloadStartOffset = data.readUnsignedByte();
-      data.skipBytes(pointerField);
+      payloadStartPosition = data.getPosition() + payloadStartOffset;
-
-      // Note: see ISO/IEC 13818-1, section 2.4.4.3 for detailed information on the format of
-      // the header.
-      data.readBytes(headerScratch, SECTION_HEADER_LENGTH);
-      data.setPosition(data.getPosition() - SECTION_HEADER_LENGTH);
-      headerScratch.skipBits(12); // table_id (8), section_syntax_indicator (1), 0 (1), reserved (2)
-      sectionLength = headerScratch.readBits(12) + SECTION_HEADER_LENGTH;
-      sectionBytesRead = 0;
-
-      sectionData.reset(sectionLength);
-    } else if (sectionLength == C.LENGTH_UNSET) {
-      // We're not already reading a section and this is not the start of a new one.
-      return;
     }
 
-    int bytesToRead = Math.min(data.bytesLeft(), sectionLength - sectionBytesRead);
+    if (waitingForPayloadStart) {
-    data.readBytes(sectionData.data, sectionBytesRead, bytesToRead);
+      if (!payloadUnitStartIndicator) {
-    sectionBytesRead += bytesToRead;
+        return;
-    if (sectionBytesRead < sectionLength) {
+      }
-      // Not yet fully read.
+      waitingForPayloadStart = false;
-      return;
+      data.setPosition(payloadStartPosition);
+      bytesRead = 0;
     }
-    sectionLength = C.LENGTH_UNSET;
+
-    if (Util.crc(sectionData.data, 0, sectionBytesRead, 0xFFFFFFFF) != 0) {
+    while (data.bytesLeft() > 0) {
-      // CRC Invalid. The section gets discarded.
+      if (bytesRead < SECTION_HEADER_LENGTH) {
-      return;
+        // Note: see ISO/IEC 13818-1, section 2.4.4.3 for detailed information on the format of
+        // the header.
+        if (bytesRead == 0) {
+          int tableId = data.readUnsignedByte();
+          data.setPosition(data.getPosition() - 1);
+          if (tableId == 0xFF /* forbidden value */) {
+            // No more sections in this ts packet.
+            waitingForPayloadStart = true;
+            return;
+          }
+        }
+        int headerBytesToRead = Math.min(data.bytesLeft(), SECTION_HEADER_LENGTH - bytesRead);
+        data.readBytes(sectionData.data, bytesRead, headerBytesToRead);
+        bytesRead += headerBytesToRead;
+        if (bytesRead == SECTION_HEADER_LENGTH) {
+          sectionData.reset(SECTION_HEADER_LENGTH);
+          sectionData.skipBytes(1); // Skip table id (8).
+          int secondHeaderByte = sectionData.readUnsignedByte();
+          int thirdHeaderByte = sectionData.readUnsignedByte();
+          sectionSyntaxIndicator = (secondHeaderByte & 0x80) != 0;
+          totalSectionLength =
+              (((secondHeaderByte & 0x0F) << 8) | thirdHeaderByte) + SECTION_HEADER_LENGTH;
+          if (sectionData.capacity() < totalSectionLength) {
+            // Ensure there is enough space to keep the whole section.
+            byte[] bytes = sectionData.data;
+            sectionData.reset(
+                Math.min(MAX_SECTION_LENGTH, Math.max(totalSectionLength, bytes.length * 2)));
+            System.arraycopy(bytes, 0, sectionData.data, 0, SECTION_HEADER_LENGTH);
+          }
+        }
+      } else {
+        // Reading the body.
+        int bodyBytesToRead = Math.min(data.bytesLeft(), totalSectionLength - bytesRead);
+        data.readBytes(sectionData.data, bytesRead, bodyBytesToRead);
+        bytesRead += bodyBytesToRead;
+        if (bytesRead == totalSectionLength) {
+          if (sectionSyntaxIndicator) {
+            // This section has common syntax as defined in ISO/IEC 13818-1, section 2.4.4.11.
+            if (Util.crc(sectionData.data, 0, totalSectionLength, 0xFFFFFFFF) != 0) {
+              // The CRC is invalid so discard the section.
+              waitingForPayloadStart = true;
+              return;
+            }
+            sectionData.reset(totalSectionLength - 4); // Exclude the CRC_32 field.
+          } else {
+            // This is a private section with private defined syntax.
+            sectionData.reset(totalSectionLength);
+          }
+          reader.consume(sectionData);
+          bytesRead = 0;
+        }
+      }
     }
-    sectionData.setLimit(sectionData.limit() - 4); // Exclude the CRC_32 field.
-    reader.consume(sectionData);
   }
 
 }