From 784431f3e08099a4f22a130eb2a3f2e1b054879d Mon Sep 17 00:00:00 2001 From: Oliver Woodman Date: Thu, 12 Feb 2015 12:48:27 +0000 Subject: [PATCH] Move EIA reordering back to the renderer (sorry for churn). Reordering in the extractor isn't going to work well with the optimizations I'm making there. This change moves sorting back to the renderer, although keeps all of the renderer simplifications. It's basically just moving where the sort happens from one place to another. --- .../hls/parser/PesPayloadReader.java | 20 ------ .../exoplayer/hls/parser/SampleQueue.java | 36 +++++------ .../exoplayer/hls/parser/SeiReader.java | 28 +-------- .../exoplayer/text/eia608/ClosedCaption.java | 18 +----- .../text/eia608/ClosedCaptionCtrl.java | 4 +- .../text/eia608/ClosedCaptionList.java | 39 ++++++++++++ .../text/eia608/ClosedCaptionText.java | 4 +- .../exoplayer/text/eia608/Eia608Parser.java | 28 ++++++--- .../text/eia608/Eia608TrackRenderer.java | 61 +++++++++++++------ 9 files changed, 125 insertions(+), 113 deletions(-) create mode 100644 library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionList.java diff --git a/library/src/main/java/com/google/android/exoplayer/hls/parser/PesPayloadReader.java b/library/src/main/java/com/google/android/exoplayer/hls/parser/PesPayloadReader.java index b6057e7717..3c3f864276 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/parser/PesPayloadReader.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/parser/PesPayloadReader.java @@ -17,33 +17,13 @@ package com.google.android.exoplayer.hls.parser; import com.google.android.exoplayer.util.ParsableByteArray; -import java.util.concurrent.ConcurrentLinkedQueue; - /** * Extracts individual samples from continuous byte stream, preserving original order. */ /* package */ abstract class PesPayloadReader extends SampleQueue { - private final ConcurrentLinkedQueue internalQueue; - protected PesPayloadReader(SamplePool samplePool) { super(samplePool); - internalQueue = new ConcurrentLinkedQueue(); - } - - @Override - protected final Sample internalPeekSample() { - return internalQueue.peek(); - } - - @Override - protected final Sample internalPollSample() { - return internalQueue.poll(); - } - - @Override - protected final void internalQueueSample(Sample sample) { - internalQueue.add(sample); } /** diff --git a/library/src/main/java/com/google/android/exoplayer/hls/parser/SampleQueue.java b/library/src/main/java/com/google/android/exoplayer/hls/parser/SampleQueue.java index 7b7916e683..1a8623468f 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/parser/SampleQueue.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/parser/SampleQueue.java @@ -18,9 +18,12 @@ package com.google.android.exoplayer.hls.parser; import com.google.android.exoplayer.MediaFormat; import com.google.android.exoplayer.util.ParsableByteArray; +import java.util.concurrent.ConcurrentLinkedQueue; + /* package */ abstract class SampleQueue { private final SamplePool samplePool; + private final ConcurrentLinkedQueue internalQueue; // Accessed only by the consuming thread. private boolean needKeyframe; @@ -33,6 +36,7 @@ import com.google.android.exoplayer.util.ParsableByteArray; protected SampleQueue(SamplePool samplePool) { this.samplePool = samplePool; + internalQueue = new ConcurrentLinkedQueue(); needKeyframe = true; lastReadTimeUs = Long.MIN_VALUE; spliceOutTimeUs = Long.MIN_VALUE; @@ -66,7 +70,7 @@ import com.google.android.exoplayer.util.ParsableByteArray; public Sample poll() { Sample head = peek(); if (head != null) { - internalPollSample(); + internalQueue.poll(); needKeyframe = false; lastReadTimeUs = head.timeUs; } @@ -79,13 +83,13 @@ import com.google.android.exoplayer.util.ParsableByteArray; * @return The next sample from the queue, or null if a sample isn't available. */ public Sample peek() { - Sample head = internalPeekSample(); + Sample head = internalQueue.peek(); if (needKeyframe) { // Peeking discard of samples until we find a keyframe or run out of available samples. while (head != null && !head.isKeyframe) { recycle(head); - internalPollSample(); - head = internalPeekSample(); + internalQueue.poll(); + head = internalQueue.peek(); } } if (head == null) { @@ -94,7 +98,7 @@ import com.google.android.exoplayer.util.ParsableByteArray; if (spliceOutTimeUs != Long.MIN_VALUE && head.timeUs >= spliceOutTimeUs) { // The sample is later than the time this queue is spliced out. recycle(head); - internalPollSample(); + internalQueue.poll(); return null; } return head; @@ -109,8 +113,8 @@ import com.google.android.exoplayer.util.ParsableByteArray; Sample head = peek(); while (head != null && head.timeUs < timeUs) { recycle(head); - internalPollSample(); - head = internalPeekSample(); + internalQueue.poll(); + head = internalQueue.peek(); // We're discarding at least one sample, so any subsequent read will need to start at // a keyframe. needKeyframe = true; @@ -122,10 +126,10 @@ import com.google.android.exoplayer.util.ParsableByteArray; * Clears the queue. */ public void release() { - Sample toRecycle = internalPollSample(); + Sample toRecycle = internalQueue.poll(); while (toRecycle != null) { recycle(toRecycle); - toRecycle = internalPollSample(); + toRecycle = internalQueue.poll(); } } @@ -150,19 +154,19 @@ import com.google.android.exoplayer.util.ParsableByteArray; return true; } long firstPossibleSpliceTime; - Sample nextSample = internalPeekSample(); + Sample nextSample = internalQueue.peek(); if (nextSample != null) { firstPossibleSpliceTime = nextSample.timeUs; } else { firstPossibleSpliceTime = lastReadTimeUs + 1; } - Sample nextQueueSample = nextQueue.internalPeekSample(); + Sample nextQueueSample = nextQueue.internalQueue.peek(); while (nextQueueSample != null && (nextQueueSample.timeUs < firstPossibleSpliceTime || !nextQueueSample.isKeyframe)) { // Discard samples from the next queue for as long as they are before the earliest possible // splice time, or not keyframes. - nextQueue.internalPollSample(); - nextQueueSample = nextQueue.internalPeekSample(); + nextQueue.internalQueue.poll(); + nextQueueSample = nextQueue.internalQueue.peek(); } if (nextQueueSample != null) { // We've found a keyframe in the next queue that can serve as the splice point. Set the @@ -203,7 +207,7 @@ import com.google.android.exoplayer.util.ParsableByteArray; protected void addSample(Sample sample) { largestParsedTimestampUs = Math.max(largestParsedTimestampUs, sample.timeUs); - internalQueueSample(sample); + internalQueue.add(sample); } protected void addToSample(Sample sample, ParsableByteArray buffer, int size) { @@ -214,8 +218,4 @@ import com.google.android.exoplayer.util.ParsableByteArray; sample.size += size; } - protected abstract Sample internalPeekSample(); - protected abstract Sample internalPollSample(); - protected abstract void internalQueueSample(Sample sample); - } diff --git a/library/src/main/java/com/google/android/exoplayer/hls/parser/SeiReader.java b/library/src/main/java/com/google/android/exoplayer/hls/parser/SeiReader.java index 0d12d60de3..6d98c50a6d 100644 --- a/library/src/main/java/com/google/android/exoplayer/hls/parser/SeiReader.java +++ b/library/src/main/java/com/google/android/exoplayer/hls/parser/SeiReader.java @@ -22,28 +22,23 @@ import com.google.android.exoplayer.util.ParsableByteArray; import android.annotation.SuppressLint; -import java.util.Comparator; -import java.util.TreeSet; - /** * Parses a SEI data from H.264 frames and extracts samples with closed captions data. * * TODO: Technically, we shouldn't allow a sample to be read from the queue until we're sure that * a sample with an earlier timestamp won't be added to it. */ -/* package */ class SeiReader extends SampleQueue implements Comparator { +/* package */ class SeiReader extends SampleQueue { // SEI data, used for Closed Captions. private static final int NAL_UNIT_TYPE_SEI = 6; private final ParsableByteArray seiBuffer; - private final TreeSet internalQueue; public SeiReader(SamplePool samplePool) { super(samplePool); setMediaFormat(MediaFormat.createEia608Format()); seiBuffer = new ParsableByteArray(); - internalQueue = new TreeSet(this); } @SuppressLint("InlinedApi") @@ -63,25 +58,4 @@ import java.util.TreeSet; } } - @Override - public int compare(Sample first, Sample second) { - // Note - We don't expect samples to have identical timestamps. - return first.timeUs <= second.timeUs ? -1 : 1; - } - - @Override - protected synchronized Sample internalPeekSample() { - return internalQueue.isEmpty() ? null : internalQueue.first(); - } - - @Override - protected synchronized Sample internalPollSample() { - return internalQueue.pollFirst(); - } - - @Override - protected synchronized void internalQueueSample(Sample sample) { - internalQueue.add(sample); - } - } diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaption.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaption.java index ab6aff54c6..1961cc7a76 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaption.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaption.java @@ -18,7 +18,7 @@ package com.google.android.exoplayer.text.eia608; /** * A Closed Caption that contains textual data associated with time indices. */ -/* package */ abstract class ClosedCaption implements Comparable { +/* package */ abstract class ClosedCaption { /** * Identifies closed captions with control characters. @@ -33,23 +33,9 @@ package com.google.android.exoplayer.text.eia608; * The type of the closed caption data. */ public final int type; - /** - * Timestamp associated with the closed caption. - */ - public final long timeUs; - protected ClosedCaption(int type, long timeUs) { + protected ClosedCaption(int type) { this.type = type; - this.timeUs = timeUs; - } - - @Override - public int compareTo(ClosedCaption another) { - long delta = this.timeUs - another.timeUs; - if (delta == 0) { - return 0; - } - return delta > 0 ? 1 : -1; } } diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionCtrl.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionCtrl.java index ceca05c919..c784f50cd9 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionCtrl.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionCtrl.java @@ -70,8 +70,8 @@ package com.google.android.exoplayer.text.eia608; public final byte cc1; public final byte cc2; - protected ClosedCaptionCtrl(byte cc1, byte cc2, long timeUs) { - super(ClosedCaption.TYPE_CTRL, timeUs); + protected ClosedCaptionCtrl(byte cc1, byte cc2) { + super(ClosedCaption.TYPE_CTRL); this.cc1 = cc1; this.cc2 = cc2; } diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionList.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionList.java new file mode 100644 index 0000000000..f47ec1f466 --- /dev/null +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionList.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer.text.eia608; + +/* package */ final class ClosedCaptionList implements Comparable { + + public final long timeUs; + public final boolean decodeOnly; + public final ClosedCaption[] captions; + + public ClosedCaptionList(long timeUs, boolean decodeOnly, ClosedCaption[] captions) { + this.timeUs = timeUs; + this.decodeOnly = decodeOnly; + this.captions = captions; + } + + @Override + public int compareTo(ClosedCaptionList other) { + long delta = timeUs - other.timeUs; + if (delta == 0) { + return 0; + } + return delta > 0 ? 1 : -1; + } + +} diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionText.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionText.java index 49fbc5af2d..98e93ea493 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionText.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/ClosedCaptionText.java @@ -19,8 +19,8 @@ package com.google.android.exoplayer.text.eia608; public final String text; - public ClosedCaptionText(String text, long timeUs) { - super(ClosedCaption.TYPE_TEXT, timeUs); + public ClosedCaptionText(String text) { + super(ClosedCaption.TYPE_TEXT); this.text = text; } diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java index bce5c5de35..a855e34839 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java @@ -15,11 +15,12 @@ */ package com.google.android.exoplayer.text.eia608; +import com.google.android.exoplayer.SampleHolder; import com.google.android.exoplayer.util.MimeTypes; import com.google.android.exoplayer.util.ParsableBitArray; import com.google.android.exoplayer.util.ParsableByteArray; -import java.util.List; +import java.util.ArrayList; /** * Facilitates the extraction and parsing of EIA-608 (a.k.a. "line 21 captions" and "CEA-608") @@ -83,23 +84,26 @@ public class Eia608Parser { private final ParsableBitArray seiBuffer; private final StringBuilder stringBuilder; + private final ArrayList captions; /* package */ Eia608Parser() { seiBuffer = new ParsableBitArray(); stringBuilder = new StringBuilder(); + captions = new ArrayList(); } /* package */ boolean canParse(String mimeType) { return mimeType.equals(MimeTypes.APPLICATION_EIA608); } - /* package */ void parse(byte[] data, int size, long timeUs, List out) { - if (size <= 0) { - return; + /* package */ ClosedCaptionList parse(SampleHolder sampleHolder) { + if (sampleHolder.size <= 0) { + return null; } + captions.clear(); stringBuilder.setLength(0); - seiBuffer.reset(data); + seiBuffer.reset(sampleHolder.data.array()); seiBuffer.skipBits(3); // reserved + process_cc_data_flag + zero_bit int ccCount = seiBuffer.readBits(5); seiBuffer.skipBits(8); @@ -135,10 +139,10 @@ public class Eia608Parser { // Control character. if (ccData1 < 0x20) { if (stringBuilder.length() > 0) { - out.add(new ClosedCaptionText(stringBuilder.toString(), timeUs)); + captions.add(new ClosedCaptionText(stringBuilder.toString())); stringBuilder.setLength(0); } - out.add(new ClosedCaptionCtrl(ccData1, ccData2, timeUs)); + captions.add(new ClosedCaptionCtrl(ccData1, ccData2)); continue; } @@ -150,8 +154,16 @@ public class Eia608Parser { } if (stringBuilder.length() > 0) { - out.add(new ClosedCaptionText(stringBuilder.toString(), timeUs)); + captions.add(new ClosedCaptionText(stringBuilder.toString())); } + + if (captions.isEmpty()) { + return null; + } + + ClosedCaption[] captionArray = new ClosedCaption[captions.size()]; + captions.toArray(captionArray); + return new ClosedCaptionList(sampleHolder.timeUs, sampleHolder.decodeOnly, captionArray); } private static char getChar(byte ccData) { diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608TrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608TrackRenderer.java index 349c1450b8..8e855bf730 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608TrackRenderer.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608TrackRenderer.java @@ -31,8 +31,7 @@ import android.os.Looper; import android.os.Message; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.TreeSet; /** * A {@link TrackRenderer} for EIA-608 closed captions in a media stream. @@ -48,6 +47,8 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { // The default number of rows to display in roll-up captions mode. private static final int DEFAULT_CAPTIONS_ROW_COUNT = 4; + // The maximum duration that captions are parsed ahead of the current position. + private static final int MAX_SAMPLE_READAHEAD_US = 5000000; private final SampleSource source; private final Eia608Parser eia608Parser; @@ -56,7 +57,7 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { private final MediaFormatHolder formatHolder; private final SampleHolder sampleHolder; private final StringBuilder captionStringBuilder; - private final List captionBuffer; + private final TreeSet pendingCaptionLists; private int trackIndex; private long currentPositionUs; @@ -85,7 +86,7 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { formatHolder = new MediaFormatHolder(); sampleHolder = new SampleHolder(SampleHolder.BUFFER_REPLACEMENT_MODE_NORMAL); captionStringBuilder = new StringBuilder(); - captionBuffer = new ArrayList(); + pendingCaptionLists = new TreeSet(); } @Override @@ -122,6 +123,7 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { private void seekToInternal(long positionUs) { currentPositionUs = positionUs; inputStreamEnded = false; + pendingCaptionLists.clear(); clearPendingSample(); captionRowCount = DEFAULT_CAPTIONS_ROW_COUNT; setCaptionMode(CC_MODE_UNKNOWN); @@ -138,10 +140,17 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { throw new ExoPlaybackException(e); } - if (!inputStreamEnded && !isSamplePending()) { + if (isSamplePending()) { + maybeParsePendingSample(); + } + + int result = inputStreamEnded ? SampleSource.END_OF_STREAM : SampleSource.SAMPLE_READ; + while (!isSamplePending() && result == SampleSource.SAMPLE_READ) { try { - int result = source.readData(trackIndex, positionUs, formatHolder, sampleHolder, false); - if (result == SampleSource.END_OF_STREAM) { + result = source.readData(trackIndex, positionUs, formatHolder, sampleHolder, false); + if (result == SampleSource.SAMPLE_READ) { + maybeParsePendingSample(); + } else if (result == SampleSource.END_OF_STREAM) { inputStreamEnded = true; } } catch (IOException e) { @@ -149,17 +158,18 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { } } - if (isSamplePending() && sampleHolder.timeUs <= currentPositionUs) { - // Parse the pending sample. - eia608Parser.parse(sampleHolder.data.array(), sampleHolder.size, sampleHolder.timeUs, - captionBuffer); - // Consume parsed captions. - consumeCaptionBuffer(); - // Update the renderer, unless the sample was marked for decoding only. - if (!sampleHolder.decodeOnly) { + while (!pendingCaptionLists.isEmpty()) { + if (pendingCaptionLists.first().timeUs > currentPositionUs) { + // We're too early to render any of the pending caption lists. + return; + } + // Remove and consume the next caption list. + ClosedCaptionList nextCaptionList = pendingCaptionLists.pollFirst(); + consumeCaptionList(nextCaptionList); + // Update the renderer, unless the caption list was marked for decoding only. + if (!nextCaptionList.decodeOnly) { invokeRenderer(caption); } - clearPendingSample(); } } @@ -221,14 +231,26 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { textRenderer.onText(text); } - private void consumeCaptionBuffer() { - int captionBufferSize = captionBuffer.size(); + private void maybeParsePendingSample() { + if (sampleHolder.timeUs > currentPositionUs + MAX_SAMPLE_READAHEAD_US) { + // We're too early to parse the sample. + return; + } + ClosedCaptionList holder = eia608Parser.parse(sampleHolder); + clearPendingSample(); + if (holder != null) { + pendingCaptionLists.add(holder); + } + } + + private void consumeCaptionList(ClosedCaptionList captionList) { + int captionBufferSize = captionList.captions.length; if (captionBufferSize == 0) { return; } for (int i = 0; i < captionBufferSize; i++) { - ClosedCaption caption = captionBuffer.get(i); + ClosedCaption caption = captionList.captions[i]; if (caption.type == ClosedCaption.TYPE_CTRL) { ClosedCaptionCtrl captionCtrl = (ClosedCaptionCtrl) caption; if (captionCtrl.isMiscCode()) { @@ -240,7 +262,6 @@ public class Eia608TrackRenderer extends TrackRenderer implements Callback { handleText((ClosedCaptionText) caption); } } - captionBuffer.clear(); if (captionMode == CC_MODE_ROLL_UP || captionMode == CC_MODE_PAINT_ON) { caption = getDisplayCaption();