From 32f0eb1278dacc1c78faf031a4a1db86ab617bac Mon Sep 17 00:00:00 2001 From: Oliver Woodman Date: Mon, 9 Feb 2015 17:25:39 +0000 Subject: [PATCH] Enhance mp4 parsing. --- .../parser/mp4/FragmentedMp4Extractor.java | 4 +- .../google/android/exoplayer/mp4/Atom.java | 2 + .../exoplayer/mp4/CommonMp4AtomParsers.java | 203 +++++++++++++++++- .../exoplayer/mp4/Mp4TrackSampleTable.java | 88 ++++++++ .../google/android/exoplayer/mp4/Mp4Util.java | 9 + .../exoplayer/source/SampleExtractor.java | 3 +- .../google/android/exoplayer/util/Util.java | 26 +++ 7 files changed, 328 insertions(+), 7 deletions(-) create mode 100644 library/src/main/java/com/google/android/exoplayer/mp4/Mp4TrackSampleTable.java diff --git a/library/src/main/java/com/google/android/exoplayer/chunk/parser/mp4/FragmentedMp4Extractor.java b/library/src/main/java/com/google/android/exoplayer/chunk/parser/mp4/FragmentedMp4Extractor.java index 01646ea66f..274a4664aa 100644 --- a/library/src/main/java/com/google/android/exoplayer/chunk/parser/mp4/FragmentedMp4Extractor.java +++ b/library/src/main/java/com/google/android/exoplayer/chunk/parser/mp4/FragmentedMp4Extractor.java @@ -86,6 +86,7 @@ public final class FragmentedMp4Extractor implements Extractor { parsedAtoms.add(Atom.TYPE_moof); parsedAtoms.add(Atom.TYPE_moov); parsedAtoms.add(Atom.TYPE_mp4a); + parsedAtoms.add(Atom.TYPE_mvhd); parsedAtoms.add(Atom.TYPE_sidx); parsedAtoms.add(Atom.TYPE_stsd); parsedAtoms.add(Atom.TYPE_tfdt); @@ -379,7 +380,8 @@ public final class FragmentedMp4Extractor implements Extractor { } ContainerAtom mvex = moov.getContainerAtomOfType(Atom.TYPE_mvex); extendsDefaults = parseTrex(mvex.getLeafAtomOfType(Atom.TYPE_trex).data); - track = CommonMp4AtomParsers.parseTrak(moov.getContainerAtomOfType(Atom.TYPE_trak)); + track = CommonMp4AtomParsers.parseTrak(moov.getContainerAtomOfType(Atom.TYPE_trak), + moov.getLeafAtomOfType(Atom.TYPE_mvhd)); } private void onMoofContainerAtomRead(ContainerAtom moof) { diff --git a/library/src/main/java/com/google/android/exoplayer/mp4/Atom.java b/library/src/main/java/com/google/android/exoplayer/mp4/Atom.java index 3451d31929..292c231087 100644 --- a/library/src/main/java/com/google/android/exoplayer/mp4/Atom.java +++ b/library/src/main/java/com/google/android/exoplayer/mp4/Atom.java @@ -39,6 +39,7 @@ public abstract class Atom { public static final int TYPE_trun = getAtomTypeInteger("trun"); public static final int TYPE_sidx = getAtomTypeInteger("sidx"); public static final int TYPE_moov = getAtomTypeInteger("moov"); + public static final int TYPE_mvhd = getAtomTypeInteger("mvhd"); public static final int TYPE_trak = getAtomTypeInteger("trak"); public static final int TYPE_mdia = getAtomTypeInteger("mdia"); public static final int TYPE_minf = getAtomTypeInteger("minf"); @@ -69,6 +70,7 @@ public abstract class Atom { public static final int TYPE_mp4v = getAtomTypeInteger("mp4v"); public static final int TYPE_stts = getAtomTypeInteger("stts"); public static final int TYPE_stss = getAtomTypeInteger("stss"); + public static final int TYPE_ctts = getAtomTypeInteger("ctts"); public static final int TYPE_stsc = getAtomTypeInteger("stsc"); public static final int TYPE_stsz = getAtomTypeInteger("stsz"); public static final int TYPE_stco = getAtomTypeInteger("stco"); diff --git a/library/src/main/java/com/google/android/exoplayer/mp4/CommonMp4AtomParsers.java b/library/src/main/java/com/google/android/exoplayer/mp4/CommonMp4AtomParsers.java index dc7e580ca5..22e5b1edcb 100644 --- a/library/src/main/java/com/google/android/exoplayer/mp4/CommonMp4AtomParsers.java +++ b/library/src/main/java/com/google/android/exoplayer/mp4/CommonMp4AtomParsers.java @@ -24,6 +24,8 @@ import com.google.android.exoplayer.util.MimeTypes; import com.google.android.exoplayer.util.ParsableByteArray; import com.google.android.exoplayer.util.Util; +import android.annotation.SuppressLint; +import android.media.MediaExtractor; import android.util.Pair; import java.util.ArrayList; @@ -40,11 +42,13 @@ public final class CommonMp4AtomParsers { 192, 224, 256, 320, 384, 448, 512, 576, 640}; /** - * Parses a trak atom (defined in 14496-12). + * Parses a trak atom (defined in 14496-12) * + * @param trak Atom to parse. + * @param mvhd Movie header atom, used to get the timescale. * @return A {@link Track} instance. */ - public static Track parseTrak(Atom.ContainerAtom trak) { + public static Track parseTrak(Atom.ContainerAtom trak, Atom.LeafAtom mvhd) { Atom.ContainerAtom mdia = trak.getContainerAtomOfType(Atom.TYPE_mdia); int trackType = parseHdlr(mdia.getLeafAtomOfType(Atom.TYPE_hdlr).data); Assertions.checkState(trackType == Track.TYPE_AUDIO || trackType == Track.TYPE_VIDEO @@ -53,22 +57,211 @@ public final class CommonMp4AtomParsers { Pair header = parseTkhd(trak.getLeafAtomOfType(Atom.TYPE_tkhd).data); int id = header.first; long duration = header.second; - long timescale = parseMdhd(mdia.getLeafAtomOfType(Atom.TYPE_mdhd).data); + long movieTimescale = parseMvhd(mvhd.data); long durationUs; if (duration == -1) { durationUs = C.UNKNOWN_TIME_US; } else { - durationUs = Util.scaleLargeTimestamp(duration, C.MICROS_PER_SECOND, timescale); + durationUs = Util.scaleLargeTimestamp(duration, C.MICROS_PER_SECOND, movieTimescale); } Atom.ContainerAtom stbl = mdia.getContainerAtomOfType(Atom.TYPE_minf) .getContainerAtomOfType(Atom.TYPE_stbl); + long mediaTimescale = parseMdhd(mdia.getLeafAtomOfType(Atom.TYPE_mdhd).data); Pair sampleDescriptions = parseStsd(stbl.getLeafAtomOfType(Atom.TYPE_stsd).data); - return new Track(id, trackType, timescale, durationUs, sampleDescriptions.first, + return new Track(id, trackType, mediaTimescale, durationUs, sampleDescriptions.first, sampleDescriptions.second); } + /** + * Parses an stbl atom (defined in 14496-12). + * + * @param track Track to which this sample table corresponds. + * @param stblAtom stbl (sample table) atom to parse. + * @return Sample table described by the stbl atom. + */ + @SuppressLint("InlinedApi") + public static Mp4TrackSampleTable parseStbl(Track track, Atom.ContainerAtom stblAtom) { + // Array of sample sizes. + ParsableByteArray stsz = stblAtom.getLeafAtomOfType(Atom.TYPE_stsz).data; + + // Entries are byte offsets of chunks. + ParsableByteArray chunkOffsets; + Atom.LeafAtom chunkOffsetsAtom = stblAtom.getLeafAtomOfType(Atom.TYPE_stco); + if (chunkOffsetsAtom == null) { + chunkOffsetsAtom = stblAtom.getLeafAtomOfType(Atom.TYPE_co64); + } + chunkOffsets = chunkOffsetsAtom.data; + // Entries are (chunk number, number of samples per chunk, sample description index). + ParsableByteArray stsc = stblAtom.getLeafAtomOfType(Atom.TYPE_stsc).data; + // Entries are (number of samples, timestamp delta between those samples). + ParsableByteArray stts = stblAtom.getLeafAtomOfType(Atom.TYPE_stts).data; + // Entries are the indices of samples that are synchronization samples. + Atom.LeafAtom stssAtom = stblAtom.getLeafAtomOfType(Atom.TYPE_stss); + ParsableByteArray stss = stssAtom != null ? stssAtom.data : null; + // Entries are (number of samples, timestamp offset). + Atom.LeafAtom cttsAtom = stblAtom.getLeafAtomOfType(Atom.TYPE_ctts); + ParsableByteArray ctts = cttsAtom != null ? cttsAtom.data : null; + + // Skip full atom. + stsz.setPosition(Mp4Util.FULL_ATOM_HEADER_SIZE); + int fixedSampleSize = stsz.readUnsignedIntToInt(); + int sampleCount = stsz.readUnsignedIntToInt(); + + int[] sizes = new int[sampleCount]; + long[] timestamps = new long[sampleCount]; + long[] offsets = new long[sampleCount]; + int[] flags = new int[sampleCount]; + + // Prepare to read chunk offsets. + chunkOffsets.setPosition(Mp4Util.FULL_ATOM_HEADER_SIZE); + int chunkCount = chunkOffsets.readUnsignedIntToInt(); + + stsc.setPosition(Mp4Util.FULL_ATOM_HEADER_SIZE); + int remainingSamplesPerChunkChanges = stsc.readUnsignedIntToInt() - 1; + Assertions.checkState(stsc.readInt() == 1, "stsc first chunk must be 1"); + int samplesPerChunk = stsc.readUnsignedIntToInt(); + stsc.skip(4); // Skip the sample description index. + int nextSamplesPerChunkChangeChunkIndex = -1; + if (remainingSamplesPerChunkChanges > 0) { + // Store the chunk index when the samples-per-chunk will next change. + nextSamplesPerChunkChangeChunkIndex = stsc.readUnsignedIntToInt() - 1; + } + + int chunkIndex = 0; + int remainingSamplesInChunk = samplesPerChunk; + + // Prepare to read sample timestamps. + stts.setPosition(Mp4Util.FULL_ATOM_HEADER_SIZE); + int remainingTimestampDeltaChanges = stts.readUnsignedIntToInt() - 1; + int remainingSamplesAtTimestampDelta = stts.readUnsignedIntToInt(); + int timestampDeltaInTimeUnits = stts.readUnsignedIntToInt(); + + // Prepare to read sample timestamp offsets, if ctts is present. + boolean cttsHasSignedOffsets = false; + int remainingSamplesAtTimestampOffset = 0; + int remainingTimestampOffsetChanges = 0; + int timestampOffset = 0; + if (ctts != null) { + ctts.setPosition(Mp4Util.ATOM_HEADER_SIZE); + cttsHasSignedOffsets = Mp4Util.parseFullAtomVersion(ctts.readInt()) == 1; + remainingTimestampOffsetChanges = ctts.readUnsignedIntToInt() - 1; + remainingSamplesAtTimestampOffset = ctts.readUnsignedIntToInt(); + timestampOffset = cttsHasSignedOffsets ? ctts.readInt() : ctts.readUnsignedIntToInt(); + } + + int nextSynchronizationSampleIndex = -1; + int remainingSynchronizationSamples = 0; + if (stss != null) { + stss.setPosition(Mp4Util.FULL_ATOM_HEADER_SIZE); + remainingSynchronizationSamples = stss.readUnsignedIntToInt(); + nextSynchronizationSampleIndex = stss.readUnsignedIntToInt() - 1; + } + + // Calculate the chunk offsets + long offsetBytes; + if (chunkOffsetsAtom.type == Atom.TYPE_stco) { + offsetBytes = chunkOffsets.readUnsignedInt(); + } else { + offsetBytes = chunkOffsets.readUnsignedLongToLong(); + } + + long timestampTimeUnits = 0; + for (int i = 0; i < sampleCount; i++) { + offsets[i] = offsetBytes; + sizes[i] = fixedSampleSize == 0 ? stsz.readUnsignedIntToInt() : fixedSampleSize; + timestamps[i] = timestampTimeUnits + timestampOffset; + + // All samples are synchronization samples if the stss is not present. + flags[i] = stss == null ? MediaExtractor.SAMPLE_FLAG_SYNC : 0; + if (i == nextSynchronizationSampleIndex) { + flags[i] = MediaExtractor.SAMPLE_FLAG_SYNC; + remainingSynchronizationSamples--; + if (remainingSynchronizationSamples > 0) { + nextSynchronizationSampleIndex = stss.readUnsignedIntToInt() - 1; + } + } + + // Add on the duration of this sample. + timestampTimeUnits += timestampDeltaInTimeUnits; + remainingSamplesAtTimestampDelta--; + if (remainingSamplesAtTimestampDelta == 0 && remainingTimestampDeltaChanges > 0) { + remainingSamplesAtTimestampDelta = stts.readUnsignedIntToInt(); + timestampDeltaInTimeUnits = stts.readUnsignedIntToInt(); + remainingTimestampDeltaChanges--; + } + + // Add on the timestamp offset if ctts is present. + if (ctts != null) { + remainingSamplesAtTimestampOffset--; + if (remainingSamplesAtTimestampOffset == 0 && remainingTimestampOffsetChanges > 0) { + remainingSamplesAtTimestampOffset = ctts.readUnsignedIntToInt(); + timestampOffset = cttsHasSignedOffsets ? ctts.readInt() : ctts.readUnsignedIntToInt(); + remainingTimestampOffsetChanges--; + } + } + + // If we're at the last sample in this chunk, move to the next chunk. + remainingSamplesInChunk--; + if (remainingSamplesInChunk == 0) { + chunkIndex++; + if (chunkIndex < chunkCount) { + if (chunkOffsetsAtom.type == Atom.TYPE_stco) { + offsetBytes = chunkOffsets.readUnsignedInt(); + } else { + offsetBytes = chunkOffsets.readUnsignedLongToLong(); + } + } + + // Change the samples-per-chunk if required. + if (chunkIndex == nextSamplesPerChunkChangeChunkIndex) { + samplesPerChunk = stsc.readUnsignedIntToInt(); + stsc.skip(4); // Skip the sample description index. + remainingSamplesPerChunkChanges--; + if (remainingSamplesPerChunkChanges > 0) { + nextSamplesPerChunkChangeChunkIndex = stsc.readUnsignedIntToInt() - 1; + } + } + + // Expect samplesPerChunk samples in the following chunk, if it's before the end. + if (chunkIndex < chunkCount) { + remainingSamplesInChunk = samplesPerChunk; + } + } else { + // The next sample follows the current one. + offsetBytes += sizes[i]; + } + } + + Util.scaleLargeTimestampsInPlace(timestamps, 1000000, track.timescale); + + // Check all the expected samples have been seen. + Assertions.checkArgument(remainingSynchronizationSamples == 0); + Assertions.checkArgument(remainingSamplesAtTimestampDelta == 0); + Assertions.checkArgument(remainingSamplesInChunk == 0); + Assertions.checkArgument(remainingTimestampDeltaChanges == 0); + Assertions.checkArgument(remainingTimestampOffsetChanges == 0); + return new Mp4TrackSampleTable(offsets, sizes, timestamps, flags); + } + + /** + * Parses a mvhd atom (defined in 14496-12), returning the timescale for the movie. + * + * @param mvhd Contents of the mvhd atom to be parsed. + * @return Timescale for the movie. + */ + private static long parseMvhd(ParsableByteArray mvhd) { + mvhd.setPosition(Mp4Util.ATOM_HEADER_SIZE); + + int fullAtom = mvhd.readInt(); + int version = Mp4Util.parseFullAtomVersion(fullAtom); + + mvhd.skip(version == 0 ? 8 : 16); + + return mvhd.readUnsignedInt(); + } + /** * Parses a tkhd atom (defined in 14496-12). * diff --git a/library/src/main/java/com/google/android/exoplayer/mp4/Mp4TrackSampleTable.java b/library/src/main/java/com/google/android/exoplayer/mp4/Mp4TrackSampleTable.java new file mode 100644 index 0000000000..325247edf1 --- /dev/null +++ b/library/src/main/java/com/google/android/exoplayer/mp4/Mp4TrackSampleTable.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer.mp4; + +import com.google.android.exoplayer.util.Assertions; +import com.google.android.exoplayer.util.Util; + +import android.media.MediaExtractor; + +/** Sample table for a track in an MP4 file. */ +public final class Mp4TrackSampleTable { + + /** Sample offsets in bytes. */ + public final long[] offsets; + /** Sample sizes in bytes. */ + public final int[] sizes; + /** Sample timestamps in microseconds. */ + public final long[] timestampsUs; + /** Sample flags. */ + public final int[] flags; + + Mp4TrackSampleTable( + long[] offsets, int[] sizes, long[] timestampsUs, int[] flags) { + Assertions.checkArgument(sizes.length == timestampsUs.length); + Assertions.checkArgument(offsets.length == timestampsUs.length); + Assertions.checkArgument(flags.length == timestampsUs.length); + + this.offsets = offsets; + this.sizes = sizes; + this.timestampsUs = timestampsUs; + this.flags = flags; + } + + /** Returns the number of samples in the table. */ + public int getSampleCount() { + return sizes.length; + } + + /** + * Returns the sample index of the closest synchronization sample at or before the given + * timestamp, if one is available. + * + * @param timeUs Timestamp adjacent to which to find a synchronization sample. + * @return Index of the synchronization sample, or {@link Mp4Util#NO_SAMPLE} if none. + */ + public int getIndexOfEarlierOrEqualSynchronizationSample(long timeUs) { + int startIndex = Util.binarySearchFloor(timestampsUs, timeUs, true, false); + for (int i = startIndex; i >= 0; i--) { + if (timestampsUs[i] <= timeUs && (flags[i] & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { + return i; + } + } + + return Mp4Util.NO_SAMPLE; + } + + /** + * Returns the sample index of the closest synchronization sample at or after the given timestamp, + * if one is available. + * + * @param timeUs Timestamp adjacent to which to find a synchronization sample. + * @return index Index of the synchronization sample, or {@link Mp4Util#NO_SAMPLE} if none. + */ + public int getIndexOfLaterOrEqualSynchronizationSample(long timeUs) { + int startIndex = Util.binarySearchCeil(timestampsUs, timeUs, true, false); + for (int i = startIndex; i < timestampsUs.length; i++) { + if (timestampsUs[i] >= timeUs && (flags[i] & MediaExtractor.SAMPLE_FLAG_SYNC) != 0) { + return i; + } + } + + return Mp4Util.NO_SAMPLE; + } + +} diff --git a/library/src/main/java/com/google/android/exoplayer/mp4/Mp4Util.java b/library/src/main/java/com/google/android/exoplayer/mp4/Mp4Util.java index 471689e24a..a78c9fb414 100644 --- a/library/src/main/java/com/google/android/exoplayer/mp4/Mp4Util.java +++ b/library/src/main/java/com/google/android/exoplayer/mp4/Mp4Util.java @@ -34,6 +34,15 @@ public final class Mp4Util { /** Size of a full atom header, in bytes. */ public static final int FULL_ATOM_HEADER_SIZE = 12; + /** Value for the first 32 bits of atomSize when the atom size is actually a long value. */ + public static final int LONG_ATOM_SIZE = 1; + + /** Sample index when no sample is available. */ + public static final int NO_SAMPLE = -1; + + /** Track index when no track is selected. */ + public static final int NO_TRACK = -1; + /** Four initial bytes that must prefix H.264/AVC NAL units for decoding. */ private static final byte[] NAL_START_CODE = new byte[] {0, 0, 0, 1}; diff --git a/library/src/main/java/com/google/android/exoplayer/source/SampleExtractor.java b/library/src/main/java/com/google/android/exoplayer/source/SampleExtractor.java index bce39cc9fc..383aa80ce2 100644 --- a/library/src/main/java/com/google/android/exoplayer/source/SampleExtractor.java +++ b/library/src/main/java/com/google/android/exoplayer/source/SampleExtractor.java @@ -91,8 +91,9 @@ public interface SampleExtractor { * {@link SampleSource#END_OF_STREAM} if the last samples in all tracks have been read, or * {@link SampleSource#NOTHING_READ} if the sample cannot be read immediately as it is not * loaded. + * @throws IOException Thrown if the source can't be read. */ - int readSample(int track, SampleHolder sampleHolder); + int readSample(int track, SampleHolder sampleHolder) throws IOException; /** Releases resources associated with this extractor. */ void release(); diff --git a/library/src/main/java/com/google/android/exoplayer/util/Util.java b/library/src/main/java/com/google/android/exoplayer/util/Util.java index ae2e014f0d..7e096cfa1b 100644 --- a/library/src/main/java/com/google/android/exoplayer/util/Util.java +++ b/library/src/main/java/com/google/android/exoplayer/util/Util.java @@ -401,6 +401,32 @@ public final class Util { return scaledTimestamps; } + /** + * Applies {@link #scaleLargeTimestamp(long, long, long)} to an array of unscaled timestamps. + * + * @param timestamps The timestamps to scale. + * @param multiplier The multiplier. + * @param divisor The divisor. + */ + public static void scaleLargeTimestampsInPlace(long[] timestamps, long multiplier, long divisor) { + if (divisor >= multiplier && (divisor % multiplier) == 0) { + long divisionFactor = divisor / multiplier; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] /= divisionFactor; + } + } else if (divisor < multiplier && (multiplier % divisor) == 0) { + long multiplicationFactor = multiplier / divisor; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] *= multiplicationFactor; + } + } else { + double multiplicationFactor = (double) multiplier / divisor; + for (int i = 0; i < timestamps.length; i++) { + timestamps[i] = (long) (timestamps[i] * multiplicationFactor); + } + } + } + /** * Converts a list of integers to a primitive array. *