sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-26 09:35:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Extract SEF slow motion cues as Metadata

Browse source 
PiperOrigin-RevId: 339307746
This commit is contained in:
samrobinson 2020-10-27 19:44:12 +00:00 committed by Oliver Woodman
parent 1c4653f7ee
commit c0a0708fc3
3 changed files with 279 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
View file

@ -48,7 +48,7 @@ public final class MetadataRetriever {
*
* <p>This is equivalent to using {@link #retrieveMetadata(MediaSourceFactory, MediaItem)} with a
* {@link DefaultMediaSourceFactory} and a {@link DefaultExtractorsFactory} with {@link
* Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} set.
* Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} and {@link Mp4Extractor#FLAG_READ_SEF_DATA} set.
*
* @param context The {@link Context}.
* @param mediaItem The {@link MediaItem} whose metadata should be retrieved.
@ -58,7 +58,8 @@ public final class MetadataRetriever {
Context context, MediaItem mediaItem) {
ExtractorsFactory extractorsFactory =
new DefaultExtractorsFactory()
.setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA);
.setMp4ExtractorFlags(
Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA | Mp4Extractor.FLAG_READ_SEF_DATA);
MediaSourceFactory mediaSourceFactory =
new DefaultMediaSourceFactory(context, extractorsFactory);
return retrieveMetadata(mediaSourceFactory, mediaItem);

63
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
View file

@ -41,6 +41,7 @@ import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.metadata.mp4.MotionPhotoMetadata;
import com.google.android.exoplayer2.metadata.mp4.SefSlowMotion;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.NalUnitUtil;
@ -65,17 +66,20 @@ public final class Mp4Extractor implements Extractor, SeekMap {
/**
* Flags controlling the behavior of the extractor. Possible flag values are {@link
* #FLAG_WORKAROUND_IGNORE_EDIT_LISTS} and {@link #FLAG_READ_MOTION_PHOTO_METADATA}.
* #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}, {@link #FLAG_READ_MOTION_PHOTO_METADATA} and {@link
* #FLAG_READ_SEF_DATA}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef(
flag = true,
value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS, FLAG_READ_MOTION_PHOTO_METADATA})
value = {
FLAG_WORKAROUND_IGNORE_EDIT_LISTS,
FLAG_READ_MOTION_PHOTO_METADATA,
FLAG_READ_SEF_DATA
})
public @interface Flags {}
/**
* Flag to ignore any edit lists in the stream.
*/
/** Flag to ignore any edit lists in the stream. */
public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1;
/**
* Flag to extract {@link MotionPhotoMetadata} from HEIC motion photos following the Google Photos
@ -85,16 +89,27 @@ public final class Mp4Extractor implements Extractor, SeekMap {
* retrieval use cases.
*/
public static final int FLAG_READ_MOTION_PHOTO_METADATA = 1 << 1;
/**
* Flag to extract {@link SefSlowMotion} metadata from Samsung Extension Format (SEF) slow motion
* videos.
*/
public static final int FLAG_READ_SEF_DATA = 1 << 2;
/** Parser states. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({STATE_READING_ATOM_HEADER, STATE_READING_ATOM_PAYLOAD, STATE_READING_SAMPLE})
@IntDef({
STATE_READING_ATOM_HEADER,
STATE_READING_ATOM_PAYLOAD,
STATE_READING_SAMPLE,
STATE_READING_SEF,
})
private @interface State {}
private static final int STATE_READING_ATOM_HEADER = 0;
private static final int STATE_READING_ATOM_PAYLOAD = 1;
private static final int STATE_READING_SAMPLE = 2;
private static final int STATE_READING_SEF = 3;
/** Supported file types. */
@Documented
@ -127,6 +142,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
private final ParsableByteArray atomHeader;
private final ArrayDeque<ContainerAtom> containerAtoms;
private final SefReader sefReader;
private final List<Metadata.Entry> slowMotionMetadataEntries;
@State private int parserState;
private int atomType;
@ -153,7 +170,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
* Creates a new extractor for unfragmented MP4 streams.
*/
public Mp4Extractor() {
this(0);
this(/* flags= */ 0);
}
/**
@ -164,6 +181,10 @@ public final class Mp4Extractor implements Extractor, SeekMap {
*/
public Mp4Extractor(@Flags int flags) {
this.flags = flags;
parserState =
((flags & FLAG_READ_SEF_DATA) != 0) ? STATE_READING_SEF : STATE_READING_ATOM_HEADER;
sefReader = new SefReader();
slowMotionMetadataEntries = new ArrayList<>();
atomHeader = new ParsableByteArray(Atom.LONG_HEADER_SIZE);
containerAtoms = new ArrayDeque<>();
nalStartCode = new ParsableByteArray(NalUnitUtil.NAL_START_CODE);
@ -192,7 +213,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
sampleBytesWritten = 0;
sampleCurrentNalBytesRemaining = 0;
if (position == 0) {
enterReadingAtomHeaderState();
// Reading the SEF data occurs before normal MP4 parsing. Therefore we can not transition to
// reading the atom header until that has completed.
if (parserState != STATE_READING_SEF) {
enterReadingAtomHeaderState();
} else {
sefReader.reset();
slowMotionMetadataEntries.clear();
}
} else if (tracks != null) {
updateSampleIndices(timeUs);
}
@ -219,6 +247,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
break;
case STATE_READING_SAMPLE:
return readSample(input, seekPosition);
case STATE_READING_SEF:
return readSefData(input, seekPosition);
default:
throw new IllegalStateException();
}
@ -396,6 +426,15 @@ public final class Mp4Extractor implements Extractor, SeekMap {
return seekRequired && parserState != STATE_READING_SAMPLE;
}
@ReadResult
private int readSefData(ExtractorInput input, PositionHolder seekPosition) throws IOException {
@ReadResult int result = sefReader.read(input, seekPosition, slowMotionMetadataEntries);
if (result == RESULT_SEEK && seekPosition.position == 0) {
enterReadingAtomHeaderState();
}
return result;
}
private void processAtomEnded(long atomEndPosition) throws ParserException {
while (!containerAtoms.isEmpty() && containerAtoms.peek().endPosition == atomEndPosition) {
Atom.ContainerAtom containerAtom = containerAtoms.pop();
@ -474,8 +513,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
float frameRate = trackSampleTable.sampleCount / (trackDurationUs / 1000000f);
formatBuilder.setFrameRate(frameRate);
}
MetadataUtil.setFormatMetadata(
track.type, udtaMetadata, mdtaMetadata, gaplessInfoHolder, formatBuilder);
track.type,
udtaMetadata,
mdtaMetadata,
gaplessInfoHolder,
formatBuilder,
/* additionalEntries...= */ slowMotionMetadataEntries.toArray(new Metadata.Entry[0]));
mp4Track.trackOutput.format(formatBuilder.build());
if (track.type == C.TRACK_TYPE_VIDEO && firstVideoTrackIndex == C.INDEX_UNSET) {

222
library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/SefReader.java Normal file
View file

@ -0,0 +1,222 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.extractor.mp4;
import static com.google.android.exoplayer2.extractor.Extractor.RESULT_SEEK;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ParserException;
import com.google.android.exoplayer2.extractor.Extractor;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import com.google.android.exoplayer2.extractor.PositionHolder;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.metadata.mp4.SefSlowMotion;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Reads Samsung Extension Format (SEF) metadata.
*
* <p>To be used in conjunction with {@link Mp4Extractor}.
*/
/* package */ final class SefReader {
/** Reader states. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
STATE_SHOULD_CHECK_FOR_SEF,
STATE_CHECKING_FOR_SEF,
STATE_READING_SDRS,
STATE_READING_SEF_DATA
})
private @interface State {}
private static final int STATE_SHOULD_CHECK_FOR_SEF = 0;
private static final int STATE_CHECKING_FOR_SEF = 1;
private static final int STATE_READING_SDRS = 2;
private static final int STATE_READING_SEF_DATA = 3;
/** Supported data types. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({TYPE_SLOW_MOTION_DATA})
private @interface DataType {}
private static final int TYPE_SLOW_MOTION_DATA = 0x0890;
private static final String TAG = "SefReader";
// Hex representation of `SEFT` (in ASCII). This is the last byte of a file that has Samsung
// Extension Format (SEF) data.
private static final int SAMSUNG_TAIL_SIGNATURE = 0x53454654;
// Start signature (4 bytes), SEF version (4 bytes), SDR count (4 bytes).
private static final int TAIL_HEADER_LENGTH = 12;
// Tail offset (4 bytes), tail signature (4 bytes).
private static final int TAIL_FOOTER_LENGTH = 8;
private static final int LENGTH_OF_ONE_SDR = 12;
private final List<DataReference> dataReferences;
@State private int readerState;
private int tailLength;
public SefReader() {
dataReferences = new ArrayList<>();
readerState = STATE_SHOULD_CHECK_FOR_SEF;
}
public void reset() {
dataReferences.clear();
readerState = STATE_SHOULD_CHECK_FOR_SEF;
}
@Extractor.ReadResult
public int read(
ExtractorInput input,
PositionHolder seekPosition,
List<Metadata.Entry> slowMotionMetadataEntries)
throws IOException {
switch (readerState) {
case STATE_SHOULD_CHECK_FOR_SEF:
long inputLength = input.getLength();
seekPosition.position =
inputLength == C.LENGTH_UNSET || inputLength < TAIL_FOOTER_LENGTH
? 0
: inputLength - TAIL_FOOTER_LENGTH;
readerState = STATE_CHECKING_FOR_SEF;
break;
case STATE_CHECKING_FOR_SEF:
checkForSefData(input, seekPosition);
break;
case STATE_READING_SDRS:
readSdrs(input, seekPosition);
break;
case STATE_READING_SEF_DATA:
readSefData(input, slowMotionMetadataEntries);
seekPosition.position = 0;
break;
default:
throw new IllegalStateException();
}
return RESULT_SEEK;
}
private void checkForSefData(ExtractorInput input, PositionHolder seekPosition)
throws IOException {
ParsableByteArray scratch = new ParsableByteArray(/* limit= */ TAIL_FOOTER_LENGTH);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ TAIL_FOOTER_LENGTH);
tailLength = scratch.readLittleEndianInt() + TAIL_FOOTER_LENGTH;
if (scratch.readInt() != SAMSUNG_TAIL_SIGNATURE) {
seekPosition.position = 0;
return;
}
// input.getPosition is at the very end of the tail, so jump forward by sefTailLength, but
// account for the tail header, which needs to be ignored.
seekPosition.position = input.getPosition() - (tailLength - TAIL_HEADER_LENGTH);
readerState = STATE_READING_SDRS;
}
private void readSdrs(ExtractorInput input, PositionHolder seekPosition) throws IOException {
long streamLength = input.getLength();
int sdrsLength = tailLength - TAIL_HEADER_LENGTH - TAIL_FOOTER_LENGTH;
ParsableByteArray scratch = new ParsableByteArray(/* limit= */ sdrsLength);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ sdrsLength);
for (int i = 0; i < sdrsLength / LENGTH_OF_ONE_SDR; i++) {
scratch.skipBytes(2); // SDR data sub info flag and reserved bits (2).
@DataType int dataType = scratch.readLittleEndianShort();
if (dataType == TYPE_SLOW_MOTION_DATA) {
// The read int is the distance from the tail info to the start of the metadata.
// Calculated as an offset from the start by working backwards.
long startOffset = streamLength - tailLength - scratch.readLittleEndianInt();
int size = scratch.readLittleEndianInt();
dataReferences.add(new DataReference(dataType, startOffset, size));
} else {
scratch.skipBytes(8); // startPosition (4), size (4).
}
}
if (dataReferences.isEmpty()) {
seekPosition.position = 0;
return;
}
Collections.sort(dataReferences, (o1, o2) -> Long.compare(o1.startOffset, o2.startOffset));
readerState = STATE_READING_SEF_DATA;
seekPosition.position = dataReferences.get(0).startOffset;
}
private void readSefData(ExtractorInput input, List<Metadata.Entry> slowMotionMetadataEntries)
throws IOException {
checkNotNull(dataReferences);
Splitter splitter = Splitter.on(':');
int totalDataLength = (int) (input.getLength() - input.getPosition() - tailLength);
ParsableByteArray scratch = new ParsableByteArray(/* limit= */ totalDataLength);
input.readFully(scratch.getData(), 0, totalDataLength);
int totalDataReferenceBytesConsumed = 0;
for (int i = 0; i < dataReferences.size(); i++) {
DataReference dataReference = dataReferences.get(i);
if (dataReference.dataType == TYPE_SLOW_MOTION_DATA) {
scratch.skipBytes(23); // data type (2), data sub info (2), name len (4), name (15).
List<SefSlowMotion.Segment> segments = new ArrayList<>();
int dataReferenceEndPosition = totalDataReferenceBytesConsumed + dataReference.size;
while (scratch.getPosition() < dataReferenceEndPosition) {
@Nullable String data = scratch.readDelimiterTerminatedString('*');
List<String> values = splitter.splitToList(checkNotNull(data));
if (values.size() != 3) {
throw new ParserException();
}
try {
int startTimeMs = Integer.parseInt(values.get(0));
int endTimeMs = Integer.parseInt(values.get(1));
int speedMode = Integer.parseInt(values.get(2));
int speedDivisor = 1 << (speedMode - 1);
segments.add(new SefSlowMotion.Segment(startTimeMs, endTimeMs, speedDivisor));
} catch (NumberFormatException e) {
throw new ParserException(e);
}
}
totalDataReferenceBytesConsumed += dataReference.size;
slowMotionMetadataEntries.add(new SefSlowMotion(segments));
}
}
}
private static final class DataReference {
@DataType public final int dataType;
public final long startOffset;
public final int size;
public DataReference(@DataType int dataType, long startOffset, int size) {
this.dataType = dataType;
this.startOffset = startOffset;
this.size = size;
}
}
}