sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-06 11:25:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support image track extraction for JPEG

Browse source 
PiperOrigin-RevId: 563746945
This commit is contained in:
tofunmi 2023-09-08 07:29:49 -07:00 committed by Copybara-Service
parent f285334a15
commit be5fa6d130
12 changed files with 522 additions and 309 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractorHelper.java → libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractor.java
View file

@ -17,8 +17,7 @@ package androidx.media3.extractor;
import static androidx.media3.common.C.BUFFER_FLAG_KEY_FRAME;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.extractor.Extractor.RESULT_CONTINUE;
import static androidx.media3.extractor.Extractor.RESULT_END_OF_INPUT;
import static androidx.media3.common.util.Assertions.checkState;
import static java.lang.annotation.ElementType.TYPE_USE;
import androidx.annotation.IntDef;
@ -35,13 +34,13 @@ import java.lang.annotation.Target;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Extracts data by loading all the bytes into one sample.
*
* <p>Used as a component in other extractors.
*/
/** Extracts data by loading all the bytes into one sample. */
@UnstableApi
public final class SingleSampleExtractorHelper {
public final class SingleSampleExtractor implements Extractor {
private final int fileSignature;
private final int fileSignatureLength;
private final String containerMimeType;
/** Parser states. */
@Documented
@ -67,36 +66,36 @@ public final class SingleSampleExtractorHelper {
private @MonotonicNonNull TrackOutput trackOutput;
/**
* Returns whether the {@link ExtractorInput} has the given {@code fileSignature}.
* Creates an instance.
*
* <p>@see Extractor#sniff(ExtractorInput)
* @param fileSignature The file signature used to {@link #sniff}, or {@link C#INDEX_UNSET} if the
* method won't be used.
* @param fileSignatureLength The length of file signature, or {@link C#LENGTH_UNSET} if the
* {@link #sniff} method won't be used.
* @param containerMimeType The mime type of the format being extracted.
*/
public boolean sniff(ExtractorInput input, int fileSignature, int fileSignatureLength)
throws IOException {
public SingleSampleExtractor(
int fileSignature, int fileSignatureLength, String containerMimeType) {
this.fileSignature = fileSignature;
this.fileSignatureLength = fileSignatureLength;
this.containerMimeType = containerMimeType;
}
@Override
public boolean sniff(ExtractorInput input) throws IOException {
checkState(fileSignature != C.INDEX_UNSET && fileSignatureLength != C.LENGTH_UNSET);
ParsableByteArray scratch = new ParsableByteArray(fileSignatureLength);
input.peekFully(scratch.getData(), /* offset= */ 0, fileSignatureLength);
return scratch.readUnsignedShort() == fileSignature;
}
/**
* See {@link Extractor#init(ExtractorOutput)}.
*
* <p>Outputs format with {@code containerMimeType}.
*/
public void init(ExtractorOutput output, String containerMimeType) {
@Override
public void init(ExtractorOutput output) {
extractorOutput = output;
outputImageTrackAndSeekMap(containerMimeType);
}
/** See {@link Extractor#seek(long, long)}. */
public void seek(long position) {
if (position == 0 || state == STATE_READING) {
state = STATE_READING;
size = 0;
}
}
/** See {@link Extractor#read(ExtractorInput, PositionHolder)}. */
@Override
public @Extractor.ReadResult int read(ExtractorInput input, PositionHolder seekPosition)
throws IOException {
switch (state) {
@ -110,6 +109,19 @@ public final class SingleSampleExtractorHelper {
}
}
@Override
public void seek(long position, long timeUs) {
if (position == 0 || state == STATE_READING) {
state = STATE_READING;
size = 0;
}
}
@Override
public void release() {
// Do Nothing
}
private void readSegment(ExtractorInput input) throws IOException {
int result =
checkNotNull(trackOutput).sampleData(input, FIXED_READ_LENGTH, /* allowEndOfInput= */ true);

14
libraries/extractor/src/main/java/androidx/media3/extractor/bmp/BmpExtractor.java
View file

@ -21,7 +21,7 @@ import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SingleSampleExtractorHelper;
import androidx.media3.extractor.SingleSampleExtractor;
import java.io.IOException;
/** Extracts data from the BMP container format. */
@ -30,21 +30,23 @@ public final class BmpExtractor implements Extractor {
private static final int BMP_FILE_SIGNATURE_LENGTH = 2;
private static final int BMP_FILE_SIGNATURE = 0x424D;
private final SingleSampleExtractorHelper imageExtractor;
private final SingleSampleExtractor imageExtractor;
/** Creates an instance. */
public BmpExtractor() {
imageExtractor = new SingleSampleExtractorHelper();
imageExtractor =
new SingleSampleExtractor(
BMP_FILE_SIGNATURE, BMP_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_BMP);
}
@Override
public boolean sniff(ExtractorInput input) throws IOException {
return imageExtractor.sniff(input, BMP_FILE_SIGNATURE, BMP_FILE_SIGNATURE_LENGTH);
return imageExtractor.sniff(input);
}
@Override
public void init(ExtractorOutput output) {
imageExtractor.init(output, MimeTypes.IMAGE_BMP);
imageExtractor.init(output);
}
@Override
@ -55,7 +57,7 @@ public final class BmpExtractor implements Extractor {
@Override
public void seek(long position, long timeUs) {
imageExtractor.seek(position);
imageExtractor.seek(position, timeUs);
}
@Override

11
libraries/extractor/src/main/java/androidx/media3/extractor/heif/HeifExtractor.java
View file

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.heif;
import androidx.media3.common.C;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
@ -22,7 +23,7 @@ import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SingleSampleExtractorHelper;
import androidx.media3.extractor.SingleSampleExtractor;
import java.io.IOException;
/** Extracts data from the HEIF (.heic) container format. */
@ -35,12 +36,12 @@ public final class HeifExtractor implements Extractor {
private static final int FILE_SIGNATURE_SEGMENT_LENGTH = 4;
private final ParsableByteArray scratch;
private final SingleSampleExtractorHelper imageExtractor;
private final SingleSampleExtractor imageExtractor;
/** Creates an instance. */
public HeifExtractor() {
scratch = new ParsableByteArray(FILE_SIGNATURE_SEGMENT_LENGTH);
imageExtractor = new SingleSampleExtractorHelper();
imageExtractor = new SingleSampleExtractor(C.INDEX_UNSET, C.LENGTH_UNSET, MimeTypes.IMAGE_HEIF);
}
@Override
@ -52,7 +53,7 @@ public final class HeifExtractor implements Extractor {
@Override
public void init(ExtractorOutput output) {
imageExtractor.init(output, MimeTypes.IMAGE_HEIF);
imageExtractor.init(output);
}
@Override
@ -63,7 +64,7 @@ public final class HeifExtractor implements Extractor {
@Override
public void seek(long position, long timeUs) {
imageExtractor.seek(position);
imageExtractor.seek(position, timeUs);
}
@Override

291
libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java
View file

@ -1,5 +1,5 @@
/*
* Copyright 2020 The Android Open Source Project
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,300 +15,91 @@
*/
package androidx.media3.extractor.jpeg;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.extractor.SingleSampleExtractorHelper.IMAGE_TRACK_ID;
import static java.lang.annotation.ElementType.TYPE_USE;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SeekMap;
import androidx.media3.extractor.TrackOutput;
import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata;
import androidx.media3.extractor.mp4.Mp4Extractor;
import androidx.media3.extractor.SingleSampleExtractor;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Extracts JPEG image using the Exif format. */
/** Extracts data from the JPEG container format. */
@UnstableApi
public final class JpegExtractor implements Extractor {
/** Parser states. */
/**
* Flags controlling the behavior of the extractor. Possible flag value is {@link
* #FLAG_READ_IMAGE}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({
STATE_READING_MARKER,
STATE_READING_SEGMENT_LENGTH,
STATE_READING_SEGMENT,
STATE_SNIFFING_MOTION_PHOTO_VIDEO,
STATE_READING_MOTION_PHOTO_VIDEO,
STATE_ENDED,
})
private @interface State {}
@IntDef(
flag = true,
value = {
FLAG_READ_IMAGE,
})
public @interface Flags {}
private static final int STATE_READING_MARKER = 0;
private static final int STATE_READING_SEGMENT_LENGTH = 1;
private static final int STATE_READING_SEGMENT = 2;
private static final int STATE_SNIFFING_MOTION_PHOTO_VIDEO = 4;
private static final int STATE_READING_MOTION_PHOTO_VIDEO = 5;
private static final int STATE_ENDED = 6;
/** Flag to load the image track instead of the video and metadata track. */
public static final int FLAG_READ_IMAGE = 1;
private static final int EXIF_ID_CODE_LENGTH = 6;
private static final long EXIF_HEADER = 0x45786966; // Exif
private static final int MARKER_SOI = 0xFFD8; // Start of image marker
private static final int MARKER_SOS = 0xFFDA; // Start of scan (image data) marker
private static final int MARKER_APP0 = 0xFFE0; // Application data 0 marker
private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker
private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/";
// Specification reference: ITU-T.81 (1992) subsection B.1.1.3
private static final int JPEG_FILE_SIGNATURE = 0xFFD8; // Start of image marker
private static final int JPEG_FILE_SIGNATURE_LENGTH = 2;
private final ParsableByteArray scratch;
private @MonotonicNonNull ExtractorOutput extractorOutput;
private @State int state;
private int marker;
private int segmentLength;
private long mp4StartPosition;
@Nullable private MotionPhotoMetadata motionPhotoMetadata;
private @MonotonicNonNull ExtractorInput lastExtractorInput;
private @MonotonicNonNull StartOffsetExtractorInput mp4ExtractorStartOffsetExtractorInput;
@Nullable private Mp4Extractor mp4Extractor;
private final Extractor extractor;
/** Creates an instance reading the video and metadata track. */
public JpegExtractor() {
scratch = new ParsableByteArray(EXIF_ID_CODE_LENGTH);
mp4StartPosition = C.INDEX_UNSET;
this(/* flags= */ 0);
}
/**
* Creates an instance.
*
* @param flags The {@link JpegExtractor.Flags} to control extractor behavior.
*/
public JpegExtractor(@JpegExtractor.Flags int flags) {
if ((flags & FLAG_READ_IMAGE) != 0) {
extractor =
new SingleSampleExtractor(
JPEG_FILE_SIGNATURE, JPEG_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_JPEG);
} else {
extractor = new JpegMotionPhotoExtractor();
}
}
@Override
public boolean sniff(ExtractorInput input) throws IOException {
// See ITU-T.81 (1992) subsection B.1.1.3 and Exif version 2.2 (2002) subsection 4.5.4.
if (peekMarker(input) != MARKER_SOI) {
return false;
}
marker = peekMarker(input);
// Even though JFIF and Exif standards are incompatible in theory, Exif files often contain a
// JFIF APP0 marker segment preceding the Exif APP1 marker segment. Skip the JFIF segment if
// present.
if (marker == MARKER_APP0) {
advancePeekPositionToNextSegment(input);
marker = peekMarker(input);
}
if (marker != MARKER_APP1) {
return false;
}
input.advancePeekPosition(2); // Unused segment length
scratch.reset(/* limit= */ EXIF_ID_CODE_LENGTH);
input.peekFully(scratch.getData(), /* offset= */ 0, EXIF_ID_CODE_LENGTH);
return scratch.readUnsignedInt() == EXIF_HEADER && scratch.readUnsignedShort() == 0; // Exif\0\0
return extractor.sniff(input);
}
@Override
public void init(ExtractorOutput output) {
extractorOutput = output;
extractor.init(output);
}
@Override
public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition)
throws IOException {
switch (state) {
case STATE_READING_MARKER:
readMarker(input);
return RESULT_CONTINUE;
case STATE_READING_SEGMENT_LENGTH:
readSegmentLength(input);
return RESULT_CONTINUE;
case STATE_READING_SEGMENT:
readSegment(input);
return RESULT_CONTINUE;
case STATE_SNIFFING_MOTION_PHOTO_VIDEO:
if (input.getPosition() != mp4StartPosition) {
seekPosition.position = mp4StartPosition;
return RESULT_SEEK;
}
sniffMotionPhotoVideo(input);
return RESULT_CONTINUE;
case STATE_READING_MOTION_PHOTO_VIDEO:
if (mp4ExtractorStartOffsetExtractorInput == null || input != lastExtractorInput) {
lastExtractorInput = input;
mp4ExtractorStartOffsetExtractorInput =
new StartOffsetExtractorInput(input, mp4StartPosition);
}
@ReadResult
int readResult =
checkNotNull(mp4Extractor).read(mp4ExtractorStartOffsetExtractorInput, seekPosition);
if (readResult == RESULT_SEEK) {
seekPosition.position += mp4StartPosition;
}
return readResult;
case STATE_ENDED:
return RESULT_END_OF_INPUT;
default:
throw new IllegalStateException();
}
return extractor.read(input, seekPosition);
}
@Override
public void seek(long position, long timeUs) {
if (position == 0) {
state = STATE_READING_MARKER;
mp4Extractor = null;
} else if (state == STATE_READING_MOTION_PHOTO_VIDEO) {
checkNotNull(mp4Extractor).seek(position, timeUs);
}
extractor.seek(position, timeUs);
}
@Override
public void release() {
if (mp4Extractor != null) {
mp4Extractor.release();
}
}
private int peekMarker(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
return scratch.readUnsignedShort();
}
private void advancePeekPositionToNextSegment(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
int segmentLength = scratch.readUnsignedShort() - 2;
input.advancePeekPosition(segmentLength);
}
private void readMarker(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
marker = scratch.readUnsignedShort();
if (marker == MARKER_SOS) { // Start of scan.
if (mp4StartPosition != C.INDEX_UNSET) {
state = STATE_SNIFFING_MOTION_PHOTO_VIDEO;
} else {
endReadingWithImageTrack();
}
} else if ((marker < 0xFFD0 || marker > 0xFFD9) && marker != 0xFF01) {
state = STATE_READING_SEGMENT_LENGTH;
}
}
private void readSegmentLength(ExtractorInput input) throws IOException {
scratch.reset(2);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
segmentLength = scratch.readUnsignedShort() - 2;
state = STATE_READING_SEGMENT;
}
private void readSegment(ExtractorInput input) throws IOException {
if (marker == MARKER_APP1) {
ParsableByteArray payload = new ParsableByteArray(segmentLength);
input.readFully(payload.getData(), /* offset= */ 0, /* length= */ segmentLength);
if (motionPhotoMetadata == null
&& HEADER_XMP_APP1.equals(payload.readNullTerminatedString())) {
@Nullable String xmpString = payload.readNullTerminatedString();
if (xmpString != null) {
motionPhotoMetadata = getMotionPhotoMetadata(xmpString, input.getLength());
if (motionPhotoMetadata != null) {
mp4StartPosition = motionPhotoMetadata.videoStartPosition;
}
}
}
} else {
input.skipFully(segmentLength);
}
state = STATE_READING_MARKER;
}
private void sniffMotionPhotoVideo(ExtractorInput input) throws IOException {
// Check if the file is truncated.
boolean peekedData =
input.peekFully(
scratch.getData(), /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true);
if (!peekedData) {
endReadingWithImageTrack();
} else {
input.resetPeekPosition();
if (mp4Extractor == null) {
mp4Extractor = new Mp4Extractor();
}
mp4ExtractorStartOffsetExtractorInput =
new StartOffsetExtractorInput(input, mp4StartPosition);
if (mp4Extractor.sniff(mp4ExtractorStartOffsetExtractorInput)) {
mp4Extractor.init(
new StartOffsetExtractorOutput(mp4StartPosition, checkNotNull(extractorOutput)));
startReadingMotionPhoto();
} else {
endReadingWithImageTrack();
}
}
}
private void startReadingMotionPhoto() {
outputImageTrack(checkNotNull(motionPhotoMetadata));
state = STATE_READING_MOTION_PHOTO_VIDEO;
}
private void endReadingWithImageTrack() {
outputImageTrack();
checkNotNull(extractorOutput).endTracks();
extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
state = STATE_ENDED;
}
private void outputImageTrack(Metadata.Entry... metadataEntries) {
TrackOutput imageTrackOutput =
checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE);
// TODO(b/289989902): Set the rotationDegrees in format so images can be decoded correctly.
imageTrackOutput.format(
new Format.Builder()
.setContainerMimeType(MimeTypes.IMAGE_JPEG)
.setMetadata(new Metadata(metadataEntries))
.build());
}
/**
* Attempts to parse the specified XMP data describing the motion photo, returning the resulting
* {@link MotionPhotoMetadata} or {@code null} if it wasn't possible to derive motion photo
* metadata.
*
* @param xmpString A string of XML containing XMP motion photo metadata to attempt to parse.
* @param inputLength The length of the input stream in bytes, or {@link C#LENGTH_UNSET} if
* unknown.
* @return The {@link MotionPhotoMetadata}, or {@code null} if it wasn't possible to derive motion
* photo metadata.
* @throws IOException If an error occurs parsing the XMP string.
*/
@Nullable
private static MotionPhotoMetadata getMotionPhotoMetadata(String xmpString, long inputLength)
throws IOException {
// Metadata defines offsets from the end of the stream, so we need the stream length to
// determine start offsets.
if (inputLength == C.LENGTH_UNSET) {
return null;
}
// Motion photos have (at least) a primary image media item and a secondary video media item.
@Nullable
MotionPhotoDescription motionPhotoDescription =
XmpMotionPhotoDescriptionParser.parse(xmpString);
if (motionPhotoDescription == null) {
return null;
}
return motionPhotoDescription.getMotionPhotoMetadata(inputLength);
extractor.release();
}
}

314
libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java Normal file
View file

@ -0,0 +1,314 @@
/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.extractor.jpeg;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.extractor.SingleSampleExtractor.IMAGE_TRACK_ID;
import static java.lang.annotation.ElementType.TYPE_USE;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SeekMap;
import androidx.media3.extractor.TrackOutput;
import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata;
import androidx.media3.extractor.mp4.Mp4Extractor;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Extracts JPEG metadata and motion photo using the Exif format. */
@UnstableApi
/* package */ final class JpegMotionPhotoExtractor implements Extractor {
/** Parser states. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({
STATE_READING_MARKER,
STATE_READING_SEGMENT_LENGTH,
STATE_READING_SEGMENT,
STATE_SNIFFING_MOTION_PHOTO_VIDEO,
STATE_READING_MOTION_PHOTO_VIDEO,
STATE_ENDED,
})
private @interface State {}
private static final int STATE_READING_MARKER = 0;
private static final int STATE_READING_SEGMENT_LENGTH = 1;
private static final int STATE_READING_SEGMENT = 2;
private static final int STATE_SNIFFING_MOTION_PHOTO_VIDEO = 4;
private static final int STATE_READING_MOTION_PHOTO_VIDEO = 5;
private static final int STATE_ENDED = 6;
private static final int EXIF_ID_CODE_LENGTH = 6;
private static final long EXIF_HEADER = 0x45786966; // Exif
private static final int MARKER_SOI = 0xFFD8; // Start of image marker
private static final int MARKER_SOS = 0xFFDA; // Start of scan (image data) marker
private static final int MARKER_APP0 = 0xFFE0; // Application data 0 marker
private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker
private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/";
private final ParsableByteArray scratch;
private @MonotonicNonNull ExtractorOutput extractorOutput;
private @State int state;
private int marker;
private int segmentLength;
private long mp4StartPosition;
@Nullable private MotionPhotoMetadata motionPhotoMetadata;
private @MonotonicNonNull ExtractorInput lastExtractorInput;
private @MonotonicNonNull StartOffsetExtractorInput mp4ExtractorStartOffsetExtractorInput;
@Nullable private Mp4Extractor mp4Extractor;
public JpegMotionPhotoExtractor() {
scratch = new ParsableByteArray(EXIF_ID_CODE_LENGTH);
mp4StartPosition = C.INDEX_UNSET;
}
@Override
public boolean sniff(ExtractorInput input) throws IOException {
// See ITU-T.81 (1992) subsection B.1.1.3 and Exif version 2.2 (2002) subsection 4.5.4.
if (peekMarker(input) != MARKER_SOI) {
return false;
}
marker = peekMarker(input);
// Even though JFIF and Exif standards are incompatible in theory, Exif files often contain a
// JFIF APP0 marker segment preceding the Exif APP1 marker segment. Skip the JFIF segment if
// present.
if (marker == MARKER_APP0) {
advancePeekPositionToNextSegment(input);
marker = peekMarker(input);
}
if (marker != MARKER_APP1) {
return false;
}
input.advancePeekPosition(2); // Unused segment length
scratch.reset(/* limit= */ EXIF_ID_CODE_LENGTH);
input.peekFully(scratch.getData(), /* offset= */ 0, EXIF_ID_CODE_LENGTH);
return scratch.readUnsignedInt() == EXIF_HEADER && scratch.readUnsignedShort() == 0; // Exif\0\0
}
@Override
public void init(ExtractorOutput output) {
extractorOutput = output;
}
@Override
public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition)
throws IOException {
switch (state) {
case STATE_READING_MARKER:
readMarker(input);
return RESULT_CONTINUE;
case STATE_READING_SEGMENT_LENGTH:
readSegmentLength(input);
return RESULT_CONTINUE;
case STATE_READING_SEGMENT:
readSegment(input);
return RESULT_CONTINUE;
case STATE_SNIFFING_MOTION_PHOTO_VIDEO:
if (input.getPosition() != mp4StartPosition) {
seekPosition.position = mp4StartPosition;
return RESULT_SEEK;
}
sniffMotionPhotoVideo(input);
return RESULT_CONTINUE;
case STATE_READING_MOTION_PHOTO_VIDEO:
if (mp4ExtractorStartOffsetExtractorInput == null || input != lastExtractorInput) {
lastExtractorInput = input;
mp4ExtractorStartOffsetExtractorInput =
new StartOffsetExtractorInput(input, mp4StartPosition);
}
@ReadResult
int readResult =
checkNotNull(mp4Extractor).read(mp4ExtractorStartOffsetExtractorInput, seekPosition);
if (readResult == RESULT_SEEK) {
seekPosition.position += mp4StartPosition;
}
return readResult;
case STATE_ENDED:
return RESULT_END_OF_INPUT;
default:
throw new IllegalStateException();
}
}
@Override
public void seek(long position, long timeUs) {
if (position == 0) {
state = STATE_READING_MARKER;
mp4Extractor = null;
} else if (state == STATE_READING_MOTION_PHOTO_VIDEO) {
checkNotNull(mp4Extractor).seek(position, timeUs);
}
}
@Override
public void release() {
if (mp4Extractor != null) {
mp4Extractor.release();
}
}
private int peekMarker(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
return scratch.readUnsignedShort();
}
private void advancePeekPositionToNextSegment(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
int segmentLength = scratch.readUnsignedShort() - 2;
input.advancePeekPosition(segmentLength);
}
private void readMarker(ExtractorInput input) throws IOException {
scratch.reset(/* limit= */ 2);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
marker = scratch.readUnsignedShort();
if (marker == MARKER_SOS) { // Start of scan.
if (mp4StartPosition != C.INDEX_UNSET) {
state = STATE_SNIFFING_MOTION_PHOTO_VIDEO;
} else {
endReadingWithImageTrack();
}
} else if ((marker < 0xFFD0 || marker > 0xFFD9) && marker != 0xFF01) {
state = STATE_READING_SEGMENT_LENGTH;
}
}
private void readSegmentLength(ExtractorInput input) throws IOException {
scratch.reset(2);
input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2);
segmentLength = scratch.readUnsignedShort() - 2;
state = STATE_READING_SEGMENT;
}
private void readSegment(ExtractorInput input) throws IOException {
if (marker == MARKER_APP1) {
ParsableByteArray payload = new ParsableByteArray(segmentLength);
input.readFully(payload.getData(), /* offset= */ 0, /* length= */ segmentLength);
if (motionPhotoMetadata == null
&& HEADER_XMP_APP1.equals(payload.readNullTerminatedString())) {
@Nullable String xmpString = payload.readNullTerminatedString();
if (xmpString != null) {
motionPhotoMetadata = getMotionPhotoMetadata(xmpString, input.getLength());
if (motionPhotoMetadata != null) {
mp4StartPosition = motionPhotoMetadata.videoStartPosition;
}
}
}
} else {
input.skipFully(segmentLength);
}
state = STATE_READING_MARKER;
}
private void sniffMotionPhotoVideo(ExtractorInput input) throws IOException {
// Check if the file is truncated.
boolean peekedData =
input.peekFully(
scratch.getData(), /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true);
if (!peekedData) {
endReadingWithImageTrack();
} else {
input.resetPeekPosition();
if (mp4Extractor == null) {
mp4Extractor = new Mp4Extractor();
}
mp4ExtractorStartOffsetExtractorInput =
new StartOffsetExtractorInput(input, mp4StartPosition);
if (mp4Extractor.sniff(mp4ExtractorStartOffsetExtractorInput)) {
mp4Extractor.init(
new StartOffsetExtractorOutput(mp4StartPosition, checkNotNull(extractorOutput)));
startReadingMotionPhoto();
} else {
endReadingWithImageTrack();
}
}
}
private void startReadingMotionPhoto() {
outputImageTrack(checkNotNull(motionPhotoMetadata));
state = STATE_READING_MOTION_PHOTO_VIDEO;
}
private void endReadingWithImageTrack() {
outputImageTrack();
checkNotNull(extractorOutput).endTracks();
extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
state = STATE_ENDED;
}
private void outputImageTrack(Metadata.Entry... metadataEntries) {
TrackOutput imageTrackOutput =
checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE);
// TODO(b/289989902): Set the rotationDegrees in format so images can be decoded correctly.
imageTrackOutput.format(
new Format.Builder()
.setContainerMimeType(MimeTypes.IMAGE_JPEG)
.setMetadata(new Metadata(metadataEntries))
.build());
}
/**
* Attempts to parse the specified XMP data describing the motion photo, returning the resulting
* {@link MotionPhotoMetadata} or {@code null} if it wasn't possible to derive motion photo
* metadata.
*
* @param xmpString A string of XML containing XMP motion photo metadata to attempt to parse.
* @param inputLength The length of the input stream in bytes, or {@link C#LENGTH_UNSET} if
* unknown.
* @return The {@link MotionPhotoMetadata}, or {@code null} if it wasn't possible to derive motion
* photo metadata.
* @throws IOException If an error occurs parsing the XMP string.
*/
@Nullable
private static MotionPhotoMetadata getMotionPhotoMetadata(String xmpString, long inputLength)
throws IOException {
// Metadata defines offsets from the end of the stream, so we need the stream length to
// determine start offsets.
if (inputLength == C.LENGTH_UNSET) {
return null;
}
// Motion photos have (at least) a primary image media item and a secondary video media item.
@Nullable
MotionPhotoDescription motionPhotoDescription =
XmpMotionPhotoDescriptionParser.parse(xmpString);
if (motionPhotoDescription == null) {
return null;
}
return motionPhotoDescription.getMotionPhotoMetadata(inputLength);
}
}

14
libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java
View file

@ -21,7 +21,7 @@ import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SingleSampleExtractorHelper;
import androidx.media3.extractor.SingleSampleExtractor;
import java.io.IOException;
/** Extracts data from the PNG container format. */
@ -32,21 +32,23 @@ public final class PngExtractor implements Extractor {
private static final int PNG_FILE_SIGNATURE = 0x8950;
private static final int PNG_FILE_SIGNATURE_LENGTH = 2;
private final SingleSampleExtractorHelper imageExtractor;
private final SingleSampleExtractor imageExtractor;
/** Creates an instance. */
public PngExtractor() {
imageExtractor = new SingleSampleExtractorHelper();
imageExtractor =
new SingleSampleExtractor(
PNG_FILE_SIGNATURE, PNG_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_PNG);
}
@Override
public boolean sniff(ExtractorInput input) throws IOException {
return imageExtractor.sniff(input, PNG_FILE_SIGNATURE, PNG_FILE_SIGNATURE_LENGTH);
return imageExtractor.sniff(input);
}
@Override
public void init(ExtractorOutput output) {
imageExtractor.init(output, MimeTypes.IMAGE_PNG);
imageExtractor.init(output);
}
@Override
@ -57,7 +59,7 @@ public final class PngExtractor implements Extractor {
@Override
public void seek(long position, long timeUs) {
imageExtractor.seek(position);
imageExtractor.seek(position, timeUs);
}
@Override

13
libraries/extractor/src/main/java/androidx/media3/extractor/webp/WebpExtractor.java
View file

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.webp;
import androidx.media3.common.C;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
@ -22,26 +23,26 @@ import androidx.media3.extractor.Extractor;
import androidx.media3.extractor.ExtractorInput;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.PositionHolder;
import androidx.media3.extractor.SingleSampleExtractorHelper;
import androidx.media3.extractor.SingleSampleExtractor;
import java.io.IOException;
/** Extracts data from the WEBP container format. */
@UnstableApi
public final class WebpExtractor implements Extractor {
// Documentation Reference:
// Documentation reference:
// https://developers.google.com/speed/webp/docs/riff_container#webp_file_header
private static final int FILE_SIGNATURE_SEGMENT_LENGTH = 4;
private static final int RIFF_FILE_SIGNATURE = 0x52494646;
private static final int WEBP_FILE_SIGNATURE = 0x57454250;
private final ParsableByteArray scratch;
private final SingleSampleExtractorHelper imageExtractor;
private final SingleSampleExtractor imageExtractor;
/** Creates an instance. */
public WebpExtractor() {
scratch = new ParsableByteArray(FILE_SIGNATURE_SEGMENT_LENGTH);
imageExtractor = new SingleSampleExtractorHelper();
imageExtractor = new SingleSampleExtractor(C.INDEX_UNSET, C.LENGTH_UNSET, MimeTypes.IMAGE_WEBP);
}
@Override
@ -62,7 +63,7 @@ public final class WebpExtractor implements Extractor {
@Override
public void init(ExtractorOutput output) {
imageExtractor.init(output, MimeTypes.IMAGE_WEBP);
imageExtractor.init(output);
}
@Override
@ -73,7 +74,7 @@ public final class WebpExtractor implements Extractor {
@Override
public void seek(long position, long timeUs) {
imageExtractor.seek(position);
imageExtractor.seek(position, timeUs);
}
@Override

44
libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/JpegExtractorTest.java
View file

@ -21,7 +21,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.ParameterizedRobolectricTestRunner;
/** Unit tests for {@link JpegExtractor}. */
/** Unit tests for {@link JpegExtractorTest}. */
@RunWith(ParameterizedRobolectricTestRunner.class)
public final class JpegExtractorTest {
@ -34,36 +34,62 @@ public final class JpegExtractorTest {
public ExtractorAsserts.SimulationConfig simulationConfig;
@Test
public void sampleNonMotionPhotoShortened() throws Exception {
public void sampleNonMotionPhotoShortened_extractImage() throws Exception {
ExtractorAsserts.assertBehavior(
() -> new JpegExtractor(JpegExtractor.FLAG_READ_IMAGE),
"media/jpeg/non-motion-photo-shortened.jpg",
new ExtractorAsserts.AssertionConfig.Builder()
.setDumpFilesPrefix(
"extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE")
.build(),
simulationConfig);
}
@Test
public void samplePixelMotionPhotoShortened_extractImage() throws Exception {
ExtractorAsserts.assertBehavior(
() -> new JpegExtractor(JpegExtractor.FLAG_READ_IMAGE),
"media/jpeg/pixel-motion-photo-shortened.jpg",
new ExtractorAsserts.AssertionConfig.Builder()
.setDumpFilesPrefix(
"extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE")
.build(),
simulationConfig);
}
@Test
public void sampleNonMotionPhotoShortened_extractMotionPhoto() throws Exception {
ExtractorAsserts.assertBehavior(
JpegExtractor::new, "media/jpeg/non-motion-photo-shortened.jpg", simulationConfig);
}
@Test
public void samplePixelMotionPhotoShortened() throws Exception {
public void samplePixelMotionPhotoShortened_extractMotionPhoto() throws Exception {
ExtractorAsserts.assertBehavior(
JpegExtractor::new, "media/jpeg/pixel-motion-photo-shortened.jpg", simulationConfig);
}
@Test
public void samplePixelMotionPhotoJfifSegmentShortened() throws Exception {
public void samplePixelMotionPhotoJfifSegmentShortened_extractMotionPhoto() throws Exception {
ExtractorAsserts.assertBehavior(
JpegExtractor::new,
JpegMotionPhotoExtractor::new,
"media/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg",
simulationConfig);
}
@Test
public void samplePixelMotionPhotoVideoRemovedShortened() throws Exception {
public void samplePixelMotionPhotoVideoRemovedShortened_extractMotionPhoto() throws Exception {
ExtractorAsserts.assertBehavior(
JpegExtractor::new,
JpegMotionPhotoExtractor::new,
"media/jpeg/pixel-motion-photo-video-removed-shortened.jpg",
simulationConfig);
}
@Test
public void sampleSsMotionPhotoShortened() throws Exception {
public void sampleSsMotionPhotoShortened_extractMotionPhoto() throws Exception {
ExtractorAsserts.assertBehavior(
JpegExtractor::new, "media/jpeg/ss-motion-photo-shortened.jpg", simulationConfig);
JpegMotionPhotoExtractor::new,
"media/jpeg/ss-motion-photo-shortened.jpg",
simulationConfig);
}
}

16
libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump Normal file
View file

@ -0,0 +1,16 @@
seekMap:
isSeekable = true
duration = UNSET TIME
getPosition(0) = [[timeUs=0, position=0]]
getPosition(1) = [[timeUs=1, position=0]]
numberOfTracks = 1
track 1024:
total output bytes = 30000
sample count = 1
format 0:
containerMimeType = image/jpeg
sample 0:
time = 0
flags = 1
data = length 30000, hash E5AA10BC
tracksEnded = true

16
libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump Normal file
View file

@ -0,0 +1,16 @@
seekMap:
isSeekable = true
duration = UNSET TIME
getPosition(0) = [[timeUs=0, position=0]]
getPosition(1) = [[timeUs=1, position=0]]
numberOfTracks = 1
track 1024:
total output bytes = 30000
sample count = 1
format 0:
containerMimeType = image/jpeg
sample 0:
time = 0
flags = 1
data = length 30000, hash E5AA10BC
tracksEnded = true

16
libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump Normal file
View file

@ -0,0 +1,16 @@
seekMap:
isSeekable = true
duration = UNSET TIME
getPosition(0) = [[timeUs=0, position=0]]
getPosition(1) = [[timeUs=1, position=0]]
numberOfTracks = 1
track 1024:
total output bytes = 140312
sample count = 1
format 0:
containerMimeType = image/jpeg
sample 0:
time = 0
flags = 1
data = length 140312, hash E1CC5149
tracksEnded = true

16
libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump Normal file
View file

@ -0,0 +1,16 @@
seekMap:
isSeekable = true
duration = UNSET TIME
getPosition(0) = [[timeUs=0, position=0]]
getPosition(1) = [[timeUs=1, position=0]]
numberOfTracks = 1
track 1024:
total output bytes = 140312
sample count = 1
format 0:
containerMimeType = image/jpeg
sample 0:
time = 0
flags = 1
data = length 140312, hash E1CC5149
tracksEnded = true