From be5fa6d1300e2782eaca183aafea422b61f665ba Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 8 Sep 2023 07:29:49 -0700 Subject: [PATCH] Support image track extraction for JPEG PiperOrigin-RevId: 563746945 --- ...Helper.java => SingleSampleExtractor.java} | 66 ++-- .../media3/extractor/bmp/BmpExtractor.java | 14 +- .../media3/extractor/heif/HeifExtractor.java | 11 +- .../media3/extractor/jpeg/JpegExtractor.java | 291 +++------------- .../jpeg/JpegMotionPhotoExtractor.java | 314 ++++++++++++++++++ .../media3/extractor/png/PngExtractor.java | 14 +- .../media3/extractor/webp/WebpExtractor.java | 13 +- .../extractor/jpeg/JpegExtractorTest.java | 44 ++- ...d.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump | 16 + ...ractor.FLAG_READ_IMAGE.unknown_length.dump | 16 + ...d.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump | 16 + ...ractor.FLAG_READ_IMAGE.unknown_length.dump | 16 + 12 files changed, 522 insertions(+), 309 deletions(-) rename libraries/extractor/src/main/java/androidx/media3/extractor/{SingleSampleExtractorHelper.java => SingleSampleExtractor.java} (75%) create mode 100644 libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java create mode 100644 libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump create mode 100644 libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump create mode 100644 libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump create mode 100644 libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractorHelper.java b/libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractor.java similarity index 75% rename from libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractorHelper.java rename to libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractor.java index 4f5da32b2a..afcdf7ee6d 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractorHelper.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/SingleSampleExtractor.java @@ -17,8 +17,7 @@ package androidx.media3.extractor; import static androidx.media3.common.C.BUFFER_FLAG_KEY_FRAME; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.extractor.Extractor.RESULT_CONTINUE; -import static androidx.media3.extractor.Extractor.RESULT_END_OF_INPUT; +import static androidx.media3.common.util.Assertions.checkState; import static java.lang.annotation.ElementType.TYPE_USE; import androidx.annotation.IntDef; @@ -35,13 +34,13 @@ import java.lang.annotation.Target; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -/** - * Extracts data by loading all the bytes into one sample. - * - *

Used as a component in other extractors. - */ +/** Extracts data by loading all the bytes into one sample. */ @UnstableApi -public final class SingleSampleExtractorHelper { +public final class SingleSampleExtractor implements Extractor { + + private final int fileSignature; + private final int fileSignatureLength; + private final String containerMimeType; /** Parser states. */ @Documented @@ -67,36 +66,36 @@ public final class SingleSampleExtractorHelper { private @MonotonicNonNull TrackOutput trackOutput; /** - * Returns whether the {@link ExtractorInput} has the given {@code fileSignature}. + * Creates an instance. * - *

@see Extractor#sniff(ExtractorInput) + * @param fileSignature The file signature used to {@link #sniff}, or {@link C#INDEX_UNSET} if the + * method won't be used. + * @param fileSignatureLength The length of file signature, or {@link C#LENGTH_UNSET} if the + * {@link #sniff} method won't be used. + * @param containerMimeType The mime type of the format being extracted. */ - public boolean sniff(ExtractorInput input, int fileSignature, int fileSignatureLength) - throws IOException { + public SingleSampleExtractor( + int fileSignature, int fileSignatureLength, String containerMimeType) { + this.fileSignature = fileSignature; + this.fileSignatureLength = fileSignatureLength; + this.containerMimeType = containerMimeType; + } + + @Override + public boolean sniff(ExtractorInput input) throws IOException { + checkState(fileSignature != C.INDEX_UNSET && fileSignatureLength != C.LENGTH_UNSET); ParsableByteArray scratch = new ParsableByteArray(fileSignatureLength); input.peekFully(scratch.getData(), /* offset= */ 0, fileSignatureLength); return scratch.readUnsignedShort() == fileSignature; } - /** - * See {@link Extractor#init(ExtractorOutput)}. - * - *

Outputs format with {@code containerMimeType}. - */ - public void init(ExtractorOutput output, String containerMimeType) { + @Override + public void init(ExtractorOutput output) { extractorOutput = output; outputImageTrackAndSeekMap(containerMimeType); } - /** See {@link Extractor#seek(long, long)}. */ - public void seek(long position) { - if (position == 0 || state == STATE_READING) { - state = STATE_READING; - size = 0; - } - } - - /** See {@link Extractor#read(ExtractorInput, PositionHolder)}. */ + @Override public @Extractor.ReadResult int read(ExtractorInput input, PositionHolder seekPosition) throws IOException { switch (state) { @@ -110,6 +109,19 @@ public final class SingleSampleExtractorHelper { } } + @Override + public void seek(long position, long timeUs) { + if (position == 0 || state == STATE_READING) { + state = STATE_READING; + size = 0; + } + } + + @Override + public void release() { + // Do Nothing + } + private void readSegment(ExtractorInput input) throws IOException { int result = checkNotNull(trackOutput).sampleData(input, FIXED_READ_LENGTH, /* allowEndOfInput= */ true); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/bmp/BmpExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/bmp/BmpExtractor.java index cac98de6ee..5196a70999 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/bmp/BmpExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/bmp/BmpExtractor.java @@ -21,7 +21,7 @@ import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; -import androidx.media3.extractor.SingleSampleExtractorHelper; +import androidx.media3.extractor.SingleSampleExtractor; import java.io.IOException; /** Extracts data from the BMP container format. */ @@ -30,21 +30,23 @@ public final class BmpExtractor implements Extractor { private static final int BMP_FILE_SIGNATURE_LENGTH = 2; private static final int BMP_FILE_SIGNATURE = 0x424D; - private final SingleSampleExtractorHelper imageExtractor; + private final SingleSampleExtractor imageExtractor; /** Creates an instance. */ public BmpExtractor() { - imageExtractor = new SingleSampleExtractorHelper(); + imageExtractor = + new SingleSampleExtractor( + BMP_FILE_SIGNATURE, BMP_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_BMP); } @Override public boolean sniff(ExtractorInput input) throws IOException { - return imageExtractor.sniff(input, BMP_FILE_SIGNATURE, BMP_FILE_SIGNATURE_LENGTH); + return imageExtractor.sniff(input); } @Override public void init(ExtractorOutput output) { - imageExtractor.init(output, MimeTypes.IMAGE_BMP); + imageExtractor.init(output); } @Override @@ -55,7 +57,7 @@ public final class BmpExtractor implements Extractor { @Override public void seek(long position, long timeUs) { - imageExtractor.seek(position); + imageExtractor.seek(position, timeUs); } @Override diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/heif/HeifExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/heif/HeifExtractor.java index c94b78037f..aca3ffc1c7 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/heif/HeifExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/heif/HeifExtractor.java @@ -15,6 +15,7 @@ */ package androidx.media3.extractor.heif; +import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; @@ -22,7 +23,7 @@ import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; -import androidx.media3.extractor.SingleSampleExtractorHelper; +import androidx.media3.extractor.SingleSampleExtractor; import java.io.IOException; /** Extracts data from the HEIF (.heic) container format. */ @@ -35,12 +36,12 @@ public final class HeifExtractor implements Extractor { private static final int FILE_SIGNATURE_SEGMENT_LENGTH = 4; private final ParsableByteArray scratch; - private final SingleSampleExtractorHelper imageExtractor; + private final SingleSampleExtractor imageExtractor; /** Creates an instance. */ public HeifExtractor() { scratch = new ParsableByteArray(FILE_SIGNATURE_SEGMENT_LENGTH); - imageExtractor = new SingleSampleExtractorHelper(); + imageExtractor = new SingleSampleExtractor(C.INDEX_UNSET, C.LENGTH_UNSET, MimeTypes.IMAGE_HEIF); } @Override @@ -52,7 +53,7 @@ public final class HeifExtractor implements Extractor { @Override public void init(ExtractorOutput output) { - imageExtractor.init(output, MimeTypes.IMAGE_HEIF); + imageExtractor.init(output); } @Override @@ -63,7 +64,7 @@ public final class HeifExtractor implements Extractor { @Override public void seek(long position, long timeUs) { - imageExtractor.seek(position); + imageExtractor.seek(position, timeUs); } @Override diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java index 142ea71e6c..be85f4f6f7 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegExtractor.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Android Open Source Project + * Copyright 2023 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,300 +15,91 @@ */ package androidx.media3.extractor.jpeg; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.extractor.SingleSampleExtractorHelper.IMAGE_TRACK_ID; import static java.lang.annotation.ElementType.TYPE_USE; import androidx.annotation.IntDef; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.Format; -import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; -import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; -import androidx.media3.extractor.SeekMap; -import androidx.media3.extractor.TrackOutput; -import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata; -import androidx.media3.extractor.mp4.Mp4Extractor; +import androidx.media3.extractor.SingleSampleExtractor; import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** Extracts JPEG image using the Exif format. */ +/** Extracts data from the JPEG container format. */ @UnstableApi public final class JpegExtractor implements Extractor { - - /** Parser states. */ + /** + * Flags controlling the behavior of the extractor. Possible flag value is {@link + * #FLAG_READ_IMAGE}. + */ @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) - @IntDef({ - STATE_READING_MARKER, - STATE_READING_SEGMENT_LENGTH, - STATE_READING_SEGMENT, - STATE_SNIFFING_MOTION_PHOTO_VIDEO, - STATE_READING_MOTION_PHOTO_VIDEO, - STATE_ENDED, - }) - private @interface State {} + @IntDef( + flag = true, + value = { + FLAG_READ_IMAGE, + }) + public @interface Flags {} - private static final int STATE_READING_MARKER = 0; - private static final int STATE_READING_SEGMENT_LENGTH = 1; - private static final int STATE_READING_SEGMENT = 2; - private static final int STATE_SNIFFING_MOTION_PHOTO_VIDEO = 4; - private static final int STATE_READING_MOTION_PHOTO_VIDEO = 5; - private static final int STATE_ENDED = 6; + /** Flag to load the image track instead of the video and metadata track. */ + public static final int FLAG_READ_IMAGE = 1; - private static final int EXIF_ID_CODE_LENGTH = 6; - private static final long EXIF_HEADER = 0x45786966; // Exif - private static final int MARKER_SOI = 0xFFD8; // Start of image marker - private static final int MARKER_SOS = 0xFFDA; // Start of scan (image data) marker - private static final int MARKER_APP0 = 0xFFE0; // Application data 0 marker - private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker - private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/"; + // Specification reference: ITU-T.81 (1992) subsection B.1.1.3 + private static final int JPEG_FILE_SIGNATURE = 0xFFD8; // Start of image marker + private static final int JPEG_FILE_SIGNATURE_LENGTH = 2; - private final ParsableByteArray scratch; - - private @MonotonicNonNull ExtractorOutput extractorOutput; - - private @State int state; - private int marker; - private int segmentLength; - private long mp4StartPosition; - - @Nullable private MotionPhotoMetadata motionPhotoMetadata; - private @MonotonicNonNull ExtractorInput lastExtractorInput; - private @MonotonicNonNull StartOffsetExtractorInput mp4ExtractorStartOffsetExtractorInput; - @Nullable private Mp4Extractor mp4Extractor; + private final Extractor extractor; + /** Creates an instance reading the video and metadata track. */ public JpegExtractor() { - scratch = new ParsableByteArray(EXIF_ID_CODE_LENGTH); - mp4StartPosition = C.INDEX_UNSET; + this(/* flags= */ 0); + } + + /** + * Creates an instance. + * + * @param flags The {@link JpegExtractor.Flags} to control extractor behavior. + */ + public JpegExtractor(@JpegExtractor.Flags int flags) { + if ((flags & FLAG_READ_IMAGE) != 0) { + extractor = + new SingleSampleExtractor( + JPEG_FILE_SIGNATURE, JPEG_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_JPEG); + } else { + extractor = new JpegMotionPhotoExtractor(); + } } @Override public boolean sniff(ExtractorInput input) throws IOException { - // See ITU-T.81 (1992) subsection B.1.1.3 and Exif version 2.2 (2002) subsection 4.5.4. - if (peekMarker(input) != MARKER_SOI) { - return false; - } - marker = peekMarker(input); - // Even though JFIF and Exif standards are incompatible in theory, Exif files often contain a - // JFIF APP0 marker segment preceding the Exif APP1 marker segment. Skip the JFIF segment if - // present. - if (marker == MARKER_APP0) { - advancePeekPositionToNextSegment(input); - marker = peekMarker(input); - } - if (marker != MARKER_APP1) { - return false; - } - input.advancePeekPosition(2); // Unused segment length - scratch.reset(/* limit= */ EXIF_ID_CODE_LENGTH); - input.peekFully(scratch.getData(), /* offset= */ 0, EXIF_ID_CODE_LENGTH); - return scratch.readUnsignedInt() == EXIF_HEADER && scratch.readUnsignedShort() == 0; // Exif\0\0 + return extractor.sniff(input); } @Override public void init(ExtractorOutput output) { - extractorOutput = output; + extractor.init(output); } @Override public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition) throws IOException { - switch (state) { - case STATE_READING_MARKER: - readMarker(input); - return RESULT_CONTINUE; - case STATE_READING_SEGMENT_LENGTH: - readSegmentLength(input); - return RESULT_CONTINUE; - case STATE_READING_SEGMENT: - readSegment(input); - return RESULT_CONTINUE; - case STATE_SNIFFING_MOTION_PHOTO_VIDEO: - if (input.getPosition() != mp4StartPosition) { - seekPosition.position = mp4StartPosition; - return RESULT_SEEK; - } - sniffMotionPhotoVideo(input); - return RESULT_CONTINUE; - case STATE_READING_MOTION_PHOTO_VIDEO: - if (mp4ExtractorStartOffsetExtractorInput == null || input != lastExtractorInput) { - lastExtractorInput = input; - mp4ExtractorStartOffsetExtractorInput = - new StartOffsetExtractorInput(input, mp4StartPosition); - } - @ReadResult - int readResult = - checkNotNull(mp4Extractor).read(mp4ExtractorStartOffsetExtractorInput, seekPosition); - if (readResult == RESULT_SEEK) { - seekPosition.position += mp4StartPosition; - } - return readResult; - case STATE_ENDED: - return RESULT_END_OF_INPUT; - default: - throw new IllegalStateException(); - } + return extractor.read(input, seekPosition); } @Override public void seek(long position, long timeUs) { - if (position == 0) { - state = STATE_READING_MARKER; - mp4Extractor = null; - } else if (state == STATE_READING_MOTION_PHOTO_VIDEO) { - checkNotNull(mp4Extractor).seek(position, timeUs); - } + extractor.seek(position, timeUs); } @Override public void release() { - if (mp4Extractor != null) { - mp4Extractor.release(); - } - } - - private int peekMarker(ExtractorInput input) throws IOException { - scratch.reset(/* limit= */ 2); - input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); - return scratch.readUnsignedShort(); - } - - private void advancePeekPositionToNextSegment(ExtractorInput input) throws IOException { - scratch.reset(/* limit= */ 2); - input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); - int segmentLength = scratch.readUnsignedShort() - 2; - input.advancePeekPosition(segmentLength); - } - - private void readMarker(ExtractorInput input) throws IOException { - scratch.reset(/* limit= */ 2); - input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); - marker = scratch.readUnsignedShort(); - if (marker == MARKER_SOS) { // Start of scan. - if (mp4StartPosition != C.INDEX_UNSET) { - state = STATE_SNIFFING_MOTION_PHOTO_VIDEO; - } else { - endReadingWithImageTrack(); - } - } else if ((marker < 0xFFD0 || marker > 0xFFD9) && marker != 0xFF01) { - state = STATE_READING_SEGMENT_LENGTH; - } - } - - private void readSegmentLength(ExtractorInput input) throws IOException { - scratch.reset(2); - input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); - segmentLength = scratch.readUnsignedShort() - 2; - state = STATE_READING_SEGMENT; - } - - private void readSegment(ExtractorInput input) throws IOException { - if (marker == MARKER_APP1) { - ParsableByteArray payload = new ParsableByteArray(segmentLength); - input.readFully(payload.getData(), /* offset= */ 0, /* length= */ segmentLength); - if (motionPhotoMetadata == null - && HEADER_XMP_APP1.equals(payload.readNullTerminatedString())) { - @Nullable String xmpString = payload.readNullTerminatedString(); - if (xmpString != null) { - motionPhotoMetadata = getMotionPhotoMetadata(xmpString, input.getLength()); - if (motionPhotoMetadata != null) { - mp4StartPosition = motionPhotoMetadata.videoStartPosition; - } - } - } - } else { - input.skipFully(segmentLength); - } - state = STATE_READING_MARKER; - } - - private void sniffMotionPhotoVideo(ExtractorInput input) throws IOException { - // Check if the file is truncated. - boolean peekedData = - input.peekFully( - scratch.getData(), /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true); - if (!peekedData) { - endReadingWithImageTrack(); - } else { - input.resetPeekPosition(); - if (mp4Extractor == null) { - mp4Extractor = new Mp4Extractor(); - } - mp4ExtractorStartOffsetExtractorInput = - new StartOffsetExtractorInput(input, mp4StartPosition); - if (mp4Extractor.sniff(mp4ExtractorStartOffsetExtractorInput)) { - mp4Extractor.init( - new StartOffsetExtractorOutput(mp4StartPosition, checkNotNull(extractorOutput))); - startReadingMotionPhoto(); - } else { - endReadingWithImageTrack(); - } - } - } - - private void startReadingMotionPhoto() { - outputImageTrack(checkNotNull(motionPhotoMetadata)); - state = STATE_READING_MOTION_PHOTO_VIDEO; - } - - private void endReadingWithImageTrack() { - outputImageTrack(); - checkNotNull(extractorOutput).endTracks(); - extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET)); - state = STATE_ENDED; - } - - private void outputImageTrack(Metadata.Entry... metadataEntries) { - TrackOutput imageTrackOutput = - checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE); - // TODO(b/289989902): Set the rotationDegrees in format so images can be decoded correctly. - imageTrackOutput.format( - new Format.Builder() - .setContainerMimeType(MimeTypes.IMAGE_JPEG) - .setMetadata(new Metadata(metadataEntries)) - .build()); - } - - /** - * Attempts to parse the specified XMP data describing the motion photo, returning the resulting - * {@link MotionPhotoMetadata} or {@code null} if it wasn't possible to derive motion photo - * metadata. - * - * @param xmpString A string of XML containing XMP motion photo metadata to attempt to parse. - * @param inputLength The length of the input stream in bytes, or {@link C#LENGTH_UNSET} if - * unknown. - * @return The {@link MotionPhotoMetadata}, or {@code null} if it wasn't possible to derive motion - * photo metadata. - * @throws IOException If an error occurs parsing the XMP string. - */ - @Nullable - private static MotionPhotoMetadata getMotionPhotoMetadata(String xmpString, long inputLength) - throws IOException { - // Metadata defines offsets from the end of the stream, so we need the stream length to - // determine start offsets. - if (inputLength == C.LENGTH_UNSET) { - return null; - } - - // Motion photos have (at least) a primary image media item and a secondary video media item. - @Nullable - MotionPhotoDescription motionPhotoDescription = - XmpMotionPhotoDescriptionParser.parse(xmpString); - if (motionPhotoDescription == null) { - return null; - } - return motionPhotoDescription.getMotionPhotoMetadata(inputLength); + extractor.release(); } } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java new file mode 100644 index 0000000000..89d7e65398 --- /dev/null +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java @@ -0,0 +1,314 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor.jpeg; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.extractor.SingleSampleExtractor.IMAGE_TRACK_ID; +import static java.lang.annotation.ElementType.TYPE_USE; + +import androidx.annotation.IntDef; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.extractor.Extractor; +import androidx.media3.extractor.ExtractorInput; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.PositionHolder; +import androidx.media3.extractor.SeekMap; +import androidx.media3.extractor.TrackOutput; +import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata; +import androidx.media3.extractor.mp4.Mp4Extractor; +import java.io.IOException; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Extracts JPEG metadata and motion photo using the Exif format. */ +@UnstableApi +/* package */ final class JpegMotionPhotoExtractor implements Extractor { + + /** Parser states. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({ + STATE_READING_MARKER, + STATE_READING_SEGMENT_LENGTH, + STATE_READING_SEGMENT, + STATE_SNIFFING_MOTION_PHOTO_VIDEO, + STATE_READING_MOTION_PHOTO_VIDEO, + STATE_ENDED, + }) + private @interface State {} + + private static final int STATE_READING_MARKER = 0; + private static final int STATE_READING_SEGMENT_LENGTH = 1; + private static final int STATE_READING_SEGMENT = 2; + private static final int STATE_SNIFFING_MOTION_PHOTO_VIDEO = 4; + private static final int STATE_READING_MOTION_PHOTO_VIDEO = 5; + private static final int STATE_ENDED = 6; + + private static final int EXIF_ID_CODE_LENGTH = 6; + private static final long EXIF_HEADER = 0x45786966; // Exif + private static final int MARKER_SOI = 0xFFD8; // Start of image marker + private static final int MARKER_SOS = 0xFFDA; // Start of scan (image data) marker + private static final int MARKER_APP0 = 0xFFE0; // Application data 0 marker + private static final int MARKER_APP1 = 0xFFE1; // Application data 1 marker + private static final String HEADER_XMP_APP1 = "http://ns.adobe.com/xap/1.0/"; + + private final ParsableByteArray scratch; + + private @MonotonicNonNull ExtractorOutput extractorOutput; + + private @State int state; + private int marker; + private int segmentLength; + private long mp4StartPosition; + + @Nullable private MotionPhotoMetadata motionPhotoMetadata; + private @MonotonicNonNull ExtractorInput lastExtractorInput; + private @MonotonicNonNull StartOffsetExtractorInput mp4ExtractorStartOffsetExtractorInput; + @Nullable private Mp4Extractor mp4Extractor; + + public JpegMotionPhotoExtractor() { + scratch = new ParsableByteArray(EXIF_ID_CODE_LENGTH); + mp4StartPosition = C.INDEX_UNSET; + } + + @Override + public boolean sniff(ExtractorInput input) throws IOException { + // See ITU-T.81 (1992) subsection B.1.1.3 and Exif version 2.2 (2002) subsection 4.5.4. + if (peekMarker(input) != MARKER_SOI) { + return false; + } + marker = peekMarker(input); + // Even though JFIF and Exif standards are incompatible in theory, Exif files often contain a + // JFIF APP0 marker segment preceding the Exif APP1 marker segment. Skip the JFIF segment if + // present. + if (marker == MARKER_APP0) { + advancePeekPositionToNextSegment(input); + marker = peekMarker(input); + } + if (marker != MARKER_APP1) { + return false; + } + input.advancePeekPosition(2); // Unused segment length + scratch.reset(/* limit= */ EXIF_ID_CODE_LENGTH); + input.peekFully(scratch.getData(), /* offset= */ 0, EXIF_ID_CODE_LENGTH); + return scratch.readUnsignedInt() == EXIF_HEADER && scratch.readUnsignedShort() == 0; // Exif\0\0 + } + + @Override + public void init(ExtractorOutput output) { + extractorOutput = output; + } + + @Override + public @ReadResult int read(ExtractorInput input, PositionHolder seekPosition) + throws IOException { + switch (state) { + case STATE_READING_MARKER: + readMarker(input); + return RESULT_CONTINUE; + case STATE_READING_SEGMENT_LENGTH: + readSegmentLength(input); + return RESULT_CONTINUE; + case STATE_READING_SEGMENT: + readSegment(input); + return RESULT_CONTINUE; + case STATE_SNIFFING_MOTION_PHOTO_VIDEO: + if (input.getPosition() != mp4StartPosition) { + seekPosition.position = mp4StartPosition; + return RESULT_SEEK; + } + sniffMotionPhotoVideo(input); + return RESULT_CONTINUE; + case STATE_READING_MOTION_PHOTO_VIDEO: + if (mp4ExtractorStartOffsetExtractorInput == null || input != lastExtractorInput) { + lastExtractorInput = input; + mp4ExtractorStartOffsetExtractorInput = + new StartOffsetExtractorInput(input, mp4StartPosition); + } + @ReadResult + int readResult = + checkNotNull(mp4Extractor).read(mp4ExtractorStartOffsetExtractorInput, seekPosition); + if (readResult == RESULT_SEEK) { + seekPosition.position += mp4StartPosition; + } + return readResult; + case STATE_ENDED: + return RESULT_END_OF_INPUT; + default: + throw new IllegalStateException(); + } + } + + @Override + public void seek(long position, long timeUs) { + if (position == 0) { + state = STATE_READING_MARKER; + mp4Extractor = null; + } else if (state == STATE_READING_MOTION_PHOTO_VIDEO) { + checkNotNull(mp4Extractor).seek(position, timeUs); + } + } + + @Override + public void release() { + if (mp4Extractor != null) { + mp4Extractor.release(); + } + } + + private int peekMarker(ExtractorInput input) throws IOException { + scratch.reset(/* limit= */ 2); + input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); + return scratch.readUnsignedShort(); + } + + private void advancePeekPositionToNextSegment(ExtractorInput input) throws IOException { + scratch.reset(/* limit= */ 2); + input.peekFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); + int segmentLength = scratch.readUnsignedShort() - 2; + input.advancePeekPosition(segmentLength); + } + + private void readMarker(ExtractorInput input) throws IOException { + scratch.reset(/* limit= */ 2); + input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); + marker = scratch.readUnsignedShort(); + if (marker == MARKER_SOS) { // Start of scan. + if (mp4StartPosition != C.INDEX_UNSET) { + state = STATE_SNIFFING_MOTION_PHOTO_VIDEO; + } else { + endReadingWithImageTrack(); + } + } else if ((marker < 0xFFD0 || marker > 0xFFD9) && marker != 0xFF01) { + state = STATE_READING_SEGMENT_LENGTH; + } + } + + private void readSegmentLength(ExtractorInput input) throws IOException { + scratch.reset(2); + input.readFully(scratch.getData(), /* offset= */ 0, /* length= */ 2); + segmentLength = scratch.readUnsignedShort() - 2; + state = STATE_READING_SEGMENT; + } + + private void readSegment(ExtractorInput input) throws IOException { + if (marker == MARKER_APP1) { + ParsableByteArray payload = new ParsableByteArray(segmentLength); + input.readFully(payload.getData(), /* offset= */ 0, /* length= */ segmentLength); + if (motionPhotoMetadata == null + && HEADER_XMP_APP1.equals(payload.readNullTerminatedString())) { + @Nullable String xmpString = payload.readNullTerminatedString(); + if (xmpString != null) { + motionPhotoMetadata = getMotionPhotoMetadata(xmpString, input.getLength()); + if (motionPhotoMetadata != null) { + mp4StartPosition = motionPhotoMetadata.videoStartPosition; + } + } + } + } else { + input.skipFully(segmentLength); + } + state = STATE_READING_MARKER; + } + + private void sniffMotionPhotoVideo(ExtractorInput input) throws IOException { + // Check if the file is truncated. + boolean peekedData = + input.peekFully( + scratch.getData(), /* offset= */ 0, /* length= */ 1, /* allowEndOfInput= */ true); + if (!peekedData) { + endReadingWithImageTrack(); + } else { + input.resetPeekPosition(); + if (mp4Extractor == null) { + mp4Extractor = new Mp4Extractor(); + } + mp4ExtractorStartOffsetExtractorInput = + new StartOffsetExtractorInput(input, mp4StartPosition); + if (mp4Extractor.sniff(mp4ExtractorStartOffsetExtractorInput)) { + mp4Extractor.init( + new StartOffsetExtractorOutput(mp4StartPosition, checkNotNull(extractorOutput))); + startReadingMotionPhoto(); + } else { + endReadingWithImageTrack(); + } + } + } + + private void startReadingMotionPhoto() { + outputImageTrack(checkNotNull(motionPhotoMetadata)); + state = STATE_READING_MOTION_PHOTO_VIDEO; + } + + private void endReadingWithImageTrack() { + outputImageTrack(); + checkNotNull(extractorOutput).endTracks(); + extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET)); + state = STATE_ENDED; + } + + private void outputImageTrack(Metadata.Entry... metadataEntries) { + TrackOutput imageTrackOutput = + checkNotNull(extractorOutput).track(IMAGE_TRACK_ID, C.TRACK_TYPE_IMAGE); + // TODO(b/289989902): Set the rotationDegrees in format so images can be decoded correctly. + imageTrackOutput.format( + new Format.Builder() + .setContainerMimeType(MimeTypes.IMAGE_JPEG) + .setMetadata(new Metadata(metadataEntries)) + .build()); + } + + /** + * Attempts to parse the specified XMP data describing the motion photo, returning the resulting + * {@link MotionPhotoMetadata} or {@code null} if it wasn't possible to derive motion photo + * metadata. + * + * @param xmpString A string of XML containing XMP motion photo metadata to attempt to parse. + * @param inputLength The length of the input stream in bytes, or {@link C#LENGTH_UNSET} if + * unknown. + * @return The {@link MotionPhotoMetadata}, or {@code null} if it wasn't possible to derive motion + * photo metadata. + * @throws IOException If an error occurs parsing the XMP string. + */ + @Nullable + private static MotionPhotoMetadata getMotionPhotoMetadata(String xmpString, long inputLength) + throws IOException { + // Metadata defines offsets from the end of the stream, so we need the stream length to + // determine start offsets. + if (inputLength == C.LENGTH_UNSET) { + return null; + } + + // Motion photos have (at least) a primary image media item and a secondary video media item. + @Nullable + MotionPhotoDescription motionPhotoDescription = + XmpMotionPhotoDescriptionParser.parse(xmpString); + if (motionPhotoDescription == null) { + return null; + } + return motionPhotoDescription.getMotionPhotoMetadata(inputLength); + } +} diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java index 45c56ef042..a5899e47c8 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/png/PngExtractor.java @@ -21,7 +21,7 @@ import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; -import androidx.media3.extractor.SingleSampleExtractorHelper; +import androidx.media3.extractor.SingleSampleExtractor; import java.io.IOException; /** Extracts data from the PNG container format. */ @@ -32,21 +32,23 @@ public final class PngExtractor implements Extractor { private static final int PNG_FILE_SIGNATURE = 0x8950; private static final int PNG_FILE_SIGNATURE_LENGTH = 2; - private final SingleSampleExtractorHelper imageExtractor; + private final SingleSampleExtractor imageExtractor; /** Creates an instance. */ public PngExtractor() { - imageExtractor = new SingleSampleExtractorHelper(); + imageExtractor = + new SingleSampleExtractor( + PNG_FILE_SIGNATURE, PNG_FILE_SIGNATURE_LENGTH, MimeTypes.IMAGE_PNG); } @Override public boolean sniff(ExtractorInput input) throws IOException { - return imageExtractor.sniff(input, PNG_FILE_SIGNATURE, PNG_FILE_SIGNATURE_LENGTH); + return imageExtractor.sniff(input); } @Override public void init(ExtractorOutput output) { - imageExtractor.init(output, MimeTypes.IMAGE_PNG); + imageExtractor.init(output); } @Override @@ -57,7 +59,7 @@ public final class PngExtractor implements Extractor { @Override public void seek(long position, long timeUs) { - imageExtractor.seek(position); + imageExtractor.seek(position, timeUs); } @Override diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/webp/WebpExtractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/webp/WebpExtractor.java index d8e58e3172..2adddb737a 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/webp/WebpExtractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/webp/WebpExtractor.java @@ -15,6 +15,7 @@ */ package androidx.media3.extractor.webp; +import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; @@ -22,26 +23,26 @@ import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.PositionHolder; -import androidx.media3.extractor.SingleSampleExtractorHelper; +import androidx.media3.extractor.SingleSampleExtractor; import java.io.IOException; /** Extracts data from the WEBP container format. */ @UnstableApi public final class WebpExtractor implements Extractor { - // Documentation Reference: + // Documentation reference: // https://developers.google.com/speed/webp/docs/riff_container#webp_file_header private static final int FILE_SIGNATURE_SEGMENT_LENGTH = 4; private static final int RIFF_FILE_SIGNATURE = 0x52494646; private static final int WEBP_FILE_SIGNATURE = 0x57454250; private final ParsableByteArray scratch; - private final SingleSampleExtractorHelper imageExtractor; + private final SingleSampleExtractor imageExtractor; /** Creates an instance. */ public WebpExtractor() { scratch = new ParsableByteArray(FILE_SIGNATURE_SEGMENT_LENGTH); - imageExtractor = new SingleSampleExtractorHelper(); + imageExtractor = new SingleSampleExtractor(C.INDEX_UNSET, C.LENGTH_UNSET, MimeTypes.IMAGE_WEBP); } @Override @@ -62,7 +63,7 @@ public final class WebpExtractor implements Extractor { @Override public void init(ExtractorOutput output) { - imageExtractor.init(output, MimeTypes.IMAGE_WEBP); + imageExtractor.init(output); } @Override @@ -73,7 +74,7 @@ public final class WebpExtractor implements Extractor { @Override public void seek(long position, long timeUs) { - imageExtractor.seek(position); + imageExtractor.seek(position, timeUs); } @Override diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/JpegExtractorTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/JpegExtractorTest.java index 0d25c6e8a5..15eeeb85b8 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/JpegExtractorTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/jpeg/JpegExtractorTest.java @@ -21,7 +21,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.ParameterizedRobolectricTestRunner; -/** Unit tests for {@link JpegExtractor}. */ +/** Unit tests for {@link JpegExtractorTest}. */ @RunWith(ParameterizedRobolectricTestRunner.class) public final class JpegExtractorTest { @@ -34,36 +34,62 @@ public final class JpegExtractorTest { public ExtractorAsserts.SimulationConfig simulationConfig; @Test - public void sampleNonMotionPhotoShortened() throws Exception { + public void sampleNonMotionPhotoShortened_extractImage() throws Exception { + ExtractorAsserts.assertBehavior( + () -> new JpegExtractor(JpegExtractor.FLAG_READ_IMAGE), + "media/jpeg/non-motion-photo-shortened.jpg", + new ExtractorAsserts.AssertionConfig.Builder() + .setDumpFilesPrefix( + "extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE") + .build(), + simulationConfig); + } + + @Test + public void samplePixelMotionPhotoShortened_extractImage() throws Exception { + ExtractorAsserts.assertBehavior( + () -> new JpegExtractor(JpegExtractor.FLAG_READ_IMAGE), + "media/jpeg/pixel-motion-photo-shortened.jpg", + new ExtractorAsserts.AssertionConfig.Builder() + .setDumpFilesPrefix( + "extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE") + .build(), + simulationConfig); + } + + @Test + public void sampleNonMotionPhotoShortened_extractMotionPhoto() throws Exception { ExtractorAsserts.assertBehavior( JpegExtractor::new, "media/jpeg/non-motion-photo-shortened.jpg", simulationConfig); } @Test - public void samplePixelMotionPhotoShortened() throws Exception { + public void samplePixelMotionPhotoShortened_extractMotionPhoto() throws Exception { ExtractorAsserts.assertBehavior( JpegExtractor::new, "media/jpeg/pixel-motion-photo-shortened.jpg", simulationConfig); } @Test - public void samplePixelMotionPhotoJfifSegmentShortened() throws Exception { + public void samplePixelMotionPhotoJfifSegmentShortened_extractMotionPhoto() throws Exception { ExtractorAsserts.assertBehavior( - JpegExtractor::new, + JpegMotionPhotoExtractor::new, "media/jpeg/pixel-motion-photo-jfif-segment-shortened.jpg", simulationConfig); } @Test - public void samplePixelMotionPhotoVideoRemovedShortened() throws Exception { + public void samplePixelMotionPhotoVideoRemovedShortened_extractMotionPhoto() throws Exception { ExtractorAsserts.assertBehavior( - JpegExtractor::new, + JpegMotionPhotoExtractor::new, "media/jpeg/pixel-motion-photo-video-removed-shortened.jpg", simulationConfig); } @Test - public void sampleSsMotionPhotoShortened() throws Exception { + public void sampleSsMotionPhotoShortened_extractMotionPhoto() throws Exception { ExtractorAsserts.assertBehavior( - JpegExtractor::new, "media/jpeg/ss-motion-photo-shortened.jpg", simulationConfig); + JpegMotionPhotoExtractor::new, + "media/jpeg/ss-motion-photo-shortened.jpg", + simulationConfig); } } diff --git a/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump b/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump new file mode 100644 index 0000000000..c10a13bffb --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump @@ -0,0 +1,16 @@ +seekMap: + isSeekable = true + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] + getPosition(1) = [[timeUs=1, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 30000 + sample count = 1 + format 0: + containerMimeType = image/jpeg + sample 0: + time = 0 + flags = 1 + data = length 30000, hash E5AA10BC +tracksEnded = true diff --git a/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump b/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump new file mode 100644 index 0000000000..c10a13bffb --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/jpeg/non-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump @@ -0,0 +1,16 @@ +seekMap: + isSeekable = true + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] + getPosition(1) = [[timeUs=1, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 30000 + sample count = 1 + format 0: + containerMimeType = image/jpeg + sample 0: + time = 0 + flags = 1 + data = length 30000, hash E5AA10BC +tracksEnded = true diff --git a/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump b/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump new file mode 100644 index 0000000000..59895bff4a --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.0.dump @@ -0,0 +1,16 @@ +seekMap: + isSeekable = true + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] + getPosition(1) = [[timeUs=1, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 140312 + sample count = 1 + format 0: + containerMimeType = image/jpeg + sample 0: + time = 0 + flags = 1 + data = length 140312, hash E1CC5149 +tracksEnded = true diff --git a/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump b/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump new file mode 100644 index 0000000000..59895bff4a --- /dev/null +++ b/libraries/test_data/src/test/assets/extractordumps/jpeg/pixel-motion-photo-shortened.jpg_JpegExtractor.FLAG_READ_IMAGE.unknown_length.dump @@ -0,0 +1,16 @@ +seekMap: + isSeekable = true + duration = UNSET TIME + getPosition(0) = [[timeUs=0, position=0]] + getPosition(1) = [[timeUs=1, position=0]] +numberOfTracks = 1 +track 1024: + total output bytes = 140312 + sample count = 1 + format 0: + containerMimeType = image/jpeg + sample 0: + time = 0 + flags = 1 + data = length 140312, hash E1CC5149 +tracksEnded = true