diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index a0905944b3..9ce93ef81d 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -99,6 +99,8 @@
* Upgrade IMA SDK dependency to 3.20.1. This brings in a fix for
companion ads rendering when targeting API 29
([#6432](https://github.com/google/ExoPlayer/issues/6432)).
+* Metadata retriever:
+ * Parse Google Photos HEIC motion photos metadata.
### 2.12.0 (2020-09-11) ###
diff --git a/library/common/src/main/java/com/google/android/exoplayer2/C.java b/library/common/src/main/java/com/google/android/exoplayer2/C.java
index 1a1543cc7e..c0baa4cbdc 100644
--- a/library/common/src/main/java/com/google/android/exoplayer2/C.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/C.java
@@ -690,12 +690,14 @@ public final class C {
public static final int TRACK_TYPE_VIDEO = 2;
/** A type constant for text tracks. */
public static final int TRACK_TYPE_TEXT = 3;
+ /** A type constant for image tracks. */
+ public static final int TRACK_TYPE_IMAGE = 4;
/** A type constant for metadata tracks. */
- public static final int TRACK_TYPE_METADATA = 4;
+ public static final int TRACK_TYPE_METADATA = 5;
/** A type constant for camera motion tracks. */
- public static final int TRACK_TYPE_CAMERA_MOTION = 5;
+ public static final int TRACK_TYPE_CAMERA_MOTION = 6;
/** A type constant for a fake or empty track. */
- public static final int TRACK_TYPE_NONE = 6;
+ public static final int TRACK_TYPE_NONE = 7;
/**
* Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or
* equal to this value.
diff --git a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
index ca1a110c61..9dfd423a7d 100644
--- a/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
+++ b/library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java
@@ -20,21 +20,23 @@ import android.os.Parcel;
import android.os.Parcelable;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.common.primitives.Longs;
/** Metadata of a motion photo file. */
public final class MotionPhoto implements Metadata.Entry {
/** The start offset of the photo data, in bytes. */
- public final int photoStartPosition;
+ public final long photoStartPosition;
/** The size of the photo data, in bytes. */
- public final int photoSize;
+ public final long photoSize;
/** The start offset of the video data, in bytes. */
- public final int videoStartPosition;
+ public final long videoStartPosition;
/** The size of the video data, in bytes. */
- public final int videoSize;
+ public final long videoSize;
/** Creates an instance. */
- public MotionPhoto(int photoStartPosition, int photoSize, int videoStartPosition, int videoSize) {
+ public MotionPhoto(
+ long photoStartPosition, long photoSize, long videoStartPosition, long videoSize) {
this.photoStartPosition = photoStartPosition;
this.photoSize = photoSize;
this.videoStartPosition = videoStartPosition;
@@ -42,10 +44,10 @@ public final class MotionPhoto implements Metadata.Entry {
}
private MotionPhoto(Parcel in) {
- photoStartPosition = in.readInt();
- photoSize = in.readInt();
- videoStartPosition = in.readInt();
- videoSize = in.readInt();
+ photoStartPosition = in.readLong();
+ photoSize = in.readLong();
+ videoStartPosition = in.readLong();
+ videoSize = in.readLong();
}
@Override
@@ -66,10 +68,10 @@ public final class MotionPhoto implements Metadata.Entry {
@Override
public int hashCode() {
int result = 17;
- result = 31 * result + photoStartPosition;
- result = 31 * result + photoSize;
- result = 31 * result + videoStartPosition;
- result = 31 * result + videoSize;
+ result = 31 * result + Longs.hashCode(photoStartPosition);
+ result = 31 * result + Longs.hashCode(photoSize);
+ result = 31 * result + Longs.hashCode(videoStartPosition);
+ result = 31 * result + Longs.hashCode(videoSize);
return result;
}
@@ -89,10 +91,10 @@ public final class MotionPhoto implements Metadata.Entry {
@Override
public void writeToParcel(Parcel dest, int flags) {
- dest.writeInt(photoStartPosition);
- dest.writeInt(photoSize);
- dest.writeInt(videoStartPosition);
- dest.writeInt(videoSize);
+ dest.writeLong(photoStartPosition);
+ dest.writeLong(photoSize);
+ dest.writeLong(videoStartPosition);
+ dest.writeLong(videoSize);
}
@Override
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java b/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
index 72f6957865..5a00cd66f8 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java
@@ -22,6 +22,9 @@ import android.content.Context;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Message;
+import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory;
+import com.google.android.exoplayer2.extractor.ExtractorsFactory;
+import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor;
import com.google.android.exoplayer2.source.DefaultMediaSourceFactory;
import com.google.android.exoplayer2.source.MediaPeriod;
import com.google.android.exoplayer2.source.MediaSource;
@@ -43,8 +46,9 @@ public final class MetadataRetriever {
/**
* Retrieves the {@link TrackGroupArray} corresponding to a {@link MediaItem}.
*
- * This is equivalent to using {@code retrieveMetadata(new DefaultMediaSourceFactory(context),
- * mediaItem)}.
+ *
This is equivalent to using {@link #retrieveMetadata(MediaSourceFactory, MediaItem)} with a
+ * {@link DefaultMediaSourceFactory} and a {@link DefaultExtractorsFactory} with {@link
+ * Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} set.
*
* @param context The {@link Context}.
* @param mediaItem The {@link MediaItem} whose metadata should be retrieved.
@@ -52,7 +56,12 @@ public final class MetadataRetriever {
*/
public static ListenableFuture retrieveMetadata(
Context context, MediaItem mediaItem) {
- return retrieveMetadata(new DefaultMediaSourceFactory(context), mediaItem);
+ ExtractorsFactory extractorsFactory =
+ new DefaultExtractorsFactory()
+ .setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA);
+ MediaSourceFactory mediaSourceFactory =
+ new DefaultMediaSourceFactory(context, extractorsFactory);
+ return retrieveMetadata(mediaSourceFactory, mediaItem);
}
/**
diff --git a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
index e666ec979d..235639f678 100644
--- a/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java
@@ -25,6 +25,7 @@ import android.net.Uri;
import android.os.SystemClock;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
import com.google.android.exoplayer2.source.TrackGroupArray;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.util.concurrent.ListenableFuture;
@@ -37,7 +38,7 @@ import org.junit.runner.RunWith;
public class MetadataRetrieverTest {
@Test
- public void retrieveMetadata_singleMediaItem() throws Exception {
+ public void retrieveMetadata_singleMediaItem_outputsExpectedMetadata() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
MediaItem mediaItem =
MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -55,7 +56,7 @@ public class MetadataRetrieverTest {
}
@Test
- public void retrieveMetadata_multipleMediaItems() throws Exception {
+ public void retrieveMetadata_multipleMediaItems_outputsExpectedMetadata() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
MediaItem mediaItem1 =
MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -84,7 +85,28 @@ public class MetadataRetrieverTest {
}
@Test
- public void retrieveMetadata_throwsErrorIfCannotLoad() {
+ public void retrieveMetadata_motionPhoto_outputsExpectedMetadata() throws Exception {
+ Context context = ApplicationProvider.getApplicationContext();
+ MediaItem mediaItem =
+ MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_MP.heic"));
+ MotionPhoto expectedMotionPhoto =
+ new MotionPhoto(
+ /* photoStartPosition= */ 0,
+ /* photoSize= */ 28_853,
+ /* videoStartPosition= */ 28_869,
+ /* videoSize= */ 28_803);
+
+ ListenableFuture trackGroupsFuture = retrieveMetadata(context, mediaItem);
+ TrackGroupArray trackGroups = waitAndGetTrackGroups(trackGroupsFuture);
+
+ assertThat(trackGroups.length).isEqualTo(1);
+ assertThat(trackGroups.get(0).length).isEqualTo(1);
+ assertThat(trackGroups.get(0).getFormat(0).metadata.length()).isEqualTo(1);
+ assertThat(trackGroups.get(0).getFormat(0).metadata.get(0)).isEqualTo(expectedMotionPhoto);
+ }
+
+ @Test
+ public void retrieveMetadata_invalidMediaItem_throwsError() {
Context context = ApplicationProvider.getApplicationContext();
MediaItem mediaItem =
MediaItem.fromUri(Uri.parse("asset://android_asset/media/does_not_exist"));
diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
index 325dc24aec..71e6c69887 100644
--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java
@@ -181,6 +181,9 @@ import java.util.List;
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_moov = 0x6d6f6f76;
+ @SuppressWarnings("ConstantCaseForConstants")
+ public static final int TYPE_mpvd = 0x6d707664;
+
@SuppressWarnings("ConstantCaseForConstants")
public static final int TYPE_mvhd = 0x6d766864;
diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
index f9e70915bc..d478eb2b4b 100644
--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java
@@ -38,6 +38,7 @@ import com.google.android.exoplayer2.extractor.SeekPoint;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.NalUnitUtil;
@@ -61,19 +62,27 @@ public final class Mp4Extractor implements Extractor, SeekMap {
public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new Mp4Extractor()};
/**
- * Flags controlling the behavior of the extractor. Possible flag value is {@link
- * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}.
+ * Flags controlling the behavior of the extractor. Possible flag values are {@link
+ * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS} and {@link #FLAG_READ_MOTION_PHOTO_METADATA}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef(
flag = true,
- value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS})
+ value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS, FLAG_READ_MOTION_PHOTO_METADATA})
public @interface Flags {}
/**
* Flag to ignore any edit lists in the stream.
*/
public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1;
+ /**
+ * Flag to extract {@link MotionPhoto} metadata from HEIC motion photos following the Google
+ * Photos Motion Photo File Format V1.1.
+ *
+ * As playback is not supported for motion photos, this flag should only be used for metadata
+ * retrieval use cases.
+ */
+ public static final int FLAG_READ_MOTION_PHOTO_METADATA = 1 << 1;
/** Parser states. */
@Documented
@@ -154,7 +163,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
@Override
public boolean sniff(ExtractorInput input) throws IOException {
- return Sniffer.sniffUnfragmented(input);
+ return Sniffer.sniffUnfragmented(
+ input, /* acceptHeic= */ (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0);
}
@Override
@@ -335,6 +345,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
this.atomData = atomData;
parserState = STATE_READING_ATOM_PAYLOAD;
} else {
+ if (atomType == Atom.TYPE_mpvd && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) {
+ // There is no need to parse the mpvd atom payload. All the necessary information is in the
+ // header.
+ processMpvdBox(
+ /* atomStartPosition= */ input.getPosition() - atomHeaderBytesRead,
+ /* atomHeaderSize= */ atomHeaderBytesRead,
+ atomSize);
+ }
atomData = null;
parserState = STATE_READING_ATOM_PAYLOAD;
}
@@ -662,6 +680,26 @@ public final class Mp4Extractor implements Extractor, SeekMap {
}
}
+ /**
+ * Processes the Motion Photo Video Data of an HEIC motion photo following the Google Photos
+ * Motion Photo File Format V1.1. This consists in adding a track with the motion photo metadata
+ * and ending playback preparation.
+ */
+ private void processMpvdBox(long atomStartPosition, int atomHeaderSize, long atomSize) {
+ ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput);
+ extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
+
+ TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
+ MotionPhoto motionPhoto =
+ new MotionPhoto(
+ /* photoStartPosition= */ 0,
+ /* photoSize= */ atomStartPosition,
+ /* videoStartPosition= */ atomStartPosition + atomHeaderSize,
+ /* videoSize= */ atomSize - atomHeaderSize);
+ trackOutput.format(new Format.Builder().setMetadata(new Metadata(motionPhoto)).build());
+ extractorOutput.endTracks();
+ }
+
/**
* For each sample of each track, calculates accumulated size of all samples which need to be read
* before this sample can be used.
diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
index 00acb29906..f830c86edb 100644
--- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
+++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java
@@ -70,7 +70,7 @@ import java.io.IOException;
* @throws IOException If an error occurs reading from the input.
*/
public static boolean sniffFragmented(ExtractorInput input) throws IOException {
- return sniffInternal(input, true);
+ return sniffInternal(input, /* fragmented= */ true, /* acceptHeic= */ false);
}
/**
@@ -82,10 +82,24 @@ import java.io.IOException;
* @throws IOException If an error occurs reading from the input.
*/
public static boolean sniffUnfragmented(ExtractorInput input) throws IOException {
- return sniffInternal(input, false);
+ return sniffInternal(input, /* fragmented= */ false, /* acceptHeic= */ false);
}
- private static boolean sniffInternal(ExtractorInput input, boolean fragmented)
+ /**
+ * Returns whether data peeked from the current position in {@code input} is consistent with the
+ * input being an unfragmented MP4 file.
+ *
+ * @param input The extractor input from which to peek data. The peek position will be modified.
+ * @param acceptHeic Whether {@code true} should be returned for HEIC photos.
+ * @return Whether the input appears to be in the unfragmented MP4 format.
+ * @throws IOException If an error occurs reading from the input.
+ */
+ public static boolean sniffUnfragmented(ExtractorInput input, boolean acceptHeic)
+ throws IOException {
+ return sniffInternal(input, /* fragmented= */ false, acceptHeic);
+ }
+
+ private static boolean sniffInternal(ExtractorInput input, boolean fragmented, boolean acceptHeic)
throws IOException {
long inputLength = input.getLength();
int bytesToSearch = (int) (inputLength == C.LENGTH_UNSET || inputLength > SEARCH_LENGTH
@@ -165,7 +179,7 @@ import java.io.IOException;
if (i == 1) {
// This index refers to the minorVersion, not a brand, so skip it.
buffer.skipBytes(4);
- } else if (isCompatibleBrand(buffer.readInt())) {
+ } else if (isCompatibleBrand(buffer.readInt(), acceptHeic)) {
foundGoodFileType = true;
break;
}
@@ -185,9 +199,12 @@ import java.io.IOException;
/**
* Returns whether {@code brand} is an ftyp atom brand that is compatible with the MP4 extractors.
*/
- private static boolean isCompatibleBrand(int brand) {
- // Accept all brands starting '3gp'.
+ private static boolean isCompatibleBrand(int brand, boolean acceptHeic) {
if (brand >>> 8 == 0x00336770) {
+ // Brand starts with '3gp'.
+ return true;
+ } else if (brand == 0x68656963 && acceptHeic) {
+ // Brand is `heic` and HEIC is supported by the extractor.
return true;
}
for (int compatibleBrand : COMPATIBLE_BRANDS) {
diff --git a/testdata/src/test/assets/media/mp4/sample_MP.heic b/testdata/src/test/assets/media/mp4/sample_MP.heic
new file mode 100644
index 0000000000..68dd4c4d6d
Binary files /dev/null and b/testdata/src/test/assets/media/mp4/sample_MP.heic differ