Read Google Photos motion photo metadata

PiperOrigin-RevId: 338436906

RELEASENOTES.md

@@ -99,6 +99,8 @@
     *   Upgrade IMA SDK dependency to 3.20.1. This brings in a fix for
         companion ads rendering when targeting API 29
         ([#6432](https://github.com/google/ExoPlayer/issues/6432)).
+*   Metadata retriever:
+    *   Parse Google Photos HEIC motion photos metadata.
 
 ### 2.12.0 (2020-09-11) ###
 

library/common/src/main/java/com/google/android/exoplayer2/C.java

@@ -690,12 +690,14 @@ public final class C {
   public static final int TRACK_TYPE_VIDEO = 2;
   /** A type constant for text tracks. */
   public static final int TRACK_TYPE_TEXT = 3;
+  /** A type constant for image tracks. */
+  public static final int TRACK_TYPE_IMAGE = 4;
   /** A type constant for metadata tracks. */
-  public static final int TRACK_TYPE_METADATA = 4;
+  public static final int TRACK_TYPE_METADATA = 5;
   /** A type constant for camera motion tracks. */
-  public static final int TRACK_TYPE_CAMERA_MOTION = 5;
+  public static final int TRACK_TYPE_CAMERA_MOTION = 6;
   /** A type constant for a fake or empty track. */
-  public static final int TRACK_TYPE_NONE = 6;
+  public static final int TRACK_TYPE_NONE = 7;
   /**
    * Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or
    * equal to this value.

library/common/src/main/java/com/google/android/exoplayer2/metadata/mp4/MotionPhoto.java

@@ -20,21 +20,23 @@ import android.os.Parcel;
 import android.os.Parcelable;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.common.primitives.Longs;
 
 /** Metadata of a motion photo file. */
 public final class MotionPhoto implements Metadata.Entry {
 
   /** The start offset of the photo data, in bytes. */
-  public final int photoStartPosition;
+  public final long photoStartPosition;
   /** The size of the photo data, in bytes. */
-  public final int photoSize;
+  public final long photoSize;
   /** The start offset of the video data, in bytes. */
-  public final int videoStartPosition;
+  public final long videoStartPosition;
   /** The size of the video data, in bytes. */
-  public final int videoSize;
+  public final long videoSize;
 
   /** Creates an instance. */
-  public MotionPhoto(int photoStartPosition, int photoSize, int videoStartPosition, int videoSize) {
+  public MotionPhoto(
+      long photoStartPosition, long photoSize, long videoStartPosition, long videoSize) {
     this.photoStartPosition = photoStartPosition;
     this.photoSize = photoSize;
     this.videoStartPosition = videoStartPosition;
@@ -42,10 +44,10 @@ public final class MotionPhoto implements Metadata.Entry {
   }
 
   private MotionPhoto(Parcel in) {
-    photoStartPosition = in.readInt();
-    photoSize = in.readInt();
-    videoStartPosition = in.readInt();
-    videoSize = in.readInt();
+    photoStartPosition = in.readLong();
+    photoSize = in.readLong();
+    videoStartPosition = in.readLong();
+    videoSize = in.readLong();
   }
 
   @Override
@@ -66,10 +68,10 @@ public final class MotionPhoto implements Metadata.Entry {
   @Override
   public int hashCode() {
     int result = 17;
-    result = 31 * result + photoStartPosition;
-    result = 31 * result + photoSize;
-    result = 31 * result + videoStartPosition;
-    result = 31 * result + videoSize;
+    result = 31 * result + Longs.hashCode(photoStartPosition);
+    result = 31 * result + Longs.hashCode(photoSize);
+    result = 31 * result + Longs.hashCode(videoStartPosition);
+    result = 31 * result + Longs.hashCode(videoSize);
     return result;
   }
 
@@ -89,10 +91,10 @@ public final class MotionPhoto implements Metadata.Entry {
 
   @Override
   public void writeToParcel(Parcel dest, int flags) {
-    dest.writeInt(photoStartPosition);
-    dest.writeInt(photoSize);
-    dest.writeInt(videoStartPosition);
-    dest.writeInt(videoSize);
+    dest.writeLong(photoStartPosition);
+    dest.writeLong(photoSize);
+    dest.writeLong(videoStartPosition);
+    dest.writeLong(videoSize);
   }
 
   @Override

library/core/src/main/java/com/google/android/exoplayer2/MetadataRetriever.java

@@ -22,6 +22,9 @@ import android.content.Context;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Message;
+import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory;
+import com.google.android.exoplayer2.extractor.ExtractorsFactory;
+import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor;
 import com.google.android.exoplayer2.source.DefaultMediaSourceFactory;
 import com.google.android.exoplayer2.source.MediaPeriod;
 import com.google.android.exoplayer2.source.MediaSource;
@@ -43,8 +46,9 @@ public final class MetadataRetriever {
   /**
    * Retrieves the {@link TrackGroupArray} corresponding to a {@link MediaItem}.
    *
-   * <p>This is equivalent to using {@code retrieveMetadata(new DefaultMediaSourceFactory(context),
-   * mediaItem)}.
+   * <p>This is equivalent to using {@link #retrieveMetadata(MediaSourceFactory, MediaItem)} with a
+   * {@link DefaultMediaSourceFactory} and a {@link DefaultExtractorsFactory} with {@link
+   * Mp4Extractor#FLAG_READ_MOTION_PHOTO_METADATA} set.
    *
    * @param context The {@link Context}.
    * @param mediaItem The {@link MediaItem} whose metadata should be retrieved.
@@ -52,7 +56,12 @@ public final class MetadataRetriever {
    */
   public static ListenableFuture<TrackGroupArray> retrieveMetadata(
       Context context, MediaItem mediaItem) {
-    return retrieveMetadata(new DefaultMediaSourceFactory(context), mediaItem);
+    ExtractorsFactory extractorsFactory =
+        new DefaultExtractorsFactory()
+            .setMp4ExtractorFlags(Mp4Extractor.FLAG_READ_MOTION_PHOTO_METADATA);
+    MediaSourceFactory mediaSourceFactory =
+        new DefaultMediaSourceFactory(context, extractorsFactory);
+    return retrieveMetadata(mediaSourceFactory, mediaItem);
   }
 
   /**

library/core/src/test/java/com/google/android/exoplayer2/MetadataRetrieverTest.java

@@ -25,6 +25,7 @@ import android.net.Uri;
 import android.os.SystemClock;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
 import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -37,7 +38,7 @@ import org.junit.runner.RunWith;
 public class MetadataRetrieverTest {
 
   @Test
-  public void retrieveMetadata_singleMediaItem() throws Exception {
+  public void retrieveMetadata_singleMediaItem_outputsExpectedMetadata() throws Exception {
     Context context = ApplicationProvider.getApplicationContext();
     MediaItem mediaItem =
         MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -55,7 +56,7 @@ public class MetadataRetrieverTest {
   }
 
   @Test
-  public void retrieveMetadata_multipleMediaItems() throws Exception {
+  public void retrieveMetadata_multipleMediaItems_outputsExpectedMetadata() throws Exception {
     Context context = ApplicationProvider.getApplicationContext();
     MediaItem mediaItem1 =
         MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample.mp4"));
@@ -84,7 +85,28 @@ public class MetadataRetrieverTest {
   }
 
   @Test
-  public void retrieveMetadata_throwsErrorIfCannotLoad() {
+  public void retrieveMetadata_motionPhoto_outputsExpectedMetadata() throws Exception {
+    Context context = ApplicationProvider.getApplicationContext();
+    MediaItem mediaItem =
+        MediaItem.fromUri(Uri.parse("asset://android_asset/media/mp4/sample_MP.heic"));
+    MotionPhoto expectedMotionPhoto =
+        new MotionPhoto(
+            /* photoStartPosition= */ 0,
+            /* photoSize= */ 28_853,
+            /* videoStartPosition= */ 28_869,
+            /* videoSize= */ 28_803);
+
+    ListenableFuture<TrackGroupArray> trackGroupsFuture = retrieveMetadata(context, mediaItem);
+    TrackGroupArray trackGroups = waitAndGetTrackGroups(trackGroupsFuture);
+
+    assertThat(trackGroups.length).isEqualTo(1);
+    assertThat(trackGroups.get(0).length).isEqualTo(1);
+    assertThat(trackGroups.get(0).getFormat(0).metadata.length()).isEqualTo(1);
+    assertThat(trackGroups.get(0).getFormat(0).metadata.get(0)).isEqualTo(expectedMotionPhoto);
+  }
+
+  @Test
+  public void retrieveMetadata_invalidMediaItem_throwsError() {
     Context context = ApplicationProvider.getApplicationContext();
     MediaItem mediaItem =
         MediaItem.fromUri(Uri.parse("asset://android_asset/media/does_not_exist"));

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Atom.java

@@ -181,6 +181,9 @@ import java.util.List;
   @SuppressWarnings("ConstantCaseForConstants")
   public static final int TYPE_moov = 0x6d6f6f76;
 
+  @SuppressWarnings("ConstantCaseForConstants")
+  public static final int TYPE_mpvd = 0x6d707664;
+
   @SuppressWarnings("ConstantCaseForConstants")
   public static final int TYPE_mvhd = 0x6d766864;
 

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Mp4Extractor.java

@@ -38,6 +38,7 @@ import com.google.android.exoplayer2.extractor.SeekPoint;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.mp4.Atom.ContainerAtom;
 import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.MotionPhoto;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
@@ -61,19 +62,27 @@ public final class Mp4Extractor implements Extractor, SeekMap {
   public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new Mp4Extractor()};
 
   /**
-   * Flags controlling the behavior of the extractor. Possible flag value is {@link
-   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}.
+   * Flags controlling the behavior of the extractor. Possible flag values are {@link
+   * #FLAG_WORKAROUND_IGNORE_EDIT_LISTS} and {@link #FLAG_READ_MOTION_PHOTO_METADATA}.
    */
   @Documented
   @Retention(RetentionPolicy.SOURCE)
   @IntDef(
       flag = true,
-      value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS})
+      value = {FLAG_WORKAROUND_IGNORE_EDIT_LISTS, FLAG_READ_MOTION_PHOTO_METADATA})
   public @interface Flags {}
   /**
    * Flag to ignore any edit lists in the stream.
    */
   public static final int FLAG_WORKAROUND_IGNORE_EDIT_LISTS = 1;
+  /**
+   * Flag to extract {@link MotionPhoto} metadata from HEIC motion photos following the Google
+   * Photos Motion Photo File Format V1.1.
+   *
+   * <p>As playback is not supported for motion photos, this flag should only be used for metadata
+   * retrieval use cases.
+   */
+  public static final int FLAG_READ_MOTION_PHOTO_METADATA = 1 << 1;
 
   /** Parser states. */
   @Documented
@@ -154,7 +163,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
 
   @Override
   public boolean sniff(ExtractorInput input) throws IOException {
-    return Sniffer.sniffUnfragmented(input);
+    return Sniffer.sniffUnfragmented(
+        input, /* acceptHeic= */ (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0);
   }
 
   @Override
@@ -335,6 +345,14 @@ public final class Mp4Extractor implements Extractor, SeekMap {
       this.atomData = atomData;
       parserState = STATE_READING_ATOM_PAYLOAD;
     } else {
+      if (atomType == Atom.TYPE_mpvd && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) {
+        // There is no need to parse the mpvd atom payload. All the necessary information is in the
+        // header.
+        processMpvdBox(
+            /* atomStartPosition= */ input.getPosition() - atomHeaderBytesRead,
+            /* atomHeaderSize= */ atomHeaderBytesRead,
+            atomSize);
+      }
       atomData = null;
       parserState = STATE_READING_ATOM_PAYLOAD;
     }
@@ -662,6 +680,26 @@ public final class Mp4Extractor implements Extractor, SeekMap {
     }
   }
 
+  /**
+   * Processes the Motion Photo Video Data of an HEIC motion photo following the Google Photos
+   * Motion Photo File Format V1.1. This consists in adding a track with the motion photo metadata
+   * and ending playback preparation.
+   */
+  private void processMpvdBox(long atomStartPosition, int atomHeaderSize, long atomSize) {
+    ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput);
+    extractorOutput.seekMap(new SeekMap.Unseekable(/* durationUs= */ C.TIME_UNSET));
+
+    TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE);
+    MotionPhoto motionPhoto =
+        new MotionPhoto(
+            /* photoStartPosition= */ 0,
+            /* photoSize= */ atomStartPosition,
+            /* videoStartPosition= */ atomStartPosition + atomHeaderSize,
+            /* videoSize= */ atomSize - atomHeaderSize);
+    trackOutput.format(new Format.Builder().setMetadata(new Metadata(motionPhoto)).build());
+    extractorOutput.endTracks();
+  }
+
   /**
    * For each sample of each track, calculates accumulated size of all samples which need to be read
    * before this sample can be used.

library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mp4/Sniffer.java

@@ -70,7 +70,7 @@ import java.io.IOException;
    * @throws IOException If an error occurs reading from the input.
    */
   public static boolean sniffFragmented(ExtractorInput input) throws IOException {
-    return sniffInternal(input, true);
+    return sniffInternal(input, /* fragmented= */ true, /* acceptHeic= */ false);
   }
 
   /**
@@ -82,10 +82,24 @@ import java.io.IOException;
    * @throws IOException If an error occurs reading from the input.
    */
   public static boolean sniffUnfragmented(ExtractorInput input) throws IOException {
-    return sniffInternal(input, false);
+    return sniffInternal(input, /* fragmented= */ false, /* acceptHeic= */ false);
   }
 
-  private static boolean sniffInternal(ExtractorInput input, boolean fragmented)
+  /**
+   * Returns whether data peeked from the current position in {@code input} is consistent with the
+   * input being an unfragmented MP4 file.
+   *
+   * @param input The extractor input from which to peek data. The peek position will be modified.
+   * @param acceptHeic Whether {@code true} should be returned for HEIC photos.
+   * @return Whether the input appears to be in the unfragmented MP4 format.
+   * @throws IOException If an error occurs reading from the input.
+   */
+  public static boolean sniffUnfragmented(ExtractorInput input, boolean acceptHeic)
+      throws IOException {
+    return sniffInternal(input, /* fragmented= */ false, acceptHeic);
+  }
+
+  private static boolean sniffInternal(ExtractorInput input, boolean fragmented, boolean acceptHeic)
       throws IOException {
     long inputLength = input.getLength();
     int bytesToSearch = (int) (inputLength == C.LENGTH_UNSET || inputLength > SEARCH_LENGTH
@@ -165,7 +179,7 @@ import java.io.IOException;
           if (i == 1) {
             // This index refers to the minorVersion, not a brand, so skip it.
             buffer.skipBytes(4);
-          } else if (isCompatibleBrand(buffer.readInt())) {
+          } else if (isCompatibleBrand(buffer.readInt(), acceptHeic)) {
             foundGoodFileType = true;
             break;
           }
@@ -185,9 +199,12 @@ import java.io.IOException;
   /**
    * Returns whether {@code brand} is an ftyp atom brand that is compatible with the MP4 extractors.
    */
-  private static boolean isCompatibleBrand(int brand) {
-    // Accept all brands starting '3gp'.
+  private static boolean isCompatibleBrand(int brand, boolean acceptHeic) {
     if (brand >>> 8 == 0x00336770) {
+      // Brand starts with '3gp'.
+      return true;
+    } else if (brand == 0x68656963 && acceptHeic) {
+      // Brand is `heic` and HEIC is supported by the extractor.
       return true;
     }
     for (int compatibleBrand : COMPATIBLE_BRANDS) {