Make SceneRenderer implement VideoFrameMetadataListener and CameraMotionListener

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=211084127

library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java

@@ -513,9 +513,7 @@ public class PlayerView extends FrameLayout {
         if (surfaceView instanceof TextureView) {
           oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
         } else if (surfaceView instanceof SphericalSurfaceView) {
-          oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
+          ((SphericalSurfaceView) surfaceView).setVideoComponent(null);
-          oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
-          oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView));
         } else if (surfaceView instanceof SurfaceView) {
           oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
         }
@@ -541,9 +539,7 @@ public class PlayerView extends FrameLayout {
         if (surfaceView instanceof TextureView) {
           newVideoComponent.setVideoTextureView((TextureView) surfaceView);
         } else if (surfaceView instanceof SphericalSurfaceView) {
-          newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
+          ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
-          newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView));
-          newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
         } else if (surfaceView instanceof SurfaceView) {
           newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
         }

library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java

@@ -21,7 +21,6 @@ import android.annotation.TargetApi;
 import android.opengl.GLES11Ext;
 import android.opengl.GLES20;
 import com.google.android.exoplayer2.C;
-import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.video.spherical.Projection;
 import java.nio.FloatBuffer;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -143,13 +142,19 @@ import org.checkerframework.checker.nullness.qual.Nullable;
   }
 
   /**
-   * Renders the mesh. This must be called on the GL thread.
+   * Renders the mesh. If the projection hasn't been set, does nothing. This must be called on the
+   * GL thread.
    *
    * @param textureId GL_TEXTURE_EXTERNAL_OES used for this mesh.
    * @param mvpMatrix The Model View Projection matrix.
    * @param eyeType An {@link EyeType} value.
    */
   /* package */ void draw(int textureId, float[] mvpMatrix, int eyeType) {
+    MeshData meshData = eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData;
+    if (meshData == null) {
+      return;
+    }
+
     // Configure shader.
     GLES20.glUseProgram(program);
     checkGlError();
@@ -174,9 +179,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
     GLES20.glUniform1i(textureHandle, 0);
     checkGlError();
 
-    MeshData meshData =
-        Assertions.checkNotNull(eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData);
-
     // Load position data.
     GLES20.glVertexAttribPointer(
         positionHandle,

library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java

@@ -21,49 +21,68 @@ import android.graphics.SurfaceTexture;
 import android.opengl.GLES20;
 import android.opengl.Matrix;
 import android.support.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.TimedValueQueue;
+import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
 import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
 import com.google.android.exoplayer2.video.spherical.Projection;
+import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
+import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
-/**
+/** Renders a GL Scene. */
- * Renders a GL Scene.
+/*package*/ class SceneRenderer implements VideoFrameMetadataListener, CameraMotionListener {
- *
- * <p>All methods should be called only on the GL thread unless GL thread is stopped.
- */
-/*package*/ final class SceneRenderer {
 
   private final AtomicBoolean frameAvailable;
+  private final AtomicBoolean resetRotationAtNextFrame;
   private final ProjectionRenderer projectionRenderer;
   private final FrameRotationQueue frameRotationQueue;
   private final TimedValueQueue<Long> sampleTimestampQueue;
+  private final TimedValueQueue<Projection> projectionQueue;
   private final float[] rotationMatrix;
   private final float[] tempMatrix;
 
+  // Used by GL thread only
   private int textureId;
   private @MonotonicNonNull SurfaceTexture surfaceTexture;
-  private @Nullable Projection pendingProjection;
-  private long pendingProjectionTimeNs;
-  private long lastFrameTimestamp;
-  private boolean resetRotationAtNextFrame;
 
-  public SceneRenderer(
+  // Used by other threads only
-      Projection projection,
+  private volatile @C.StreamType int defaultStereoMode;
-      FrameRotationQueue frameRotationQueue,
+  private @C.StreamType int lastStereoMode;
-      TimedValueQueue<Long> sampleTimestampQueue) {
+  private @Nullable byte[] lastProjectionData;
-    this.frameRotationQueue = frameRotationQueue;
+
-    this.sampleTimestampQueue = sampleTimestampQueue;
+  // Methods called on any thread.
+
+  public SceneRenderer() {
     frameAvailable = new AtomicBoolean();
+    resetRotationAtNextFrame = new AtomicBoolean(true);
     projectionRenderer = new ProjectionRenderer();
-    projectionRenderer.setProjection(projection);
+    frameRotationQueue = new FrameRotationQueue();
+    sampleTimestampQueue = new TimedValueQueue<>();
+    projectionQueue = new TimedValueQueue<>();
     rotationMatrix = new float[16];
     tempMatrix = new float[16];
-    resetRotation();
+    defaultStereoMode = C.STEREO_MODE_MONO;
+    lastStereoMode = Format.NO_VALUE;
   }
 
+  /**
+   * Sets the default stereo mode. If the played video doesn't contain a stereo mode the default one
+   * is used.
+   *
+   * @param stereoMode A {@link C.StereoMode} value.
+   */
+  public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
+    defaultStereoMode = stereoMode;
+  }
+
+  // Methods called on GL thread.
+
   /** Initializes the renderer. */
   public SurfaceTexture init() {
     // Set the background frame color. This is only visible if the display mesh isn't a full sphere.
@@ -79,21 +98,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     return surfaceTexture;
   }
 
-  public void resetRotation() {
-    resetRotationAtNextFrame = true;
-  }
-
-  /** Sets a {@link Projection} to be used to display video. */
-  public void setProjection(Projection projection, long timeNs) {
-    pendingProjection = projection;
-    pendingProjectionTimeNs = timeNs;
-  }
-
   /**
    * Draws the scene with a given eye pose and type.
    *
    * @param viewProjectionMatrix 16 element GL matrix.
-   * @param eyeType an {@link EyeType} value
+   * @param eyeType An {@link EyeType} value
    */
   public void drawFrame(float[] viewProjectionMatrix, int eyeType) {
     // glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve
@@ -104,20 +113,73 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     if (frameAvailable.compareAndSet(true, false)) {
       Assertions.checkNotNull(surfaceTexture).updateTexImage();
       checkGlError();
-      if (resetRotationAtNextFrame) {
+      if (resetRotationAtNextFrame.compareAndSet(true, false)) {
         Matrix.setIdentityM(rotationMatrix, 0);
       }
-      lastFrameTimestamp = surfaceTexture.getTimestamp();
+      long lastFrameTimestampNs = surfaceTexture.getTimestamp();
-      Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp);
+      Long sampleTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs);
-      if (sampleTimestamp != null) {
+      if (sampleTimestampUs != null) {
-        frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp);
+        frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestampUs);
+      }
+      Projection projection = projectionQueue.pollFloor(lastFrameTimestampNs);
+      if (projection != null) {
+        projectionRenderer.setProjection(projection);
       }
-    }
-    if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) {
-      projectionRenderer.setProjection(pendingProjection);
-      pendingProjection = null;
     }
     Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
     projectionRenderer.draw(textureId, tempMatrix, eyeType);
   }
+
+  // Methods called on playback thread.
+
+  // VideoFrameMetadataListener implementation.
+
+  @Override
+  public void onVideoFrameAboutToBeRendered(
+      long presentationTimeUs, long releaseTimeNs, Format format) {
+    sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
+    setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
+  }
+
+  // CameraMotionListener implementation.
+
+  @Override
+  public void onCameraMotion(long timeUs, float[] rotation) {
+    frameRotationQueue.setRotation(timeUs, rotation);
+  }
+
+  @Override
+  public void onCameraMotionReset() {
+    sampleTimestampQueue.clear();
+    frameRotationQueue.reset();
+    resetRotationAtNextFrame.set(true);
+  }
+
+  /**
+   * Sets projection data and stereo mode of the media to be played.
+   *
+   * @param projectionData Contains the projection data to be rendered.
+   * @param stereoMode A {@link C.StereoMode} value.
+   * @param timeNs When then new projection should be used.
+   */
+  private void setProjection(
+      @Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
+    byte[] oldProjectionData = lastProjectionData;
+    int oldStereoMode = lastStereoMode;
+    lastProjectionData = projectionData;
+    lastStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
+    if (oldStereoMode == lastStereoMode && Arrays.equals(oldProjectionData, lastProjectionData)) {
+      return;
+    }
+
+    Projection projectionFromData = null;
+    if (lastProjectionData != null) {
+      projectionFromData = ProjectionDecoder.decode(lastProjectionData, lastStereoMode);
+    }
+    Projection projection =
+        projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
+            ? projectionFromData
+            : Projection.createEquirectangular(lastStereoMode);
+    projectionQueue.add(timeNs, projection);
+  }
 }

library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java

@@ -37,17 +37,10 @@ import android.view.Display;
 import android.view.Surface;
 import android.view.WindowManager;
 import com.google.android.exoplayer2.C;
-import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.Player;
 import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
 import com.google.android.exoplayer2.util.Assertions;
-import com.google.android.exoplayer2.util.TimedValueQueue;
 import com.google.android.exoplayer2.util.Util;
-import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
-import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
-import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
-import com.google.android.exoplayer2.video.spherical.Projection;
-import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
-import java.util.Arrays;
 import javax.microedition.khronos.egl.EGLConfig;
 import javax.microedition.khronos.opengles.GL10;
 
@@ -62,8 +55,7 @@ import javax.microedition.khronos.opengles.GL10;
  * match what they expect.
  */
 @TargetApi(15)
-public final class SphericalSurfaceView extends GLSurfaceView
+public final class SphericalSurfaceView extends GLSurfaceView {
-    implements VideoFrameMetadataListener, CameraMotionListener {
 
   /**
    * This listener can be used to be notified when the {@link Surface} associated with this view is
@@ -94,15 +86,12 @@ public final class SphericalSurfaceView extends GLSurfaceView
   private final PhoneOrientationListener phoneOrientationListener;
   private final Renderer renderer;
   private final Handler mainHandler;
-  private final TimedValueQueue<Long> sampleTimestampQueue;
-  private final FrameRotationQueue frameRotationQueue;
   private final TouchTracker touchTracker;
+  private final SceneRenderer scene;
   private @Nullable SurfaceListener surfaceListener;
   private @Nullable SurfaceTexture surfaceTexture;
   private @Nullable Surface surface;
-  private @C.StreamType int defaultStereoMode;
+  private @Nullable Player.VideoComponent videoComponent;
-  private @C.StreamType int currentStereoMode;
-  private @Nullable byte[] currentProjectionData;
 
   public SphericalSurfaceView(Context context) {
     this(context, null);
@@ -122,12 +111,7 @@ public final class SphericalSurfaceView extends GLSurfaceView
     int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR;
     orientationSensor = sensorManager.getDefaultSensor(type);
 
-    defaultStereoMode = C.STEREO_MODE_MONO;
+    scene = new SceneRenderer();
-    currentStereoMode = defaultStereoMode;
-    Projection projection = Projection.createEquirectangular(defaultStereoMode);
-    frameRotationQueue = new FrameRotationQueue();
-    sampleTimestampQueue = new TimedValueQueue<>();
-    SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue);
     renderer = new Renderer(scene);
 
     touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES);
@@ -147,12 +131,27 @@ public final class SphericalSurfaceView extends GLSurfaceView
    * @param stereoMode A {@link C.StereoMode} value.
    */
   public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
-    defaultStereoMode = stereoMode;
+    scene.setDefaultStereoMode(stereoMode);
   }
 
-  /** Returns the {@link Surface} associated with this view. */
+  /** Sets the {@link Player.VideoComponent} to use. */
-  public @Nullable Surface getSurface() {
+  public void setVideoComponent(@Nullable Player.VideoComponent newVideoComponent) {
-    return surface;
+    if (newVideoComponent == videoComponent) {
+      return;
+    }
+    if (videoComponent != null) {
+      if (surface != null) {
+        videoComponent.clearVideoSurface(surface);
+      }
+      videoComponent.clearVideoFrameMetadataListener(scene);
+      videoComponent.clearCameraMotionListener(scene);
+    }
+    videoComponent = newVideoComponent;
+    if (videoComponent != null) {
+      videoComponent.setVideoFrameMetadataListener(scene);
+      videoComponent.setCameraMotionListener(scene);
+      videoComponent.setVideoSurface(surface);
+    }
   }
 
   /**
@@ -169,29 +168,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
     touchTracker.setSingleTapListener(listener);
   }
 
-  // VideoFrameMetadataListener implementation.
-
-  @Override
-  public void onVideoFrameAboutToBeRendered(
-      long presentationTimeUs, long releaseTimeNs, Format format) {
-    sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
-    setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
-  }
-
-  // CameraMotionListener implementation.
-
-  @Override
-  public void onCameraMotion(long timeUs, float[] rotation) {
-    frameRotationQueue.setRotation(timeUs, rotation);
-  }
-
-  @Override
-  public void onCameraMotionReset() {
-    sampleTimestampQueue.clear();
-    frameRotationQueue.reset();
-    queueEvent(renderer.scene::resetRotation);
-  }
-
   @Override
   public void onResume() {
     super.onResume();
@@ -253,35 +229,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
     }
   }
 
-  /**
-   * Sets projection data and stereo mode of the media to be played.
-   *
-   * @param projectionData Contains the projection data to be rendered.
-   * @param stereoMode A {@link C.StereoMode} value.
-   * @param timeNs When then new projection should be used.
-   */
-  private void setProjection(
-      @Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
-    byte[] oldProjectionData = currentProjectionData;
-    int oldStereoMode = currentStereoMode;
-    currentProjectionData = projectionData;
-    currentStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
-    if (oldStereoMode == currentStereoMode
-        && Arrays.equals(oldProjectionData, currentProjectionData)) {
-      return;
-    }
-
-    Projection projectionFromData = null;
-    if (currentProjectionData != null) {
-      projectionFromData = ProjectionDecoder.decode(currentProjectionData, currentStereoMode);
-    }
-    Projection projection =
-        projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
-            ? projectionFromData
-            : Projection.createEquirectangular(currentStereoMode);
-    queueEvent(() -> renderer.scene.setProjection(projection, timeNs));
-  }
-
   /** Detects sensor events and saves them as a matrix. */
   private static class PhoneOrientationListener implements SensorEventListener {
     private final float[] phoneInWorldSpaceMatrix = new float[16];