diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java index 142b251a61..9cc042129a 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java @@ -513,9 +513,7 @@ public class PlayerView extends FrameLayout { if (surfaceView instanceof TextureView) { oldVideoComponent.clearVideoTextureView((TextureView) surfaceView); } else if (surfaceView instanceof SphericalSurfaceView) { - oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); - oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); - oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView)); + ((SphericalSurfaceView) surfaceView).setVideoComponent(null); } else if (surfaceView instanceof SurfaceView) { oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView); } @@ -541,9 +539,7 @@ public class PlayerView extends FrameLayout { if (surfaceView instanceof TextureView) { newVideoComponent.setVideoTextureView((TextureView) surfaceView); } else if (surfaceView instanceof SphericalSurfaceView) { - newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); - newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView)); - newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); + ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent); } else if (surfaceView instanceof SurfaceView) { newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView); } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java index 3b3e921253..a52a25d085 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java @@ -21,7 +21,6 @@ import android.annotation.TargetApi; import android.opengl.GLES11Ext; import android.opengl.GLES20; import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.video.spherical.Projection; import java.nio.FloatBuffer; import org.checkerframework.checker.nullness.qual.Nullable; @@ -143,13 +142,19 @@ import org.checkerframework.checker.nullness.qual.Nullable; } /** - * Renders the mesh. This must be called on the GL thread. + * Renders the mesh. If the projection hasn't been set, does nothing. This must be called on the + * GL thread. * * @param textureId GL_TEXTURE_EXTERNAL_OES used for this mesh. * @param mvpMatrix The Model View Projection matrix. * @param eyeType An {@link EyeType} value. */ /* package */ void draw(int textureId, float[] mvpMatrix, int eyeType) { + MeshData meshData = eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData; + if (meshData == null) { + return; + } + // Configure shader. GLES20.glUseProgram(program); checkGlError(); @@ -174,9 +179,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; GLES20.glUniform1i(textureHandle, 0); checkGlError(); - MeshData meshData = - Assertions.checkNotNull(eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData); - // Load position data. GLES20.glVertexAttribPointer( positionHandle, diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java index 023d68f988..a8d4abcf07 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java @@ -21,49 +21,68 @@ import android.graphics.SurfaceTexture; import android.opengl.GLES20; import android.opengl.Matrix; import android.support.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.TimedValueQueue; +import com.google.android.exoplayer2.video.VideoFrameMetadataListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionListener; import com.google.android.exoplayer2.video.spherical.FrameRotationQueue; import com.google.android.exoplayer2.video.spherical.Projection; +import com.google.android.exoplayer2.video.spherical.ProjectionDecoder; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicBoolean; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** - * Renders a GL Scene. - * - *

All methods should be called only on the GL thread unless GL thread is stopped. - */ -/*package*/ final class SceneRenderer { +/** Renders a GL Scene. */ +/*package*/ class SceneRenderer implements VideoFrameMetadataListener, CameraMotionListener { private final AtomicBoolean frameAvailable; + private final AtomicBoolean resetRotationAtNextFrame; private final ProjectionRenderer projectionRenderer; private final FrameRotationQueue frameRotationQueue; private final TimedValueQueue sampleTimestampQueue; + private final TimedValueQueue projectionQueue; private final float[] rotationMatrix; private final float[] tempMatrix; + // Used by GL thread only private int textureId; private @MonotonicNonNull SurfaceTexture surfaceTexture; - private @Nullable Projection pendingProjection; - private long pendingProjectionTimeNs; - private long lastFrameTimestamp; - private boolean resetRotationAtNextFrame; - public SceneRenderer( - Projection projection, - FrameRotationQueue frameRotationQueue, - TimedValueQueue sampleTimestampQueue) { - this.frameRotationQueue = frameRotationQueue; - this.sampleTimestampQueue = sampleTimestampQueue; + // Used by other threads only + private volatile @C.StreamType int defaultStereoMode; + private @C.StreamType int lastStereoMode; + private @Nullable byte[] lastProjectionData; + + // Methods called on any thread. + + public SceneRenderer() { frameAvailable = new AtomicBoolean(); + resetRotationAtNextFrame = new AtomicBoolean(true); projectionRenderer = new ProjectionRenderer(); - projectionRenderer.setProjection(projection); + frameRotationQueue = new FrameRotationQueue(); + sampleTimestampQueue = new TimedValueQueue<>(); + projectionQueue = new TimedValueQueue<>(); rotationMatrix = new float[16]; tempMatrix = new float[16]; - resetRotation(); + defaultStereoMode = C.STEREO_MODE_MONO; + lastStereoMode = Format.NO_VALUE; } + /** + * Sets the default stereo mode. If the played video doesn't contain a stereo mode the default one + * is used. + * + * @param stereoMode A {@link C.StereoMode} value. + */ + public void setDefaultStereoMode(@C.StereoMode int stereoMode) { + defaultStereoMode = stereoMode; + } + + // Methods called on GL thread. + /** Initializes the renderer. */ public SurfaceTexture init() { // Set the background frame color. This is only visible if the display mesh isn't a full sphere. @@ -79,21 +98,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return surfaceTexture; } - public void resetRotation() { - resetRotationAtNextFrame = true; - } - - /** Sets a {@link Projection} to be used to display video. */ - public void setProjection(Projection projection, long timeNs) { - pendingProjection = projection; - pendingProjectionTimeNs = timeNs; - } - /** * Draws the scene with a given eye pose and type. * * @param viewProjectionMatrix 16 element GL matrix. - * @param eyeType an {@link EyeType} value + * @param eyeType An {@link EyeType} value */ public void drawFrame(float[] viewProjectionMatrix, int eyeType) { // glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve @@ -104,20 +113,73 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (frameAvailable.compareAndSet(true, false)) { Assertions.checkNotNull(surfaceTexture).updateTexImage(); checkGlError(); - if (resetRotationAtNextFrame) { + if (resetRotationAtNextFrame.compareAndSet(true, false)) { Matrix.setIdentityM(rotationMatrix, 0); } - lastFrameTimestamp = surfaceTexture.getTimestamp(); - Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp); - if (sampleTimestamp != null) { - frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp); + long lastFrameTimestampNs = surfaceTexture.getTimestamp(); + Long sampleTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs); + if (sampleTimestampUs != null) { + frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestampUs); + } + Projection projection = projectionQueue.pollFloor(lastFrameTimestampNs); + if (projection != null) { + projectionRenderer.setProjection(projection); } - } - if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) { - projectionRenderer.setProjection(pendingProjection); - pendingProjection = null; } Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0); projectionRenderer.draw(textureId, tempMatrix, eyeType); } + + // Methods called on playback thread. + + // VideoFrameMetadataListener implementation. + + @Override + public void onVideoFrameAboutToBeRendered( + long presentationTimeUs, long releaseTimeNs, Format format) { + sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs); + setProjection(format.projectionData, format.stereoMode, releaseTimeNs); + } + + // CameraMotionListener implementation. + + @Override + public void onCameraMotion(long timeUs, float[] rotation) { + frameRotationQueue.setRotation(timeUs, rotation); + } + + @Override + public void onCameraMotionReset() { + sampleTimestampQueue.clear(); + frameRotationQueue.reset(); + resetRotationAtNextFrame.set(true); + } + + /** + * Sets projection data and stereo mode of the media to be played. + * + * @param projectionData Contains the projection data to be rendered. + * @param stereoMode A {@link C.StereoMode} value. + * @param timeNs When then new projection should be used. + */ + private void setProjection( + @Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) { + byte[] oldProjectionData = lastProjectionData; + int oldStereoMode = lastStereoMode; + lastProjectionData = projectionData; + lastStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode; + if (oldStereoMode == lastStereoMode && Arrays.equals(oldProjectionData, lastProjectionData)) { + return; + } + + Projection projectionFromData = null; + if (lastProjectionData != null) { + projectionFromData = ProjectionDecoder.decode(lastProjectionData, lastStereoMode); + } + Projection projection = + projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData) + ? projectionFromData + : Projection.createEquirectangular(lastStereoMode); + projectionQueue.add(timeNs, projection); + } } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java index 30995aca5f..c6ddb8148b 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java @@ -37,17 +37,10 @@ import android.view.Display; import android.view.Surface; import android.view.WindowManager; import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.Player; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.util.Assertions; -import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.Util; -import com.google.android.exoplayer2.video.VideoFrameMetadataListener; -import com.google.android.exoplayer2.video.spherical.CameraMotionListener; -import com.google.android.exoplayer2.video.spherical.FrameRotationQueue; -import com.google.android.exoplayer2.video.spherical.Projection; -import com.google.android.exoplayer2.video.spherical.ProjectionDecoder; -import java.util.Arrays; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; @@ -62,8 +55,7 @@ import javax.microedition.khronos.opengles.GL10; * match what they expect. */ @TargetApi(15) -public final class SphericalSurfaceView extends GLSurfaceView - implements VideoFrameMetadataListener, CameraMotionListener { +public final class SphericalSurfaceView extends GLSurfaceView { /** * This listener can be used to be notified when the {@link Surface} associated with this view is @@ -94,15 +86,12 @@ public final class SphericalSurfaceView extends GLSurfaceView private final PhoneOrientationListener phoneOrientationListener; private final Renderer renderer; private final Handler mainHandler; - private final TimedValueQueue sampleTimestampQueue; - private final FrameRotationQueue frameRotationQueue; private final TouchTracker touchTracker; + private final SceneRenderer scene; private @Nullable SurfaceListener surfaceListener; private @Nullable SurfaceTexture surfaceTexture; private @Nullable Surface surface; - private @C.StreamType int defaultStereoMode; - private @C.StreamType int currentStereoMode; - private @Nullable byte[] currentProjectionData; + private @Nullable Player.VideoComponent videoComponent; public SphericalSurfaceView(Context context) { this(context, null); @@ -122,12 +111,7 @@ public final class SphericalSurfaceView extends GLSurfaceView int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR; orientationSensor = sensorManager.getDefaultSensor(type); - defaultStereoMode = C.STEREO_MODE_MONO; - currentStereoMode = defaultStereoMode; - Projection projection = Projection.createEquirectangular(defaultStereoMode); - frameRotationQueue = new FrameRotationQueue(); - sampleTimestampQueue = new TimedValueQueue<>(); - SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue); + scene = new SceneRenderer(); renderer = new Renderer(scene); touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES); @@ -147,12 +131,27 @@ public final class SphericalSurfaceView extends GLSurfaceView * @param stereoMode A {@link C.StereoMode} value. */ public void setDefaultStereoMode(@C.StereoMode int stereoMode) { - defaultStereoMode = stereoMode; + scene.setDefaultStereoMode(stereoMode); } - /** Returns the {@link Surface} associated with this view. */ - public @Nullable Surface getSurface() { - return surface; + /** Sets the {@link Player.VideoComponent} to use. */ + public void setVideoComponent(@Nullable Player.VideoComponent newVideoComponent) { + if (newVideoComponent == videoComponent) { + return; + } + if (videoComponent != null) { + if (surface != null) { + videoComponent.clearVideoSurface(surface); + } + videoComponent.clearVideoFrameMetadataListener(scene); + videoComponent.clearCameraMotionListener(scene); + } + videoComponent = newVideoComponent; + if (videoComponent != null) { + videoComponent.setVideoFrameMetadataListener(scene); + videoComponent.setCameraMotionListener(scene); + videoComponent.setVideoSurface(surface); + } } /** @@ -169,29 +168,6 @@ public final class SphericalSurfaceView extends GLSurfaceView touchTracker.setSingleTapListener(listener); } - // VideoFrameMetadataListener implementation. - - @Override - public void onVideoFrameAboutToBeRendered( - long presentationTimeUs, long releaseTimeNs, Format format) { - sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs); - setProjection(format.projectionData, format.stereoMode, releaseTimeNs); - } - - // CameraMotionListener implementation. - - @Override - public void onCameraMotion(long timeUs, float[] rotation) { - frameRotationQueue.setRotation(timeUs, rotation); - } - - @Override - public void onCameraMotionReset() { - sampleTimestampQueue.clear(); - frameRotationQueue.reset(); - queueEvent(renderer.scene::resetRotation); - } - @Override public void onResume() { super.onResume(); @@ -253,35 +229,6 @@ public final class SphericalSurfaceView extends GLSurfaceView } } - /** - * Sets projection data and stereo mode of the media to be played. - * - * @param projectionData Contains the projection data to be rendered. - * @param stereoMode A {@link C.StereoMode} value. - * @param timeNs When then new projection should be used. - */ - private void setProjection( - @Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) { - byte[] oldProjectionData = currentProjectionData; - int oldStereoMode = currentStereoMode; - currentProjectionData = projectionData; - currentStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode; - if (oldStereoMode == currentStereoMode - && Arrays.equals(oldProjectionData, currentProjectionData)) { - return; - } - - Projection projectionFromData = null; - if (currentProjectionData != null) { - projectionFromData = ProjectionDecoder.decode(currentProjectionData, currentStereoMode); - } - Projection projection = - projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData) - ? projectionFromData - : Projection.createEquirectangular(currentStereoMode); - queueEvent(() -> renderer.scene.setProjection(projection, timeNs)); - } - /** Detects sensor events and saves them as a matrix. */ private static class PhoneOrientationListener implements SensorEventListener { private final float[] phoneInWorldSpaceMatrix = new float[16];