From 65adbbb745db9942916bbf4ce5bb8a369853ceee Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 14 Jan 2022 12:27:18 +0000 Subject: [PATCH] Transformer GL: Clarify variables and comments. Simplifying and clarifying variables, and adding comments. Tested by confirming demo-gl and demo-transformer both correctly display videos PiperOrigin-RevId: 421792079 --- .../demo/gl/BitmapOverlayVideoProcessor.java | 18 ++------------- .../androidx/media3/common/util/GlUtil.java | 23 +++++++++++++++++++ .../main/assets/shaders/vertex_shader.glsl | 4 ++-- .../media3/transformer/FrameEditor.java | 19 +++------------ .../transformer/TransformationRequest.java | 7 ++++++ .../transformer/VideoSamplePipeline.java | 21 +++++++++-------- 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java index b83fe2bf46..65472412b0 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java @@ -86,23 +86,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException(e); } program.setBufferAttribute( - "a_position", - new float[] { - -1, -1, 0, 1, - 1, -1, 0, 1, - -1, 1, 0, 1, - 1, 1, 0, 1 - }, - 4); + "a_position", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); program.setBufferAttribute( - "a_texcoord", - new float[] { - 0, 0, 0, 1, - 1, 0, 0, 1, - 0, 1, 0, 1, - 1, 1, 0, 1 - }, - 4); + "a_texcoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); GLES20.glGenTextures(1, textures, 0); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index f08f0549fe..2269101d34 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -203,6 +203,9 @@ public final class GlUtil { /** Whether to throw a {@link GlException} in case of an OpenGL error. */ public static boolean glAssertionsEnabled = false; + /** Number of vertices in a rectangle. */ + public static final int RECTANGLE_VERTICES_COUNT = 4; + private static final String TAG = "GlUtil"; private static final String EXTENSION_PROTECTED_CONTENT = "EGL_EXT_protected_content"; @@ -211,6 +214,26 @@ public final class GlUtil { /** Class only contains static methods. */ private GlUtil() {} + /** Bounds of normalized device coordinates, commonly used for defining viewport boundaries. */ + public static float[] getNormalizedCoordinateBounds() { + return new float[] { + -1, -1, 0, 1, + 1, -1, 0, 1, + -1, 1, 0, 1, + 1, 1, 0, 1 + }; + } + + /** Typical bounds used for sampling from textures. */ + public static float[] getTextureCoordinateBounds() { + return new float[] { + 0, 0, 0, 1, + 1, 0, 0, 1, + 0, 1, 0, 1, + 1, 1, 0, 1 + }; + } + /** * Returns whether creating a GL context with {@value #EXTENSION_PROTECTED_CONTENT} is possible. * If {@code true}, the device supports a protected output path for DRM content when using GL. diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl index 3fd3e553fc..4f5e883390 100644 --- a/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl +++ b/libraries/transformer/src/main/assets/shaders/vertex_shader.glsl @@ -11,12 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -attribute vec4 aPosition; +attribute vec4 aFramePosition; attribute vec4 aTexCoords; uniform mat4 uTexTransform; uniform mat4 uTransformationMatrix; varying vec2 vTexCoords; void main() { - gl_Position = uTransformationMatrix * aPosition; + gl_Position = uTransformationMatrix * aFramePosition; vTexCoords = (uTexTransform * aTexCoords).xy; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java index 169b0671f4..929a0c7c10 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java @@ -131,24 +131,11 @@ import java.util.concurrent.atomic.AtomicInteger; GlUtil.Program glProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( - "aPosition", - new float[] { - -1.0f, -1.0f, 0.0f, 1.0f, - 1.0f, -1.0f, 0.0f, 1.0f, - -1.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); glProgram.setBufferAttribute( - "aTexCoords", - new float[] { - 0.0f, 0.0f, 0.0f, 1.0f, - 1.0f, 0.0f, 0.0f, 1.0f, - 0.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); + "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); glProgram.setSamplerTexIdUniform("uTexSampler", textureId, /* unit= */ 0); float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index 9f371280ff..728800af8c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -63,6 +63,9 @@ public final class TransformationRequest { *

This can be used to perform operations supported by {@link Matrix}, like scaling and * rotating the video. * + *

The video dimensions will be on the x axis, from -aspectRatio to aspectRatio, and on the y + * axis, from -1 to 1. + * *

For now, resolution will not be affected by this method. * * @param transformationMatrix The transformation to apply to video frames. @@ -73,6 +76,10 @@ public final class TransformationRequest { // allow transformations to change the resolution, by scaling to the appropriate min/max // values. This will also be required to create the VertexTransformation class, in order to // have aspect ratio helper methods (which require resolution to change). + + // TODO(b/213198690): Consider changing how transformationMatrix is applied, so that + // dimensions will be from -1 to 1 on both x and y axes, but transformations will be applied + // in a predictable manner. this.transformationMatrix = transformationMatrix; return this; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index d1f7819929..f448093999 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -63,12 +63,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Scale width and height to desired transformationRequest.outputHeight, preserving aspect // ratio. // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. - float inputAspectRatio = (float) inputFormat.width / inputFormat.height; + float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height; int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; if (transformationRequest.outputHeight != C.LENGTH_UNSET && transformationRequest.outputHeight != inputFormat.height) { - outputWidth = Math.round(inputAspectRatio * transformationRequest.outputHeight); + outputWidth = Math.round(inputFormatAspectRatio * transformationRequest.outputHeight); outputHeight = transformationRequest.outputHeight; } @@ -82,14 +82,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } else { outputRotationDegrees = inputFormat.rotationDegrees; } - if ((inputFormat.rotationDegrees % 180) != 0) { - inputAspectRatio = 1.0f / inputAspectRatio; - } + float displayAspectRatio = + (inputFormat.rotationDegrees % 180) == 0 + ? inputFormatAspectRatio + : 1.0f / inputFormatAspectRatio; - // Scale frames by input aspect ratio, to account for FrameEditor normalized device coordinates - // (-1 to 1) and preserve frame relative dimensions during transformations (ex. rotations). - transformationRequest.transformationMatrix.preScale(inputAspectRatio, 1); - transformationRequest.transformationMatrix.postScale(1.0f / inputAspectRatio, 1); + // Scale frames by input aspect ratio, to account for FrameEditor's square normalized device + // coordinates (-1 to 1) and preserve frame relative dimensions during transformations + // (ex. rotations). After this scaling, transformationMatrix operations operate on a rectangle + // for x from -displayAspectRatio to displayAspectRatio, and y from -1 to 1 + transformationRequest.transformationMatrix.preScale(displayAspectRatio, 1); + transformationRequest.transformationMatrix.postScale(1.0f / displayAspectRatio, 1); // The decoder rotates videos to their intended display orientation. The frameEditor rotates // them back for improved encoder compatibility.