sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-25 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Transformer GL: Clarify variables and comments.

Browse source 
Simplifying and clarifying variables, and adding comments.

Tested by confirming demo-gl and demo-transformer both
correctly display videos

PiperOrigin-RevId: 421792079
This commit is contained in:
huangdarwin 2022-01-14 12:27:18 +00:00 committed by Ian Baker
parent b1533980a8
commit 65adbbb745
6 changed files with 49 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java
View file

@ -86,23 +86,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new IllegalStateException(e);
}
program.setBufferAttribute(
"a_position",
new float[] {
-1, -1, 0, 1,
1, -1, 0, 1,
-1, 1, 0, 1,
1, 1, 0, 1
},
4);
"a_position", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
program.setBufferAttribute(
"a_texcoord",
new float[] {
0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1
},
4);
"a_texcoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]);
GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST);

23
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View file

@ -203,6 +203,9 @@ public final class GlUtil {
/** Whether to throw a {@link GlException} in case of an OpenGL error. */
public static boolean glAssertionsEnabled = false;
/** Number of vertices in a rectangle. */
public static final int RECTANGLE_VERTICES_COUNT = 4;
private static final String TAG = "GlUtil";
private static final String EXTENSION_PROTECTED_CONTENT = "EGL_EXT_protected_content";
@ -211,6 +214,26 @@ public final class GlUtil {
/** Class only contains static methods. */
private GlUtil() {}
/** Bounds of normalized device coordinates, commonly used for defining viewport boundaries. */
public static float[] getNormalizedCoordinateBounds() {
return new float[] {
-1, -1, 0, 1,
1, -1, 0, 1,
-1, 1, 0, 1,
1, 1, 0, 1
};
}
/** Typical bounds used for sampling from textures. */
public static float[] getTextureCoordinateBounds() {
return new float[] {
0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1
};
}
/**
* Returns whether creating a GL context with {@value #EXTENSION_PROTECTED_CONTENT} is possible.
* If {@code true}, the device supports a protected output path for DRM content when using GL.

4
libraries/transformer/src/main/assets/shaders/vertex_shader.glsl
View file

@ -11,12 +11,12 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
attribute vec4 aPosition;
attribute vec4 aFramePosition;
attribute vec4 aTexCoords;
uniform mat4 uTexTransform;
uniform mat4 uTransformationMatrix;
varying vec2 vTexCoords;
void main() {
gl_Position = uTransformationMatrix * aPosition;
gl_Position = uTransformationMatrix * aFramePosition;
vTexCoords = (uTexTransform * aTexCoords).xy;
}

19
libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
View file

@ -131,24 +131,11 @@ import java.util.concurrent.atomic.AtomicInteger;
GlUtil.Program glProgram =
new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aPosition",
new float[] {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexCoords",
new float[] {
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
"aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setSamplerTexIdUniform("uTexSampler", textureId, /* unit= */ 0);
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);

7
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java
View file

@ -63,6 +63,9 @@ public final class TransformationRequest {
* <p>This can be used to perform operations supported by {@link Matrix}, like scaling and
* rotating the video.
*
* <p>The video dimensions will be on the x axis, from -aspectRatio to aspectRatio, and on the y
* axis, from -1 to 1.
*
* <p>For now, resolution will not be affected by this method.
*
* @param transformationMatrix The transformation to apply to video frames.
@ -73,6 +76,10 @@ public final class TransformationRequest {
// allow transformations to change the resolution, by scaling to the appropriate min/max
// values. This will also be required to create the VertexTransformation class, in order to
// have aspect ratio helper methods (which require resolution to change).
// TODO(b/213198690): Consider changing how transformationMatrix is applied, so that
// dimensions will be from -1 to 1 on both x and y axes, but transformations will be applied
// in a predictable manner.
this.transformationMatrix = transformationMatrix;
return this;
}

21
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
View file

@ -63,12 +63,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Scale width and height to desired transformationRequest.outputHeight, preserving aspect
// ratio.
// TODO(internal b/209781577): Think about which edge length should be set for portrait videos.
float inputAspectRatio = (float) inputFormat.width / inputFormat.height;
float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height;
int outputWidth = inputFormat.width;
int outputHeight = inputFormat.height;
if (transformationRequest.outputHeight != C.LENGTH_UNSET
&& transformationRequest.outputHeight != inputFormat.height) {
outputWidth = Math.round(inputAspectRatio * transformationRequest.outputHeight);
outputWidth = Math.round(inputFormatAspectRatio * transformationRequest.outputHeight);
outputHeight = transformationRequest.outputHeight;
}
@ -82,14 +82,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} else {
outputRotationDegrees = inputFormat.rotationDegrees;
}
if ((inputFormat.rotationDegrees % 180) != 0) {
inputAspectRatio = 1.0f / inputAspectRatio;
}
float displayAspectRatio =
(inputFormat.rotationDegrees % 180) == 0
? inputFormatAspectRatio
: 1.0f / inputFormatAspectRatio;
// Scale frames by input aspect ratio, to account for FrameEditor normalized device coordinates
// (-1 to 1) and preserve frame relative dimensions during transformations (ex. rotations).
transformationRequest.transformationMatrix.preScale(inputAspectRatio, 1);
transformationRequest.transformationMatrix.postScale(1.0f / inputAspectRatio, 1);
// Scale frames by input aspect ratio, to account for FrameEditor's square normalized device
// coordinates (-1 to 1) and preserve frame relative dimensions during transformations
// (ex. rotations). After this scaling, transformationMatrix operations operate on a rectangle
// for x from -displayAspectRatio to displayAspectRatio, and y from -1 to 1
transformationRequest.transformationMatrix.preScale(displayAspectRatio, 1);
transformationRequest.transformationMatrix.postScale(1.0f / displayAspectRatio, 1);
// The decoder rotates videos to their intended display orientation. The frameEditor rotates
// them back for improved encoder compatibility.