From fa06f0b4ccc3b37daa7462b847c7801cb9951911 Mon Sep 17 00:00:00 2001 From: sofijajvc Date: Fri, 4 Oct 2019 19:10:31 +0100 Subject: [PATCH] Fix video decoder GL rendering Calculate stride and width ratio per each plane. PiperOrigin-RevId: 272916423 --- .../video/VideoDecoderRenderer.java | 139 +++++++++++------- 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java index ab338a0af5..c85dff0c26 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java @@ -17,6 +17,7 @@ package com.google.android.exoplayer2.video; import android.opengl.GLES20; import android.opengl.GLSurfaceView; +import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.GlUtil; import java.nio.FloatBuffer; import java.util.concurrent.atomic.AtomicReference; @@ -48,50 +49,64 @@ import javax.microedition.khronos.opengles.GL10; }; private static final String VERTEX_SHADER = - "varying vec2 interp_tc;\n" - + "attribute vec4 in_pos;\n" - + "attribute vec2 in_tc;\n" - + "void main() {\n" - + " gl_Position = in_pos;\n" - + " interp_tc = in_tc;\n" - + "}\n"; + "varying vec2 interp_tc_y;\n" + + "varying vec2 interp_tc_u;\n" + + "varying vec2 interp_tc_v;\n" + + "attribute vec4 in_pos;\n" + + "attribute vec2 in_tc_y;\n" + + "attribute vec2 in_tc_u;\n" + + "attribute vec2 in_tc_v;\n" + + "void main() {\n" + + " gl_Position = in_pos;\n" + + " interp_tc_y = in_tc_y;\n" + + " interp_tc_u = in_tc_u;\n" + + " interp_tc_v = in_tc_v;\n" + + "}\n"; private static final String[] TEXTURE_UNIFORMS = {"y_tex", "u_tex", "v_tex"}; private static final String FRAGMENT_SHADER = "precision mediump float;\n" - + "varying vec2 interp_tc;\n" - + "uniform sampler2D y_tex;\n" - + "uniform sampler2D u_tex;\n" - + "uniform sampler2D v_tex;\n" - + "uniform mat3 mColorConversion;\n" - + "void main() {\n" - + " vec3 yuv;\n" - + " yuv.x = texture2D(y_tex, interp_tc).r - 0.0625;\n" - + " yuv.y = texture2D(u_tex, interp_tc).r - 0.5;\n" - + " yuv.z = texture2D(v_tex, interp_tc).r - 0.5;\n" - + " gl_FragColor = vec4(mColorConversion * yuv, 1.0);\n" - + "}\n"; + + "varying vec2 interp_tc_y;\n" + + "varying vec2 interp_tc_u;\n" + + "varying vec2 interp_tc_v;\n" + + "uniform sampler2D y_tex;\n" + + "uniform sampler2D u_tex;\n" + + "uniform sampler2D v_tex;\n" + + "uniform mat3 mColorConversion;\n" + + "void main() {\n" + + " vec3 yuv;\n" + + " yuv.x = texture2D(y_tex, interp_tc_y).r - 0.0625;\n" + + " yuv.y = texture2D(u_tex, interp_tc_u).r - 0.5;\n" + + " yuv.z = texture2D(v_tex, interp_tc_v).r - 0.5;\n" + + " gl_FragColor = vec4(mColorConversion * yuv, 1.0);\n" + + "}\n"; private static final FloatBuffer TEXTURE_VERTICES = GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f}); private final int[] yuvTextures = new int[3]; private final AtomicReference pendingOutputBufferReference; - // Kept in a field rather than a local variable so that it doesn't get garbage collected before + // Kept in field rather than a local variable in order not to get garbage collected before // glDrawArrays uses it. @SuppressWarnings("FieldCanBeLocal") - private FloatBuffer textureCoords; + private FloatBuffer[] textureCoords; + private int program; - private int texLocation; + private int[] texLocations; private int colorMatrixLocation; - private int previousWidth; - private int previousStride; + private int[] previousWidths; + private int[] previousStrides; private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread. public VideoDecoderRenderer() { - previousWidth = -1; - previousStride = -1; pendingOutputBufferReference = new AtomicReference<>(); + textureCoords = new FloatBuffer[3]; + texLocations = new int[3]; + previousWidths = new int[3]; + previousStrides = new int[3]; + for (int i = 0; i < 3; i++) { + previousWidths[i] = previousStrides[i] = -1; + } } /** @@ -115,10 +130,13 @@ import javax.microedition.khronos.opengles.GL10; GLES20.glUseProgram(program); int posLocation = GLES20.glGetAttribLocation(program, "in_pos"); GLES20.glEnableVertexAttribArray(posLocation); - GLES20.glVertexAttribPointer( - posLocation, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES); - texLocation = GLES20.glGetAttribLocation(program, "in_tc"); - GLES20.glEnableVertexAttribArray(texLocation); + GLES20.glVertexAttribPointer(posLocation, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES); + texLocations[0] = GLES20.glGetAttribLocation(program, "in_tc_y"); + GLES20.glEnableVertexAttribArray(texLocations[0]); + texLocations[1] = GLES20.glGetAttribLocation(program, "in_tc_u"); + GLES20.glEnableVertexAttribArray(texLocations[1]); + texLocations[2] = GLES20.glGetAttribLocation(program, "in_tc_v"); + GLES20.glEnableVertexAttribArray(texLocations[2]); GlUtil.checkGlError(); colorMatrixLocation = GLES20.glGetUniformLocation(program, "mColorConversion"); GlUtil.checkGlError(); @@ -165,22 +183,41 @@ import javax.microedition.khronos.opengles.GL10; GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, - outputBuffer.yuvStrides[i], h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + GLES20.glTexImage2D( + GLES20.GL_TEXTURE_2D, + 0, + GLES20.GL_LUMINANCE, + outputBuffer.yuvStrides[i], + h, + 0, + GLES20.GL_LUMINANCE, + GLES20.GL_UNSIGNED_BYTE, outputBuffer.yuvPlanes[i]); } - // Set cropping of stride if either width or stride has changed. - if (previousWidth != outputBuffer.width || previousStride != outputBuffer.yuvStrides[0]) { - float crop = (float) outputBuffer.width / outputBuffer.yuvStrides[0]; - // This buffer is consumed during each call to glDrawArrays. It needs to be a member variable - // rather than a local variable to ensure that it doesn't get garbage collected. - textureCoords = - GlUtil.createBuffer(new float[] {0.0f, 0.0f, 0.0f, 1.0f, crop, 0.0f, crop, 1.0f}); - GLES20.glVertexAttribPointer( - texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords); - previousWidth = outputBuffer.width; - previousStride = outputBuffer.yuvStrides[0]; + + int[] widths = new int[3]; + widths[0] = outputBuffer.width; + // TODO: Handle streams where chroma channels are not stored at half width and height + // compared to luma channel. See [Internal: b/142097774]. + // U and V planes are being stored at half width compared to Y. + widths[1] = widths[2] = (widths[0] + 1) / 2; + for (int i = 0; i < 3; i++) { + // Set cropping of stride if either width or stride has changed. + if (previousWidths[i] != widths[i] || previousStrides[i] != outputBuffer.yuvStrides[i]) { + Assertions.checkState(outputBuffer.yuvStrides[i] != 0); + float widthRatio = (float) widths[i] / outputBuffer.yuvStrides[i]; + // These buffers are consumed during each call to glDrawArrays. They need to be member + // variables rather than local variables in order not to get garbage collected. + textureCoords[i] = + GlUtil.createBuffer( + new float[] {0.0f, 0.0f, 0.0f, 1.0f, widthRatio, 0.0f, widthRatio, 1.0f}); + GLES20.glVertexAttribPointer( + texLocations[i], 2, GLES20.GL_FLOAT, false, 0, textureCoords[i]); + previousWidths[i] = widths[i]; + previousStrides[i] = outputBuffer.yuvStrides[i]; + } } + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GlUtil.checkGlError(); @@ -188,18 +225,16 @@ import javax.microedition.khronos.opengles.GL10; private void setupTextures() { GLES20.glGenTextures(3, yuvTextures, 0); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 3; i++) { GLES20.glUniform1i(GLES20.glGetUniformLocation(program, TEXTURE_UNIFORMS[i]), i); GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); - GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, - GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); - GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, - GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); - GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, - GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); - GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, - GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameterf( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); } GlUtil.checkGlError(); }