From f3b80fd54d9511b76b49c855b61723f8d9bcb9db Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 7 Apr 2022 11:28:46 +0100 Subject: [PATCH] Add bitmap overlay effect to transformer demo. The new demo GlFrameProcessor is based on BitmapOverlayVideoProcessor from the gl-demo. The demo-only GlFrameProcessor can be deleted once Transformer supports this functionality. PiperOrigin-RevId: 440059735 --- .../fragment_shader_bitmap_overlay_es2.glsl | 37 ++++ .../BitmapOverlayFrameProcessor.java | 163 ++++++++++++++++++ .../transformer/ConfigurationActivity.java | 2 +- .../demo/transformer/TransformerActivity.java | 3 + 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 demos/transformer/src/main/assets/fragment_shader_bitmap_overlay_es2.glsl create mode 100644 demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java diff --git a/demos/transformer/src/main/assets/fragment_shader_bitmap_overlay_es2.glsl b/demos/transformer/src/main/assets/fragment_shader_bitmap_overlay_es2.glsl new file mode 100644 index 0000000000..90ff827132 --- /dev/null +++ b/demos/transformer/src/main/assets/fragment_shader_bitmap_overlay_es2.glsl @@ -0,0 +1,37 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that overlays the bitmap from uTexSampler1 over a video +// frame from uTexSampler0. + +precision mediump float; +// Texture containing an input video frame. +uniform sampler2D uTexSampler0; +// Texture containing the overlap bitmap. +uniform sampler2D uTexSampler1; +// Horizontal scaling factor for the overlap bitmap. +uniform float uScaleX; +// Vertical scaling factory for the overlap bitmap. +uniform float uScaleY; +varying vec2 vTexSamplingCoord; +void main() { + vec4 videoColor = texture2D(uTexSampler0, vTexSamplingCoord); + vec4 overlayColor = texture2D(uTexSampler1, + vec2(vTexSamplingCoord.x * uScaleX, + vTexSamplingCoord.y * uScaleY)); + // Blend the video decoder output and the overlay bitmap. + gl_FragColor = videoColor * (1.0 - overlayColor.a) + + overlayColor * overlayColor.a; +} diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java new file mode 100644 index 0000000000..dd7b9539c6 --- /dev/null +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java @@ -0,0 +1,163 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.transformer; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.drawable.BitmapDrawable; +import android.opengl.GLES20; +import android.opengl.GLUtils; +import android.util.Size; +import androidx.media3.common.C; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import androidx.media3.transformer.GlFrameProcessor; +import java.io.IOException; +import java.util.Locale; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A {@link GlFrameProcessor} that overlays a bitmap with a logo and timer on each frame. + * + *

The bitmap is drawn using an Android {@link Canvas}. + */ +// TODO(b/227625365): Delete this class and use a frame processor from the Transformer library, once +// overlaying a bitmap and text is supported in Transformer. +/* package */ final class BitmapOverlayFrameProcessor implements GlFrameProcessor { + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "fragment_shader_bitmap_overlay_es2.glsl"; + + private static final int BITMAP_WIDTH_HEIGHT = 512; + + private final Context context; + private final Paint paint; + private final Bitmap overlayBitmap; + private final Bitmap logoBitmap; + private final Canvas overlayCanvas; + + private float bitmapScaleX; + private float bitmapScaleY; + private int bitmapTexId; + private @MonotonicNonNull Size outputSize; + private @MonotonicNonNull GlProgram glProgram; + + public BitmapOverlayFrameProcessor(Context context) { + this.context = context; + paint = new Paint(); + paint.setTextSize(64); + paint.setAntiAlias(true); + paint.setARGB(0xFF, 0xFF, 0xFF, 0xFF); + paint.setColor(Color.GRAY); + overlayBitmap = + Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888); + overlayCanvas = new Canvas(overlayBitmap); + try { + logoBitmap = + ((BitmapDrawable) + context.getPackageManager().getApplicationIcon(context.getPackageName())) + .getBitmap(); + } catch (PackageManager.NameNotFoundException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void initialize(int inputTexId, int inputWidth, int inputHeight) throws IOException { + if (inputWidth > inputHeight) { + bitmapScaleX = inputWidth / (float) inputHeight; + bitmapScaleY = 1f; + } else { + bitmapScaleX = 1f; + bitmapScaleY = inputHeight / (float) inputWidth; + } + outputSize = new Size(inputWidth, inputHeight); + + bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0); + + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0); + glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1); + glProgram.setFloatUniform("uScaleX", bitmapScaleX); + glProgram.setFloatUniform("uScaleY", bitmapScaleY); + } + + @Override + public Size getOutputSize() { + return checkStateNotNull(outputSize); + } + + @Override + public void updateProgramAndDraw(long presentationTimeUs) { + checkStateNotNull(glProgram); + glProgram.use(); + + // Draw to the canvas and store it in a texture. + String text = + String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); + overlayBitmap.eraseColor(Color.TRANSPARENT); + overlayCanvas.drawBitmap(logoBitmap, /* left= */ 3, /* top= */ 378, paint); + overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); + GLUtils.texSubImage2D( + GLES20.GL_TEXTURE_2D, + /* level= */ 0, + /* xoffset= */ 0, + /* yoffset= */ 0, + flipBitmapVertically(overlayBitmap)); + GlUtil.checkGlError(); + + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } + + @Override + public void release() { + if (glProgram != null) { + glProgram.delete(); + } + } + + private static Bitmap flipBitmapVertically(Bitmap bitmap) { + Matrix flip = new Matrix(); + flip.postScale(1f, -1f); + return Bitmap.createBitmap( + bitmap, + /* x= */ 0, + /* y= */ 0, + bitmap.getWidth(), + bitmap.getHeight(), + flip, + /* filter= */ true); + } +} diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 3dc0dc0cd3..93d17b51bd 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -88,7 +88,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "MP4 with HDR (HDR10) H265 video (encoding may fail)", }; private static final String[] DEMO_FRAME_PROCESSORS = { - "Dizzy crop", "Periodic vignette", "3D spin", "Zoom in start" + "Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start" }; private static final int PERIODIC_VIGNETTE_INDEX = 1; private static final String SAME_AS_INPUT_OPTION = "same as input"; diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 51c2a33ade..03e603b313 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -266,6 +266,9 @@ public final class TransformerActivity extends AppCompatActivity { AdvancedFrameProcessorFactory.createSpin3dFrameProcessor(/* context= */ this)); } if (selectedFrameProcessors[3]) { + frameProcessors.add(new BitmapOverlayFrameProcessor(/* context= */ this)); + } + if (selectedFrameProcessors[4]) { frameProcessors.add( AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor( /* context= */ this));