Add bitmap overlay effect to transformer demo.

The new demo GlFrameProcessor is based on BitmapOverlayVideoProcessor
from the gl-demo. The demo-only GlFrameProcessor can be deleted once
Transformer supports this functionality.

PiperOrigin-RevId: 440059735
This commit is contained in:
hschlueter 2022-04-07 11:28:46 +01:00 committed by Ian Baker
parent 7d0034748f
commit f3b80fd54d
4 changed files with 204 additions and 1 deletions

View file

@ -0,0 +1,37 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that overlays the bitmap from uTexSampler1 over a video
// frame from uTexSampler0.
precision mediump float;
// Texture containing an input video frame.
uniform sampler2D uTexSampler0;
// Texture containing the overlap bitmap.
uniform sampler2D uTexSampler1;
// Horizontal scaling factor for the overlap bitmap.
uniform float uScaleX;
// Vertical scaling factory for the overlap bitmap.
uniform float uScaleY;
varying vec2 vTexSamplingCoord;
void main() {
vec4 videoColor = texture2D(uTexSampler0, vTexSamplingCoord);
vec4 overlayColor = texture2D(uTexSampler1,
vec2(vTexSamplingCoord.x * uScaleX,
vTexSamplingCoord.y * uScaleY));
// Blend the video decoder output and the overlay bitmap.
gl_FragColor = videoColor * (1.0 - overlayColor.a)
+ overlayColor * overlayColor.a;
}

View file

@ -0,0 +1,163 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.demo.transformer;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.drawable.BitmapDrawable;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.util.Size;
import androidx.media3.common.C;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.GlFrameProcessor;
import java.io.IOException;
import java.util.Locale;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link GlFrameProcessor} that overlays a bitmap with a logo and timer on each frame.
*
* <p>The bitmap is drawn using an Android {@link Canvas}.
*/
// TODO(b/227625365): Delete this class and use a frame processor from the Transformer library, once
// overlaying a bitmap and text is supported in Transformer.
/* package */ final class BitmapOverlayFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "fragment_shader_bitmap_overlay_es2.glsl";
private static final int BITMAP_WIDTH_HEIGHT = 512;
private final Context context;
private final Paint paint;
private final Bitmap overlayBitmap;
private final Bitmap logoBitmap;
private final Canvas overlayCanvas;
private float bitmapScaleX;
private float bitmapScaleY;
private int bitmapTexId;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull GlProgram glProgram;
public BitmapOverlayFrameProcessor(Context context) {
this.context = context;
paint = new Paint();
paint.setTextSize(64);
paint.setAntiAlias(true);
paint.setARGB(0xFF, 0xFF, 0xFF, 0xFF);
paint.setColor(Color.GRAY);
overlayBitmap =
Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888);
overlayCanvas = new Canvas(overlayBitmap);
try {
logoBitmap =
((BitmapDrawable)
context.getPackageManager().getApplicationIcon(context.getPackageName()))
.getBitmap();
} catch (PackageManager.NameNotFoundException e) {
throw new IllegalStateException(e);
}
}
@Override
public void initialize(int inputTexId, int inputWidth, int inputHeight) throws IOException {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
outputSize = new Size(inputWidth, inputHeight);
bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setBufferAttribute(
"aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT);
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1);
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void updateProgramAndDraw(long presentationTimeUs) {
checkStateNotNull(glProgram);
glProgram.use();
// Draw to the canvas and store it in a texture.
String text =
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND);
overlayBitmap.eraseColor(Color.TRANSPARENT);
overlayCanvas.drawBitmap(logoBitmap, /* left= */ 3, /* top= */ 378, paint);
overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texSubImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
/* xoffset= */ 0,
/* yoffset= */ 0,
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
@Override
public void release() {
if (glProgram != null) {
glProgram.delete();
}
}
private static Bitmap flipBitmapVertically(Bitmap bitmap) {
Matrix flip = new Matrix();
flip.postScale(1f, -1f);
return Bitmap.createBitmap(
bitmap,
/* x= */ 0,
/* y= */ 0,
bitmap.getWidth(),
bitmap.getHeight(),
flip,
/* filter= */ true);
}
}

View file

@ -88,7 +88,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"MP4 with HDR (HDR10) H265 video (encoding may fail)",
};
private static final String[] DEMO_FRAME_PROCESSORS = {
"Dizzy crop", "Periodic vignette", "3D spin", "Zoom in start"
"Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start"
};
private static final int PERIODIC_VIGNETTE_INDEX = 1;
private static final String SAME_AS_INPUT_OPTION = "same as input";

View file

@ -266,6 +266,9 @@ public final class TransformerActivity extends AppCompatActivity {
AdvancedFrameProcessorFactory.createSpin3dFrameProcessor(/* context= */ this));
}
if (selectedFrameProcessors[3]) {
frameProcessors.add(new BitmapOverlayFrameProcessor(/* context= */ this));
}
if (selectedFrameProcessors[4]) {
frameProcessors.add(
AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor(
/* context= */ this));