diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/BitmapTestUtil.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/BitmapTestUtil.java index d8fed8623e..427fa0727f 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/BitmapTestUtil.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/BitmapTestUtil.java @@ -24,7 +24,6 @@ import android.content.Context; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.Color; -import android.graphics.Matrix; import android.graphics.PixelFormat; import android.media.Image; import android.opengl.GLES20; @@ -211,7 +210,7 @@ public class BitmapTestUtil { // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888. bitmap.copyPixelsFromBuffer(rgba8888Buffer); // Flip the bitmap as its positive y-axis points down while OpenGL's positive y-axis points up. - return flipBitmapVertically(bitmap); + return BitmapUtil.flipBitmapVertically(bitmap); } /** @@ -227,23 +226,10 @@ public class BitmapTestUtil { bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false); // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down // while OpenGL's positive y-axis points up. - GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, BitmapUtil.flipBitmapVertically(bitmap), 0); GlUtil.checkGlError(); return texId; } - private static Bitmap flipBitmapVertically(Bitmap bitmap) { - Matrix flip = new Matrix(); - flip.postScale(1f, -1f); - return Bitmap.createBitmap( - bitmap, - /* x= */ 0, - /* y= */ 0, - bitmap.getWidth(), - bitmap.getHeight(), - flip, - /* filter= */ true); - } - private BitmapTestUtil() {} } diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java new file mode 100644 index 0000000000..c770dbc9f3 --- /dev/null +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java @@ -0,0 +1,175 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.createGlTextureFromBitmap; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory; +import static com.google.android.exoplayer2.effect.BitmapTestUtil.readBitmap; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLSurface; +import android.opengl.Matrix; +import android.util.Pair; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.util.FrameProcessingException; +import com.google.android.exoplayer2.util.GlUtil; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Pixel test for texture processing via {@link OverlayTextureProcessor}. + * + *

Expected bitmaps are taken from an emulator, so tests on different emulators or physical + * devices may fail. To test on other devices, please increase the {@link + * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps + * as recommended in {@link GlEffectsFrameProcessorPixelTest}. + */ +@RunWith(AndroidJUnit4.class) +public class OverlayTextureProcessorPixelTest { + public static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/overlay/100winners.png"; + public static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; + public static final String OVERLAY_BITMAP_DEFAULT = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png"; + public static final String OVERLAY_BITMAP_SCALED = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png"; + + private final Context context = getApplicationContext(); + + private @MonotonicNonNull EGLDisplay eglDisplay; + private @MonotonicNonNull EGLContext eglContext; + private @MonotonicNonNull SingleFrameGlTextureProcessor overlayTextureProcessor; + private @MonotonicNonNull EGLSurface placeholderEglSurface; + private int inputTexId; + private int inputWidth; + private int inputHeight; + + @Before + public void createGlObjects() throws IOException, GlUtil.GlException { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = GlUtil.createEglContext(eglDisplay); + placeholderEglSurface = GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); + + Bitmap inputBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + inputWidth = inputBitmap.getWidth(); + inputHeight = inputBitmap.getHeight(); + inputTexId = createGlTextureFromBitmap(inputBitmap); + } + + @After + public void release() throws GlUtil.GlException, FrameProcessingException { + if (overlayTextureProcessor != null) { + overlayTextureProcessor.release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + } + + @Test + public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception { + String testId = "drawFrame_noOverlays"; + overlayTextureProcessor = + new OverlayEffect(/* textureOverlays= */ ImmutableList.of()) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_bitmapOverlay_blendsBitmapIntoFrame() throws Exception { + String testId = "drawFrame_bitmapOverlay"; + Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + BitmapOverlay scaledBitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap); + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(scaledBitmapOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_DEFAULT); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_scaledBitmapOverlay_blendsBitmapIntoFrame() throws Exception { + String testId = "drawFrame_scaledBitmapOverlay"; + Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH); + float[] scaleMatrix = GlUtil.create4x4IdentityMatrix(); + Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1); + OverlaySettings overlaySettings = new OverlaySettings.Builder().setMatrix(scaleMatrix).build(); + BitmapOverlay staticBitmapOverlay = + BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings); + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(staticBitmapOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_SCALED); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { + int outputTexId = + GlUtil.createTexture( + outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false); + int frameBuffer = GlUtil.createFboForTexture(outputTexId); + GlUtil.focusFramebuffer( + checkNotNull(eglDisplay), + checkNotNull(eglContext), + checkNotNull(placeholderEglSurface), + frameBuffer, + outputWidth, + outputHeight); + } +} diff --git a/library/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl b/library/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl new file mode 100644 index 0000000000..436edb4555 --- /dev/null +++ b/library/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl @@ -0,0 +1,52 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that overlays a bitmap over a video frame. + +precision mediump float; +// Texture containing an input video frame. +uniform sampler2D uVideoTexSampler0; +// Texture containing the overlay bitmap. +uniform sampler2D uOverlayTexSampler1; +varying vec2 vVideoTexSamplingCoord; +varying vec2 vOverlayTexSamplingCoord1; + +// Manually implementing the CLAMP_TO_BORDER texture wrapping option +// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2. +vec4 getClampToBorderOverlayColor() { + if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0 + || vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0){ + return vec4(0.0, 0.0, 0.0, 0.0); + } else { + return vec4(texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1)); + } +} + +float getMixAlpha(float videoAlpha, float overlayAlpha) { + if (videoAlpha == 0.0){ + return 1.0; + } else { + return clamp(overlayAlpha/videoAlpha, 0.0, 1.0); + } +} + +void main() { + vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord)); + vec4 overlayColor = getClampToBorderOverlayColor(); + + // Blend the video decoder output and the overlay bitmap. + gl_FragColor = mix( + videoColor, overlayColor, getMixAlpha(videoColor.a, overlayColor.a)); +} diff --git a/library/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl b/library/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl new file mode 100644 index 0000000000..2aa40bfe9a --- /dev/null +++ b/library/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl @@ -0,0 +1,37 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 vertex shader that leaves the frame coordinates unchanged +// and applies matrix transformations to the texture coordinates. + +uniform mat4 uAspectRatioMatrix; +uniform mat4 uOverlayMatrix; +attribute vec4 aFramePosition; +varying vec2 vVideoTexSamplingCoord; +varying vec2 vOverlayTexSamplingCoord1; + + +vec2 getTexSamplingCoord(vec2 ndcPosition) { + return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5); +} + +void main() { + gl_Position = aFramePosition; + vec4 aOverlayPosition = uAspectRatioMatrix * uOverlayMatrix * aFramePosition; + vOverlayTexSamplingCoord1 = getTexSamplingCoord(aOverlayPosition.xy); + vVideoTexSamplingCoord = getTexSamplingCoord(aFramePosition.xy); +} + + diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapLoader.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapLoader.java new file mode 100644 index 0000000000..a3f7aad6f8 --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapLoader.java @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.google.android.exoplayer2.effect; + +import android.graphics.Bitmap; +import android.net.Uri; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.MediaMetadata; +import com.google.common.util.concurrent.ListenableFuture; + +// TODO(b/258685047): delete this copy once session/BitmapLoader.java is moved to common +// (b/194284041, b/258658893). + +/** Loads images. */ +public interface BitmapLoader { + /** Decodes an image from compressed binary data. */ + ListenableFuture decodeBitmap(byte[] data); + + /** Loads an image from {@code uri}. */ + ListenableFuture loadBitmap(Uri uri); + + /** + * Loads an image from {@link MediaMetadata}. Returns null if {@code metadata} doesn't contain + * bitmap information. + * + *

By default, the method will try to decode an image from {@link MediaMetadata#artworkData} if + * it is present. Otherwise, the method will try to load an image from {@link + * MediaMetadata#artworkUri} if it is present. The method will return null if neither {@link + * MediaMetadata#artworkData} nor {@link MediaMetadata#artworkUri} is present. + */ + @Nullable + default ListenableFuture loadBitmapFromMetadata(MediaMetadata metadata) { + @Nullable ListenableFuture future; + if (metadata.artworkData != null) { + future = decodeBitmap(metadata.artworkData); + } else if (metadata.artworkUri != null) { + future = loadBitmap(metadata.artworkUri); + } else { + future = null; + } + return future; + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapOverlay.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapOverlay.java new file mode 100644 index 0000000000..b1957700cf --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapOverlay.java @@ -0,0 +1,153 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + +import android.graphics.Bitmap; +import android.net.Uri; +import android.opengl.GLES20; +import android.opengl.GLUtils; +import android.util.Pair; +import com.google.android.exoplayer2.util.FrameProcessingException; +import com.google.android.exoplayer2.util.GlUtil; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.ExecutionException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Creates {@link TextureOverlay}s from {@link Bitmap}s. + * + *

Useful for overlaying images and animated images (e.g. GIFs). + */ +public abstract class BitmapOverlay extends TextureOverlay { + private int lastTextureId; + private @MonotonicNonNull Bitmap lastBitmap; + + /** + * Returns the overlay bitmap displayed at the specified timestamp. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + * @throws FrameProcessingException If an error occurs while processing or drawing the frame. + */ + public abstract Bitmap getBitmap(long presentationTimeUs) throws FrameProcessingException; + + /** + * {@inheritDoc} + * + *

Gets the width and height of the cached bitmap. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + @Override + public Pair getTextureSize(long presentationTimeUs) { + return Pair.create(checkNotNull(lastBitmap).getWidth(), checkNotNull(lastBitmap).getHeight()); + } + + @Override + public int getTextureId(long presentationTimeUs) throws FrameProcessingException { + Bitmap bitmap = getBitmap(presentationTimeUs); + if (bitmap != lastBitmap) { + try { + lastBitmap = bitmap; + lastTextureId = + GlUtil.createTexture( + bitmap.getWidth(), + bitmap.getHeight(), + /* useHighPrecisionColorComponents= */ false); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, lastTextureId); + GLUtils.texImage2D( + GLES20.GL_TEXTURE_2D, + /* level= */ 0, + BitmapUtil.flipBitmapVertically(lastBitmap), + /* border= */ 0); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } + return lastTextureId; + } + + /** + * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same position and + * size throughout the whole video. + */ + public static BitmapOverlay createStaticBitmapOverlay(Bitmap overlayBitmap) { + return new BitmapOverlay() { + @Override + public Bitmap getBitmap(long presentationTimeUs) { + return overlayBitmap; + } + }; + } + + /** + * Creates a {@link BitmapOverlay} that shows the {@code overlayBitmap} in the same {@link + * OverlaySettings} throughout the whole video. + * + * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on + * the frames. + */ + public static BitmapOverlay createStaticBitmapOverlay( + Bitmap overlayBitmap, OverlaySettings overlaySettings) { + return new BitmapOverlay() { + @Override + public Bitmap getBitmap(long presentationTimeUs) { + return overlayBitmap; + } + + @Override + public OverlaySettings getOverlaySettings(long presentationTimeUs) { + return overlaySettings; + } + }; + } + + /** + * Creates a {@link BitmapOverlay} that shows the input at {@code overlayBitmapUri} with the same + * {@link OverlaySettings} throughout the whole video. + * + * @param overlayBitmapUri The {@link Uri} pointing to the resource to be converted into a bitmap. + * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on + * the frames. + */ + public static BitmapOverlay createStaticBitmapOverlay( + Uri overlayBitmapUri, OverlaySettings overlaySettings) { + return new BitmapOverlay() { + private @MonotonicNonNull Bitmap lastBitmap; + + @Override + public Bitmap getBitmap(long presentationTimeUs) throws FrameProcessingException { + if (lastBitmap == null) { + BitmapLoader bitmapLoader = new SimpleBitmapLoader(); + ListenableFuture future = bitmapLoader.loadBitmap(overlayBitmapUri); + try { + lastBitmap = future.get(); + } catch (ExecutionException | InterruptedException e) { + throw new FrameProcessingException(e); + } + } + return lastBitmap; + } + + @Override + public OverlaySettings getOverlaySettings(long presentationTimeUs) { + return overlaySettings; + } + }; + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapUtil.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapUtil.java new file mode 100644 index 0000000000..57d6c35aab --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapUtil.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import android.graphics.Bitmap; +import android.graphics.Matrix; + +/** Utility functions for working with {@link Bitmap}. */ +/* package */ final class BitmapUtil { + static Bitmap flipBitmapVertically(Bitmap bitmap) { + Matrix flip = new Matrix(); + flip.postScale(1f, -1f); + return Bitmap.createBitmap( + bitmap, + /* x= */ 0, + /* y= */ 0, + bitmap.getWidth(), + bitmap.getHeight(), + flip, + /* filter= */ true); + } + + /** Class only contains static methods. */ + private BitmapUtil() {} +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayEffect.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayEffect.java new file mode 100644 index 0000000000..93b8b886fe --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayEffect.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import android.content.Context; +import com.google.android.exoplayer2.util.FrameProcessingException; +import com.google.common.collect.ImmutableList; + +/** + * Applies a list of {@link TextureOverlay}s to a frame in FIFO order (the last overlay in the list + * is displayed on top). + */ +public final class OverlayEffect implements GlEffect { + + private final ImmutableList overlays; + + /** + * Creates a new instance for the given list of {@link TextureOverlay}s. + * + * @param textureOverlays The {@link TextureOverlay}s to be blended into the frame. + */ + public OverlayEffect(ImmutableList textureOverlays) { + this.overlays = textureOverlays; + } + + @Override + public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + return new OverlayTextureProcessor(context, useHdr, overlays); + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java new file mode 100644 index 0000000000..bbc0bef93e --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java @@ -0,0 +1,69 @@ +package com.google.android.exoplayer2.effect; +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.google.android.exoplayer2.util.GlUtil; +import com.google.errorprone.annotations.CanIgnoreReturnValue; + +/** Contains information to control how an {@link TextureOverlay} is displayed on the screen. */ +public final class OverlaySettings { + public final boolean useHdr; + public final float[] matrix; + + private OverlaySettings(boolean useHdr, float[] matrix) { + this.useHdr = useHdr; + this.matrix = matrix; + } + + /** A builder for {@link OverlaySettings} instances. */ + public static final class Builder { + private boolean useHdr; + private float[] matrix; + + /** Creates a new {@link Builder}. */ + public Builder() { + matrix = GlUtil.create4x4IdentityMatrix(); + } + + /** + * Sets whether input overlay comes from an HDR source. If {@code true}, colors will be in + * linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * + *

Set to {@code false} by default. + */ + @CanIgnoreReturnValue + public Builder setUsesHdr(boolean useHdr) { + this.useHdr = useHdr; + return this; + } + + /** + * Sets the {@link android.opengl.Matrix} used to transform the overlay before applying it to a + * frame. + * + *

Set to always return the identity matrix by default. + */ + @CanIgnoreReturnValue + public Builder setMatrix(float[] matrix) { + this.matrix = matrix; + return this; + } + + /** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */ + public OverlaySettings build() { + return new OverlaySettings(useHdr, matrix); + } + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java new file mode 100644 index 0000000000..eaa38d4b6a --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java @@ -0,0 +1,147 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import static com.google.android.exoplayer2.util.Assertions.checkArgument; + +import android.content.Context; +import android.opengl.GLES20; +import android.opengl.Matrix; +import android.util.Pair; +import com.google.android.exoplayer2.util.FrameProcessingException; +import com.google.android.exoplayer2.util.GlProgram; +import com.google.android.exoplayer2.util.GlUtil; +import com.google.common.collect.ImmutableList; +import java.io.IOException; + +/** Applies one or more {@link TextureOverlay}s onto each frame. */ +/* package */ final class OverlayTextureProcessor extends SingleFrameGlTextureProcessor { + + private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_overlay_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_overlay_es2.glsl"; + private static final int MATRIX_OFFSET = 0; + private static final int TRANSPARENT_TEXTURE_WIDTH_HEIGHT = 1; + + private final GlProgram glProgram; + private final ImmutableList overlays; + private final float[] aspectRatioMatrix; + private final float[] overlayMatrix; + + private int videoWidth; + private int videoHeight; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @throws FrameProcessingException If a problem occurs while reading shader files. + */ + public OverlayTextureProcessor( + Context context, boolean useHdr, ImmutableList overlays) + throws FrameProcessingException { + super(useHdr); + checkArgument(!useHdr, "OverlayTextureProcessor does not support HDR colors yet."); + checkArgument( + overlays.size() <= 1, + "OverlayTextureProcessor does not support multiple overlays in the same processor yet."); + this.overlays = overlays; + aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); + overlayMatrix = GlUtil.create4x4IdentityMatrix(); + + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (GlUtil.GlException | IOException e) { + throw new FrameProcessingException(e); + } + + glProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + } + + @Override + public Pair configure(int inputWidth, int inputHeight) { + videoWidth = inputWidth; + videoHeight = inputHeight; + return Pair.create(inputWidth, inputHeight); + } + + @Override + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + try { + glProgram.use(); + if (!overlays.isEmpty()) { + TextureOverlay overlay = overlays.get(0); + glProgram.setSamplerTexIdUniform( + "uOverlayTexSampler1", overlay.getTextureId(presentationTimeUs), /* texUnitIndex= */ 1); + Pair overlayTextureSize = overlay.getTextureSize(presentationTimeUs); + GlUtil.setToIdentity(aspectRatioMatrix); + Matrix.scaleM( + aspectRatioMatrix, + MATRIX_OFFSET, + videoWidth / (float) overlayTextureSize.first, + videoHeight / (float) overlayTextureSize.second, + /* z= */ 1); + glProgram.setFloatsUniform("uAspectRatioMatrix", aspectRatioMatrix); + + Matrix.invertM( + overlayMatrix, + MATRIX_OFFSET, + overlay.getOverlaySettings(presentationTimeUs).matrix, + MATRIX_OFFSET); + glProgram.setFloatsUniform("uOverlayMatrix", overlayMatrix); + + } else { + glProgram.setSamplerTexIdUniform( + "uOverlayTexSampler1", createTransparentTexture(), /* texUnitIndex= */ 1); + } + glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e, presentationTimeUs); + } + } + + private int createTransparentTexture() throws FrameProcessingException { + try { + int textureId = + GlUtil.createTexture( + TRANSPARENT_TEXTURE_WIDTH_HEIGHT, + TRANSPARENT_TEXTURE_WIDTH_HEIGHT, + /* useHighPrecisionColorComponents= */ false); + GlUtil.checkGlError(); + return textureId; + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } + + @Override + public void release() throws FrameProcessingException { + super.release(); + try { + glProgram.delete(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/SimpleBitmapLoader.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/SimpleBitmapLoader.java new file mode 100644 index 0000000000..ccefb887fa --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/SimpleBitmapLoader.java @@ -0,0 +1,119 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package com.google.android.exoplayer2.effect; + +import static com.google.android.exoplayer2.util.Assertions.checkArgument; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.net.Uri; +import androidx.annotation.Nullable; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.io.ByteStreams; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +// TODO(b/258685047): delete this copy once substitute is created in common +// (b/194284041, b/258658893) + +/** + * A {@link SimpleBitmapLoader} that delegates all tasks to an executor and supports fetching images + * from URIs with {@code file}, {@code http} and {@code https} schemes. + * + *

Loading tasks are delegated to an {@link ExecutorService} (or {@link + * ListeningExecutorService}) defined during construction. If no executor service is defined, all + * tasks are delegated to a single-thread executor service that is shared between instances of this + * class. + * + *

For HTTP(S) transfers, this class reads a resource only when the endpoint responds with an + * {@code HTTP 200} after sending the HTTP request. + */ +public final class SimpleBitmapLoader implements BitmapLoader { + + private static final String FILE_URI_EXCEPTION_MESSAGE = "Could not read image from file"; + + private static final Supplier DEFAULT_EXECUTOR_SERVICE = + Suppliers.memoize( + () -> MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor())); + + private final ListeningExecutorService executorService; + + /** + * Creates an instance that delegates all load tasks to a single-thread executor service shared + * between instances. + */ + public SimpleBitmapLoader() { + this(checkStateNotNull(DEFAULT_EXECUTOR_SERVICE.get())); + } + + /** Creates an instance that delegates loading tasks to the {@code executorService}. */ + public SimpleBitmapLoader(ExecutorService executorService) { + this.executorService = MoreExecutors.listeningDecorator(executorService); + } + + @Override + public ListenableFuture decodeBitmap(byte[] data) { + return executorService.submit(() -> decode(data)); + } + + @Override + public ListenableFuture loadBitmap(Uri uri) { + return executorService.submit(() -> load(uri)); + } + + private static Bitmap decode(byte[] data) { + @Nullable Bitmap bitmap = BitmapFactory.decodeByteArray(data, /* offset= */ 0, data.length); + checkArgument(bitmap != null, "Could not decode image data"); + return bitmap; + } + + private static Bitmap load(Uri uri) throws IOException { + if ("file".equals(uri.getScheme())) { + @Nullable String path = uri.getPath(); + if (path == null) { + throw new IllegalArgumentException(FILE_URI_EXCEPTION_MESSAGE); + } + @Nullable Bitmap bitmap = BitmapFactory.decodeFile(path); + if (bitmap == null) { + throw new IllegalArgumentException(FILE_URI_EXCEPTION_MESSAGE); + } + return bitmap; + } + URLConnection connection = new URL(uri.toString()).openConnection(); + if (!(connection instanceof HttpURLConnection)) { + throw new UnsupportedOperationException("Unsupported scheme: " + uri.getScheme()); + } + HttpURLConnection httpConnection = (HttpURLConnection) connection; + httpConnection.connect(); + int responseCode = httpConnection.getResponseCode(); + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new IOException("Invalid response status code: " + responseCode); + } + try (InputStream inputStream = httpConnection.getInputStream()) { + return decode(ByteStreams.toByteArray(inputStream)); + } + } +} diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureOverlay.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureOverlay.java new file mode 100644 index 0000000000..868b98c8af --- /dev/null +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureOverlay.java @@ -0,0 +1,51 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import android.util.Pair; +import com.google.android.exoplayer2.util.FrameProcessingException; + +/** Creates overlays from OpenGL textures. */ +public abstract class TextureOverlay { + /** + * Returns the overlay texture identifier displayed at the specified timestamp. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + * @throws FrameProcessingException If an error occurs while processing or drawing the frame. + */ + public abstract int getTextureId(long presentationTimeUs) throws FrameProcessingException; + + // This method is required to find the size of a texture given a texture identifier using OpenGL + // ES 2.0. OpenGL ES 3.1 can do this with glGetTexLevelParameteriv(). + /** + * Returns the pixel width and height of the overlay texture displayed at the specified timestamp. + * + *

This method must be called after {@link #getTextureId(long)}. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + public abstract Pair getTextureSize(long presentationTimeUs); + + /** + * Returns the {@link OverlaySettings} controlling how the overlay is displayed at the specified + * timestamp. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + public OverlaySettings getOverlaySettings(long presentationTimeUs) { + return new OverlaySettings.Builder().build(); + } +} diff --git a/testdata/src/test/assets/media/bitmap/overlay/100winners.png b/testdata/src/test/assets/media/bitmap/overlay/100winners.png new file mode 100644 index 0000000000..d11046a9c0 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/overlay/100winners.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png new file mode 100644 index 0000000000..b9294aaaa8 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png new file mode 100644 index 0000000000..6cab7f796c Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png differ