From ee93a9832ecf37413d09899b81e661dcb80a1c56 Mon Sep 17 00:00:00 2001 From: dancho Date: Thu, 22 Aug 2024 09:15:28 -0700 Subject: [PATCH] Add ByteBufferGlEffect for easier non-GPU video effects * new ByteBufferGlEffect GlEffect that enables API users to implement an effect that accesses video frame data via CPU-mapped ByteBuffer * ByteBufferConcurrentEffect responsible for reading video frame data in CPU-accessible ByteBuffer PiperOrigin-RevId: 666375594 --- .../media3/effect/ByteBufferGlEffectTest.java | 217 ++++++++++++++++++ .../effect/QueuingGlShaderProgramTest.java | 4 +- .../effect/ByteBufferConcurrentEffect.java | 121 ++++++++++ .../media3/effect/ByteBufferGlEffect.java | 139 +++++++++++ .../media3/effect/QueuingGlShaderProgram.java | 8 +- .../ByteBufferGlEffectTest/pts_0.png | Bin 0 -> 591 bytes .../ByteBufferGlEffectTest/pts_333333.png | Bin 0 -> 1319 bytes .../ByteBufferGlEffectTest/pts_666667.png | Bin 0 -> 1434 bytes .../ByteBufferGlEffectTest_input/pts_0.png | Bin 0 -> 407 bytes .../pts_333333.png | Bin 0 -> 1175 bytes .../pts_666667.png | Bin 0 -> 1092 bytes .../ByteBufferGlEffectTest_output/pts_0.png | Bin 0 -> 307 bytes .../pts_333333.png | Bin 0 -> 564 bytes .../pts_666667.png | Bin 0 -> 607 bytes 14 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 libraries/effect/src/androidTest/java/androidx/media3/effect/ByteBufferGlEffectTest.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_0.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_333333.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_666667.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_0.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_333333.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_666667.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_output/pts_0.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_output/pts_333333.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_output/pts_666667.png diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/ByteBufferGlEffectTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/ByteBufferGlEffectTest.java new file mode 100644 index 0000000000..f09104542d --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/ByteBufferGlEffectTest.java @@ -0,0 +1,217 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import android.graphics.Color; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.util.Consumer; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link ByteBufferGlEffect}. */ +@RunWith(AndroidJUnit4.class) +public class ByteBufferGlEffectTest { + + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "test-generated-goldens/ByteBufferGlEffectTest"; + + private static final int INPUT_FRAME_WIDTH = 100; + private static final int INPUT_FRAME_HEIGHT = 50; + private static final int EFFECT_INPUT_FRAME_WIDTH = 75; + private static final int EFFECT_INPUT_FRAME_HEIGHT = 30; + private static final int EFFECT_OUTPUT_FRAME_WIDTH = 50; + private static final int EFFECT_OUTPUT_FRAME_HEIGHT = 20; + private static final Consumer TEXT_SPAN_CONSUMER = + (text) -> { + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private String testId; + + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + @Test + public void byteBufferEffectImplementation_receivesCorrectBitmapData() throws Exception { + List effectInputBitmaps = new ArrayList<>(); + List effectOutputBitmaps = new ArrayList<>(); + ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + INPUT_FRAME_WIDTH, + INPUT_FRAME_HEIGHT, + frameTimesUs, + new ByteBufferGlEffect<>( + new TestByteBufferProcessor(effectInputBitmaps, effectOutputBitmaps)), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder(); + + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + assertBitmapsMatchExpected( + effectInputBitmaps, actualPresentationTimesUs, testId, /* suffix= */ "_input"); + assertBitmapsMatchExpected( + effectOutputBitmaps, actualPresentationTimesUs, testId, /* suffix= */ "_output"); + } + + private static class TestByteBufferProcessor implements ByteBufferGlEffect.Processor { + + private final List inputBitmaps; + private final List outputBitmaps; + private final ListeningExecutorService drawingService; + + TestByteBufferProcessor(List inputBitmaps, List outputBitmaps) { + drawingService = + MoreExecutors.listeningDecorator( + Util.newSingleThreadExecutor(/* threadName= */ "TestByteBufferEffect")); + this.inputBitmaps = inputBitmaps; + this.outputBitmaps = outputBitmaps; + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + checkState(inputWidth == INPUT_FRAME_WIDTH); + checkState(inputHeight == INPUT_FRAME_HEIGHT); + return new Size(EFFECT_INPUT_FRAME_WIDTH, EFFECT_INPUT_FRAME_HEIGHT); + } + + @Override + public Rect getScaledRegion(long presentationTimeUs) { + return new Rect( + /* left= */ 0, + /* top= */ 0, + /* right= */ INPUT_FRAME_WIDTH, + /* bottom= */ INPUT_FRAME_HEIGHT); + } + + @Override + public ListenableFuture processPixelBuffer( + ByteBuffer pixelBuffer, long presentationTimeUs) { + // TODO: b/361286064 - Add helper functions for easier conversion to Bitmap. + // The memory layout of pixels differs between OpenGL and Android Bitmap. + // The first pixel in OpenGL is in the lower left corner, and the first + // pixel in Android Bitmap is in the top left corner. + // Mirror the Bitmap's Y axis. + Bitmap bitmapInGlMemoryLayout = + Bitmap.createBitmap( + EFFECT_INPUT_FRAME_WIDTH, EFFECT_INPUT_FRAME_HEIGHT, Bitmap.Config.ARGB_8888); + bitmapInGlMemoryLayout.copyPixelsFromBuffer(pixelBuffer); + Matrix glToAndroidTransformation = new Matrix(); + glToAndroidTransformation.setScale(/* sx= */ 1, /* sy= */ -1); + Bitmap inputBitmap = + Bitmap.createBitmap( + bitmapInGlMemoryLayout, + /* x= */ 0, + /* y= */ 0, + bitmapInGlMemoryLayout.getWidth(), + bitmapInGlMemoryLayout.getHeight(), + glToAndroidTransformation, + /* filter= */ true); + inputBitmaps.add(inputBitmap); + return drawingService.submit( + () -> + Bitmap.createScaledBitmap( + inputBitmap, + EFFECT_OUTPUT_FRAME_WIDTH, + EFFECT_OUTPUT_FRAME_HEIGHT, + /* filter= */ true)); + } + + @Override + public void finishProcessingAndBlend( + GlTextureInfo outputFrame, long presentationTimeUs, Bitmap result) { + outputBitmaps.add(result); + } + + @Override + public void release() {} + } + + private static void assertBitmapsMatchExpected( + List bitmaps, List presentationTimesUs, String testId, String suffix) + throws IOException { + checkState(bitmaps.size() == presentationTimesUs.size()); + for (int i = 0; i < presentationTimesUs.size(); i++) { + long presentationTimeUs = presentationTimesUs.get(i); + Bitmap actualBitmap = bitmaps.get(i); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ presentationTimeUs + suffix, actualBitmap, /* path= */ null); + Bitmap expectedBitmap = + readBitmap( + Util.formatInvariant("%s/pts_%d.png", ASSET_PATH + suffix, presentationTimeUs)); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId + "_" + i); + // Golden bitmaps were generated with ffmpeg, use a higher threshold. + // TODO: b/361286064 - Use PSNR for quality computations. + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); + } + } +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java index 0af36dc595..a4d838022a 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java @@ -29,6 +29,7 @@ import android.text.style.AbsoluteSizeSpan; import android.text.style.ForegroundColorSpan; import android.text.style.TypefaceSpan; import android.util.Pair; +import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.util.Consumer; import androidx.media3.test.utils.TextureBitmapReader; @@ -162,7 +163,8 @@ public class QueuingGlShaderProgramTest { } @Override - public Future queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs) { + public Future queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) { checkState(textureInfo.width == BLANK_FRAME_WIDTH); checkState(textureInfo.height == BLANK_FRAME_HEIGHT); events.add(Pair.create("queueInputFrame", presentationTimeUs)); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java new file mode 100644 index 0000000000..c038954767 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferConcurrentEffect.java @@ -0,0 +1,121 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; + +import android.graphics.Rect; +import android.opengl.GLES20; +import android.opengl.GLES30; +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; +import java.nio.ByteBuffer; +import java.util.concurrent.Future; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A {@link QueuingGlShaderProgram.ConcurrentEffect} implementation which wraps a {@link + * ByteBufferGlEffect.Processor}. + * + *

This class is responsible for asynchronously transferring texture frame data to a + * CPU-accessible {@link ByteBuffer} that will be used by the wrapped {@link + * ByteBufferGlEffect.Processor}. + */ +/* package */ class ByteBufferConcurrentEffect + implements QueuingGlShaderProgram.ConcurrentEffect { + + private static final int BYTES_PER_PIXEL = 4; + + private final ByteBufferGlEffect.Processor processor; + + private int inputWidth; + private int inputHeight; + private @MonotonicNonNull GlTextureInfo effectInputTexture; + + /** + * Creates an instance. + * + * @param processor The {@linkplain ByteBufferGlEffect.Processor effect}. + */ + public ByteBufferConcurrentEffect(ByteBufferGlEffect.Processor processor) { + this.processor = processor; + inputWidth = C.LENGTH_UNSET; + inputHeight = C.LENGTH_UNSET; + } + + @Override + public Future queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs) { + try { + if (effectInputTexture == null + || textureInfo.width != inputWidth + || textureInfo.height != inputHeight) { + inputWidth = textureInfo.width; + inputHeight = textureInfo.height; + Size effectInputSize = processor.configure(inputWidth, inputHeight); + if (effectInputTexture != null) { + effectInputTexture.release(); + } + int texId = + GlUtil.createTexture( + effectInputSize.getWidth(), + effectInputSize.getHeight(), + /* useHighPrecisionColorComponents= */ false); + effectInputTexture = + glObjectsProvider.createBuffersForTexture( + texId, effectInputSize.getWidth(), effectInputSize.getHeight()); + } + + GlUtil.blitFrameBuffer( + textureInfo.fboId, + processor.getScaledRegion(presentationTimeUs), + effectInputTexture.fboId, + new Rect( + /* left= */ 0, /* top= */ 0, effectInputTexture.width, effectInputTexture.height)); + + GlUtil.focusFramebufferUsingCurrentContext( + effectInputTexture.fboId, effectInputTexture.width, effectInputTexture.height); + ByteBuffer pixelBuffer = + ByteBuffer.allocateDirect(texturePixelBufferSize(effectInputTexture)); + GLES20.glReadPixels( + /* x= */ 0, + /* y= */ 0, + effectInputTexture.width, + effectInputTexture.height, + GLES30.GL_RGBA, + GLES30.GL_UNSIGNED_BYTE, + pixelBuffer); + GlUtil.checkGlError(); + return processor.processPixelBuffer(pixelBuffer, presentationTimeUs); + } catch (GlUtil.GlException | VideoFrameProcessingException e) { + return immediateFailedFuture(e); + } + } + + @Override + public void finishProcessingAndBlend(GlTextureInfo textureInfo, long presentationTimeUs, T result) + throws VideoFrameProcessingException { + processor.finishProcessingAndBlend(textureInfo, presentationTimeUs, result); + } + + private static int texturePixelBufferSize(GlTextureInfo textureInfo) { + return textureInfo.width * textureInfo.height * BYTES_PER_PIXEL; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java new file mode 100644 index 0000000000..3efc23027a --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ByteBufferGlEffect.java @@ -0,0 +1,139 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.content.Context; +import android.graphics.Rect; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; +import com.google.common.util.concurrent.ListenableFuture; +import java.nio.ByteBuffer; +import java.util.concurrent.Future; + +/** + * A {@link GlEffect} implementation that runs an asynchronous {@link Processor} on video frame data + * passed in as a {@link ByteBuffer}. + * + *

This effect can be used to apply CPU-based effects. Or the provided {@link ByteBuffer} can be + * passed to other heterogeneous compute components that are available such as another GPU context, + * FPGAs, or NPUs. + */ +@UnstableApi +/* package */ class ByteBufferGlEffect implements GlEffect { + + private static final int DEFAULT_QUEUE_SIZE = 6; + + /** + * A processor that takes in {@link ByteBuffer ByteBuffers} that represent input image data, and + * produces results of type {@code }. + * + *

All methods are called on the GL thread. + * + * @param The result type of running the processor. + */ + public interface Processor { + + /** + * Configures the instance and returns the dimensions of the image required by {@link + * #processPixelBuffer}. + * + *

When the returned dimensions differ from {@code inputWidth} and {@code inputHeight}, the + * image will be scaled based on {@link #getScaledRegion}. + * + * @param inputWidth The input width in pixels. + * @param inputHeight The input height in pixels. + * @return The size in pixels of the image data accepted by {@link #processPixelBuffer}. + * @throws VideoFrameProcessingException On error. + */ + Size configure(int inputWidth, int inputHeight) throws VideoFrameProcessingException; + + /** + * Selects a region of the input texture that will be scaled to fill the image given that is + * given to {@link #processPixelBuffer}. + * + *

Called once per input frame. + * + *

The contents are scaled to fit the image dimensions returned by {@link #configure}. + * + * @param presentationTimeUs The presentation time in microseconds. + * @return The rectangular region of the input image that will be scaled to fill the effect + * input image. + */ + // TODO: b/b/361286064 - This method misuses android.graphics.Rect for OpenGL coordinates. + // Implement a custom GlUtils.Rect to correctly label lower left corner as (0, 0). + Rect getScaledRegion(long presentationTimeUs); + + /** + * Processing the image data in the {@code pixelBuffer}. + * + *

Accessing {@code pixelBuffer} after the returned future is {@linkplain Future#isDone() + * done} or {@linkplain Future#isCancelled() cancelled} can lead to undefined behaviour. + * + * @param pixelBuffer The image data. + * @param presentationTimeUs The presentation time in microseconds. + * @return A {@link ListenableFuture} of the result. + */ + // TODO: b/361286064 - Add helper functions for easier conversion to Bitmap. + ListenableFuture processPixelBuffer(ByteBuffer pixelBuffer, long presentationTimeUs); + + /** + * Finishes processing the frame at {@code presentationTimeUs}. Use this method to perform + * custom drawing on the output frame. + * + *

The {@linkplain GlTextureInfo outputFrame} contains the image data corresponding to the + * frame at {@code presentationTimeUs} when this method is invoked. + * + * @param outputFrame The texture info of the frame. + * @param presentationTimeUs The presentation timestamp of the frame, in microseconds. + * @param result The result of the asynchronous computation in {@link #processPixelBuffer}. + */ + void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result) + throws VideoFrameProcessingException; + + /** + * Releases all resources. + * + * @throws VideoFrameProcessingException If an error occurs while releasing resources. + */ + void release() throws VideoFrameProcessingException; + } + + private final Processor processor; + + /** + * Creates an instance. + * + * @param processor The effect to apply. + */ + public ByteBufferGlEffect(Processor processor) { + this.processor = processor; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + // TODO: b/361286064 - Implement HDR support. + checkArgument(!useHdr, "HDR support not yet implemented."); + return new QueuingGlShaderProgram<>( + /* useHighPrecisionColorComponents= */ useHdr, + /* queueSize= */ DEFAULT_QUEUE_SIZE, + new ByteBufferConcurrentEffect<>(processor)); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java index 3473ebfa12..c64f14f220 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java @@ -92,7 +92,8 @@ import java.util.concurrent.TimeUnit; * @param presentationTimeUs The presentation timestamp of the input frame, in microseconds. * @return A {@link Future} representing pending completion of the task. */ - Future queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs); + Future queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo textureInfo, long presentationTimeUs); /** * Finishes processing the frame at {@code presentationTimeUs}. This method optionally allows @@ -172,6 +173,8 @@ import java.util.concurrent.TimeUnit; if (inputWidth != inputTexture.width || inputHeight != inputTexture.height || !outputTexturePool.isConfigured()) { + // Output all pending frames before processing a format change. + while (outputOneFrame()) {} inputWidth = inputTexture.width; inputHeight = inputTexture.height; outputTexturePool.ensureConfigured(glObjectsProvider, inputWidth, inputHeight); @@ -189,7 +192,8 @@ import java.util.concurrent.TimeUnit; new Rect( /* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight)); - Future task = concurrentEffect.queueInputFrame(outputTexture, presentationTimeUs); + Future task = + concurrentEffect.queueInputFrame(glObjectsProvider, outputTexture, presentationTimeUs); frameQueue.add(new TimedTextureInfo(outputTexture, presentationTimeUs, task)); inputListener.onInputFrameProcessed(inputTexture); diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_0.png new file mode 100644 index 0000000000000000000000000000000000000000..4ffb8030fd0cc9248247d2c2c83f484fb229831d GIT binary patch literal 591 zcmeAS@N?(olHy`uVBq!ia0vp^DL`z*!3HE(nbz$CQjEnx?oJHr&dIz4a#)I;JVQ8u zpoSx*11R^?)5S5QV$R!}w%H+$B5V)TUxyk>KNc72?CKJ15ozjJ(R(A}rZz{}wSR2& zP4$P41?6nrD8Rz0)U+bB!)XzlebS3;VG6qKheysqBKbz&QsZWv1Az>#t~QpK&ZI`K8vW()Kq8 zRPP+9?3wadiJMGoSdQ8@ z_}&}z{r%e;7T?F3nJf{%@j`OlN}1f3AI<}nX>O2QT@{}le)?kM?5Q%`mp*-ezx>U; zNK3Jl1y3U;n7&l_fBCxom91vlK#|B@_r$8^3%16~c1*2ny`^C(_N2RH_p2@5*QE81 zr%Ud;RIGENx@JP$i}pYF`lS!Yt=wsG_kPA#pgECyCfs_LUa%xPZ}t_vnMb_$ysO?d ztLV))J@5BDoGW)*Ywh(uR>&B>(LKxR-tp~eNnw-k+U~t_%r)k4;mbV{6Y5rp6^d{F zxC$6LPoo4b9kkW_`@b;a*4xh&UG7!icmAui6koDPx#1am@3R0s$N2z&@+hyVZt(Md!>RCt{2+3Rlk zkX>Iq&ty+e&&CEI1p*}0)PL{Olpe$ASK{*U?p&()0iKD&A=B-{9zKa0$AvoZZ-aa z6g5kme{0{8MFwFromSS#Jb3{g+ zs$(wFI-OpfwmK$Mrwur@j_HgrQ<8Z|4;TX;1TF!V0jq&~wIN^Xey-_O^AK>UXU5db z_sr%LbB|*dIOYW4;|wzjY(qx1Z${Nz=9%9dQ*z8lTIXTU-0hf!9QTJxY)1&+NbvEkpWYuXUa)cZfv$N&{U@I_~vb`B;@Sob4D)7EzLf6!0{rtnL^Z68* zl4m|dZoq@cNz79xk0~MN;9t!mq@lyW@|>m>If=u-xshfFIFEe2uodYPL%{hFW{a8& zBFtxwiMk3E86j?tMSg_C^?Ms|9PkA2RfcIrW+jQ413VdNjt8CsdI^t)+1j>g`n_H2 zJRM=$fVpb6MVe={PG|&8Syln>A+wZkkp{oO_!(Bnyc;m{bDABVy_; zGqlb~N}V(_H_}X5nt-E_p?WehHT(uyt*(eL{dG)ZhFL|pCx%R4q*)7WSF^f+xsYNr z*-1+oxC7}Sk3^aZ@F@9)c^SCfF?kL;F$aLh$ai%p1Gga~Fvi@f=5ZqL(?xFTry;%k zJLEx{n+h^3sRCzGoAxe7rh-`pU}>bOItDdO!1o!ZQotm2%E%9zn4QR_$~Rl}^9&;I zBSnVOb--Q7L%&d(^z*NPNo!!EW2%n1Gs7fx_BnsMR_nMYyk6U4OwzV%91}W+p*p!z zWQ3G~HORW*YsUAG7T_xV-s73e9kVB=S)===nG&^?v|A@J!=AZP_xU7XMj}nSV}?9) zh3@ky;i+&zTViq!tJAw7cP`(e?t!6K>=vzkZ$!KCakY__Xf%}17 zj#;k8uhZew37NAq%mDBJ@UwF~Zkz$0HXuWG9WrEh zAx}43kj2_|;CY~n+B=c5n)S#fXm%da6w$ax$mVSS%Kc#bU8oEEbE!VzF2( d7K`PO%|C_w@g`BaI9>n%002ovPDHLkV1hQoaw7l$ literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_666667.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest/pts_666667.png new file mode 100644 index 0000000000000000000000000000000000000000..cd7cfdcbd1f2f2945b73b05757da9cb19aa7168e GIT binary patch literal 1434 zcmV;L1!ek)P)Px#1am@3R0s$N2z&@+hyVZuL`g(JRCt{2nMrI_RT#&A9c&9$5uYSggFqWKv7iP7 zhlR6k72`m`DelA}YMjuG3M||i$0!RGjF`9)!Nk~_pfQ%}f-ZC;tyN5oR60;1B4Qaz zUHrexJKlHiefJGEhMZs0G`;u!?m71i-}%ldcsw4D$K&yMJRXn7L^ z22wa{P(yA*BW?f~A-)5XoVX5b23DKXZBFn#u5$=@0fn{>qOJ#*eI(3vpj@)S-IccOLYMM@j=sc8ic0$&01QaUZbUqv_v z%xNsp=|by38g2mobevK``6(!*jQ|&y_^#BfN8zH~{;mT~C3!TwgGTLgRLTF!dE1;7 z1vqb^*>ML7MQNNk+R$L#XzwYAVtTI*=ttq^8kAuzK&jwCU^|N2`$NtTC=6VS)}OG` z4g3*u`p}5G4#oI1PA_WAz?l!!n$u%-3p+c28Nl^uF%CQ3z`m4i^8uQrx1br9l+oo- zOouc9JJEvGiPnOnC}y4y>;g6eZ=x^|I6nfvMmXmIgE3COy{{8kf<|%JSx2_>8?BtM za|AfooGn(4>-5{b&w#}-&N}|xkx^U$bfCeq!ugie6f-5yvfmlhuv<``eHe|bhkz%6 zpHQX(a3<>fD5p8(Y)0WQ?0js_p^&rF%9)I)TiAIJc*2}b5zdkb=KAbC56ULoIC&DP{3Ke4Ywh|{^dWnIyJDPU5zd4;ccYv>%IR|MEk!dg z&Uwu4yUtCHb59;;dTgs`Pc{sE3(QJ*uIwyF4ZfA=BZLNEFB)NU>{>f&yscy#JnW1C z<7h(`I1}bXEshPS5y#19VC|^!K2LB)%n7ooWsWl~r4yzCc*)-LDCu|cIV5dv;t0-l zMo^p$GmJt)4GKH1GfMPX(l81MwPZt8TBjz$Nvd1mG}yJ^xukN!-xH<@4Tb^WQwm(j zU5e?DQTsh7;hFFbdep9+Lwq6;{s_06d`?haAx=QB<4>wv(s>yu49`ZV6@w|BFDtXq zN9;u(5hjB|$=}ZXGk_s;qB=5z8gtya?>cj1oDnN$p5yE`Cu;MVK_fnJD$MyUk27RW zBe|V+HcGXJQCeQoMzEL}GKMmt(}DX@RuNEzrfvrKn(P_X7|N_>0xQuz!F86QhUrEh z@0OF#`6lFS0Zv01&y0|ByW4+(36}hRX>+Fsw zXBD}Jnw997JnRgZ^K^s*T04%Qg}ju*$m5g}@2Y@HfU8jUQ3d=?_UPn50ZtFf^W&UF zz}1ek&nYKrH(ZJn)U7&&(}GThyX>dr*-?&@$V#-1RG|h<^Dok3oqE(z|2N0u@pwEQ okH_Qjcsw4D$K&yMJRXOC0PWql%6UBgg#Z8m07*qoM6N<$g2vyQ-~a#s literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_0.png new file mode 100644 index 0000000000000000000000000000000000000000..54db58ae0d4ccd3b5c01e10238b573631e54cd9b GIT binary patch literal 407 zcmeAS@N?(olHy`uVBq!ia0vp^-astJ!3HE#cpt9=Qk(@Ik;On71Q;1wD#bw@#^NA% zC&rs6b?Si}g=CK)Uj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G% zcl32+VA$Bt{U?zXWT2;uV@SoEx7Q54odPA=KZ^S>EsSXrV~-V@^@pvlwZ%ZjsLMe% zX7O3ytjtvl73Zw6^AK3+cuoA3<(#KQJ3rgL|4{#6`<~0EAI>>#8tkRHbV`tyXqe~T zhVNw;kMi*EIL^SXIsMZOzGFX*F$JI5cu1M=^4e$XlWU@ER{TBlM{2*svWQdJwrdI} z&fdQ9Xz)vE%XdQY7AMb2i{)Rd2J-Z5wij&f;?usPBOA6%yIcNqT;96bp%+#jon1O_ zwtS?`n&+3yi(=TNyX8M+rPa0B_XO!Y-TJ|~s$2d}WKExN%>&W1DYGqe`D^z);Pt+< rU}}-(`_}Xbcipc_V5hE{^N-PUy0V|RTc{8)Y#2OU{an^LB{Ts5VttwN literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_333333.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_333333.png new file mode 100644 index 0000000000000000000000000000000000000000..ee006f7a0644835ba4da8ce1aeb50c793a696d1c GIT binary patch literal 1175 zcmV;I1Zew-P)0015c1^@s6WDVo400009a7bBm00000 z000010EBrLa{vGU0drDELIK&yEPnt103c&XQcVB=dL{q>fP?@5`Tzg`fam}Kbua(` z>RI+y?e7jT@qQ9J+u00d`2O+f$vv5yPn&X;Bj$=M|QAcn#=+mbAa5V0UhrzwGzGv8q5%gm$%X<{nViKP)r(fZPx zy`cD3lp2aCens*&eP*6}%P*6}% zP|*KS-9ctM;xa9^E%j*Ud)pJ&)!x!pAy@=98MR;%=;q=|z&nw+rC_9sq1k7_TCf?s z59WaZF0L<_3pQBXdQbr#aB;)I8_rv)q$cDE@V4_ds`Y7iTob^L;9IZ^ybeAE=fRvv z+-&d%IAG>$z*pcGGfFX0tH592L-3kta(@M%gMJp55B7tTU=4WHCitgl(h?O^48AjO z%gx(v8~>$+s=i;V-f50pD{E%qXWVtw21d~-_&!b!L)@a`H z&0BE`-dMRGNoqUkw74qc!9=hGoD%IRUW3fs(_kz3$^PT@uxNs(gF~X*aG;Cn2_6&O z#;iKFgE~n~Voth$-qJ7IJH7#*KPJma1T=&~XaNP`Z z#ki$bx)`qbH{b_*F0;>7E{>DB2DaEUuN2LqeJHAGa7Xl-LTxv1{=ld+Z;yyJc`jIO zj5nT|+>*WrrmYu5kN*ZUTW&ADD4HCuK9jmyv^sd%WdC7**dcl!tG2P&N7&3ygT-K` zXq9j>+}4TYa66r}$?$}nA$o^mOp_%05tH^e*d=;e109?|B`mpmAWVJR~yJHcLy-3FdC@APZ(p`u$y`;h34|xCp002ovPDHLkV1l>tCldew literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_666667.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_input/pts_666667.png new file mode 100644 index 0000000000000000000000000000000000000000..9cdab8bc5ff3023dafb470a108e701acf9841ed5 GIT binary patch literal 1092 zcmV-K1iSl*P)0015c1^@s6WDVo400009a7bBm00000 z000010EBrLa{vGU0drDELIK&yEPnt103c&XQcVB=dL{q>fP?@5`Tzg`fam}Kbua(` z>RI+y?e7jT@qQ9J+u00d`2O+f$vv5yPq4b8enq z@4a{AmeTXDUo-ce|NGwaocEmbJR{=e<>lq&<>lq&<@JB7B^U+ffuT`6r9ek89xMc% zqWCI;9$+fS19hVKYJ)*whIvay@wEUW!CWxRx!wdU0CNK58T0Mlgee(J0oTDnuvs)| zK~)E9z!k6;tOs2}d~Lx2a31Ug%R#*mUk11UPJ(8`yC0!+fqed#I zY2TX?@Ua>nh*nNHC$?y<>S`Le2bvebTV(YIKfn^v%Gn3bfVH5N<68{gfLUO-Xq%lD zO-`zLih+yZhG_EnEGC3iHb#Cel9lup>;lWocLMAIy`1l4n|=pNo!CxWY;9t!sVR6W zny`c_TQpgm2LnMGNEdCT0#Ma_+swzA#lg)M8*4$WHct(3SG1*bM2CMn(Zszpzu5I} zqH~DX+KbM<9imkk*)+23I}eL)0*tLMxMZ=_jIyS6;HJfsP_Z%!Enrs}7APx^ZL=TT z7HvUS+!4QuR!alX#cm9ESaIjTRr{UQ^AU^-_}F&uM5~RO4qkx{0bjbsmKn01?xHg; zY=io1ai@6=8V9a76WvKB*k``=t0DJuJ7?fbHmi^`W@U(v3FK_#kmlk)fFU71R_1p5 zot4F8H4S+9naR!$T<6R?0#1Q4A^(h@u>aKW`4 z4;R5XRv=oX<-tSI)-LY&c+UGQy8Tm3>U+@_>T0;?Eb9{zGhck~kaZ?RYzwv`cMhI& z`iUlnEyr_IS@W@Kn25`w)y7uf;gySa7sJ8JnZolL509LK8}0c$9p7MZM|8e$haCZq zgR`P%r$}r$c5ki|+bys`bcc+HE&N#Yj2PU>6Dl73xUj7TTaIV1DghsNJ)W}JCaek` z5Hmx9ZgH~2huLDhh-oBcg4_EHlkE}DnZ z#kt?h%gf8l%gf8_ALu9Mt-pOP9t=SM0000< KMNUMnLSTa5eK@{3dtTpz6=aiY77hwEes65fIb(8Sj}~j~<(+%J$lx6J`GRfvKLVfYoZhr%ogmX1gV1|Qd5zjdF&~^(?DB}{ zF;{y#_t7$Q_qaCa_xBbRoh_CQNqO#QTvrdK9oE=5xw|`P_a#6NUX}4 v<9z&gChmCVZFXbd|NfcZI8JZ+w}gMfdj8-448o2B-O1qT>gTe~DWM4f-=1xr literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_output/pts_333333.png b/libraries/test_data/src/test/assets/test-generated-goldens/ByteBufferGlEffectTest_output/pts_333333.png new file mode 100644 index 0000000000000000000000000000000000000000..f1da5c353814d214fe692b742f26a50ab1c9115b GIT binary patch literal 564 zcmV-40?Yl0P)fP?@5`Tzg`fam}Kbua(` z>RI+y?e7jT@qQ9J+u00d`2O+f$vv5yPq4nB&EeMI)6B3WTRDz)43wVUYYeB*WAIpC-r*Sf-O=rPE?mzh@ z)0y+Tow;XniTwES9lB%HiobaKuxxCT4m7$iXucwir-z&5x9 z5q6^B3hc6@>Ppy28cvj*h{f?N&IRwuIa^?a$A#b%41hMU#~W0c2ai171f#{!;Q;(Sm{y0vj|3z|etHE1XO2G@j_c48K%N;vBooEsjS zPEt6|sUzLi7jOWI#M%@%Bi*UK49+2MGM!1`^o#l0EY3s*C#R@x)X;Yu1K0Z(wfd|un@8$_u;RC9Ttn!>%Q&*vboiP8F zPJyG75Kh8yI$aMTQw~Zk^B%ALmE-ia^5e(PKg}0Qj#OfvP05)60000fP?@5`Tzg`fam}Kbua(` z>RI+y?e7jT@qQ9J+u00d`2O+f$vv5yP0Q;{C2u|L95Uv6m8ekV zC_zTT6Hs^Y8DG%gtY_ghq8LZI)4PLD0WukX$=uVmCcMK0-eLy7bhw178aw*K{^?C( zR_v+8%QPa@>zNI`$2D)g3O{h)%cM9=Jr#SX3D+1l(Tgdx>iTv3#!I}x2+FiK`j5<^ zmnqlYh?jXNXAiDWhpkO!kZ)MWP1Iu-EvUgU9*e!ZgV@4le8mRpG_#|_n9Lt^X=Xq( z=1NArjM;Ffw-K|3A@8Bw!|0POoEFVTBcz$O*$-Rqc?)D=+{6002ovPDHLkV1kg~2gCpX literal 0 HcmV?d00001