mirror of
https://github.com/samsonjs/media.git
synced 2026-04-27 15:07:40 +00:00
Rename FrameProcessorChain to GlEffectsFrameProcessor.
This change is just renaming. There is no functional change intended. The FrameProcessor interface will be created in a follow-up. PiperOrigin-RevId: 456741628
This commit is contained in:
parent
19bdff96ba
commit
216fefd669
12 changed files with 91 additions and 86 deletions
|
|
@ -39,7 +39,7 @@ import java.io.InputStream;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link
|
* Utilities for instrumentation tests for the {@link GlEffectsFrameProcessor} and {@link
|
||||||
* SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors}.
|
* SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors}.
|
||||||
*/
|
*/
|
||||||
public class BitmapTestUtil {
|
public class BitmapTestUtil {
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ import org.junit.runner.RunWith;
|
||||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||||
* devices may fail. To test on other devices, please increase the {@link
|
* devices may fail. To test on other devices, please increase the {@link
|
||||||
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
||||||
* as recommended in {@link FrameProcessorChainPixelTest}.
|
* as recommended in {@link GlEffectsFrameProcessorPixelTest}.
|
||||||
*/
|
*/
|
||||||
@RunWith(AndroidJUnit4.class)
|
@RunWith(AndroidJUnit4.class)
|
||||||
public final class CropPixelTest {
|
public final class CropPixelTest {
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ import org.junit.Test;
|
||||||
import org.junit.runner.RunWith;
|
import org.junit.runner.RunWith;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Pixel test for frame processing via {@link FrameProcessorChain}.
|
* Pixel test for frame processing via {@link GlEffectsFrameProcessor}.
|
||||||
*
|
*
|
||||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||||
* devices may fail. To test on other devices, please increase the {@link
|
* devices may fail. To test on other devices, please increase the {@link
|
||||||
|
|
@ -54,7 +54,7 @@ import org.junit.runner.RunWith;
|
||||||
* bitmaps.
|
* bitmaps.
|
||||||
*/
|
*/
|
||||||
@RunWith(AndroidJUnit4.class)
|
@RunWith(AndroidJUnit4.class)
|
||||||
public final class FrameProcessorChainPixelTest {
|
public final class GlEffectsFrameProcessorPixelTest {
|
||||||
public static final String ORIGINAL_PNG_ASSET_PATH =
|
public static final String ORIGINAL_PNG_ASSET_PATH =
|
||||||
"media/bitmap/sample_mp4_first_frame/original.png";
|
"media/bitmap/sample_mp4_first_frame/original.png";
|
||||||
public static final String SCALE_WIDE_PNG_ASSET_PATH =
|
public static final String SCALE_WIDE_PNG_ASSET_PATH =
|
||||||
|
|
@ -79,8 +79,9 @@ public final class FrameProcessorChainPixelTest {
|
||||||
/** Timeout for dequeueing buffers from the codec, in microseconds. */
|
/** Timeout for dequeueing buffers from the codec, in microseconds. */
|
||||||
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
|
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
|
||||||
/**
|
/**
|
||||||
* Time to wait for the decoded frame to populate the {@link FrameProcessorChain}'s input surface
|
* Time to wait for the decoded frame to populate the {@link GlEffectsFrameProcessor} instance's
|
||||||
* and the {@link FrameProcessorChain} to finish processing the frame, in milliseconds.
|
* input surface and the {@link GlEffectsFrameProcessor} to finish processing the frame, in
|
||||||
|
* milliseconds.
|
||||||
*/
|
*/
|
||||||
private static final int FRAME_PROCESSING_WAIT_MS = 5000;
|
private static final int FRAME_PROCESSING_WAIT_MS = 5000;
|
||||||
/** The ratio of width over height, for each pixel in a frame. */
|
/** The ratio of width over height, for each pixel in a frame. */
|
||||||
|
|
@ -90,14 +91,14 @@ public final class FrameProcessorChainPixelTest {
|
||||||
new AtomicReference<>();
|
new AtomicReference<>();
|
||||||
|
|
||||||
private @MonotonicNonNull MediaFormat mediaFormat;
|
private @MonotonicNonNull MediaFormat mediaFormat;
|
||||||
private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
|
private @MonotonicNonNull GlEffectsFrameProcessor glEffectsFrameProcessor;
|
||||||
private volatile @MonotonicNonNull ImageReader outputImageReader;
|
private volatile @MonotonicNonNull ImageReader outputImageReader;
|
||||||
private volatile boolean frameProcessingEnded;
|
private volatile boolean frameProcessingEnded;
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void release() {
|
public void release() {
|
||||||
if (frameProcessorChain != null) {
|
if (glEffectsFrameProcessor != null) {
|
||||||
frameProcessorChain.release();
|
glEffectsFrameProcessor.release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -321,8 +322,8 @@ public final class FrameProcessorChainPixelTest {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set up and prepare the first frame from an input video, as well as relevant test
|
* Set up and prepare the first frame from an input video, as well as relevant test
|
||||||
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
|
* infrastructure. The frame will be sent towards the {@link GlEffectsFrameProcessor}, and output
|
||||||
* accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
|
* may be accessed on the {@code outputImageReader}.
|
||||||
*
|
*
|
||||||
* @param pixelWidthHeightRatio The ratio of width over height for each pixel.
|
* @param pixelWidthHeightRatio The ratio of width over height for each pixel.
|
||||||
* @param effects The {@link GlEffect GlEffects} to apply to the input frame.
|
* @param effects The {@link GlEffect GlEffects} to apply to the input frame.
|
||||||
|
|
@ -350,11 +351,11 @@ public final class FrameProcessorChainPixelTest {
|
||||||
|
|
||||||
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
|
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
|
||||||
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
|
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
|
||||||
frameProcessorChain =
|
glEffectsFrameProcessor =
|
||||||
checkNotNull(
|
checkNotNull(
|
||||||
FrameProcessorChain.create(
|
GlEffectsFrameProcessor.create(
|
||||||
context,
|
context,
|
||||||
new FrameProcessorChain.Listener() {
|
new GlEffectsFrameProcessor.Listener() {
|
||||||
@Override
|
@Override
|
||||||
public void onFrameProcessingError(FrameProcessingException exception) {
|
public void onFrameProcessingError(FrameProcessingException exception) {
|
||||||
frameProcessingException.set(exception);
|
frameProcessingException.set(exception);
|
||||||
|
|
@ -382,13 +383,16 @@ public final class FrameProcessorChainPixelTest {
|
||||||
},
|
},
|
||||||
Transformer.DebugViewProvider.NONE,
|
Transformer.DebugViewProvider.NONE,
|
||||||
/* enableExperimentalHdrEditing= */ false));
|
/* enableExperimentalHdrEditing= */ false));
|
||||||
frameProcessorChain.registerInputFrame();
|
glEffectsFrameProcessor.registerInputFrame();
|
||||||
|
|
||||||
// Queue the first video frame from the extractor.
|
// Queue the first video frame from the extractor.
|
||||||
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
|
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
|
||||||
mediaCodec = MediaCodec.createDecoderByType(mimeType);
|
mediaCodec = MediaCodec.createDecoderByType(mimeType);
|
||||||
mediaCodec.configure(
|
mediaCodec.configure(
|
||||||
mediaFormat, frameProcessorChain.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
|
mediaFormat,
|
||||||
|
glEffectsFrameProcessor.getInputSurface(),
|
||||||
|
/* crypto= */ null,
|
||||||
|
/* flags= */ 0);
|
||||||
mediaCodec.start();
|
mediaCodec.start();
|
||||||
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
|
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
|
||||||
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
|
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
|
||||||
|
|
@ -429,15 +433,15 @@ public final class FrameProcessorChainPixelTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
private Bitmap processFirstFrameAndEnd() throws InterruptedException {
|
private Bitmap processFirstFrameAndEnd() throws InterruptedException {
|
||||||
checkNotNull(frameProcessorChain).signalEndOfInputStream();
|
checkNotNull(glEffectsFrameProcessor).signalEndOfInputStream();
|
||||||
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
|
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
|
||||||
assertThat(frameProcessingEnded).isTrue();
|
assertThat(frameProcessingEnded).isTrue();
|
||||||
assertThat(frameProcessingException.get()).isNull();
|
assertThat(frameProcessingException.get()).isNull();
|
||||||
|
|
||||||
Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
|
Image frameProcessorOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
|
||||||
Bitmap actualBitmap =
|
Bitmap actualBitmap =
|
||||||
BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(frameProcessorChainOutputImage);
|
BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(frameProcessorOutputImage);
|
||||||
frameProcessorChainOutputImage.close();
|
frameProcessorOutputImage.close();
|
||||||
return actualBitmap;
|
return actualBitmap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -475,11 +479,11 @@ public final class FrameProcessorChainPixelTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wraps a {@link GlEffect} to prevent the {@link FrameProcessorChain} from detecting its class
|
* Wraps a {@link GlEffect} to prevent the {@link GlEffectsFrameProcessor} from detecting its
|
||||||
* and optimizing it.
|
* class and optimizing it.
|
||||||
*
|
*
|
||||||
* <p>This ensures that {@link FrameProcessorChain} uses a separate {@link GlTextureProcessor} for
|
* <p>This ensures that {@link GlEffectsFrameProcessor} uses a separate {@link GlTextureProcessor}
|
||||||
* the wrapped {@link GlEffect} rather than merging it with preceding or subsequent {@link
|
* for the wrapped {@link GlEffect} rather than merging it with preceding or subsequent {@link
|
||||||
* GlEffect} instances and applying them in one combined {@link GlTextureProcessor}.
|
* GlEffect} instances and applying them in one combined {@link GlTextureProcessor}.
|
||||||
*/
|
*/
|
||||||
private static final class GlEffectWrapper implements GlEffect {
|
private static final class GlEffectWrapper implements GlEffect {
|
||||||
|
|
@ -40,7 +40,7 @@ import org.junit.runner.RunWith;
|
||||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||||
* devices may fail. To test on other devices, please increase the {@link
|
* devices may fail. To test on other devices, please increase the {@link
|
||||||
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
||||||
* as recommended in {@link FrameProcessorChainPixelTest}.
|
* as recommended in {@link GlEffectsFrameProcessorPixelTest}.
|
||||||
*/
|
*/
|
||||||
@RunWith(AndroidJUnit4.class)
|
@RunWith(AndroidJUnit4.class)
|
||||||
public final class MatrixTransformationProcessorPixelTest {
|
public final class MatrixTransformationProcessorPixelTest {
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ import org.junit.runner.RunWith;
|
||||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||||
* devices may fail. To test on other devices, please increase the {@link
|
* devices may fail. To test on other devices, please increase the {@link
|
||||||
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
||||||
* as recommended in {@link FrameProcessorChainPixelTest}.
|
* as recommended in {@link GlEffectsFrameProcessorPixelTest}.
|
||||||
*/
|
*/
|
||||||
@RunWith(AndroidJUnit4.class)
|
@RunWith(AndroidJUnit4.class)
|
||||||
public final class PresentationPixelTest {
|
public final class PresentationPixelTest {
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ import java.util.Queue;
|
||||||
@Nullable private final GlTextureProcessor previousGlTextureProcessor;
|
@Nullable private final GlTextureProcessor previousGlTextureProcessor;
|
||||||
@Nullable private final GlTextureProcessor nextGlTextureProcessor;
|
@Nullable private final GlTextureProcessor nextGlTextureProcessor;
|
||||||
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
|
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
|
||||||
private final FrameProcessorChain.Listener frameProcessorChainListener;
|
private final GlEffectsFrameProcessor.Listener frameProcessorListener;
|
||||||
private final Queue<Pair<TextureInfo, Long>> pendingFrames;
|
private final Queue<Pair<TextureInfo, Long>> pendingFrames;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -45,18 +45,18 @@ import java.util.Queue;
|
||||||
* OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by
|
* OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by
|
||||||
* the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link
|
* the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link
|
||||||
* FrameProcessingTaskExecutor}.
|
* FrameProcessingTaskExecutor}.
|
||||||
* @param frameProcessorChainListener The {@link FrameProcessorChain.Listener} to forward
|
* @param frameProcessorListener The {@link GlEffectsFrameProcessor.Listener} to forward
|
||||||
* exceptions to.
|
* exceptions to.
|
||||||
*/
|
*/
|
||||||
public ChainingGlTextureProcessorListener(
|
public ChainingGlTextureProcessorListener(
|
||||||
@Nullable GlTextureProcessor previousGlTextureProcessor,
|
@Nullable GlTextureProcessor previousGlTextureProcessor,
|
||||||
@Nullable GlTextureProcessor nextGlTextureProcessor,
|
@Nullable GlTextureProcessor nextGlTextureProcessor,
|
||||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
||||||
FrameProcessorChain.Listener frameProcessorChainListener) {
|
GlEffectsFrameProcessor.Listener frameProcessorListener) {
|
||||||
this.previousGlTextureProcessor = previousGlTextureProcessor;
|
this.previousGlTextureProcessor = previousGlTextureProcessor;
|
||||||
this.nextGlTextureProcessor = nextGlTextureProcessor;
|
this.nextGlTextureProcessor = nextGlTextureProcessor;
|
||||||
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
|
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
|
||||||
this.frameProcessorChainListener = frameProcessorChainListener;
|
this.frameProcessorListener = frameProcessorListener;
|
||||||
pendingFrames = new ArrayDeque<>();
|
pendingFrames = new ArrayDeque<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -101,6 +101,6 @@ import java.util.Queue;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFrameProcessingError(FrameProcessingException e) {
|
public void onFrameProcessingError(FrameProcessingException e) {
|
||||||
frameProcessorChainListener.onFrameProcessingError(e);
|
frameProcessorListener.onFrameProcessingError(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
* dimensions specified by the provided {@link SurfaceInfo}.
|
* dimensions specified by the provided {@link SurfaceInfo}.
|
||||||
*
|
*
|
||||||
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
|
* <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
|
||||||
* GlTextureProcessor} instances used by {@link FrameProcessorChain}.
|
* GlTextureProcessor} instances used by {@link GlEffectsFrameProcessor}.
|
||||||
*/
|
*/
|
||||||
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
|
/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
|
||||||
|
|
||||||
|
|
@ -60,7 +60,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
private final SurfaceInfo.Provider outputSurfaceProvider;
|
private final SurfaceInfo.Provider outputSurfaceProvider;
|
||||||
private final long streamOffsetUs;
|
private final long streamOffsetUs;
|
||||||
private final Transformer.DebugViewProvider debugViewProvider;
|
private final Transformer.DebugViewProvider debugViewProvider;
|
||||||
private final FrameProcessorChain.Listener frameProcessorChainListener;
|
private final GlEffectsFrameProcessor.Listener frameProcessorListener;
|
||||||
private final boolean enableExperimentalHdrEditing;
|
private final boolean enableExperimentalHdrEditing;
|
||||||
|
|
||||||
private int inputWidth;
|
private int inputWidth;
|
||||||
|
|
@ -78,7 +78,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
ImmutableList<GlMatrixTransformation> matrixTransformations,
|
ImmutableList<GlMatrixTransformation> matrixTransformations,
|
||||||
SurfaceInfo.Provider outputSurfaceProvider,
|
SurfaceInfo.Provider outputSurfaceProvider,
|
||||||
long streamOffsetUs,
|
long streamOffsetUs,
|
||||||
FrameProcessorChain.Listener listener,
|
GlEffectsFrameProcessor.Listener frameProcessorListener,
|
||||||
Transformer.DebugViewProvider debugViewProvider,
|
Transformer.DebugViewProvider debugViewProvider,
|
||||||
boolean enableExperimentalHdrEditing) {
|
boolean enableExperimentalHdrEditing) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
|
|
@ -88,7 +88,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
this.outputSurfaceProvider = outputSurfaceProvider;
|
this.outputSurfaceProvider = outputSurfaceProvider;
|
||||||
this.streamOffsetUs = streamOffsetUs;
|
this.streamOffsetUs = streamOffsetUs;
|
||||||
this.debugViewProvider = debugViewProvider;
|
this.debugViewProvider = debugViewProvider;
|
||||||
this.frameProcessorChainListener = listener;
|
this.frameProcessorListener = frameProcessorListener;
|
||||||
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
|
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
*
|
*
|
||||||
* <p>The {@code FinalMatrixTransformationProcessorWrapper} will only call {@link
|
* <p>The {@code FinalMatrixTransformationProcessorWrapper} will only call {@link
|
||||||
* Listener#onInputFrameProcessed(TextureInfo)}. Other events are handled via the {@link
|
* Listener#onInputFrameProcessed(TextureInfo)}. Other events are handled via the {@link
|
||||||
* FrameProcessorChain.Listener} passed to the constructor.
|
* GlEffectsFrameProcessor.Listener} passed to the constructor.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void setListener(Listener listener) {
|
public void setListener(Listener listener) {
|
||||||
|
|
@ -130,7 +130,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
/* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
|
/* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
|
||||||
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
|
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
|
||||||
} catch (FrameProcessingException | GlUtil.GlException e) {
|
} catch (FrameProcessingException | GlUtil.GlException e) {
|
||||||
frameProcessorChainListener.onFrameProcessingError(
|
frameProcessorListener.onFrameProcessingError(
|
||||||
FrameProcessingException.from(e, presentationTimeUs));
|
FrameProcessingException.from(e, presentationTimeUs));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -247,7 +247,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void signalEndOfInputStream() {
|
public void signalEndOfInputStream() {
|
||||||
frameProcessorChainListener.onFrameProcessingEnded();
|
frameProcessorListener.onFrameProcessingEnded();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
||||||
|
|
@ -30,19 +30,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
* instances.
|
* instances.
|
||||||
*
|
*
|
||||||
* <p>The wrapper handles calling {@link
|
* <p>The wrapper handles calling {@link
|
||||||
* FrameProcessorChain.Listener#onFrameProcessingError(FrameProcessingException)} for errors that
|
* GlEffectsFrameProcessor.Listener#onFrameProcessingError(FrameProcessingException)} for errors
|
||||||
* occur during these tasks.
|
* that occur during these tasks.
|
||||||
*/
|
*/
|
||||||
/* package */ final class FrameProcessingTaskExecutor {
|
/* package */ final class FrameProcessingTaskExecutor {
|
||||||
|
|
||||||
private final ExecutorService singleThreadExecutorService;
|
private final ExecutorService singleThreadExecutorService;
|
||||||
private final FrameProcessorChain.Listener listener;
|
private final GlEffectsFrameProcessor.Listener listener;
|
||||||
private final ConcurrentLinkedQueue<Future<?>> futures;
|
private final ConcurrentLinkedQueue<Future<?>> futures;
|
||||||
private final AtomicBoolean shouldCancelTasks;
|
private final AtomicBoolean shouldCancelTasks;
|
||||||
|
|
||||||
/** Creates a new instance. */
|
/** Creates a new instance. */
|
||||||
public FrameProcessingTaskExecutor(
|
public FrameProcessingTaskExecutor(
|
||||||
ExecutorService singleThreadExecutorService, FrameProcessorChain.Listener listener) {
|
ExecutorService singleThreadExecutorService, GlEffectsFrameProcessor.Listener listener) {
|
||||||
this.singleThreadExecutorService = singleThreadExecutorService;
|
this.singleThreadExecutorService = singleThreadExecutorService;
|
||||||
this.listener = listener;
|
this.listener = listener;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -37,7 +37,7 @@ import java.util.concurrent.Future;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@code FrameProcessorChain} applies changes to individual video frames.
|
* {@code GlEffectsFrameProcessor} applies changes to individual video frames.
|
||||||
*
|
*
|
||||||
* <p>Input becomes available on its {@linkplain #getInputSurface() input surface} asynchronously
|
* <p>Input becomes available on its {@linkplain #getInputSurface() input surface} asynchronously
|
||||||
* and is processed on a background thread as it becomes available. All input frames should be
|
* and is processed on a background thread as it becomes available. All input frames should be
|
||||||
|
|
@ -47,20 +47,21 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
* Listener, float, int, int, long, List, SurfaceInfo.Provider, Transformer.DebugViewProvider,
|
* Listener, float, int, int, long, List, SurfaceInfo.Provider, Transformer.DebugViewProvider,
|
||||||
* boolean) output surface}.
|
* boolean) output surface}.
|
||||||
*/
|
*/
|
||||||
// TODO(b/227625423): Factor out FrameProcessor interface and rename this class to GlFrameProcessor.
|
// TODO(b/227625423): Factor out FrameProcessor interface
|
||||||
/* package */ final class FrameProcessorChain {
|
/* package */ final class GlEffectsFrameProcessor {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Listener for asynchronous frame processing events.
|
* Listener for asynchronous frame processing events.
|
||||||
*
|
*
|
||||||
* <p>This listener is only called from the {@link FrameProcessorChain}'s background thread.
|
* <p>This listener is only called from the {@link GlEffectsFrameProcessor} instance's background
|
||||||
|
* thread.
|
||||||
*/
|
*/
|
||||||
public interface Listener {
|
public interface Listener {
|
||||||
/**
|
/**
|
||||||
* Called when an exception occurs during asynchronous frame processing.
|
* Called when an exception occurs during asynchronous frame processing.
|
||||||
*
|
*
|
||||||
* <p>If an error occurred, consuming and producing further frames will not work as expected and
|
* <p>If an error occurred, consuming and producing further frames will not work as expected and
|
||||||
* the {@link FrameProcessorChain} should be released.
|
* the {@link GlEffectsFrameProcessor} should be released.
|
||||||
*/
|
*/
|
||||||
void onFrameProcessingError(FrameProcessingException exception);
|
void onFrameProcessingError(FrameProcessingException exception);
|
||||||
|
|
||||||
|
|
@ -86,9 +87,9 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
|
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
|
||||||
* creating and configuring the OpenGL components.
|
* creating and configuring the OpenGL components.
|
||||||
*/
|
*/
|
||||||
public static FrameProcessorChain create(
|
public static GlEffectsFrameProcessor create(
|
||||||
Context context,
|
Context context,
|
||||||
FrameProcessorChain.Listener listener,
|
GlEffectsFrameProcessor.Listener listener,
|
||||||
float pixelWidthHeightRatio,
|
float pixelWidthHeightRatio,
|
||||||
int inputWidth,
|
int inputWidth,
|
||||||
int inputHeight,
|
int inputHeight,
|
||||||
|
|
@ -103,10 +104,10 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
|
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
|
||||||
|
|
||||||
Future<FrameProcessorChain> frameProcessorChainFuture =
|
Future<GlEffectsFrameProcessor> glFrameProcessorFuture =
|
||||||
singleThreadExecutorService.submit(
|
singleThreadExecutorService.submit(
|
||||||
() ->
|
() ->
|
||||||
createOpenGlObjectsAndFrameProcessorChain(
|
createOpenGlObjectsAndFrameProcessor(
|
||||||
context,
|
context,
|
||||||
listener,
|
listener,
|
||||||
pixelWidthHeightRatio,
|
pixelWidthHeightRatio,
|
||||||
|
|
@ -120,7 +121,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
singleThreadExecutorService));
|
singleThreadExecutorService));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return frameProcessorChainFuture.get();
|
return glFrameProcessorFuture.get();
|
||||||
} catch (ExecutionException e) {
|
} catch (ExecutionException e) {
|
||||||
throw new FrameProcessingException(e);
|
throw new FrameProcessingException(e);
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
|
|
@ -132,15 +133,15 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
/**
|
/**
|
||||||
* Creates the OpenGL context, surfaces, textures, and framebuffers, initializes {@link
|
* Creates the OpenGL context, surfaces, textures, and framebuffers, initializes {@link
|
||||||
* GlTextureProcessor} instances corresponding to the {@link GlEffect} instances, and returns a
|
* GlTextureProcessor} instances corresponding to the {@link GlEffect} instances, and returns a
|
||||||
* new {@code FrameProcessorChain}.
|
* new {@code GlEffectsFrameProcessor}.
|
||||||
*
|
*
|
||||||
* <p>This method must be executed using the {@code singleThreadExecutorService}, as later OpenGL
|
* <p>This method must be executed using the {@code singleThreadExecutorService}, as later OpenGL
|
||||||
* commands will be called on that thread.
|
* commands will be called on that thread.
|
||||||
*/
|
*/
|
||||||
@WorkerThread
|
@WorkerThread
|
||||||
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
|
private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor(
|
||||||
Context context,
|
Context context,
|
||||||
FrameProcessorChain.Listener listener,
|
GlEffectsFrameProcessor.Listener listener,
|
||||||
float pixelWidthHeightRatio,
|
float pixelWidthHeightRatio,
|
||||||
int inputWidth,
|
int inputWidth,
|
||||||
int inputHeight,
|
int inputHeight,
|
||||||
|
|
@ -196,7 +197,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
chainTextureProcessorsWithListeners(
|
chainTextureProcessorsWithListeners(
|
||||||
externalTextureProcessor, textureProcessors, frameProcessingTaskExecutor, listener);
|
externalTextureProcessor, textureProcessors, frameProcessingTaskExecutor, listener);
|
||||||
|
|
||||||
return new FrameProcessorChain(
|
return new GlEffectsFrameProcessor(
|
||||||
eglDisplay,
|
eglDisplay,
|
||||||
eglContext,
|
eglContext,
|
||||||
frameProcessingTaskExecutor,
|
frameProcessingTaskExecutor,
|
||||||
|
|
@ -244,7 +245,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder,
|
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder,
|
||||||
SurfaceInfo.Provider outputSurfaceProvider,
|
SurfaceInfo.Provider outputSurfaceProvider,
|
||||||
long streamOffsetUs,
|
long streamOffsetUs,
|
||||||
FrameProcessorChain.Listener listener,
|
GlEffectsFrameProcessor.Listener listener,
|
||||||
Transformer.DebugViewProvider debugViewProvider,
|
Transformer.DebugViewProvider debugViewProvider,
|
||||||
boolean enableExperimentalHdrEditing)
|
boolean enableExperimentalHdrEditing)
|
||||||
throws FrameProcessingException {
|
throws FrameProcessingException {
|
||||||
|
|
@ -289,7 +290,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
ExternalTextureProcessor externalTextureProcessor,
|
ExternalTextureProcessor externalTextureProcessor,
|
||||||
ImmutableList<GlTextureProcessor> textureProcessors,
|
ImmutableList<GlTextureProcessor> textureProcessors,
|
||||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
||||||
FrameProcessorChain.Listener listener) {
|
GlEffectsFrameProcessor.Listener listener) {
|
||||||
externalTextureProcessor.setListener(
|
externalTextureProcessor.setListener(
|
||||||
new ChainingGlTextureProcessorListener(
|
new ChainingGlTextureProcessorListener(
|
||||||
/* previousGlTextureProcessor= */ null,
|
/* previousGlTextureProcessor= */ null,
|
||||||
|
|
@ -312,8 +313,8 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final String TAG = "FrameProcessorChain";
|
private static final String TAG = "GlEffectsFrameProcessor";
|
||||||
private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
|
private static final String THREAD_NAME = "Transformer:GlEffectsFrameProcessor";
|
||||||
private static final long RELEASE_WAIT_TIME_MS = 100;
|
private static final long RELEASE_WAIT_TIME_MS = 100;
|
||||||
|
|
||||||
private final EGLDisplay eglDisplay;
|
private final EGLDisplay eglDisplay;
|
||||||
|
|
@ -341,7 +342,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
private boolean inputStreamEnded;
|
private boolean inputStreamEnded;
|
||||||
|
|
||||||
private FrameProcessorChain(
|
private GlEffectsFrameProcessor(
|
||||||
EGLDisplay eglDisplay,
|
EGLDisplay eglDisplay,
|
||||||
EGLContext eglContext,
|
EGLContext eglContext,
|
||||||
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
|
||||||
|
|
@ -374,9 +375,9 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Informs the {@code FrameProcessorChain} that a frame will be queued to its input surface.
|
* Informs the {@code GlEffectsFrameProcessor} that a frame will be queued to its input surface.
|
||||||
*
|
*
|
||||||
* <p>Must be called before rendering a frame to the frame processor chain's input surface.
|
* <p>Must be called before rendering a frame to the frame processor's input surface.
|
||||||
*
|
*
|
||||||
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
|
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
|
||||||
*/
|
*/
|
||||||
|
|
@ -394,7 +395,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Informs the {@code FrameProcessorChain} that no further input frames should be accepted.
|
* Informs the {@code GlEffectsFrameProcessor} that no further input frames should be accepted.
|
||||||
*
|
*
|
||||||
* @throws IllegalStateException If called more than once.
|
* @throws IllegalStateException If called more than once.
|
||||||
*/
|
*/
|
||||||
|
|
@ -407,12 +408,12 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||||
/**
|
/**
|
||||||
* Releases all resources.
|
* Releases all resources.
|
||||||
*
|
*
|
||||||
* <p>If the frame processor chain is released before it has {@linkplain
|
* <p>If the frame processor is released before it has {@linkplain
|
||||||
* Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames
|
* Listener#onFrameProcessingEnded() ended}, it will attempt to cancel processing any input frames
|
||||||
* that have already become available. Input frames that become available after release are
|
* that have already become available. Input frames that become available after release are
|
||||||
* ignored.
|
* ignored.
|
||||||
*
|
*
|
||||||
* <p>This method blocks until all OpenGL resources are released or releasing times out.
|
* <p>This method blocks until all resources are released or releasing times out.
|
||||||
*/
|
*/
|
||||||
public void release() {
|
public void release() {
|
||||||
try {
|
try {
|
||||||
|
|
@ -272,15 +272,15 @@ public final class TransformationException extends Exception {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an instance for a {@link FrameProcessorChain} related exception.
|
* Creates an instance for a {@link GlEffectsFrameProcessor} related exception.
|
||||||
*
|
*
|
||||||
* @param cause The cause of the failure.
|
* @param cause The cause of the failure.
|
||||||
* @param errorCode See {@link #errorCode}.
|
* @param errorCode See {@link #errorCode}.
|
||||||
* @return The created instance.
|
* @return The created instance.
|
||||||
*/
|
*/
|
||||||
/* package */ static TransformationException createForFrameProcessorChain(
|
/* package */ static TransformationException createForFrameProcessingException(
|
||||||
Throwable cause, int errorCode) {
|
FrameProcessingException cause, int errorCode) {
|
||||||
return new TransformationException("FrameProcessorChain error", cause, errorCode);
|
return new TransformationException("Frame processing error", cause, errorCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -44,7 +44,7 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
private final Codec decoder;
|
private final Codec decoder;
|
||||||
private final ArrayList<Long> decodeOnlyPresentationTimestamps;
|
private final ArrayList<Long> decodeOnlyPresentationTimestamps;
|
||||||
|
|
||||||
private final FrameProcessorChain frameProcessorChain;
|
private final GlEffectsFrameProcessor frameProcessor;
|
||||||
|
|
||||||
private final EncoderWrapper encoderWrapper;
|
private final EncoderWrapper encoderWrapper;
|
||||||
private final DecoderInputBuffer encoderOutputBuffer;
|
private final DecoderInputBuffer encoderOutputBuffer;
|
||||||
|
|
@ -99,14 +99,14 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
asyncErrorListener);
|
asyncErrorListener);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
frameProcessorChain =
|
frameProcessor =
|
||||||
FrameProcessorChain.create(
|
GlEffectsFrameProcessor.create(
|
||||||
context,
|
context,
|
||||||
new FrameProcessorChain.Listener() {
|
new GlEffectsFrameProcessor.Listener() {
|
||||||
@Override
|
@Override
|
||||||
public void onFrameProcessingError(FrameProcessingException exception) {
|
public void onFrameProcessingError(FrameProcessingException exception) {
|
||||||
asyncErrorListener.onTransformationException(
|
asyncErrorListener.onTransformationException(
|
||||||
TransformationException.createForFrameProcessorChain(
|
TransformationException.createForFrameProcessingException(
|
||||||
exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED));
|
exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -128,14 +128,14 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
debugViewProvider,
|
debugViewProvider,
|
||||||
transformationRequest.enableHdrEditing);
|
transformationRequest.enableHdrEditing);
|
||||||
} catch (FrameProcessingException e) {
|
} catch (FrameProcessingException e) {
|
||||||
throw TransformationException.createForFrameProcessorChain(
|
throw TransformationException.createForFrameProcessingException(
|
||||||
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
|
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
|
||||||
}
|
}
|
||||||
|
|
||||||
decoder =
|
decoder =
|
||||||
decoderFactory.createForVideoDecoding(
|
decoderFactory.createForVideoDecoding(
|
||||||
inputFormat,
|
inputFormat,
|
||||||
frameProcessorChain.getInputSurface(),
|
frameProcessor.getInputSurface(),
|
||||||
transformationRequest.enableRequestSdrToneMapping);
|
transformationRequest.enableRequestSdrToneMapping);
|
||||||
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
|
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
|
||||||
}
|
}
|
||||||
|
|
@ -165,7 +165,7 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
processedData = true;
|
processedData = true;
|
||||||
}
|
}
|
||||||
if (decoder.isEnded()) {
|
if (decoder.isEnded()) {
|
||||||
frameProcessorChain.signalEndOfInputStream();
|
frameProcessor.signalEndOfInputStream();
|
||||||
}
|
}
|
||||||
// If the decoder produced output, signal that it may be possible to process data again.
|
// If the decoder produced output, signal that it may be possible to process data again.
|
||||||
return processedData;
|
return processedData;
|
||||||
|
|
@ -202,7 +202,7 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void release() {
|
public void release() {
|
||||||
frameProcessorChain.release();
|
frameProcessor.release();
|
||||||
decoder.release();
|
decoder.release();
|
||||||
encoderWrapper.release();
|
encoderWrapper.release();
|
||||||
}
|
}
|
||||||
|
|
@ -255,11 +255,11 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT
|
if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT
|
||||||
&& frameProcessorChain.getPendingInputFrameCount() == maxPendingFrameCount) {
|
&& frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
frameProcessorChain.registerInputFrame();
|
frameProcessor.registerInputFrame();
|
||||||
decoder.releaseOutputBuffer(/* render= */ true);
|
decoder.releaseOutputBuffer(/* render= */ true);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -31,11 +31,11 @@ import org.junit.runner.RunWith;
|
||||||
public final class ChainingGlTextureProcessorListenerTest {
|
public final class ChainingGlTextureProcessorListenerTest {
|
||||||
private static final long EXECUTOR_WAIT_TIME_MS = 100;
|
private static final long EXECUTOR_WAIT_TIME_MS = 100;
|
||||||
|
|
||||||
private final FrameProcessorChain.Listener mockFrameProcessorChainListener =
|
private final GlEffectsFrameProcessor.Listener mockframeProcessorListener =
|
||||||
mock(FrameProcessorChain.Listener.class);
|
mock(GlEffectsFrameProcessor.Listener.class);
|
||||||
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor =
|
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor =
|
||||||
new FrameProcessingTaskExecutor(
|
new FrameProcessingTaskExecutor(
|
||||||
Util.newSingleThreadExecutor("Test"), mockFrameProcessorChainListener);
|
Util.newSingleThreadExecutor("Test"), mockframeProcessorListener);
|
||||||
private final GlTextureProcessor mockPreviousGlTextureProcessor = mock(GlTextureProcessor.class);
|
private final GlTextureProcessor mockPreviousGlTextureProcessor = mock(GlTextureProcessor.class);
|
||||||
private final FakeGlTextureProcessor fakeNextGlTextureProcessor =
|
private final FakeGlTextureProcessor fakeNextGlTextureProcessor =
|
||||||
spy(new FakeGlTextureProcessor());
|
spy(new FakeGlTextureProcessor());
|
||||||
|
|
@ -44,7 +44,7 @@ public final class ChainingGlTextureProcessorListenerTest {
|
||||||
mockPreviousGlTextureProcessor,
|
mockPreviousGlTextureProcessor,
|
||||||
fakeNextGlTextureProcessor,
|
fakeNextGlTextureProcessor,
|
||||||
frameProcessingTaskExecutor,
|
frameProcessingTaskExecutor,
|
||||||
mockFrameProcessorChainListener);
|
mockframeProcessorListener);
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void release() throws InterruptedException {
|
public void release() throws InterruptedException {
|
||||||
|
|
@ -57,7 +57,7 @@ public final class ChainingGlTextureProcessorListenerTest {
|
||||||
|
|
||||||
chainingGlTextureProcessorListener.onFrameProcessingError(exception);
|
chainingGlTextureProcessorListener.onFrameProcessingError(exception);
|
||||||
|
|
||||||
verify(mockFrameProcessorChainListener, times(1)).onFrameProcessingError(exception);
|
verify(mockframeProcessorListener, times(1)).onFrameProcessingError(exception);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue