mirror of
https://github.com/samsonjs/media.git
synced 2026-04-27 15:07:40 +00:00
Effect: Multiple Texture output
Allow the VideoFrameProcessor to output multiple textures at a time, so that lifetime of textures is up to the consumer calling VFP.releaseOutputFrame. The FinalShaderProgramWrapper also has a new maxCapacity limit added, to ensure the a reasonable amount of textures is used and avoid using up memory. PiperOrigin-RevId: 532094256
This commit is contained in:
parent
96dd0ae583
commit
07ec1eaa48
5 changed files with 188 additions and 62 deletions
|
|
@ -145,7 +145,7 @@ public interface VideoFrameProcessor {
|
||||||
*/
|
*/
|
||||||
void onError(VideoFrameProcessingException exception);
|
void onError(VideoFrameProcessingException exception);
|
||||||
|
|
||||||
/** Called after the {@link VideoFrameProcessor} has produced its final output frame. */
|
/** Called after the {@link VideoFrameProcessor} has rendered its final output frame. */
|
||||||
void onEnded();
|
void onEnded();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -291,6 +291,18 @@ public interface VideoFrameProcessor {
|
||||||
*/
|
*/
|
||||||
void renderOutputFrame(long renderTimeNs);
|
void renderOutputFrame(long renderTimeNs);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Releases resources associated with all output frames with presentation time less than or equal
|
||||||
|
* to {@code presentationTimeUs}.
|
||||||
|
*
|
||||||
|
* <p>Not needed for outputting to an {@linkplain #setOutputSurfaceInfo output surface}, but may
|
||||||
|
* be required for other outputs.
|
||||||
|
*
|
||||||
|
* @param presentationTimeUs The presentation time where all frames before and at this time should
|
||||||
|
* be released, in microseconds.
|
||||||
|
*/
|
||||||
|
void releaseOutputFrame(long presentationTimeUs);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
|
* Informs the {@code VideoFrameProcessor} that no further input frames should be accepted.
|
||||||
*
|
*
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ import android.opengl.GLES20;
|
||||||
import android.opengl.GLES30;
|
import android.opengl.GLES30;
|
||||||
import android.view.Surface;
|
import android.view.Surface;
|
||||||
import androidx.annotation.GuardedBy;
|
import androidx.annotation.GuardedBy;
|
||||||
|
import androidx.annotation.IntRange;
|
||||||
import androidx.annotation.Nullable;
|
import androidx.annotation.Nullable;
|
||||||
import androidx.annotation.VisibleForTesting;
|
import androidx.annotation.VisibleForTesting;
|
||||||
import androidx.media3.common.C;
|
import androidx.media3.common.C;
|
||||||
|
|
@ -73,7 +74,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
public interface TextureOutputListener {
|
public interface TextureOutputListener {
|
||||||
/** Called when a texture has been rendered to. */
|
/** Called when a texture has been rendered to. */
|
||||||
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
|
void onTextureRendered(GlTextureInfo outputTexture, long presentationTimeUs)
|
||||||
throws GlUtil.GlException, VideoFrameProcessingException;
|
throws VideoFrameProcessingException;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
|
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
|
||||||
|
|
@ -84,6 +85,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
private boolean enableColorTransfers;
|
private boolean enableColorTransfers;
|
||||||
private GlObjectsProvider glObjectsProvider;
|
private GlObjectsProvider glObjectsProvider;
|
||||||
@Nullable private TextureOutputListener textureOutputListener;
|
@Nullable private TextureOutputListener textureOutputListener;
|
||||||
|
private int textureOutputCapacity;
|
||||||
|
|
||||||
/** Creates an instance. */
|
/** Creates an instance. */
|
||||||
public Builder() {
|
public Builder() {
|
||||||
|
|
@ -114,39 +116,55 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sets the {@link TextureOutputListener}.
|
* Sets texture output settings.
|
||||||
*
|
*
|
||||||
* <p>If set, the {@link VideoFrameProcessor} will output to an OpenGL texture, accessible via
|
* <p>If set, the {@link VideoFrameProcessor} will output to OpenGL textures, accessible via
|
||||||
* {@link TextureOutputListener#onTextureRendered}. Otherwise, no texture will be rendered to.
|
* {@link TextureOutputListener#onTextureRendered}. Textures will stop being output when
|
||||||
|
* {@code textureOutputCapacity} is reached, until they're released via {@link
|
||||||
|
* #releaseOutputFrame}. Output textures must be released using {@link #releaseOutputFrame}.
|
||||||
*
|
*
|
||||||
* <p>If an {@linkplain #setOutputSurfaceInfo output surface} is set, the texture output will
|
* <p>If not set, there will be no texture output.
|
||||||
* be be adjusted as needed, to match the output surface's output.
|
*
|
||||||
|
* <p>This must not be set if the {@linkplain #setOutputSurfaceInfo output surface info} is
|
||||||
|
* also set.
|
||||||
|
*
|
||||||
|
* @param textureOutputListener The {@link TextureOutputListener}.
|
||||||
|
* @param textureOutputCapacity The amount of output textures that may be allocated at a time
|
||||||
|
* before texture output blocks. Must be greater than or equal to 1.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
@CanIgnoreReturnValue
|
@CanIgnoreReturnValue
|
||||||
public Builder setOnTextureRenderedListener(TextureOutputListener textureOutputListener) {
|
public Builder setTextureOutput(
|
||||||
|
TextureOutputListener textureOutputListener,
|
||||||
|
@IntRange(from = 1) int textureOutputCapacity) {
|
||||||
|
// TODO: http://b/262694346 - Add tests for multiple texture output.
|
||||||
this.textureOutputListener = textureOutputListener;
|
this.textureOutputListener = textureOutputListener;
|
||||||
|
checkArgument(textureOutputCapacity >= 1);
|
||||||
|
this.textureOutputCapacity = textureOutputCapacity;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
|
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
|
||||||
public DefaultVideoFrameProcessor.Factory build() {
|
public DefaultVideoFrameProcessor.Factory build() {
|
||||||
return new DefaultVideoFrameProcessor.Factory(
|
return new DefaultVideoFrameProcessor.Factory(
|
||||||
enableColorTransfers, glObjectsProvider, textureOutputListener);
|
enableColorTransfers, glObjectsProvider, textureOutputListener, textureOutputCapacity);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private final boolean enableColorTransfers;
|
private final boolean enableColorTransfers;
|
||||||
private final GlObjectsProvider glObjectsProvider;
|
private final GlObjectsProvider glObjectsProvider;
|
||||||
@Nullable private final TextureOutputListener textureOutputListener;
|
@Nullable private final TextureOutputListener textureOutputListener;
|
||||||
|
private final int textureOutputCapacity;
|
||||||
|
|
||||||
private Factory(
|
private Factory(
|
||||||
boolean enableColorTransfers,
|
boolean enableColorTransfers,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable TextureOutputListener textureOutputListener) {
|
@Nullable TextureOutputListener textureOutputListener,
|
||||||
|
int textureOutputCapacity) {
|
||||||
this.enableColorTransfers = enableColorTransfers;
|
this.enableColorTransfers = enableColorTransfers;
|
||||||
this.glObjectsProvider = glObjectsProvider;
|
this.glObjectsProvider = glObjectsProvider;
|
||||||
this.textureOutputListener = textureOutputListener;
|
this.textureOutputListener = textureOutputListener;
|
||||||
|
this.textureOutputCapacity = textureOutputCapacity;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -231,7 +249,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
listenerExecutor,
|
listenerExecutor,
|
||||||
listener,
|
listener,
|
||||||
glObjectsProvider,
|
glObjectsProvider,
|
||||||
textureOutputListener));
|
textureOutputListener,
|
||||||
|
textureOutputCapacity));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return defaultVideoFrameProcessorFuture.get();
|
return defaultVideoFrameProcessorFuture.get();
|
||||||
|
|
@ -411,11 +430,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
return checkNotNull(textureManager).getPendingFrameCount();
|
return checkNotNull(textureManager).getPendingFrameCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* <p>This must not be set on an instance where {@linkplain Factory.Builder#setTextureOutput
|
||||||
|
* texture output} is set.
|
||||||
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
|
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
|
||||||
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
|
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* <p>This may also be used for rendering from an output texture, if a {@link
|
||||||
|
* TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set}
|
||||||
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void renderOutputFrame(long renderTimeNs) {
|
public void renderOutputFrame(long renderTimeNs) {
|
||||||
checkState(
|
checkState(
|
||||||
|
|
@ -425,6 +456,21 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
|
() -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* <p>If a {@link TextureOutputListener} {@linkplain Factory.Builder#setTextureOutput is set},
|
||||||
|
* this must be called to release the output information stored in the {@link GlTextureInfo}
|
||||||
|
* instances.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void releaseOutputFrame(long presentationTimeUs) {
|
||||||
|
// TODO(b/262694346): Add Compositor system tests exercising this code path after GL texture
|
||||||
|
// input is possible.
|
||||||
|
videoFrameProcessingTaskExecutor.submit(
|
||||||
|
() -> finalShaderProgramWrapper.releaseOutputFrame(presentationTimeUs));
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void signalEndOfInput() {
|
public void signalEndOfInput() {
|
||||||
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
|
DebugTraceUtil.recordVideoFrameProcessorReceiveDecoderEos();
|
||||||
|
|
@ -511,7 +557,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
Executor executor,
|
Executor executor,
|
||||||
Listener listener,
|
Listener listener,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable TextureOutputListener textureOutputListener)
|
@Nullable TextureOutputListener textureOutputListener,
|
||||||
|
int textureOutputCapacity)
|
||||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||||
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
|
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
|
||||||
|
|
||||||
|
|
@ -568,7 +615,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
executor,
|
executor,
|
||||||
listener,
|
listener,
|
||||||
glObjectsProvider,
|
glObjectsProvider,
|
||||||
textureOutputListener);
|
textureOutputListener,
|
||||||
|
textureOutputCapacity);
|
||||||
|
|
||||||
inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
|
inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
|
||||||
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
|
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
|
||||||
|
|
@ -618,7 +666,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
Executor executor,
|
Executor executor,
|
||||||
Listener listener,
|
Listener listener,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable TextureOutputListener textureOutputListener)
|
@Nullable TextureOutputListener textureOutputListener,
|
||||||
|
int textureOutputCapacity)
|
||||||
throws VideoFrameProcessingException {
|
throws VideoFrameProcessingException {
|
||||||
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
|
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
|
||||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||||
|
|
@ -670,7 +719,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
||||||
executor,
|
executor,
|
||||||
listener,
|
listener,
|
||||||
glObjectsProvider,
|
glObjectsProvider,
|
||||||
textureOutputListener));
|
textureOutputListener,
|
||||||
|
textureOutputCapacity));
|
||||||
return shaderProgramListBuilder.build();
|
return shaderProgramListBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -50,8 +50,8 @@ import java.util.concurrent.Executor;
|
||||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around a {@link DefaultShaderProgram} that renders to the provided output surface or
|
* Wrapper around a {@link DefaultShaderProgram} that renders to either the provided output surface
|
||||||
* texture.
|
* or texture.
|
||||||
*
|
*
|
||||||
* <p>Also renders to a debug surface, if provided.
|
* <p>Also renders to a debug surface, if provided.
|
||||||
*
|
*
|
||||||
|
|
@ -87,17 +87,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
private final Executor videoFrameProcessorListenerExecutor;
|
private final Executor videoFrameProcessorListenerExecutor;
|
||||||
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
|
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
|
||||||
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
||||||
|
private final Queue<Pair<GlTextureInfo, Long>> outputTextures;
|
||||||
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
|
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
|
||||||
|
private final int textureOutputCapacity;
|
||||||
|
|
||||||
private int inputWidth;
|
private int inputWidth;
|
||||||
private int inputHeight;
|
private int inputHeight;
|
||||||
|
private int outputWidth;
|
||||||
|
private int outputHeight;
|
||||||
@Nullable private DefaultShaderProgram defaultShaderProgram;
|
@Nullable private DefaultShaderProgram defaultShaderProgram;
|
||||||
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
|
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
|
||||||
private GlObjectsProvider glObjectsProvider;
|
private GlObjectsProvider glObjectsProvider;
|
||||||
private InputListener inputListener;
|
private InputListener inputListener;
|
||||||
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
|
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
|
||||||
@Nullable private SurfaceView debugSurfaceView;
|
@Nullable private SurfaceView debugSurfaceView;
|
||||||
@Nullable private GlTextureInfo outputTexture;
|
|
||||||
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
|
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
|
||||||
private boolean frameProcessingStarted;
|
private boolean frameProcessingStarted;
|
||||||
|
|
||||||
|
|
@ -125,7 +128,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
Executor videoFrameProcessorListenerExecutor,
|
Executor videoFrameProcessorListenerExecutor,
|
||||||
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
||||||
GlObjectsProvider glObjectsProvider,
|
GlObjectsProvider glObjectsProvider,
|
||||||
@Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener) {
|
@Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
|
||||||
|
int textureOutputCapacity) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.matrixTransformations = matrixTransformations;
|
this.matrixTransformations = matrixTransformations;
|
||||||
this.rgbMatrices = rgbMatrices;
|
this.rgbMatrices = rgbMatrices;
|
||||||
|
|
@ -139,9 +143,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
this.videoFrameProcessorListener = videoFrameProcessorListener;
|
this.videoFrameProcessorListener = videoFrameProcessorListener;
|
||||||
this.glObjectsProvider = glObjectsProvider;
|
this.glObjectsProvider = glObjectsProvider;
|
||||||
this.textureOutputListener = textureOutputListener;
|
this.textureOutputListener = textureOutputListener;
|
||||||
|
this.textureOutputCapacity = textureOutputCapacity;
|
||||||
|
|
||||||
inputListener = new InputListener() {};
|
inputListener = new InputListener() {};
|
||||||
availableFrames = new ConcurrentLinkedQueue<>();
|
availableFrames = new ConcurrentLinkedQueue<>();
|
||||||
|
outputTextures = new ConcurrentLinkedQueue<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -155,7 +161,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
@Override
|
@Override
|
||||||
public void setInputListener(InputListener inputListener) {
|
public void setInputListener(InputListener inputListener) {
|
||||||
this.inputListener = inputListener;
|
this.inputListener = inputListener;
|
||||||
inputListener.onReadyToAcceptInputFrame();
|
maybeOnReadyToAcceptInputFrame();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -193,20 +199,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
frameProcessingStarted = true;
|
frameProcessingStarted = true;
|
||||||
videoFrameProcessorListenerExecutor.execute(
|
videoFrameProcessorListenerExecutor.execute(
|
||||||
() -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs));
|
() -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs));
|
||||||
if (renderFramesAutomatically) {
|
if (textureOutputListener == null) {
|
||||||
renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
|
if (renderFramesAutomatically) {
|
||||||
|
renderFrame(
|
||||||
|
inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
|
||||||
|
} else {
|
||||||
|
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
|
checkState(outputTextures.size() < textureOutputCapacity);
|
||||||
|
renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
|
||||||
}
|
}
|
||||||
inputListener.onReadyToAcceptInputFrame();
|
maybeOnReadyToAcceptInputFrame();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||||
// The final shader program writes to a surface so there is no texture to release.
|
// FinalShaderProgramWrapper cannot release output textures using GlTextureInfo.
|
||||||
throw new UnsupportedOperationException();
|
throw new UnsupportedOperationException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void releaseOutputFrame(long presentationTimeUs) throws VideoFrameProcessingException {
|
||||||
|
while (!outputTextures.isEmpty()
|
||||||
|
&& checkNotNull(outputTextures.peek()).second <= presentationTimeUs) {
|
||||||
|
GlTextureInfo outputTexture = outputTextures.remove().first;
|
||||||
|
try {
|
||||||
|
GlUtil.deleteTexture(outputTexture.texId);
|
||||||
|
GlUtil.deleteFbo(outputTexture.fboId);
|
||||||
|
} catch (GlUtil.GlException exception) {
|
||||||
|
throw new VideoFrameProcessingException(exception);
|
||||||
|
}
|
||||||
|
maybeOnReadyToAcceptInputFrame();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void renderOutputFrame(long renderTimeNs) {
|
public void renderOutputFrame(long renderTimeNs) {
|
||||||
frameProcessingStarted = true;
|
frameProcessingStarted = true;
|
||||||
checkState(!renderFramesAutomatically);
|
checkState(!renderFramesAutomatically);
|
||||||
|
|
@ -226,7 +252,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
defaultShaderProgram.flush();
|
defaultShaderProgram.flush();
|
||||||
}
|
}
|
||||||
inputListener.onFlush();
|
inputListener.onFlush();
|
||||||
inputListener.onReadyToAcceptInputFrame();
|
maybeOnReadyToAcceptInputFrame();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -235,8 +261,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
defaultShaderProgram.release();
|
defaultShaderProgram.release();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
if (outputTexture != null) {
|
while (!outputTextures.isEmpty()) {
|
||||||
GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
|
GlTextureInfo outputTexture = outputTextures.remove().first;
|
||||||
GlUtil.deleteTexture(outputTexture.texId);
|
GlUtil.deleteTexture(outputTexture.texId);
|
||||||
GlUtil.deleteFbo(outputTexture.fboId);
|
GlUtil.deleteFbo(outputTexture.fboId);
|
||||||
}
|
}
|
||||||
|
|
@ -252,6 +278,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
* @see VideoFrameProcessor#setOutputSurfaceInfo(SurfaceInfo)
|
* @see VideoFrameProcessor#setOutputSurfaceInfo(SurfaceInfo)
|
||||||
*/
|
*/
|
||||||
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
|
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
|
||||||
|
checkState(textureOutputListener == null);
|
||||||
if (Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
|
if (Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -276,18 +303,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
this.outputSurfaceInfo = outputSurfaceInfo;
|
this.outputSurfaceInfo = outputSurfaceInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void maybeOnReadyToAcceptInputFrame() {
|
||||||
|
if (textureOutputListener == null || outputTextures.size() < textureOutputCapacity) {
|
||||||
|
inputListener.onReadyToAcceptInputFrame();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private synchronized void renderFrame(
|
private synchronized void renderFrame(
|
||||||
GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) {
|
GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) {
|
||||||
try {
|
try {
|
||||||
if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|
if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|
||||||
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
|
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
|
||||||
inputListener.onInputFrameProcessed(inputTexture);
|
inputListener.onInputFrameProcessed(inputTexture);
|
||||||
return; // Drop frames when requested, or there is no output surface.
|
return; // Drop frames when requested, or there is no output surface and output texture.
|
||||||
}
|
}
|
||||||
if (outputSurfaceInfo != null) {
|
if (outputSurfaceInfo != null) {
|
||||||
renderFrameToOutputSurface(inputTexture, presentationTimeUs, renderTimeNs);
|
renderFrameToOutputSurface(inputTexture, presentationTimeUs, renderTimeNs);
|
||||||
}
|
} else if (textureOutputListener != null) {
|
||||||
if (textureOutputListener != null) {
|
|
||||||
renderFrameToOutputTexture(inputTexture, presentationTimeUs);
|
renderFrameToOutputTexture(inputTexture, presentationTimeUs);
|
||||||
}
|
}
|
||||||
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
||||||
|
|
@ -331,12 +363,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
|
|
||||||
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
|
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
|
||||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||||
GlTextureInfo outputTexture = checkNotNull(this.outputTexture);
|
// TODO(b/262694346): Use a texture pool instead of creating a new texture on every frame.
|
||||||
|
int outputTexId =
|
||||||
|
GlUtil.createTexture(
|
||||||
|
outputWidth,
|
||||||
|
outputHeight,
|
||||||
|
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
|
||||||
|
GlTextureInfo outputTexture =
|
||||||
|
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
|
||||||
|
|
||||||
GlUtil.focusFramebufferUsingCurrentContext(
|
GlUtil.focusFramebufferUsingCurrentContext(
|
||||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||||
GlUtil.clearOutputFrame();
|
GlUtil.clearOutputFrame();
|
||||||
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs);
|
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs);
|
||||||
|
// TODO(b/262694346): If Compositor's VFPs all use the same context, media3 should be able to
|
||||||
|
// avoid calling glFinish, and require the onTextureRendered listener to decide whether to
|
||||||
|
// glFinish. Consider removing glFinish and requiring onTextureRendered to handle
|
||||||
|
// synchronization.
|
||||||
GLES20.glFinish();
|
GLES20.glFinish();
|
||||||
|
outputTextures.add(Pair.create(outputTexture, presentationTimeUs));
|
||||||
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
|
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -381,11 +426,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
int outputWidth =
|
outputWidth =
|
||||||
outputSurfaceInfo == null
|
outputSurfaceInfo == null
|
||||||
? outputSizeBeforeSurfaceTransformation.getWidth()
|
? outputSizeBeforeSurfaceTransformation.getWidth()
|
||||||
: outputSurfaceInfo.width;
|
: outputSurfaceInfo.width;
|
||||||
int outputHeight =
|
outputHeight =
|
||||||
outputSurfaceInfo == null
|
outputSurfaceInfo == null
|
||||||
? outputSizeBeforeSurfaceTransformation.getHeight()
|
? outputSizeBeforeSurfaceTransformation.getHeight()
|
||||||
: outputSurfaceInfo.height;
|
: outputSurfaceInfo.height;
|
||||||
|
|
@ -411,16 +456,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
}
|
}
|
||||||
this.debugSurfaceView = debugSurfaceView;
|
this.debugSurfaceView = debugSurfaceView;
|
||||||
|
|
||||||
if (textureOutputListener != null) {
|
|
||||||
int outputTexId =
|
|
||||||
GlUtil.createTexture(
|
|
||||||
outputWidth,
|
|
||||||
outputHeight,
|
|
||||||
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
|
|
||||||
outputTexture =
|
|
||||||
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
|
if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
|
||||||
defaultShaderProgram.release();
|
defaultShaderProgram.release();
|
||||||
defaultShaderProgram = null;
|
defaultShaderProgram = null;
|
||||||
|
|
|
||||||
|
|
@ -285,13 +285,14 @@ public final class VideoFrameProcessorTestRunner {
|
||||||
new VideoFrameProcessor.Listener() {
|
new VideoFrameProcessor.Listener() {
|
||||||
@Override
|
@Override
|
||||||
public void onOutputSizeChanged(int width, int height) {
|
public void onOutputSizeChanged(int width, int height) {
|
||||||
|
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
|
||||||
@Nullable
|
@Nullable
|
||||||
Surface outputSurface =
|
Surface outputSurface =
|
||||||
bitmapReader.getSurface(
|
bitmapReader.getSurface(
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(
|
useHighPrecisionColorComponents,
|
||||||
outputColorInfo));
|
checkNotNull(videoFrameProcessor)::releaseOutputFrame);
|
||||||
if (outputSurface != null) {
|
if (outputSurface != null) {
|
||||||
checkNotNull(videoFrameProcessor)
|
checkNotNull(videoFrameProcessor)
|
||||||
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
|
.setOutputSurfaceInfo(new SurfaceInfo(outputSurface, width, height));
|
||||||
|
|
@ -406,10 +407,18 @@ public final class VideoFrameProcessorTestRunner {
|
||||||
|
|
||||||
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
|
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
|
||||||
public interface BitmapReader {
|
public interface BitmapReader {
|
||||||
|
/** Wraps a callback for {@link VideoFrameProcessor#releaseOutputFrame}. */
|
||||||
|
interface ReleaseOutputFrameListener {
|
||||||
|
void releaseOutputFrame(long releaseTimeUs);
|
||||||
|
}
|
||||||
|
|
||||||
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
|
/** Returns the {@link VideoFrameProcessor} output {@link Surface}, if one is needed. */
|
||||||
@Nullable
|
@Nullable
|
||||||
Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents);
|
Surface getSurface(
|
||||||
|
int width,
|
||||||
|
int height,
|
||||||
|
boolean useHighPrecisionColorComponents,
|
||||||
|
ReleaseOutputFrameListener listener);
|
||||||
|
|
||||||
/** Returns the output {@link Bitmap}. */
|
/** Returns the output {@link Bitmap}. */
|
||||||
Bitmap getBitmap();
|
Bitmap getBitmap();
|
||||||
|
|
@ -429,7 +438,11 @@ public final class VideoFrameProcessorTestRunner {
|
||||||
@Override
|
@Override
|
||||||
@SuppressLint("WrongConstant")
|
@SuppressLint("WrongConstant")
|
||||||
@Nullable
|
@Nullable
|
||||||
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
|
public Surface getSurface(
|
||||||
|
int width,
|
||||||
|
int height,
|
||||||
|
boolean useHighPrecisionColorComponents,
|
||||||
|
ReleaseOutputFrameListener listener) {
|
||||||
imageReader =
|
imageReader =
|
||||||
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
|
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
|
||||||
return imageReader.getSurface();
|
return imageReader.getSurface();
|
||||||
|
|
|
||||||
|
|
@ -138,10 +138,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
|
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
|
||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setOnTextureRenderedListener(
|
.setTextureOutput(
|
||||||
(outputTexture, presentationTimeUs) ->
|
(outputTexture, presentationTimeUs) ->
|
||||||
inputTextureIntoVideoFrameProcessor(
|
inputTextureIntoVideoFrameProcessor(
|
||||||
testId, consumersBitmapReader, outputTexture, presentationTimeUs))
|
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
|
||||||
|
/* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
|
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
|
||||||
new VideoFrameProcessorTestRunner.Builder()
|
new VideoFrameProcessorTestRunner.Builder()
|
||||||
|
|
@ -206,10 +207,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
|
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
|
||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setOnTextureRenderedListener(
|
.setTextureOutput(
|
||||||
(outputTexture, presentationTimeUs) ->
|
(outputTexture, presentationTimeUs) ->
|
||||||
inputTextureIntoVideoFrameProcessor(
|
inputTextureIntoVideoFrameProcessor(
|
||||||
testId, consumersBitmapReader, outputTexture, presentationTimeUs))
|
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
|
||||||
|
/* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
|
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
|
||||||
new VideoFrameProcessorTestRunner.Builder()
|
new VideoFrameProcessorTestRunner.Builder()
|
||||||
|
|
@ -380,7 +382,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
|
new DefaultGlObjectsProvider(GlUtil.getCurrentContext());
|
||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setOnTextureRenderedListener(bitmapReader::readBitmapFromTexture)
|
.setTextureOutput(bitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
|
||||||
.setGlObjectsProvider(contextSharingGlObjectsProvider)
|
.setGlObjectsProvider(contextSharingGlObjectsProvider)
|
||||||
.build();
|
.build();
|
||||||
videoFrameProcessorTestRunner =
|
videoFrameProcessorTestRunner =
|
||||||
|
|
@ -405,7 +407,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
|
TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
|
||||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||||
new DefaultVideoFrameProcessor.Factory.Builder()
|
new DefaultVideoFrameProcessor.Factory.Builder()
|
||||||
.setOnTextureRenderedListener(textureBitmapReader::readBitmapFromTexture)
|
.setTextureOutput(
|
||||||
|
textureBitmapReader::readBitmapFromTexture, /* textureOutputCapacity= */ 1)
|
||||||
.build();
|
.build();
|
||||||
return new VideoFrameProcessorTestRunner.Builder()
|
return new VideoFrameProcessorTestRunner.Builder()
|
||||||
.setTestId(testId)
|
.setTestId(testId)
|
||||||
|
|
@ -422,13 +425,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
private static final class TextureBitmapReader implements BitmapReader {
|
private static final class TextureBitmapReader implements BitmapReader {
|
||||||
// TODO(b/239172735): This outputs an incorrect black output image on emulators.
|
// TODO(b/239172735): This outputs an incorrect black output image on emulators.
|
||||||
private boolean useHighPrecisionColorComponents;
|
private boolean useHighPrecisionColorComponents;
|
||||||
|
private @MonotonicNonNull ReleaseOutputFrameListener releaseOutputFrameListener;
|
||||||
|
|
||||||
private @MonotonicNonNull Bitmap outputBitmap;
|
private @MonotonicNonNull Bitmap outputBitmap;
|
||||||
|
|
||||||
@Override
|
|
||||||
@Nullable
|
@Nullable
|
||||||
public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) {
|
@Override
|
||||||
|
public Surface getSurface(
|
||||||
|
int width,
|
||||||
|
int height,
|
||||||
|
boolean useHighPrecisionColorComponents,
|
||||||
|
ReleaseOutputFrameListener releaseOutputFrameListener) {
|
||||||
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
|
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
|
||||||
|
this.releaseOutputFrameListener = releaseOutputFrameListener;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -438,12 +447,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
|
public void readBitmapFromTexture(GlTextureInfo outputTexture, long presentationTimeUs)
|
||||||
throws GlUtil.GlException {
|
throws VideoFrameProcessingException {
|
||||||
GlUtil.focusFramebufferUsingCurrentContext(
|
try {
|
||||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
GlUtil.focusFramebufferUsingCurrentContext(
|
||||||
outputBitmap =
|
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||||
createBitmapFromCurrentGlFrameBuffer(
|
outputBitmap =
|
||||||
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
|
createBitmapFromCurrentGlFrameBuffer(
|
||||||
|
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
|
||||||
|
GlUtil.deleteTexture(outputTexture.texId);
|
||||||
|
GlUtil.deleteFbo(outputTexture.fboId);
|
||||||
|
} catch (GlUtil.GlException e) {
|
||||||
|
throw new VideoFrameProcessingException(e);
|
||||||
|
}
|
||||||
|
checkNotNull(releaseOutputFrameListener).releaseOutputFrame(presentationTimeUs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Bitmap createBitmapFromCurrentGlFrameBuffer(
|
private static Bitmap createBitmapFromCurrentGlFrameBuffer(
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue