mirror of
https://github.com/samsonjs/media.git
synced 2026-04-02 10:45:51 +00:00
Add an input switcher to switch between input types.
Also make FinalShaderProgramWrapper always receive internal texture. This means it does not sample from a input texture, and its input color is always linear, hence the input type does not matter. PiperOrigin-RevId: 527869045
This commit is contained in:
claincly
Marc Baechinger
parent
4b75397f4e
commit
349eca7ae2
6 changed files with 366 additions and 156 deletions
|
|
@ -38,7 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
* <p>Public methods in this class can be called from any thread.
|
||||
*/
|
||||
@UnstableApi
|
||||
/* package */ final class BitmapTextureManager implements InputHandler {
|
||||
/* package */ final class BitmapTextureManager implements TextureManager {
|
||||
private final GlShaderProgram shaderProgram;
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
|
||||
|
|
|
|||
|
|
@ -251,11 +251,15 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
|
||||
private final EGLDisplay eglDisplay;
|
||||
private final EGLContext eglContext;
|
||||
private final InputSwitcher inputSwitcher;
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
private final InputHandler inputHandler;
|
||||
// TODO(b/274109008) Use InputSwither to interact with texture manager.
|
||||
// Owned and released by inputSwitcher.
|
||||
private final TextureManager textureManager;
|
||||
private final boolean renderFramesAutomatically;
|
||||
private final FinalShaderProgramWrapper finalShaderProgramWrapper;
|
||||
private final ImmutableList<GlShaderProgram> allShaderPrograms;
|
||||
// Shader programs that apply Effects.
|
||||
private final ImmutableList<GlShaderProgram> effectsShaderPrograms;
|
||||
// A queue of input streams that have not been fully processed identified by their input types.
|
||||
private final Queue<@InputType Integer> unprocessedInputStreams;
|
||||
|
||||
|
|
@ -268,41 +272,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
private DefaultVideoFrameProcessor(
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
InputSwitcher inputSwitcher,
|
||||
@InputType int inputType,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
ImmutableList<GlShaderProgram> shaderPrograms,
|
||||
boolean renderFramesAutomatically)
|
||||
throws VideoFrameProcessingException {
|
||||
|
||||
ImmutableList<GlShaderProgram> effectsShaderPrograms,
|
||||
boolean renderFramesAutomatically) {
|
||||
this.eglDisplay = eglDisplay;
|
||||
this.eglContext = eglContext;
|
||||
this.inputSwitcher = inputSwitcher;
|
||||
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
|
||||
this.renderFramesAutomatically = renderFramesAutomatically;
|
||||
this.unprocessedInputStreams = new ConcurrentLinkedQueue<>();
|
||||
|
||||
checkState(!shaderPrograms.isEmpty());
|
||||
checkState(getLast(shaderPrograms) instanceof FinalShaderProgramWrapper);
|
||||
checkState(!effectsShaderPrograms.isEmpty());
|
||||
checkState(getLast(effectsShaderPrograms) instanceof FinalShaderProgramWrapper);
|
||||
|
||||
GlShaderProgram inputShaderProgram = shaderPrograms.get(0);
|
||||
|
||||
switch (inputType) {
|
||||
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
|
||||
checkState(inputShaderProgram instanceof ExternalShaderProgram);
|
||||
inputHandler =
|
||||
new ExternalTextureManager(
|
||||
(ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
break;
|
||||
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
|
||||
inputHandler =
|
||||
new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
break;
|
||||
case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
|
||||
default:
|
||||
throw new VideoFrameProcessingException("Input type not supported yet");
|
||||
}
|
||||
inputShaderProgram.setInputListener(inputHandler);
|
||||
|
||||
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
|
||||
textureManager = inputSwitcher.switchToInput(inputType);
|
||||
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(effectsShaderPrograms);
|
||||
finalShaderProgramWrapper.setOnInputStreamProcessedListener(
|
||||
() -> {
|
||||
@InputType int currentInputType = unprocessedInputStreams.remove();
|
||||
|
|
@ -319,7 +305,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
}
|
||||
return inputStreamEnded && unprocessedInputStreams.isEmpty();
|
||||
});
|
||||
allShaderPrograms = shaderPrograms;
|
||||
this.effectsShaderPrograms = effectsShaderPrograms;
|
||||
}
|
||||
|
||||
/** Returns the task executor that runs video frame processing tasks. */
|
||||
|
|
@ -344,7 +330,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
* @param height The default height for input buffers, in pixels.
|
||||
*/
|
||||
public void setInputDefaultBufferSize(int width, int height) {
|
||||
inputHandler.setDefaultBufferSize(width, height);
|
||||
textureManager.setDefaultBufferSize(width, height);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -352,7 +338,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
checkState(
|
||||
hasRefreshedNextInputFrameInfo,
|
||||
"setInputFrameInfo must be called before queueing another bitmap");
|
||||
inputHandler.queueInputBitmap(
|
||||
textureManager.queueInputBitmap(
|
||||
inputBitmap,
|
||||
durationUs,
|
||||
checkNotNull(nextInputFrameInfo).offsetToAddUs,
|
||||
|
|
@ -363,13 +349,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
|
||||
@Override
|
||||
public Surface getInputSurface() {
|
||||
return inputHandler.getInputSurface();
|
||||
return textureManager.getInputSurface();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerInputStream(@InputType int inputType) {
|
||||
if (!unprocessedInputStreams.isEmpty()) {
|
||||
inputHandler.signalEndOfCurrentInputStream();
|
||||
textureManager.signalEndOfCurrentInputStream();
|
||||
// Wait until the current video is processed before continuing to the next input.
|
||||
if (checkNotNull(unprocessedInputStreams.peek()) == INPUT_TYPE_SURFACE) {
|
||||
latch = new CountDownLatch(1);
|
||||
|
|
@ -396,13 +382,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
checkStateNotNull(
|
||||
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
|
||||
|
||||
inputHandler.registerInputFrame(nextInputFrameInfo);
|
||||
textureManager.registerInputFrame(nextInputFrameInfo);
|
||||
hasRefreshedNextInputFrameInfo = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getPendingInputFrameCount() {
|
||||
return inputHandler.getPendingFrameCount();
|
||||
return textureManager.getPendingFrameCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -423,8 +409,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
public void signalEndOfInput() {
|
||||
checkState(!inputStreamEnded);
|
||||
inputStreamEnded = true;
|
||||
inputHandler.signalEndOfCurrentInputStream();
|
||||
inputHandler.signalEndOfInput();
|
||||
textureManager.signalEndOfCurrentInputStream();
|
||||
inputSwitcher.signalEndOfInput();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -432,10 +418,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
try {
|
||||
videoFrameProcessingTaskExecutor.flush();
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
inputHandler.setOnFlushCompleteListener(latch::countDown);
|
||||
textureManager.setOnFlushCompleteListener(latch::countDown);
|
||||
videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
|
||||
latch.await();
|
||||
inputHandler.setOnFlushCompleteListener(null);
|
||||
textureManager.setOnFlushCompleteListener(null);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
|
|
@ -450,7 +436,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException(unexpected);
|
||||
}
|
||||
inputHandler.release();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -526,35 +511,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
throw new VideoFrameProcessingException("BT.2020 PQ OpenGL output isn't supported.");
|
||||
}
|
||||
}
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
|
||||
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
|
||||
ColorInfo linearColorInfo =
|
||||
outputColorInfo
|
||||
.buildUpon()
|
||||
.setColorTransfer(C.COLOR_TRANSFER_LINEAR)
|
||||
.setHdrStaticInfo(null)
|
||||
.build();
|
||||
InputSwitcher inputSwitcher =
|
||||
new InputSwitcher(
|
||||
context,
|
||||
inputColorInfo,
|
||||
/* outputColorInfo= */ linearColorInfo,
|
||||
glObjectsProvider,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
enableColorTransfers);
|
||||
|
||||
ImmutableList<GlShaderProgram> shaderPrograms =
|
||||
ImmutableList<GlShaderProgram> effectsShaderPrograms =
|
||||
getGlShaderProgramsForGlEffects(
|
||||
context,
|
||||
effects,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
debugViewProvider,
|
||||
inputColorInfo,
|
||||
/* inputColorInfo= */ linearColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType,
|
||||
renderFramesAutomatically,
|
||||
executor,
|
||||
listener,
|
||||
glObjectsProvider,
|
||||
textureOutputListener);
|
||||
setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
|
||||
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
|
||||
|
||||
// TODO(b/274109008): Register both image and video input.
|
||||
inputSwitcher.registerInput(inputType);
|
||||
inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0));
|
||||
|
||||
setGlObjectProviderOnShaderPrograms(effectsShaderPrograms, glObjectsProvider);
|
||||
chainShaderProgramsWithListeners(
|
||||
shaderPrograms, videoFrameProcessingTaskExecutor, listener, executor);
|
||||
effectsShaderPrograms, videoFrameProcessingTaskExecutor, listener, executor);
|
||||
|
||||
return new DefaultVideoFrameProcessor(
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
inputSwitcher,
|
||||
inputType,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
shaderPrograms,
|
||||
effectsShaderPrograms,
|
||||
renderFramesAutomatically);
|
||||
}
|
||||
|
||||
|
|
@ -566,8 +570,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
|
||||
*
|
||||
* @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. The
|
||||
* first is an {@link ExternalShaderProgram} and the last is a {@link
|
||||
* FinalShaderProgramWrapper}.
|
||||
* last is a {@link FinalShaderProgramWrapper}.
|
||||
*/
|
||||
private static ImmutableList<GlShaderProgram> getGlShaderProgramsForGlEffects(
|
||||
Context context,
|
||||
|
|
@ -578,7 +581,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
ColorInfo inputColorInfo,
|
||||
ColorInfo outputColorInfo,
|
||||
boolean enableColorTransfers,
|
||||
@InputType int inputType,
|
||||
boolean renderFramesAutomatically,
|
||||
Executor executor,
|
||||
Listener listener,
|
||||
|
|
@ -589,13 +591,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||
new ImmutableList.Builder<>();
|
||||
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
|
||||
boolean sampleFromInputTexture = true;
|
||||
ColorInfo linearColorInfo =
|
||||
outputColorInfo
|
||||
.buildUpon()
|
||||
.setColorTransfer(C.COLOR_TRANSFER_LINEAR)
|
||||
.setHdrStaticInfo(null)
|
||||
.build();
|
||||
for (int i = 0; i < effects.size(); i++) {
|
||||
Effect effect = effects.get(i);
|
||||
checkArgument(
|
||||
|
|
@ -617,38 +612,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
matrixTransformationListBuilder.build();
|
||||
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
|
||||
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
|
||||
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) {
|
||||
DefaultShaderProgram defaultShaderProgram;
|
||||
if (sampleFromInputTexture) {
|
||||
if (inputType == INPUT_TYPE_SURFACE) {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createWithExternalSampler(
|
||||
context,
|
||||
matrixTransformations,
|
||||
rgbMatrices,
|
||||
inputColorInfo,
|
||||
linearColorInfo,
|
||||
enableColorTransfers);
|
||||
} else {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createWithInternalSampler(
|
||||
context,
|
||||
matrixTransformations,
|
||||
rgbMatrices,
|
||||
inputColorInfo,
|
||||
linearColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType);
|
||||
}
|
||||
} else {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.create(
|
||||
context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
|
||||
}
|
||||
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) {
|
||||
DefaultShaderProgram defaultShaderProgram =
|
||||
DefaultShaderProgram.create(
|
||||
context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
|
||||
shaderProgramListBuilder.add(defaultShaderProgram);
|
||||
matrixTransformationListBuilder = new ImmutableList.Builder<>();
|
||||
rgbMatrixListBuilder = new ImmutableList.Builder<>();
|
||||
sampleFromInputTexture = false;
|
||||
}
|
||||
shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr));
|
||||
}
|
||||
|
|
@ -661,11 +631,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
matrixTransformationListBuilder.build(),
|
||||
rgbMatrixListBuilder.build(),
|
||||
debugViewProvider,
|
||||
/* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
sampleFromInputTexture,
|
||||
inputType,
|
||||
renderFramesAutomatically,
|
||||
executor,
|
||||
listener,
|
||||
|
|
@ -712,12 +680,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
*/
|
||||
private void releaseShaderProgramsAndDestroyGlContext() {
|
||||
try {
|
||||
for (int i = 0; i < allShaderPrograms.size(); i++) {
|
||||
try {
|
||||
allShaderPrograms.get(i).release();
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error releasing shader program", e);
|
||||
try {
|
||||
inputSwitcher.release();
|
||||
for (int i = 0; i < effectsShaderPrograms.size(); i++) {
|
||||
effectsShaderPrograms.get(i).release();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error releasing shader program", e);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
|
|
@ -35,7 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
|
||||
* {@link ExternalShaderProgram} for consumption.
|
||||
*/
|
||||
/* package */ final class ExternalTextureManager implements InputHandler {
|
||||
/* package */ final class ExternalTextureManager implements TextureManager {
|
||||
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
private final ExternalShaderProgram externalShaderProgram;
|
||||
|
|
@ -167,6 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
*/
|
||||
@Override
|
||||
public void registerInputFrame(FrameInfo frame) {
|
||||
checkState(!inputStreamEnded);
|
||||
pendingFrames.add(frame);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
|
||||
|
|
@ -63,7 +62,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
* <p>This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of
|
||||
* {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}.
|
||||
*/
|
||||
/* package */ final class FinalShaderProgramWrapper implements ExternalShaderProgram {
|
||||
/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram {
|
||||
|
||||
/** Listener interface for the current input stream ending. */
|
||||
interface OnInputStreamProcessedListener {
|
||||
|
|
@ -82,15 +81,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
private final EGLDisplay eglDisplay;
|
||||
private final EGLContext eglContext;
|
||||
private final DebugViewProvider debugViewProvider;
|
||||
private final boolean sampleFromInputTexture;
|
||||
private final @VideoFrameProcessor.InputType int inputType;
|
||||
private final ColorInfo inputColorInfo;
|
||||
private final ColorInfo outputColorInfo;
|
||||
private final boolean enableColorTransfers;
|
||||
private final boolean renderFramesAutomatically;
|
||||
private final Executor videoFrameProcessorListenerExecutor;
|
||||
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
|
||||
private final float[] textureTransformMatrix;
|
||||
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
||||
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
|
||||
|
||||
|
|
@ -127,8 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
ColorInfo inputColorInfo,
|
||||
ColorInfo outputColorInfo,
|
||||
boolean enableColorTransfers,
|
||||
boolean sampleFromInputTexture,
|
||||
@VideoFrameProcessor.InputType int inputType,
|
||||
boolean renderFramesAutomatically,
|
||||
Executor videoFrameProcessorListenerExecutor,
|
||||
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
||||
|
|
@ -140,9 +133,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
this.eglDisplay = eglDisplay;
|
||||
this.eglContext = eglContext;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
this.sampleFromInputTexture = sampleFromInputTexture;
|
||||
this.inputType = inputType;
|
||||
this.inputColorInfo = inputColorInfo;
|
||||
this.outputColorInfo = outputColorInfo;
|
||||
this.enableColorTransfers = enableColorTransfers;
|
||||
this.renderFramesAutomatically = renderFramesAutomatically;
|
||||
|
|
@ -151,7 +141,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.textureOutputListener = textureOutputListener;
|
||||
|
||||
textureTransformMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
inputListener = new InputListener() {};
|
||||
availableFrames = new ConcurrentLinkedQueue<>();
|
||||
}
|
||||
|
|
@ -240,20 +229,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
inputListener.onReadyToAcceptInputFrame();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
|
||||
System.arraycopy(
|
||||
/* src= */ textureTransformMatrix,
|
||||
/* srcPos= */ 0,
|
||||
/* dest= */ this.textureTransformMatrix,
|
||||
/* destPost= */ 0,
|
||||
/* length= */ textureTransformMatrix.length);
|
||||
|
||||
if (defaultShaderProgram != null) {
|
||||
defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void release() throws VideoFrameProcessingException {
|
||||
if (defaultShaderProgram != null) {
|
||||
|
|
@ -479,38 +454,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
DefaultShaderProgram defaultShaderProgram;
|
||||
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
|
||||
matrixTransformationListBuilder.build();
|
||||
if (sampleFromInputTexture) {
|
||||
if (inputType == INPUT_TYPE_SURFACE) {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createWithExternalSampler(
|
||||
context,
|
||||
expandedMatrixTransformations,
|
||||
rgbMatrices,
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers);
|
||||
} else {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createWithInternalSampler(
|
||||
context,
|
||||
expandedMatrixTransformations,
|
||||
rgbMatrices,
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType);
|
||||
}
|
||||
} else {
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createApplyingOetf(
|
||||
context,
|
||||
expandedMatrixTransformations,
|
||||
rgbMatrices,
|
||||
outputColorInfo,
|
||||
enableColorTransfers);
|
||||
}
|
||||
defaultShaderProgram =
|
||||
DefaultShaderProgram.createApplyingOetf(
|
||||
context,
|
||||
expandedMatrixTransformations,
|
||||
rgbMatrices,
|
||||
outputColorInfo,
|
||||
enableColorTransfers);
|
||||
|
||||
defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
|
||||
Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
|
||||
if (outputSurfaceInfo != null) {
|
||||
SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,288 @@
|
|||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
|
||||
import android.content.Context;
|
||||
import android.util.SparseArray;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* A switcher to switch between {@linkplain TextureManager texture managers} of different
|
||||
* {@linkplain VideoFrameProcessor.InputType input types}.
|
||||
*/
|
||||
/* package */ final class InputSwitcher {
|
||||
private final Context context;
|
||||
private final ColorInfo inputColorInfo;
|
||||
private final ColorInfo outputColorInfo;
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
private final SparseArray<Input> inputs;
|
||||
private final boolean enableColorTransfers;
|
||||
|
||||
private @MonotonicNonNull GlShaderProgram downstreamShaderProgram;
|
||||
private boolean inputEnded;
|
||||
private int activeInputType;
|
||||
|
||||
public InputSwitcher(
|
||||
Context context,
|
||||
ColorInfo inputColorInfo,
|
||||
ColorInfo outputColorInfo,
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
boolean enableColorTransfers) {
|
||||
this.context = context;
|
||||
this.inputColorInfo = inputColorInfo;
|
||||
this.outputColorInfo = outputColorInfo;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
|
||||
this.inputs = new SparseArray<>();
|
||||
this.enableColorTransfers = enableColorTransfers;
|
||||
activeInputType = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers for a new {@link VideoFrameProcessor.InputType input}.
|
||||
*
|
||||
* <p>Can be called multiple times on the same {@link VideoFrameProcessor.InputType inputType},
|
||||
* with the new inputs overwriting the old ones. For example, a new instance of {@link
|
||||
* ExternalTextureManager} is created following each call to this method with {@link
|
||||
* VideoFrameProcessor#INPUT_TYPE_SURFACE}. Effectively, the {@code inputSwitcher} keeps exactly
|
||||
* one {@link TextureManager} per {@linkplain VideoFrameProcessor.InputType input type}.
|
||||
*
|
||||
* <p>Creates an {@link TextureManager} and an appropriate {@linkplain DefaultShaderProgram
|
||||
* sampler} to sample from the input.
|
||||
*/
|
||||
public void registerInput(@VideoFrameProcessor.InputType int inputType)
|
||||
throws VideoFrameProcessingException {
|
||||
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
|
||||
DefaultShaderProgram samplingShaderProgram;
|
||||
TextureManager textureManager;
|
||||
// TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling.
|
||||
switch (inputType) {
|
||||
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
|
||||
samplingShaderProgram =
|
||||
DefaultShaderProgram.createWithExternalSampler(
|
||||
context,
|
||||
/* matrixTransformations= */ ImmutableList.of(),
|
||||
/* rgbMatrices= */ ImmutableList.of(),
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers);
|
||||
samplingShaderProgram.setGlObjectsProvider(glObjectsProvider);
|
||||
textureManager =
|
||||
new ExternalTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
inputs.put(inputType, new Input(textureManager, samplingShaderProgram));
|
||||
break;
|
||||
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
|
||||
samplingShaderProgram =
|
||||
DefaultShaderProgram.createWithInternalSampler(
|
||||
context,
|
||||
/* matrixTransformations= */ ImmutableList.of(),
|
||||
/* rgbMatrices= */ ImmutableList.of(),
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
inputType);
|
||||
samplingShaderProgram.setGlObjectsProvider(glObjectsProvider);
|
||||
textureManager =
|
||||
new BitmapTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
inputs.put(inputType, new Input(textureManager, samplingShaderProgram));
|
||||
break;
|
||||
case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
|
||||
default:
|
||||
throw new VideoFrameProcessingException("Unsupported input type " + inputType);
|
||||
}
|
||||
}
|
||||
|
||||
public void setDownstreamShaderProgram(GlShaderProgram downstreamShaderProgram) {
|
||||
this.downstreamShaderProgram = downstreamShaderProgram;
|
||||
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
Input input = inputs.get(inputType);
|
||||
input.setChainingListener(
|
||||
new GatedChainingListenerWrapper(
|
||||
input.samplingGlShaderProgram,
|
||||
this.downstreamShaderProgram,
|
||||
videoFrameProcessingTaskExecutor));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches to a new source of input.
|
||||
*
|
||||
* <p>Blocks until the current input stream is processed.
|
||||
*
|
||||
* <p>Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput
|
||||
* registered}.
|
||||
*
|
||||
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
|
||||
* @return The {@link TextureManager} associated with the {@code newInputType}.
|
||||
*/
|
||||
public TextureManager switchToInput(@VideoFrameProcessor.InputType int newInputType) {
|
||||
checkStateNotNull(downstreamShaderProgram);
|
||||
checkState(inputs.indexOfKey(newInputType) >= 0, "Input type not registered: " + newInputType);
|
||||
|
||||
if (newInputType == activeInputType) {
|
||||
return inputs.get(activeInputType).textureManager;
|
||||
}
|
||||
|
||||
@Nullable TextureManager activeTextureManager = null;
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
Input input = inputs.get(inputType);
|
||||
if (inputType == newInputType) {
|
||||
input.setActive(true);
|
||||
downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper));
|
||||
activeTextureManager = input.textureManager;
|
||||
} else {
|
||||
input.setActive(false);
|
||||
}
|
||||
}
|
||||
|
||||
activeInputType = newInputType;
|
||||
return checkNotNull(activeTextureManager);
|
||||
}
|
||||
|
||||
/** Signals end of input to all {@linkplain #registerInput registered inputs}. */
|
||||
public void signalEndOfInput() {
|
||||
checkState(!inputEnded);
|
||||
inputEnded = true;
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
inputs.get(inputType).signalEndOfInput();
|
||||
}
|
||||
}
|
||||
|
||||
/** Releases the resources. */
|
||||
public void release() throws VideoFrameProcessingException {
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
inputs.get(inputs.keyAt(i)).release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a {@link TextureManager} and an appropriate {@linkplain GlShaderProgram sampling shader
|
||||
* program}.
|
||||
*
|
||||
* <p>The output is always an internal GL texture.
|
||||
*/
|
||||
private static final class Input {
|
||||
public final TextureManager textureManager;
|
||||
public final GlShaderProgram samplingGlShaderProgram;
|
||||
|
||||
private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper;
|
||||
|
||||
public Input(TextureManager textureManager, GlShaderProgram samplingGlShaderProgram) {
|
||||
this.textureManager = textureManager;
|
||||
this.samplingGlShaderProgram = samplingGlShaderProgram;
|
||||
samplingGlShaderProgram.setInputListener(textureManager);
|
||||
}
|
||||
|
||||
public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) {
|
||||
this.gatedChainingListenerWrapper = gatedChainingListenerWrapper;
|
||||
samplingGlShaderProgram.setOutputListener(gatedChainingListenerWrapper);
|
||||
}
|
||||
|
||||
public void setActive(boolean active) {
|
||||
checkStateNotNull(gatedChainingListenerWrapper);
|
||||
gatedChainingListenerWrapper.setActive(active);
|
||||
}
|
||||
|
||||
public void signalEndOfInput() {
|
||||
textureManager.signalEndOfInput();
|
||||
}
|
||||
|
||||
public void release() throws VideoFrameProcessingException {
|
||||
textureManager.release();
|
||||
samplingGlShaderProgram.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a {@link ChainingGlShaderProgramListener}, with the ability to turn off the event
|
||||
* listening.
|
||||
*/
|
||||
private static final class GatedChainingListenerWrapper
|
||||
implements GlShaderProgram.OutputListener, GlShaderProgram.InputListener {
|
||||
|
||||
private final ChainingGlShaderProgramListener chainingGlShaderProgramListener;
|
||||
private boolean isActive = false;
|
||||
|
||||
public GatedChainingListenerWrapper(
|
||||
GlShaderProgram producingGlShaderProgram,
|
||||
GlShaderProgram consumingGlShaderProgram,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
|
||||
this.chainingGlShaderProgramListener =
|
||||
new ChainingGlShaderProgramListener(
|
||||
producingGlShaderProgram, consumingGlShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onReadyToAcceptInputFrame() {
|
||||
if (isActive) {
|
||||
chainingGlShaderProgramListener.onReadyToAcceptInputFrame();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onInputFrameProcessed(GlTextureInfo inputTexture) {
|
||||
if (isActive) {
|
||||
chainingGlShaderProgramListener.onInputFrameProcessed(inputTexture);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void onFlush() {
|
||||
if (isActive) {
|
||||
chainingGlShaderProgramListener.onFlush();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void onOutputFrameAvailable(
|
||||
GlTextureInfo outputTexture, long presentationTimeUs) {
|
||||
if (isActive) {
|
||||
chainingGlShaderProgramListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void onCurrentOutputStreamEnded() {
|
||||
if (isActive) {
|
||||
chainingGlShaderProgramListener.onCurrentOutputStreamEnded();
|
||||
}
|
||||
}
|
||||
|
||||
public void setActive(boolean isActive) {
|
||||
this.isActive = isActive;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -25,7 +25,7 @@ import androidx.media3.common.FrameInfo;
|
|||
import androidx.media3.common.VideoFrameProcessor;
|
||||
|
||||
/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */
|
||||
/* package */ interface InputHandler extends GlShaderProgram.InputListener {
|
||||
/* package */ interface TextureManager extends GlShaderProgram.InputListener {
|
||||
|
||||
/**
|
||||
* See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
|
||||
|
|
@ -61,7 +61,7 @@ import androidx.media3.common.VideoFrameProcessor;
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/** Informs the {@code InputHandler} that a frame will be queued. */
|
||||
/** Informs the {@code TextureManager} that a frame will be queued. */
|
||||
default void registerInputFrame(FrameInfo frameInfo) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
Loading…
Reference in a new issue