mirror of
https://github.com/samsonjs/media.git
synced 2026-06-28 05:29:33 +00:00
Allow using different video effects per MediaItem
PiperOrigin-RevId: 539596345
This commit is contained in:
claincly
Tofunmi Adigun-Hameed
parent
1c18503ad0
commit
5f43180a68
10 changed files with 210 additions and 114 deletions
|
|
@ -204,9 +204,15 @@ public interface VideoFrameProcessor {
|
|||
*
|
||||
* <p>Call {@link #setInputFrameInfo} before this method if the {@link FrameInfo} of the new input
|
||||
* stream differs from that of the current input stream.
|
||||
*
|
||||
* @param inputType The {@link InputType} of the new input stream.
|
||||
* @param effects The list of {@link Effect effects} to apply to the new input stream. The list is
|
||||
* ignored for the first input stream registered after {@linkplain Factory#create creating the
|
||||
* VideoFrameProcessor}. The first input stream will use the effects passed in in {@link
|
||||
* Factory#create}.
|
||||
*/
|
||||
// TODO(b/274109008) Merge this and setInputFrameInfo.
|
||||
void registerInputStream(@InputType int inputType);
|
||||
// TODO(b/286032822) Merge this and setInputFrameInfo.
|
||||
void registerInputStream(@InputType int inputType, List<Effect> effects);
|
||||
|
||||
/**
|
||||
* Sets information about the input frames.
|
||||
|
|
|
|||
|
|
@ -347,7 +347,8 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
|
|||
blankFrameProducer.configureGlObjects();
|
||||
// A frame needs to be registered despite not queuing any external input to ensure
|
||||
// that the video frame processor knows about the stream offset.
|
||||
checkNotNull(defaultVideoFrameProcessor).registerInputStream(INPUT_TYPE_SURFACE);
|
||||
checkNotNull(defaultVideoFrameProcessor)
|
||||
.registerInputStream(INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of());
|
||||
defaultVideoFrameProcessor.setInputFrameInfo(
|
||||
new FrameInfo.Builder(WIDTH, HEIGHT).build());
|
||||
blankFrameProducer.produceBlankFramesAndQueueEndOfStream(inputPresentationTimesUs);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import static androidx.media3.common.util.Assertions.checkArgument;
|
|||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
import static com.google.common.collect.Iterables.getFirst;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.Bitmap;
|
||||
|
|
@ -52,6 +52,7 @@ import androidx.media3.common.util.Util;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
|
|
@ -285,6 +286,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
private static final String THREAD_NAME = "Effect:GlThread";
|
||||
private static final long RELEASE_WAIT_TIME_MS = 500;
|
||||
|
||||
private final Context context;
|
||||
private final EGLDisplay eglDisplay;
|
||||
private final EGLContext eglContext;
|
||||
private final InputSwitcher inputSwitcher;
|
||||
|
|
@ -293,13 +295,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
private final Executor listenerExecutor;
|
||||
private final boolean renderFramesAutomatically;
|
||||
private final FinalShaderProgramWrapper finalShaderProgramWrapper;
|
||||
|
||||
// Shader programs that apply Effects.
|
||||
private final ImmutableList<GlShaderProgram> effectsShaderPrograms;
|
||||
private final List<GlShaderProgram> intermediateGlShaderPrograms;
|
||||
// A queue of input streams that have not been fully processed identified by their input types.
|
||||
@GuardedBy("lock")
|
||||
private final Queue<@InputType Integer> unprocessedInputStreams;
|
||||
|
||||
private final List<Effect> activeEffects;
|
||||
private final Object lock;
|
||||
private final ColorInfo outputColorInfo;
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
|
||||
// CountDownLatch to wait for the current input stream to finish processing.
|
||||
private volatile @MonotonicNonNull CountDownLatch latch;
|
||||
|
|
@ -308,14 +314,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
private volatile boolean hasRefreshedNextInputFrameInfo;
|
||||
|
||||
private DefaultVideoFrameProcessor(
|
||||
Context context,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
InputSwitcher inputSwitcher,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
VideoFrameProcessor.Listener listener,
|
||||
Listener listener,
|
||||
Executor listenerExecutor,
|
||||
ImmutableList<GlShaderProgram> effectsShaderPrograms,
|
||||
boolean renderFramesAutomatically) {
|
||||
ImmutableList<GlShaderProgram> intermediateGlShaderPrograms,
|
||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||
boolean renderFramesAutomatically,
|
||||
ColorInfo outputColorInfo,
|
||||
GlObjectsProvider glObjectsProvider) {
|
||||
this.context = context;
|
||||
this.eglDisplay = eglDisplay;
|
||||
this.eglContext = eglContext;
|
||||
this.inputSwitcher = inputSwitcher;
|
||||
|
|
@ -324,12 +335,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
this.listenerExecutor = listenerExecutor;
|
||||
this.renderFramesAutomatically = renderFramesAutomatically;
|
||||
this.unprocessedInputStreams = new ConcurrentLinkedQueue<>();
|
||||
this.activeEffects = new ArrayList<>();
|
||||
this.lock = new Object();
|
||||
|
||||
checkState(!effectsShaderPrograms.isEmpty());
|
||||
checkState(getLast(effectsShaderPrograms) instanceof FinalShaderProgramWrapper);
|
||||
|
||||
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(effectsShaderPrograms);
|
||||
this.outputColorInfo = outputColorInfo;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
||||
finalShaderProgramWrapper.setOnInputStreamProcessedListener(
|
||||
() -> {
|
||||
synchronized (lock) {
|
||||
|
|
@ -340,7 +350,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
return inputStreamEnded && unprocessedInputStreams.isEmpty();
|
||||
}
|
||||
});
|
||||
this.effectsShaderPrograms = effectsShaderPrograms;
|
||||
this.intermediateGlShaderPrograms = new ArrayList<>(intermediateGlShaderPrograms);
|
||||
}
|
||||
|
||||
/** Returns the task executor that runs video frame processing tasks. */
|
||||
|
|
@ -400,11 +410,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void registerInputStream(@InputType int inputType) {
|
||||
public void registerInputStream(@InputType int inputType, List<Effect> effects) {
|
||||
synchronized (lock) {
|
||||
if (unprocessedInputStreams.isEmpty()) {
|
||||
inputSwitcher.switchToInput(inputType);
|
||||
unprocessedInputStreams.add(inputType);
|
||||
activeEffects.clear();
|
||||
activeEffects.addAll(effects);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -418,10 +430,47 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
Thread.currentThread().interrupt();
|
||||
listenerExecutor.execute(() -> listener.onError(VideoFrameProcessingException.from(e)));
|
||||
}
|
||||
inputSwitcher.switchToInput(inputType);
|
||||
|
||||
synchronized (lock) {
|
||||
unprocessedInputStreams.add(inputType);
|
||||
}
|
||||
|
||||
if (!activeEffects.equals(effects)) {
|
||||
// TODO(b/269424561) Investigate non blocking re-configuration.
|
||||
// Shader program recreation must be on GL thread. Currently the calling thread is blocked
|
||||
// until all shader programs are recreated, so that DefaultVideoFrameProcessor doesn't receive
|
||||
// a new frame from the new input stream prematurely.
|
||||
videoFrameProcessingTaskExecutor.submitAndBlock(
|
||||
() -> {
|
||||
try {
|
||||
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
|
||||
intermediateGlShaderPrograms.get(i).release();
|
||||
}
|
||||
intermediateGlShaderPrograms.clear();
|
||||
intermediateGlShaderPrograms.addAll(
|
||||
createGlShaderPrograms(
|
||||
context, effects, outputColorInfo, finalShaderProgramWrapper));
|
||||
} catch (VideoFrameProcessingException e) {
|
||||
listenerExecutor.execute(() -> listener.onError(e));
|
||||
return;
|
||||
}
|
||||
|
||||
inputSwitcher.setDownstreamShaderProgram(
|
||||
getFirst(
|
||||
intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper));
|
||||
setGlObjectProviderOnShaderPrograms(intermediateGlShaderPrograms, glObjectsProvider);
|
||||
chainShaderProgramsWithListeners(
|
||||
intermediateGlShaderPrograms,
|
||||
finalShaderProgramWrapper,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
listener,
|
||||
listenerExecutor);
|
||||
|
||||
activeEffects.clear();
|
||||
activeEffects.addAll(effects);
|
||||
});
|
||||
}
|
||||
inputSwitcher.switchToInput(inputType);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -506,7 +555,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
public void release() {
|
||||
try {
|
||||
videoFrameProcessingTaskExecutor.release(
|
||||
/* releaseTask= */ this::releaseShaderProgramsAndDestroyGlContext, RELEASE_WAIT_TIME_MS);
|
||||
/* releaseTask= */ this::releaseGlObjects, RELEASE_WAIT_TIME_MS);
|
||||
} catch (InterruptedException unexpected) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException(unexpected);
|
||||
|
|
@ -555,7 +604,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
boolean enableColorTransfers,
|
||||
boolean renderFramesAutomatically,
|
||||
ExecutorService singleThreadExecutorService,
|
||||
Executor executor,
|
||||
Executor videoFrameProcessorListenerExecutor,
|
||||
Listener listener,
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
@Nullable TextureOutputListener textureOutputListener,
|
||||
|
|
@ -602,10 +651,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
videoFrameProcessingTaskExecutor,
|
||||
enableColorTransfers);
|
||||
|
||||
ImmutableList<GlShaderProgram> effectsShaderPrograms =
|
||||
getGlShaderProgramsForGlEffects(
|
||||
FinalShaderProgramWrapper finalShaderProgramWrapper =
|
||||
new FinalShaderProgramWrapper(
|
||||
context,
|
||||
effects,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
debugViewProvider,
|
||||
|
|
@ -613,12 +661,18 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
enableColorTransfers,
|
||||
renderFramesAutomatically,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
executor,
|
||||
videoFrameProcessorListenerExecutor,
|
||||
listener,
|
||||
glObjectsProvider,
|
||||
textureOutputListener,
|
||||
textureOutputCapacity);
|
||||
|
||||
// TODO(b/269424561): Move effect creation to registerInputStream().
|
||||
// The GlShaderPrograms that should be inserted in between InputSwitcher and
|
||||
// FinalShaderProgramWrapper.
|
||||
ImmutableList<GlShaderProgram> intermediateGlShaderPrograms =
|
||||
createGlShaderPrograms(context, effects, outputColorInfo, finalShaderProgramWrapper);
|
||||
|
||||
inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_SURFACE);
|
||||
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
|
||||
// HDR bitmap input is not supported. Bitmaps are always sRGB/Full range/BT.709.
|
||||
|
|
@ -628,48 +682,53 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
// Image and textureId concatenation not supported.
|
||||
inputSwitcher.registerInput(inputColorInfo, INPUT_TYPE_TEXTURE_ID);
|
||||
}
|
||||
inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0));
|
||||
|
||||
setGlObjectProviderOnShaderPrograms(effectsShaderPrograms, glObjectsProvider);
|
||||
inputSwitcher.setDownstreamShaderProgram(
|
||||
getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper));
|
||||
|
||||
setGlObjectProviderOnShaderPrograms(intermediateGlShaderPrograms, glObjectsProvider);
|
||||
chainShaderProgramsWithListeners(
|
||||
effectsShaderPrograms, videoFrameProcessingTaskExecutor, listener, executor);
|
||||
intermediateGlShaderPrograms,
|
||||
finalShaderProgramWrapper,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
listener,
|
||||
videoFrameProcessorListenerExecutor);
|
||||
|
||||
return new DefaultVideoFrameProcessor(
|
||||
context,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
inputSwitcher,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
listener,
|
||||
executor,
|
||||
effectsShaderPrograms,
|
||||
renderFramesAutomatically);
|
||||
videoFrameProcessorListenerExecutor,
|
||||
intermediateGlShaderPrograms,
|
||||
finalShaderProgramWrapper,
|
||||
renderFramesAutomatically,
|
||||
outputColorInfo,
|
||||
glObjectsProvider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Combines consecutive {@link GlMatrixTransformation} and {@link RgbMatrix} instances into a
|
||||
* single {@link DefaultShaderProgram} and converts all other {@link GlEffect} instances to
|
||||
* separate {@link GlShaderProgram} instances.
|
||||
* Combines consecutive {@link GlMatrixTransformation GlMatrixTransformations} and {@link
|
||||
* RgbMatrix RgbMatrices} instances into a single {@link DefaultShaderProgram} and converts all
|
||||
* other {@link GlEffect} instances to separate {@link GlShaderProgram} instances.
|
||||
*
|
||||
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
|
||||
*
|
||||
* @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. The
|
||||
* last is a {@link FinalShaderProgramWrapper}.
|
||||
* @param context The {@link Context}.
|
||||
* @param effects The list of {@link GlEffect effects}.
|
||||
* @param outputColorInfo The {@link ColorInfo} on {@code DefaultVideoFrameProcessor} output.
|
||||
* @param finalShaderProgramWrapper The {@link FinalShaderProgramWrapper} to apply the {@link
|
||||
* GlMatrixTransformation GlMatrixTransformations} and {@link RgbMatrix RgbMatrices} after all
|
||||
* other {@link GlEffect GlEffects}.
|
||||
* @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order.
|
||||
*/
|
||||
private static ImmutableList<GlShaderProgram> getGlShaderProgramsForGlEffects(
|
||||
private static ImmutableList<GlShaderProgram> createGlShaderPrograms(
|
||||
Context context,
|
||||
List<Effect> effects,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
DebugViewProvider debugViewProvider,
|
||||
ColorInfo outputColorInfo,
|
||||
boolean enableColorTransfers,
|
||||
boolean renderFramesAutomatically,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
Executor executor,
|
||||
Listener listener,
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
@Nullable TextureOutputListener textureOutputListener,
|
||||
int textureOutputCapacity)
|
||||
FinalShaderProgramWrapper finalShaderProgramWrapper)
|
||||
throws VideoFrameProcessingException {
|
||||
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
|
||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||
|
|
@ -707,29 +766,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr));
|
||||
}
|
||||
|
||||
shaderProgramListBuilder.add(
|
||||
new FinalShaderProgramWrapper(
|
||||
context,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
matrixTransformationListBuilder.build(),
|
||||
rgbMatrixListBuilder.build(),
|
||||
debugViewProvider,
|
||||
outputColorInfo,
|
||||
enableColorTransfers,
|
||||
renderFramesAutomatically,
|
||||
videoFrameProcessingTaskExecutor,
|
||||
executor,
|
||||
listener,
|
||||
glObjectsProvider,
|
||||
textureOutputListener,
|
||||
textureOutputCapacity));
|
||||
finalShaderProgramWrapper.setMatrixTransformations(
|
||||
matrixTransformationListBuilder.build(), rgbMatrixListBuilder.build());
|
||||
return shaderProgramListBuilder.build();
|
||||
}
|
||||
|
||||
/** Sets the {@link GlObjectsProvider} on all of the {@linkplain GlShaderProgram}s provided. */
|
||||
private static void setGlObjectProviderOnShaderPrograms(
|
||||
ImmutableList<GlShaderProgram> shaderPrograms, GlObjectsProvider glObjectsProvider) {
|
||||
List<GlShaderProgram> shaderPrograms, GlObjectsProvider glObjectsProvider) {
|
||||
for (int i = 0; i < shaderPrograms.size() - 1; i++) {
|
||||
GlShaderProgram shaderProgram = shaderPrograms.get(i);
|
||||
shaderProgram.setGlObjectsProvider(glObjectsProvider);
|
||||
|
|
@ -741,13 +785,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
* ChainingGlShaderProgramListener} instances.
|
||||
*/
|
||||
private static void chainShaderProgramsWithListeners(
|
||||
ImmutableList<GlShaderProgram> shaderPrograms,
|
||||
List<GlShaderProgram> shaderPrograms,
|
||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||
Listener videoFrameProcessorListener,
|
||||
Executor videoFrameProcessorListenerExecutor) {
|
||||
for (int i = 0; i < shaderPrograms.size() - 1; i++) {
|
||||
GlShaderProgram producingGlShaderProgram = shaderPrograms.get(i);
|
||||
GlShaderProgram consumingGlShaderProgram = shaderPrograms.get(i + 1);
|
||||
ArrayList<GlShaderProgram> shaderProgramsToChain = new ArrayList<>(shaderPrograms);
|
||||
shaderProgramsToChain.add(finalShaderProgramWrapper);
|
||||
for (int i = 0; i < shaderProgramsToChain.size() - 1; i++) {
|
||||
GlShaderProgram producingGlShaderProgram = shaderProgramsToChain.get(i);
|
||||
GlShaderProgram consumingGlShaderProgram = shaderProgramsToChain.get(i + 1);
|
||||
ChainingGlShaderProgramListener chainingGlShaderProgramListener =
|
||||
new ChainingGlShaderProgramListener(
|
||||
producingGlShaderProgram, consumingGlShaderProgram, videoFrameProcessingTaskExecutor);
|
||||
|
|
@ -763,12 +810,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
*
|
||||
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
|
||||
*/
|
||||
private void releaseShaderProgramsAndDestroyGlContext() {
|
||||
private void releaseGlObjects() {
|
||||
try {
|
||||
try {
|
||||
inputSwitcher.release();
|
||||
for (int i = 0; i < effectsShaderPrograms.size(); i++) {
|
||||
effectsShaderPrograms.get(i).release();
|
||||
for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) {
|
||||
intermediateGlShaderPrograms.get(i).release();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Error releasing shader program", e);
|
||||
|
|
|
|||
|
|
@ -45,6 +45,8 @@ import androidx.media3.common.util.Size;
|
|||
import androidx.media3.common.util.Util;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.Executor;
|
||||
|
|
@ -77,8 +79,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
private static final String TAG = "FinalShaderWrapper";
|
||||
|
||||
private final Context context;
|
||||
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
|
||||
private final ImmutableList<RgbMatrix> rgbMatrices;
|
||||
private final List<GlMatrixTransformation> matrixTransformations;
|
||||
private final List<RgbMatrix> rgbMatrices;
|
||||
private final EGLDisplay eglDisplay;
|
||||
private final EGLContext eglContext;
|
||||
private final DebugViewProvider debugViewProvider;
|
||||
|
|
@ -105,6 +107,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
@Nullable private SurfaceView debugSurfaceView;
|
||||
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
|
||||
private boolean frameProcessingStarted;
|
||||
private boolean matrixTransformationsChanged;
|
||||
|
||||
@GuardedBy("this")
|
||||
private boolean outputSurfaceInfoChanged;
|
||||
|
|
@ -122,8 +125,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
Context context,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
ImmutableList<GlMatrixTransformation> matrixTransformations,
|
||||
ImmutableList<RgbMatrix> rgbMatrices,
|
||||
DebugViewProvider debugViewProvider,
|
||||
ColorInfo outputColorInfo,
|
||||
boolean enableColorTransfers,
|
||||
|
|
@ -135,8 +136,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
@Nullable DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener,
|
||||
int textureOutputCapacity) {
|
||||
this.context = context;
|
||||
this.matrixTransformations = matrixTransformations;
|
||||
this.rgbMatrices = rgbMatrices;
|
||||
this.matrixTransformations = new ArrayList<>();
|
||||
this.rgbMatrices = new ArrayList<>();
|
||||
this.eglDisplay = eglDisplay;
|
||||
this.eglContext = eglContext;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
|
|
@ -227,6 +228,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the list of {@link GlMatrixTransformation GlMatrixTransformations} and list of {@link
|
||||
* RgbMatrix RgbMatrices} to apply to the next {@linkplain #queueInputFrame queued} frame.
|
||||
*
|
||||
* <p>The new transformations will be applied to the next {@linkplain #queueInputFrame queued}
|
||||
* frame.
|
||||
*/
|
||||
public void setMatrixTransformations(
|
||||
List<GlMatrixTransformation> matrixTransformations, List<RgbMatrix> rgbMatrices) {
|
||||
this.matrixTransformations.clear();
|
||||
this.matrixTransformations.addAll(matrixTransformations);
|
||||
this.rgbMatrices.clear();
|
||||
this.rgbMatrices.addAll(rgbMatrices);
|
||||
matrixTransformationsChanged = true;
|
||||
}
|
||||
|
||||
public void releaseOutputFrame(long presentationTimeUs) {
|
||||
videoFrameProcessingTaskExecutor.submit(() -> releaseOutputFrameInternal(presentationTimeUs));
|
||||
}
|
||||
|
|
@ -456,10 +473,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
}
|
||||
this.debugSurfaceView = debugSurfaceView;
|
||||
|
||||
if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged)) {
|
||||
if (defaultShaderProgram != null
|
||||
&& (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) {
|
||||
defaultShaderProgram.release();
|
||||
defaultShaderProgram = null;
|
||||
outputSurfaceInfoChanged = false;
|
||||
matrixTransformationsChanged = false;
|
||||
}
|
||||
|
||||
if (defaultShaderProgram == null) {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
|||
|
||||
import android.content.Context;
|
||||
import android.util.SparseArray;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
|
|
@ -47,7 +46,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
private @MonotonicNonNull GlShaderProgram downstreamShaderProgram;
|
||||
private @MonotonicNonNull TextureManager activeTextureManager;
|
||||
private boolean inputEnded;
|
||||
private int activeInputType;
|
||||
|
||||
public InputSwitcher(
|
||||
Context context,
|
||||
|
|
@ -61,7 +59,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
|
||||
this.inputs = new SparseArray<>();
|
||||
this.enableColorTransfers = enableColorTransfers;
|
||||
activeInputType = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -136,25 +133,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
}
|
||||
}
|
||||
|
||||
/** Sets the {@link GlShaderProgram} that {@code InputSwitcher} outputs to. */
|
||||
public void setDownstreamShaderProgram(GlShaderProgram downstreamShaderProgram) {
|
||||
this.downstreamShaderProgram = downstreamShaderProgram;
|
||||
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
Input input = inputs.get(inputType);
|
||||
input.setChainingListener(
|
||||
new GatedChainingListenerWrapper(
|
||||
input.samplingGlShaderProgram,
|
||||
this.downstreamShaderProgram,
|
||||
videoFrameProcessingTaskExecutor));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches to a new source of input.
|
||||
*
|
||||
* <p>Blocks until the current input stream is processed.
|
||||
*
|
||||
* <p>Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput
|
||||
* registered}.
|
||||
*
|
||||
|
|
@ -164,14 +150,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
checkStateNotNull(downstreamShaderProgram);
|
||||
checkState(inputs.indexOfKey(newInputType) >= 0, "Input type not registered: " + newInputType);
|
||||
|
||||
if (newInputType == activeInputType) {
|
||||
activeTextureManager = inputs.get(activeInputType).textureManager;
|
||||
}
|
||||
|
||||
for (int i = 0; i < inputs.size(); i++) {
|
||||
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
|
||||
Input input = inputs.get(inputType);
|
||||
if (inputType == newInputType) {
|
||||
input.setChainingListener(
|
||||
new GatedChainingListenerWrapper(
|
||||
input.samplingGlShaderProgram,
|
||||
this.downstreamShaderProgram,
|
||||
videoFrameProcessingTaskExecutor));
|
||||
input.setActive(true);
|
||||
downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper));
|
||||
activeTextureManager = input.textureManager;
|
||||
|
|
@ -179,7 +166,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
input.setActive(false);
|
||||
}
|
||||
}
|
||||
activeInputType = newInputType;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -240,7 +226,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
}
|
||||
|
||||
public void setActive(boolean active) {
|
||||
checkStateNotNull(gatedChainingListenerWrapper);
|
||||
if (gatedChainingListenerWrapper == null) {
|
||||
return;
|
||||
}
|
||||
gatedChainingListenerWrapper.setActive(active);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import android.opengl.Matrix;
|
|||
import androidx.media3.common.util.Size;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/** Utility functions for working with matrices, vertices, and polygons. */
|
||||
/* package */ final class MatrixUtils {
|
||||
|
|
@ -223,9 +224,7 @@ import java.util.Arrays;
|
|||
* GlMatrixTransformations} to an input frame with the given size.
|
||||
*/
|
||||
public static Size configureAndGetOutputSize(
|
||||
int inputWidth,
|
||||
int inputHeight,
|
||||
ImmutableList<GlMatrixTransformation> matrixTransformations) {
|
||||
int inputWidth, int inputHeight, List<GlMatrixTransformation> matrixTransformations) {
|
||||
checkArgument(inputWidth > 0, "inputWidth must be positive");
|
||||
checkArgument(inputHeight > 0, "inputHeight must be positive");
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,9 @@ import androidx.media3.common.VideoFrameProcessingException;
|
|||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
|
|
@ -52,7 +54,7 @@ import java.util.concurrent.RejectedExecutionException;
|
|||
private final Object lock;
|
||||
|
||||
@GuardedBy("lock")
|
||||
private final ArrayDeque<VideoFrameProcessingTask> highPriorityTasks;
|
||||
private final Queue<VideoFrameProcessingTask> highPriorityTasks;
|
||||
|
||||
@GuardedBy("lock")
|
||||
private boolean shouldCancelTasks;
|
||||
|
|
@ -89,6 +91,28 @@ import java.util.concurrent.RejectedExecutionException;
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submits the given {@link VideoFrameProcessingTask} to execute, and returns after the task is
|
||||
* executed.
|
||||
*/
|
||||
public void submitAndBlock(VideoFrameProcessingTask task) {
|
||||
synchronized (lock) {
|
||||
if (shouldCancelTasks) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Future<?> future = wrapTaskAndSubmitToExecutorService(task, /* isFlushOrReleaseTask= */ false);
|
||||
try {
|
||||
future.get();
|
||||
} catch (ExecutionException e) {
|
||||
handleException(e);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
handleException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Submits the given {@link VideoFrameProcessingTask} to be executed after the currently running
|
||||
* task and all previously submitted high-priority tasks have completed.
|
||||
|
|
|
|||
|
|
@ -2120,7 +2120,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
|
|||
throw new IllegalStateException();
|
||||
}
|
||||
});
|
||||
videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE);
|
||||
|
||||
videoFrameProcessor.registerInputStream(
|
||||
VideoFrameProcessor.INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of());
|
||||
this.initialStreamOffsetUs = initialStreamOffsetUs;
|
||||
} catch (Exception e) {
|
||||
throw renderer.createRendererException(
|
||||
|
|
|
|||
|
|
@ -312,7 +312,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||
videoFrameProcessingEnded = true;
|
||||
}
|
||||
});
|
||||
videoFrameProcessor.registerInputStream(inputType);
|
||||
videoFrameProcessor.registerInputStream(inputType, /* effects= */ ImmutableList.of());
|
||||
}
|
||||
|
||||
public void processFirstFrameAndEnd() throws Exception {
|
||||
|
|
@ -327,7 +327,8 @@ public final class VideoFrameProcessorTestRunner {
|
|||
mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
|
||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||
.build());
|
||||
videoFrameProcessor.registerInputStream(INPUT_TYPE_SURFACE);
|
||||
videoFrameProcessor.registerInputStream(
|
||||
INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of());
|
||||
videoFrameProcessor.registerInputFrame();
|
||||
}
|
||||
|
||||
|
|
@ -347,7 +348,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||
.setOffsetToAddUs(offsetToAddUs)
|
||||
.build());
|
||||
videoFrameProcessor.registerInputStream(INPUT_TYPE_BITMAP);
|
||||
videoFrameProcessor.registerInputStream(INPUT_TYPE_BITMAP, /* effects= */ ImmutableList.of());
|
||||
videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
|
||||
}
|
||||
|
||||
|
|
@ -356,7 +357,8 @@ public final class VideoFrameProcessorTestRunner {
|
|||
new FrameInfo.Builder(inputTexture.getWidth(), inputTexture.getHeight())
|
||||
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
|
||||
.build());
|
||||
videoFrameProcessor.registerInputStream(INPUT_TYPE_TEXTURE_ID);
|
||||
videoFrameProcessor.registerInputStream(
|
||||
INPUT_TYPE_TEXTURE_ID, /* effects= */ ImmutableList.of());
|
||||
videoFrameProcessor.setOnInputFrameProcessedListener(
|
||||
texId -> {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
|
@ -74,6 +73,7 @@ import org.checkerframework.dataflow.qual.Pure;
|
|||
private final ColorInfo videoFrameProcessorInputColor;
|
||||
private final EncoderWrapper encoderWrapper;
|
||||
private final DecoderInputBuffer encoderOutputBuffer;
|
||||
@Nullable final Presentation presentation;
|
||||
|
||||
private volatile boolean encoderExpectsTimestampZero;
|
||||
/**
|
||||
|
|
@ -149,15 +149,12 @@ import org.checkerframework.dataflow.qual.Pure;
|
|||
videoFrameProcessorOutputColor = videoFrameProcessorInputColor;
|
||||
}
|
||||
|
||||
List<Effect> effectsWithPresentation = new ArrayList<>(effects);
|
||||
if (presentation != null) {
|
||||
effectsWithPresentation.add(presentation);
|
||||
}
|
||||
this.presentation = presentation;
|
||||
try {
|
||||
videoFrameProcessor =
|
||||
videoFrameProcessorFactory.create(
|
||||
context,
|
||||
effectsWithPresentation,
|
||||
createEffectListWithPresentation(effects, presentation),
|
||||
debugViewProvider,
|
||||
videoFrameProcessorInputColor,
|
||||
videoFrameProcessorOutputColor,
|
||||
|
|
@ -216,7 +213,8 @@ import org.checkerframework.dataflow.qual.Pure;
|
|||
if (trackFormat != null) {
|
||||
Size decodedSize = getDecodedSize(trackFormat);
|
||||
videoFrameProcessor.registerInputStream(
|
||||
getInputType(checkNotNull(trackFormat.sampleMimeType)));
|
||||
getInputType(checkNotNull(trackFormat.sampleMimeType)),
|
||||
createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation));
|
||||
videoFrameProcessor.setInputFrameInfo(
|
||||
new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight())
|
||||
.setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio)
|
||||
|
|
@ -335,6 +333,16 @@ import org.checkerframework.dataflow.qual.Pure;
|
|||
return new Size(decodedWidth, decodedHeight);
|
||||
}
|
||||
|
||||
private static ImmutableList<Effect> createEffectListWithPresentation(
|
||||
List<Effect> effects, @Nullable Presentation presentation) {
|
||||
if (presentation == null) {
|
||||
return ImmutableList.copyOf(effects);
|
||||
}
|
||||
ImmutableList.Builder<Effect> effectsWithPresentationBuilder = new ImmutableList.Builder<>();
|
||||
effectsWithPresentationBuilder.addAll(effects).add(presentation);
|
||||
return effectsWithPresentationBuilder.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}.
|
||||
*
|
||||
|
|
|
|||
Loading…
Reference in a new issue