sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-27 15:07:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Converging BlankFrameProducer

Browse source 
`BlankFrameProducer` was originally internal to
`DefaultVideoFrameProcessorVideoFrameRenderingTest`. We later copied it in
c221958889. This CL converges the usages.

PiperOrigin-RevId: 555953620
This commit is contained in:
claincly 2023-08-11 15:01:38 +00:00 committed by Tianyi Feng
parent b24f7678bb
commit 2980c30ace
1 changed files with 35 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

145
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java
View file

@ -25,25 +25,21 @@ import android.graphics.PixelFormat;
import android.media.Image; import android.media.Image;
import android.media.ImageReader; import android.media.ImageReader;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo; import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider; import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.FrameInfo; import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.SurfaceInfo; import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.NullableType; import androidx.media3.common.util.NullableType;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.primitives.Longs; import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
@ -88,7 +84,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
long originalPresentationTimeUs = 1234; long originalPresentationTimeUs = 1234;
AtomicLong actualPresentationTimeUs = new AtomicLong(); AtomicLong actualPresentationTimeUs = new AtomicLong();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* inputPresentationTimesUs= */ ImmutableList.of(originalPresentationTimeUs),
/* onFrameAvailableListener= */ actualPresentationTimeUs::set, /* onFrameAvailableListener= */ actualPresentationTimeUs::set,
/* renderFramesAutomatically= */ true); /* renderFramesAutomatically= */ true);
@ -100,7 +96,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
@Test @Test
public void automaticFrameRendering_withThreeFrames_reusesInputTimestamps() throws Exception { public void automaticFrameRendering_withThreeFrames_reusesInputTimestamps() throws Exception {
long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; ImmutableList<Long> originalPresentationTimesUs = ImmutableList.of(1234L, 3456L, 4567L);
ArrayList<Long> actualPresentationTimesUs = new ArrayList<>(); ArrayList<Long> actualPresentationTimesUs = new ArrayList<>();
processFramesToEndOfStream( processFramesToEndOfStream(
originalPresentationTimesUs, originalPresentationTimesUs,
@ -119,18 +115,14 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
/* renderFramesAutomatically= */ true); /* renderFramesAutomatically= */ true);
assertThat(actualPresentationTimesUs) assertThat(actualPresentationTimesUs)
.containsExactly( .containsExactlyElementsIn(originalPresentationTimesUs)
originalPresentationTimesUs[0],
originalPresentationTimesUs[1],
originalPresentationTimesUs[2])
.inOrder(); .inOrder();
ImmutableList<Long> actualRenderTimesNs = ImmutableList<Long> actualRenderTimesNs =
waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3); waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3);
assertThat(actualRenderTimesNs) assertThat(actualRenderTimesNs)
.containsExactly( .containsExactlyElementsIn(
MICROS_TO_NANOS * originalPresentationTimesUs[0], // Convert microseconds to nanoseconds.
MICROS_TO_NANOS * originalPresentationTimesUs[1], Iterables.transform(originalPresentationTimesUs, input -> MICROS_TO_NANOS * input))
MICROS_TO_NANOS * originalPresentationTimesUs[2])
.inOrder(); .inOrder();
} }
@ -140,7 +132,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
long renderTimesNs = System.nanoTime() + 345678; long renderTimesNs = System.nanoTime() + 345678;
AtomicLong actualPresentationTimeUs = new AtomicLong(); AtomicLong actualPresentationTimeUs = new AtomicLong();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* inputPresentationTimesUs= */ ImmutableList.of(originalPresentationTimeUs),
/* onFrameAvailableListener= */ presentationTimeUs -> { /* onFrameAvailableListener= */ presentationTimeUs -> {
actualPresentationTimeUs.set(presentationTimeUs); actualPresentationTimeUs.set(presentationTimeUs);
checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs); checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs);
@ -159,7 +151,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
long renderTimesNs = VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY; long renderTimesNs = VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY;
AtomicLong actualPresentationTimeUs = new AtomicLong(); AtomicLong actualPresentationTimeUs = new AtomicLong();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* inputPresentationTimesUs= */ ImmutableList.of(originalPresentationTimeUs),
/* onFrameAvailableListener= */ presentationTimeUs -> { /* onFrameAvailableListener= */ presentationTimeUs -> {
actualPresentationTimeUs.set(presentationTimeUs); actualPresentationTimeUs.set(presentationTimeUs);
checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs); checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs);
@ -179,7 +171,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
long renderTimeBeforeCurrentTimeNs = System.nanoTime() - 345678; long renderTimeBeforeCurrentTimeNs = System.nanoTime() - 345678;
AtomicLong actualPresentationTimeUs = new AtomicLong(); AtomicLong actualPresentationTimeUs = new AtomicLong();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* inputPresentationTimesUs= */ ImmutableList.of(originalPresentationTimeUs),
/* onFrameAvailableListener= */ presentationTimeUs -> { /* onFrameAvailableListener= */ presentationTimeUs -> {
actualPresentationTimeUs.set(presentationTimeUs); actualPresentationTimeUs.set(presentationTimeUs);
checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimeBeforeCurrentTimeNs); checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimeBeforeCurrentTimeNs);
@ -198,7 +190,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
long originalPresentationTimeUs = 1234; long originalPresentationTimeUs = 1234;
AtomicLong actualPresentationTimeUs = new AtomicLong(); AtomicLong actualPresentationTimeUs = new AtomicLong();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* inputPresentationTimesUs= */ ImmutableList.of(originalPresentationTimeUs),
/* onFrameAvailableListener= */ presentationTimeNs -> { /* onFrameAvailableListener= */ presentationTimeNs -> {
actualPresentationTimeUs.set(presentationTimeNs); actualPresentationTimeUs.set(presentationTimeNs);
checkNotNull(defaultVideoFrameProcessor) checkNotNull(defaultVideoFrameProcessor)
@ -212,9 +204,10 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
@Test @Test
public void controlledFrameRendering_withThreeIndividualFrames_usesGivenTimestamps() public void controlledFrameRendering_withThreeIndividualFrames_usesGivenTimestamps()
throws Exception { throws Exception {
long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; ImmutableList<Long> originalPresentationTimesUs = ImmutableList.of(1234L, 3456L, 4567L);
long offsetNs = System.nanoTime(); long offsetNs = System.nanoTime();
long[] renderTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; ImmutableList<Long> renderTimesNs =
ImmutableList.of(offsetNs + 123456, offsetNs + 234567, offsetNs + 345678);
ArrayList<Long> actualPresentationTimesUs = new ArrayList<>(); ArrayList<Long> actualPresentationTimesUs = new ArrayList<>();
AtomicInteger frameIndex = new AtomicInteger(); AtomicInteger frameIndex = new AtomicInteger();
processFramesToEndOfStream( processFramesToEndOfStream(
@ -222,7 +215,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
/* onFrameAvailableListener= */ presentationTimeUs -> { /* onFrameAvailableListener= */ presentationTimeUs -> {
actualPresentationTimesUs.add(presentationTimeUs); actualPresentationTimesUs.add(presentationTimeUs);
checkNotNull(defaultVideoFrameProcessor) checkNotNull(defaultVideoFrameProcessor)
.renderOutputFrame(renderTimesNs[frameIndex.getAndIncrement()]); .renderOutputFrame(renderTimesNs.get(frameIndex.getAndIncrement()));
try { try {
// TODO(b/264252759): Investigate output frames being dropped and remove sleep. // TODO(b/264252759): Investigate output frames being dropped and remove sleep.
// Frames can be dropped silently between EGL and the ImageReader. Sleep after each call // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call
@ -236,24 +229,22 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
/* renderFramesAutomatically= */ false); /* renderFramesAutomatically= */ false);
assertThat(actualPresentationTimesUs) assertThat(actualPresentationTimesUs)
.containsExactly( .containsExactlyElementsIn(originalPresentationTimesUs)
originalPresentationTimesUs[0],
originalPresentationTimesUs[1],
originalPresentationTimesUs[2])
.inOrder(); .inOrder();
int actualFrameCount = frameIndex.get(); int actualFrameCount = frameIndex.get();
assertThat(actualFrameCount).isEqualTo(originalPresentationTimesUs.length); assertThat(actualFrameCount).isEqualTo(originalPresentationTimesUs.size());
long[] actualRenderTimesNs = ImmutableList<Long> actualRenderTimesNs =
Longs.toArray(waitForFrameRenderingAndGetRenderTimesNs(actualFrameCount)); waitForFrameRenderingAndGetRenderTimesNs(actualFrameCount);
assertThat(actualRenderTimesNs).isEqualTo(renderTimesNs); assertThat(actualRenderTimesNs).containsExactlyElementsIn(renderTimesNs).inOrder();
} }
@Test @Test
public void controlledFrameRendering_withThreeFramesAtOnce_usesGivenTimestamps() public void controlledFrameRendering_withThreeFramesAtOnce_usesGivenTimestamps()
throws Exception { throws Exception {
long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; ImmutableList<Long> originalPresentationTimesUs = ImmutableList.of(1234L, 3456L, 4567L);
long offsetNs = System.nanoTime(); long offsetNs = System.nanoTime();
long[] renderTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; ImmutableList<Long> renderTimesNs =
ImmutableList.of(offsetNs + 123456, offsetNs + 234567, offsetNs + 345678);
ArrayList<Long> actualPresentationTimesUs = new ArrayList<>(); ArrayList<Long> actualPresentationTimesUs = new ArrayList<>();
processFramesToEndOfStream( processFramesToEndOfStream(
/* inputPresentationTimesUs= */ originalPresentationTimesUs, /* inputPresentationTimesUs= */ originalPresentationTimesUs,
@ -263,22 +254,19 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
// TODO(b/264252759): Investigate output frames being dropped and remove sleep. // TODO(b/264252759): Investigate output frames being dropped and remove sleep.
// Frames can be dropped silently between EGL and the ImageReader. Sleep after each call // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call
// to swap buffers, to avoid this behavior. // to swap buffers, to avoid this behavior.
defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[0]); defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(0));
Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS);
defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[1]); defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(1));
Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS);
defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[2]); defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(2));
Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS);
assertThat(actualPresentationTimesUs) assertThat(actualPresentationTimesUs)
.containsExactly( .containsExactlyElementsIn(originalPresentationTimesUs)
originalPresentationTimesUs[0],
originalPresentationTimesUs[1],
originalPresentationTimesUs[2])
.inOrder(); .inOrder();
long[] actualRenderTimesNs = ImmutableList<Long> actualRenderTimesNs =
Longs.toArray(waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3)); waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3);
assertThat(actualRenderTimesNs).isEqualTo(renderTimesNs); assertThat(actualRenderTimesNs).containsExactlyElementsIn(renderTimesNs).inOrder();
} }
private interface OnOutputFrameAvailableForRenderingListener { private interface OnOutputFrameAvailableForRenderingListener {
@ -287,13 +275,13 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
@EnsuresNonNull("defaultVideoFrameProcessor") @EnsuresNonNull("defaultVideoFrameProcessor")
private void processFramesToEndOfStream( private void processFramesToEndOfStream(
long[] inputPresentationTimesUs, List<Long> inputPresentationTimesUs,
OnOutputFrameAvailableForRenderingListener onFrameAvailableListener, OnOutputFrameAvailableForRenderingListener onFrameAvailableListener,
boolean renderFramesAutomatically) boolean renderFramesAutomatically)
throws Exception { throws Exception {
AtomicReference<@NullableType VideoFrameProcessingException> AtomicReference<@NullableType VideoFrameProcessingException>
videoFrameProcessingExceptionReference = new AtomicReference<>(); videoFrameProcessingExceptionReference = new AtomicReference<>();
BlankFrameProducer blankFrameProducer = new BlankFrameProducer(); BlankFrameProducer blankFrameProducer = new BlankFrameProducer(WIDTH, HEIGHT);
CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1); CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1);
defaultVideoFrameProcessor = defaultVideoFrameProcessor =
checkNotNull( checkNotNull(
@ -314,7 +302,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
width, width,
height, height,
PixelFormat.RGBA_8888, PixelFormat.RGBA_8888,
/* maxImages= */ inputPresentationTimesUs.length); /* maxImages= */ inputPresentationTimesUs.size());
checkNotNull(defaultVideoFrameProcessor) checkNotNull(defaultVideoFrameProcessor)
.setOutputSurfaceInfo( .setOutputSurfaceInfo(
new SurfaceInfo(outputImageReader.getSurface(), width, height)); new SurfaceInfo(outputImageReader.getSurface(), width, height));
@ -352,7 +340,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
INPUT_TYPE_SURFACE, INPUT_TYPE_SURFACE,
/* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer), /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer),
new FrameInfo.Builder(WIDTH, HEIGHT).build()); new FrameInfo.Builder(WIDTH, HEIGHT).build());
blankFrameProducer.produceBlankFramesAndQueueEndOfStream(inputPresentationTimesUs); blankFrameProducer.produceBlankFrames(inputPresentationTimesUs);
defaultVideoFrameProcessor.signalEndOfInput(); defaultVideoFrameProcessor.signalEndOfInput();
videoFrameProcessingEndedCountDownLatch.await(); videoFrameProcessingEndedCountDownLatch.await();
@Nullable @Nullable
@ -373,67 +361,4 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest {
assertThat(outputRenderTimesNs).isEmpty(); assertThat(outputRenderTimesNs).isEmpty();
return listBuilder.build(); return listBuilder.build();
} }
/** Produces blank frames with the given timestamps. */
private static final class BlankFrameProducer implements GlShaderProgram {
private @MonotonicNonNull GlTextureInfo blankTexture;
private @MonotonicNonNull OutputListener outputListener;
public void configureGlObjects() throws VideoFrameProcessingException {
try {
int texId =
GlUtil.createTexture(WIDTH, HEIGHT, /* useHighPrecisionColorComponents= */ false);
int fboId = GlUtil.createFboForTexture(texId);
blankTexture = new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, WIDTH, HEIGHT);
GlUtil.focusFramebufferUsingCurrentContext(fboId, WIDTH, HEIGHT);
GlUtil.clearFocusedBuffers();
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
}
public void produceBlankFramesAndQueueEndOfStream(long[] presentationTimesUs) {
checkNotNull(outputListener);
for (long presentationTimeUs : presentationTimesUs) {
outputListener.onOutputFrameAvailable(checkNotNull(blankTexture), presentationTimeUs);
}
}
@Override
public void setInputListener(InputListener inputListener) {}
@Override
public void setOutputListener(OutputListener outputListener) {
this.outputListener = outputListener;
}
@Override
public void setErrorListener(Executor executor, ErrorListener errorListener) {}
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
// No input is queued in these tests. The BlankFrameProducer is used to produce frames.
throw new UnsupportedOperationException();
}
@Override
public void releaseOutputFrame(GlTextureInfo outputTexture) {}
@Override
public void signalEndOfCurrentInputStream() {
checkNotNull(outputListener).onCurrentOutputStreamEnded();
}
@Override
public void flush() {
throw new UnsupportedOperationException();
}
@Override
public void release() {
// Do nothing as destroying the OpenGL context destroys the texture.
}
}
} }