sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-12 12:25:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow setting individual offset for bitmaps.

Browse source 
PiperOrigin-RevId: 527001582
This commit is contained in:
claincly 2023-04-25 18:10:59 +01:00 committed by Ian Baker
parent 8612d2820d
commit 19b979d817
7 changed files with 116 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
View file

@ -66,11 +66,20 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2);
readBitmap(ORIGINAL_PNG_ASSET_PATH),
C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(SCALE_WIDE_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3);
readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
2 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 3);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4);
readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH),
3 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 4);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
int actualFrameCount = framesProduced.get();
@ -87,6 +96,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 1);
}
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
@ -95,6 +105,63 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
assertThat(actualFrameCount).isEqualTo(/* expected= */ 20);
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs()
throws Exception {
String testId =
"imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs";
Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
.build();
long offsetUs = 1_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
assertThat(actualPresentationTimesUs)
.containsExactly(offsetUs, offsetUs + C.MICROS_PER_SECOND / 2)
.inOrder();
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs()
throws Exception {
String testId = "imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs";
Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
.build();
long offsetUs1 = 1_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs1,
/* frameRate= */ 2);
long offsetUs2 = 2_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs2,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
assertThat(actualPresentationTimesUs)
.containsExactly(
offsetUs1,
offsetUs1 + C.MICROS_PER_SECOND / 2,
offsetUs2,
offsetUs2 + C.MICROS_PER_SECOND / 2)
.inOrder();
}
@RequiresNonNull("framesProduced")
@Test
public void
@ -111,11 +178,13 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ 2 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 3);
assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder();

4
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
View file

@ -120,7 +120,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.queueInputBitmap(
originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
@ -148,7 +148,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
Bitmap expectedBitmap = readBitmap(IMAGE_TO_CROPPED_VIDEO_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.queueInputBitmap(
originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.

15
libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java
View file

@ -79,9 +79,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
videoFrameProcessingTaskExecutor.submit(
() -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
() -> setupBitmap(inputBitmap, durationUs, offsetUs, frameRate, useHdr));
}
@Override
@ -116,7 +116,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Methods that must be called on the GL thread.
private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr)
private void setupBitmap(
Bitmap bitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr)
throws VideoFrameProcessingException {
this.useHdr = useHdr;
if (inputEnded) {
@ -124,7 +125,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
double frameDurationUs = C.MICROS_PER_SECOND / frameRate;
pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, frameDurationUs, framesToAdd));
pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, offsetUs, frameDurationUs, framesToAdd));
maybeQueueToShaderProgram();
}
@ -138,6 +139,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (framesToQueueForCurrentBitmap == 0) {
Bitmap bitmap = currentBitmapInfo.bitmap;
framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames;
currentPresentationTimeUs = currentBitmapInfo.offsetUs;
int currentTexId;
try {
if (currentGlTextureInfo != null) {
@ -189,11 +191,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Information to generate all the frames associated with a specific {@link Bitmap}. */
private static final class BitmapFrameSequenceInfo {
public final Bitmap bitmap;
public final long offsetUs;
public final double frameDurationUs;
public final int numberOfFrames;
public BitmapFrameSequenceInfo(Bitmap bitmap, double frameDurationUs, int numberOfFrames) {
public BitmapFrameSequenceInfo(
Bitmap bitmap, long offsetUs, double frameDurationUs, int numberOfFrames) {
this.bitmap = bitmap;
this.offsetUs = offsetUs;
this.frameDurationUs = frameDurationUs;
this.numberOfFrames = numberOfFrames;
}

15
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View file

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
@ -253,6 +254,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
private volatile boolean inputStreamEnded;
private volatile boolean hasRefreshedNextInputFrameInfo;
private DefaultVideoFrameProcessor(
EGLDisplay eglDisplay,
@ -321,7 +323,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
@Override
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
checkState(
hasRefreshedNextInputFrameInfo,
"setInputFrameInfo must be called before queueing another bitmap");
inputHandler.queueInputBitmap(
inputBitmap,
durationUs,
checkNotNull(nextInputFrameInfo).offsetToAddUs,
frameRate,
/* useHdr= */ false);
hasRefreshedNextInputFrameInfo = false;
}
@Override
@ -332,6 +343,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
@Override
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
hasRefreshedNextInputFrameInfo = true;
}
@Override
@ -341,6 +353,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
inputHandler.registerInputFrame(nextInputFrameInfo);
hasRefreshedNextInputFrameInfo = false;
}
@Override

7
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
View file

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.Nullable;
@ -109,6 +110,12 @@ import java.util.concurrent.atomic.AtomicInteger;
surfaceTexture.setDefaultBufferSize(width, height);
}
@Override
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
throw new UnsupportedOperationException();
}
@Override
public Surface getInputSurface() {
return surface;

10
libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java
View file

@ -39,10 +39,16 @@ import androidx.media3.common.VideoFrameProcessor;
/**
* Provides an input {@link Bitmap} to put into the video frames.
*
* @see VideoFrameProcessor#queueInputBitmap
* @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}.
* @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
* @param offsetUs The offset, from the start of the input stream, to apply for the {@code
* inputBitmap} in microseconds.
* @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
* second.
* @param useHdr Whether input and/or output colors are HDR.
*/
default void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
throw new UnsupportedOperationException();
}

4
libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
View file

@ -333,10 +333,12 @@ public final class VideoFrameProcessorTestRunner {
return endFrameProcessingAndGetImage();
}
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) {
videoFrameProcessor.setInputFrameInfo(
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
.setOffsetToAddUs(offsetToAddUs)
.build());
videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
}