mirror of
https://github.com/samsonjs/media.git
synced 2026-03-25 09:25:53 +00:00
HDR: Implement DefaultVideoFrameProcessor texture output for tests.
Previously, we always used ImageReader to read from the output of DefaultVideoFrameProcessor, for pixel tests. This has a limitation of not being able to read HDR contents, so that we couldn't support HDR pixel tests. Reading from a texture allows us to use glReadPixels to read from DefaultVideoFrameProcessor, and build upon this to implement HDR pixel tests. We do still want tests for surface output though, because real use-cases only will output to Surfaces. Also, add some tests for outputting to textures, since this test infrastructure is a bit complex. PiperOrigin-RevId: 519786535
This commit is contained in:
huangdarwin
Tianyi Feng
parent
26aee812d5
commit
a0838771d3
7 changed files with 404 additions and 59 deletions
|
|
@ -37,6 +37,7 @@ import androidx.media3.common.DebugViewProvider;
|
|||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.FrameInfo;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.SurfaceInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
|
|
@ -64,6 +65,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
|
||||
public static final class Factory implements VideoFrameProcessor.Factory {
|
||||
private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
|
||||
private boolean outputToTexture;
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
|
|
@ -77,6 +79,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets whether to output to a texture for testing.
|
||||
*
|
||||
* <p>Must be called before {@link #create}.
|
||||
*
|
||||
* <p>The default value is {@code false}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
public Factory setOutputToTexture(boolean outputToTexture) {
|
||||
this.outputToTexture = outputToTexture;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*
|
||||
|
|
@ -153,7 +168,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
singleThreadExecutorService,
|
||||
listenerExecutor,
|
||||
listener,
|
||||
glObjectsProvider));
|
||||
glObjectsProvider,
|
||||
outputToTexture));
|
||||
|
||||
try {
|
||||
return defaultVideoFrameProcessorFuture.get();
|
||||
|
|
@ -226,7 +242,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
|
||||
/** Returns the task executor that runs video frame processing tasks. */
|
||||
@VisibleForTesting
|
||||
/* package */ VideoFrameProcessingTaskExecutor getTaskExecutor() {
|
||||
public VideoFrameProcessingTaskExecutor getTaskExecutor() {
|
||||
return videoFrameProcessingTaskExecutor;
|
||||
}
|
||||
|
||||
|
|
@ -289,6 +305,20 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
finalShaderProgramWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the output {@link GlTextureInfo}.
|
||||
*
|
||||
* <p>Should only be called if {@code outputToTexture} is true, and after a frame is available, as
|
||||
* reported by the output {@linkplain #setOutputSurfaceInfo surface}'s {@link
|
||||
* SurfaceTexture#setOnFrameAvailableListener}. Returns {@code null} if an output texture is not
|
||||
* yet available.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
@Nullable
|
||||
public GlTextureInfo getOutputTextureInfo() {
|
||||
return finalShaderProgramWrapper.getOutputTextureInfo();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(long releaseTimeNs) {
|
||||
checkState(
|
||||
|
|
@ -382,7 +412,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
ExecutorService singleThreadExecutorService,
|
||||
Executor executor,
|
||||
Listener listener,
|
||||
GlObjectsProvider glObjectsProvider)
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
boolean outputToTexture)
|
||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
|
||||
|
||||
|
|
@ -425,7 +456,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
releaseFramesAutomatically,
|
||||
executor,
|
||||
listener,
|
||||
glObjectsProvider);
|
||||
glObjectsProvider,
|
||||
outputToTexture);
|
||||
setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
|
||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
|
||||
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
|
||||
|
|
@ -464,7 +496,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
boolean releaseFramesAutomatically,
|
||||
Executor executor,
|
||||
Listener listener,
|
||||
GlObjectsProvider glObjectsProvider)
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
boolean outputToTexture)
|
||||
throws VideoFrameProcessingException {
|
||||
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
|
||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||
|
|
@ -538,7 +571,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||
releaseFramesAutomatically,
|
||||
executor,
|
||||
listener,
|
||||
glObjectsProvider));
|
||||
glObjectsProvider,
|
||||
outputToTexture));
|
||||
return shaderProgramListBuilder.build();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import android.view.SurfaceHolder;
|
|||
import android.view.SurfaceView;
|
||||
import androidx.annotation.GuardedBy;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.annotation.VisibleForTesting;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
|
|
@ -52,8 +53,8 @@ import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
|
|||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* Wrapper around a {@link DefaultShaderProgram} that writes to the provided output surface and
|
||||
* optional debug surface view.
|
||||
* Wrapper around a {@link DefaultShaderProgram} that writes to the provided output surface and if
|
||||
* provided, the optional debug surface view or output texture.
|
||||
*
|
||||
* <p>The wrapped {@link DefaultShaderProgram} applies the {@link GlMatrixTransformation} and {@link
|
||||
* RgbMatrix} instances passed to the constructor, followed by any transformations needed to convert
|
||||
|
|
@ -82,6 +83,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
private final float[] textureTransformMatrix;
|
||||
private final Queue<Long> streamOffsetUsQueue;
|
||||
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
||||
private final boolean outputToTexture;
|
||||
|
||||
private int inputWidth;
|
||||
private int inputHeight;
|
||||
|
|
@ -91,6 +93,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
private InputListener inputListener;
|
||||
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
|
||||
@Nullable private SurfaceView debugSurfaceView;
|
||||
private @MonotonicNonNull GlTextureInfo outputTexture;
|
||||
private boolean frameProcessingStarted;
|
||||
|
||||
private volatile boolean outputChanged;
|
||||
|
|
@ -117,7 +120,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
boolean releaseFramesAutomatically,
|
||||
Executor videoFrameProcessorListenerExecutor,
|
||||
VideoFrameProcessor.Listener videoFrameProcessorListener,
|
||||
GlObjectsProvider glObjectsProvider) {
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
boolean outputToTexture) {
|
||||
this.context = context;
|
||||
this.matrixTransformations = matrixTransformations;
|
||||
this.rgbMatrices = rgbMatrices;
|
||||
|
|
@ -132,6 +136,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
|
||||
this.videoFrameProcessorListener = videoFrameProcessorListener;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.outputToTexture = outputToTexture;
|
||||
|
||||
textureTransformMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
streamOffsetUsQueue = new ConcurrentLinkedQueue<>();
|
||||
|
|
@ -202,7 +207,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
videoFrameProcessorListenerExecutor.execute(
|
||||
() -> videoFrameProcessorListener.onOutputFrameAvailable(offsetPresentationTimeUs));
|
||||
if (releaseFramesAutomatically) {
|
||||
renderFrameToSurfaces(
|
||||
renderFrame(
|
||||
inputTexture, presentationTimeUs, /* releaseTimeNs= */ offsetPresentationTimeUs * 1000);
|
||||
} else {
|
||||
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
|
||||
|
|
@ -220,7 +225,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
frameProcessingStarted = true;
|
||||
checkState(!releaseFramesAutomatically);
|
||||
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
|
||||
renderFrameToSurfaces(
|
||||
renderFrame(
|
||||
/* inputTexture= */ oldestAvailableFrame.first,
|
||||
/* presentationTimeUs= */ oldestAvailableFrame.second,
|
||||
releaseTimeNs);
|
||||
|
|
@ -258,6 +263,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
defaultShaderProgram.release();
|
||||
}
|
||||
try {
|
||||
if (outputTexture != null) {
|
||||
GlUtil.deleteTexture(outputTexture.texId);
|
||||
GlUtil.deleteFbo(outputTexture.fboId);
|
||||
}
|
||||
GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw new VideoFrameProcessingException(e);
|
||||
|
|
@ -294,17 +303,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
this.outputSurfaceInfo = outputSurfaceInfo;
|
||||
}
|
||||
|
||||
private void renderFrameToSurfaces(
|
||||
private void renderFrame(
|
||||
GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
|
||||
try {
|
||||
maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
|
||||
if (outputToTexture && defaultShaderProgram != null) {
|
||||
renderFrameToOutputTexture(inputTexture, presentationTimeUs);
|
||||
}
|
||||
|
||||
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
||||
videoFrameProcessorListenerExecutor.execute(
|
||||
() ->
|
||||
videoFrameProcessorListener.onError(
|
||||
VideoFrameProcessingException.from(e, presentationTimeUs)));
|
||||
}
|
||||
maybeRenderFrameToDebugSurface(inputTexture, presentationTimeUs);
|
||||
if (debugSurfaceViewWrapper != null && defaultShaderProgram != null) {
|
||||
renderFrameToDebugSurface(inputTexture, presentationTimeUs);
|
||||
}
|
||||
|
||||
inputListener.onInputFrameProcessed(inputTexture);
|
||||
}
|
||||
|
||||
|
|
@ -338,6 +354,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
|
||||
}
|
||||
|
||||
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
|
||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||
checkNotNull(outputTexture);
|
||||
GlUtil.focusFramebufferUsingCurrentContext(
|
||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||
GlUtil.clearOutputFrame();
|
||||
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
@Nullable
|
||||
/* package */ GlTextureInfo getOutputTextureInfo() {
|
||||
return outputTexture;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the instance is configured.
|
||||
*
|
||||
|
|
@ -408,7 +439,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
outputChanged = false;
|
||||
}
|
||||
if (defaultShaderProgram == null) {
|
||||
defaultShaderProgram = createDefaultShaderProgramForOutputSurface(outputSurfaceInfo);
|
||||
DefaultShaderProgram defaultShaderProgram =
|
||||
createDefaultShaderProgramForOutputSurface(outputSurfaceInfo);
|
||||
if (outputToTexture) {
|
||||
configureOutputTexture(
|
||||
checkNotNull(outputSizeBeforeSurfaceTransformation).getWidth(),
|
||||
checkNotNull(outputSizeBeforeSurfaceTransformation).getHeight());
|
||||
}
|
||||
this.defaultShaderProgram = defaultShaderProgram;
|
||||
}
|
||||
|
||||
this.outputSurfaceInfo = outputSurfaceInfo;
|
||||
|
|
@ -416,6 +454,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
return true;
|
||||
}
|
||||
|
||||
private void configureOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
|
||||
if (outputTexture != null) {
|
||||
GlUtil.deleteTexture(outputTexture.texId);
|
||||
GlUtil.deleteFbo(outputTexture.fboId);
|
||||
}
|
||||
int outputTexId =
|
||||
GlUtil.createTexture(
|
||||
outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
|
||||
outputTexture =
|
||||
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
|
||||
}
|
||||
|
||||
private DefaultShaderProgram createDefaultShaderProgramForOutputSurface(
|
||||
SurfaceInfo outputSurfaceInfo) throws VideoFrameProcessingException {
|
||||
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
|
||||
|
|
@ -464,24 +514,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
return defaultShaderProgram;
|
||||
}
|
||||
|
||||
private void maybeRenderFrameToDebugSurface(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
if (debugSurfaceViewWrapper == null || this.defaultShaderProgram == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
DefaultShaderProgram defaultShaderProgram = this.defaultShaderProgram;
|
||||
private void renderFrameToDebugSurface(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
|
||||
SurfaceViewWrapper debugSurfaceViewWrapper = checkNotNull(this.debugSurfaceViewWrapper);
|
||||
try {
|
||||
debugSurfaceViewWrapper.maybeRenderToSurfaceView(
|
||||
() -> {
|
||||
GlUtil.clearOutputFrame();
|
||||
@C.ColorTransfer
|
||||
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
|
||||
defaultShaderProgram.setOutputColorTransfer(
|
||||
checkNotNull(debugSurfaceViewWrapper).outputColorTransfer);
|
||||
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
|
||||
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
|
||||
},
|
||||
glObjectsProvider);
|
||||
checkNotNull(debugSurfaceViewWrapper)
|
||||
.maybeRenderToSurfaceView(
|
||||
() -> {
|
||||
GlUtil.clearOutputFrame();
|
||||
@C.ColorTransfer
|
||||
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
|
||||
defaultShaderProgram.setOutputColorTransfer(
|
||||
debugSurfaceViewWrapper.outputColorTransfer);
|
||||
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
|
||||
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
|
||||
},
|
||||
glObjectsProvider);
|
||||
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
||||
Log.d(TAG, "Error rendering to debug preview", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,20 @@
|
|||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
|
||||
|
||||
import androidx.annotation.VisibleForTesting;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/**
|
||||
* Interface for tasks that may throw a {@link GlUtil.GlException} or {@link
|
||||
* VideoFrameProcessingException}.
|
||||
*/
|
||||
/* package */ interface VideoFrameProcessingTask {
|
||||
@UnstableApi
|
||||
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
|
||||
public interface VideoFrameProcessingTask {
|
||||
/** Runs the task. */
|
||||
void run() throws VideoFrameProcessingException, GlUtil.GlException;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,12 +15,15 @@
|
|||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
|
||||
import androidx.annotation.GuardedBy;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.annotation.VisibleForTesting;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
|
@ -44,7 +47,9 @@ import java.util.concurrent.RejectedExecutionException;
|
|||
* executed before {@linkplain #submit(VideoFrameProcessingTask) default priority tasks}. Tasks with
|
||||
* equal priority are executed in FIFO order.
|
||||
*/
|
||||
/* package */ final class VideoFrameProcessingTaskExecutor {
|
||||
@UnstableApi
|
||||
@VisibleForTesting(otherwise = PACKAGE_PRIVATE)
|
||||
public final class VideoFrameProcessingTaskExecutor {
|
||||
|
||||
private final ExecutorService singleThreadExecutorService;
|
||||
private final VideoFrameProcessor.Listener listener;
|
||||
|
|
|
|||
|
|
@ -52,19 +52,35 @@ public class BitmapPixelTestUtil {
|
|||
|
||||
/**
|
||||
* Maximum allowed average pixel difference between the expected and actual edited images in pixel
|
||||
* difference-based tests. The value is chosen so that differences in decoder behavior across
|
||||
* emulator versions don't affect whether the test passes for most emulators, but substantial
|
||||
* distortions introduced by changes in tested components will cause the test to fail.
|
||||
* difference-based tests, between emulators.
|
||||
*
|
||||
* <p>To run pixel difference-based tests on physical devices, please use a value of 5f, rather
|
||||
* than 0.5f. This higher value will ignore some very small errors, but will allow for some
|
||||
* differences caused by graphics implementations to be ignored. When the difference is close to
|
||||
* the threshold, manually inspect expected/actual bitmaps to confirm failure, as it's possible
|
||||
* this is caused by a difference in the codec or graphics implementation as opposed to an issue
|
||||
* in the tested component.
|
||||
* <p>The value is chosen so that differences in decoder behavior across emulator versions don't
|
||||
* affect whether the test passes, but substantial distortions introduced by changes in tested
|
||||
* components will cause the test to fail.
|
||||
*
|
||||
* <p>When the difference is close to the threshold, manually inspect expected/actual bitmaps to
|
||||
* confirm failure, as it's possible this is caused by a difference in the codec or graphics
|
||||
* implementation as opposed to an issue in the tested component.
|
||||
*/
|
||||
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 1.f;
|
||||
|
||||
/**
|
||||
* Maximum allowed average pixel difference between the expected and actual edited images in pixel
|
||||
* difference-based tests, between devices, or devices and emulators.
|
||||
*
|
||||
* <p>The value is chosen so that differences in decoder behavior across devices don't affect
|
||||
* whether the test passes, but substantial distortions introduced by changes in tested components
|
||||
* will cause the test to fail.
|
||||
*
|
||||
* <p>When the difference is close to the threshold, manually inspect expected/actual bitmaps to
|
||||
* confirm failure, as it's possible this is caused by a difference in the codec or graphics
|
||||
* implementation as opposed to an issue in the tested component.
|
||||
*
|
||||
* <p>This value is larger than {@link #MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} to support the
|
||||
* larger variance in decoder outputs between different physical devices and emulators.
|
||||
*/
|
||||
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE = 5.f;
|
||||
|
||||
/**
|
||||
* Reads a bitmap from the specified asset location.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import android.graphics.PixelFormat;
|
|||
import android.media.Image;
|
||||
import android.media.ImageReader;
|
||||
import android.media.MediaFormat;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.annotation.RequiresApi;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
|
|
@ -57,6 +58,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||
|
||||
private @MonotonicNonNull String testId;
|
||||
private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
|
||||
private BitmapReader.@MonotonicNonNull Factory bitmapReaderFactory;
|
||||
private @MonotonicNonNull String videoAssetPath;
|
||||
private @MonotonicNonNull String outputFileLabel;
|
||||
private @MonotonicNonNull ImmutableList<Effect> effects;
|
||||
|
|
@ -96,6 +98,17 @@ public final class VideoFrameProcessorTestRunner {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the {@link BitmapReader.Factory}.
|
||||
*
|
||||
* <p>The default value is {@link SurfaceBitmapReader.Factory}.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder setBitmapReaderFactory(BitmapReader.Factory bitmapReaderFactory) {
|
||||
this.bitmapReaderFactory = bitmapReaderFactory;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the input video asset path.
|
||||
*
|
||||
|
|
@ -205,6 +218,7 @@ public final class VideoFrameProcessorTestRunner {
|
|||
return new VideoFrameProcessorTestRunner(
|
||||
testId,
|
||||
videoFrameProcessorFactory,
|
||||
bitmapReaderFactory == null ? new SurfaceBitmapReader.Factory() : bitmapReaderFactory,
|
||||
videoAssetPath,
|
||||
outputFileLabel == null ? "" : outputFileLabel,
|
||||
effects == null ? ImmutableList.of() : effects,
|
||||
|
|
@ -227,15 +241,16 @@ public final class VideoFrameProcessorTestRunner {
|
|||
private final String outputFileLabel;
|
||||
private final float pixelWidthHeightRatio;
|
||||
private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException;
|
||||
|
||||
private final VideoFrameProcessor videoFrameProcessor;
|
||||
|
||||
private volatile @MonotonicNonNull ImageReader outputImageReader;
|
||||
private @MonotonicNonNull BitmapReader bitmapReader;
|
||||
|
||||
private volatile boolean videoFrameProcessingEnded;
|
||||
|
||||
private VideoFrameProcessorTestRunner(
|
||||
String testId,
|
||||
VideoFrameProcessor.Factory videoFrameProcessorFactory,
|
||||
BitmapReader.Factory bitmapReaderFactory,
|
||||
@Nullable String videoAssetPath,
|
||||
String outputFileLabel,
|
||||
ImmutableList<Effect> effects,
|
||||
|
|
@ -262,15 +277,13 @@ public final class VideoFrameProcessorTestRunner {
|
|||
/* releaseFramesAutomatically= */ true,
|
||||
MoreExecutors.directExecutor(),
|
||||
new VideoFrameProcessor.Listener() {
|
||||
@SuppressLint("WrongConstant")
|
||||
@Override
|
||||
public void onOutputSizeChanged(int width, int height) {
|
||||
outputImageReader =
|
||||
ImageReader.newInstance(
|
||||
width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
|
||||
checkNotNull(videoFrameProcessor)
|
||||
.setOutputSurfaceInfo(
|
||||
new SurfaceInfo(outputImageReader.getSurface(), width, height));
|
||||
bitmapReader =
|
||||
bitmapReaderFactory.create(checkNotNull(videoFrameProcessor), width, height);
|
||||
Surface outputSurface = bitmapReader.getSurface();
|
||||
videoFrameProcessor.setOutputSurfaceInfo(
|
||||
new SurfaceInfo(outputSurface, width, height));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -291,7 +304,6 @@ public final class VideoFrameProcessorTestRunner {
|
|||
});
|
||||
}
|
||||
|
||||
@RequiresApi(19)
|
||||
public Bitmap processFirstFrameAndEnd() throws Exception {
|
||||
DecodeOneFrameUtil.decodeOneAssetFileFrame(
|
||||
checkNotNull(videoAssetPath),
|
||||
|
|
@ -324,19 +336,16 @@ public final class VideoFrameProcessorTestRunner {
|
|||
videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
|
||||
}
|
||||
|
||||
@RequiresApi(19)
|
||||
public Bitmap endFrameProcessingAndGetImage() throws Exception {
|
||||
videoFrameProcessor.signalEndOfInput();
|
||||
Thread.sleep(VIDEO_FRAME_PROCESSING_WAIT_MS);
|
||||
|
||||
assertThat(videoFrameProcessingEnded).isTrue();
|
||||
assertThat(videoFrameProcessingException.get()).isNull();
|
||||
assertThat(videoFrameProcessingEnded).isTrue();
|
||||
|
||||
Image videoFrameProcessorOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
|
||||
Bitmap actualBitmap = createArgb8888BitmapFromRgba8888Image(videoFrameProcessorOutputImage);
|
||||
videoFrameProcessorOutputImage.close();
|
||||
maybeSaveTestBitmap(testId, /* bitmapLabel= */ outputFileLabel, actualBitmap, /* path= */ null);
|
||||
return actualBitmap;
|
||||
Bitmap outputBitmap = checkNotNull(bitmapReader).getBitmap();
|
||||
maybeSaveTestBitmap(testId, /* bitmapLabel= */ outputFileLabel, outputBitmap, /* path= */ null);
|
||||
return outputBitmap;
|
||||
}
|
||||
|
||||
public void release() {
|
||||
|
|
@ -348,4 +357,56 @@ public final class VideoFrameProcessorTestRunner {
|
|||
public interface OnOutputFrameAvailableListener {
|
||||
void onFrameAvailable(long presentationTimeUs);
|
||||
}
|
||||
|
||||
/** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */
|
||||
public interface BitmapReader {
|
||||
interface Factory {
|
||||
BitmapReader create(VideoFrameProcessor videoFrameProcessor, int width, int height);
|
||||
}
|
||||
|
||||
/** Returns the {@link VideoFrameProcessor} output {@link Surface}. */
|
||||
Surface getSurface();
|
||||
|
||||
/** Returns the output {@link Bitmap}. */
|
||||
Bitmap getBitmap();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*
|
||||
* <p>Reads from a {@link Surface}. Only supports SDR input.
|
||||
*/
|
||||
public static final class SurfaceBitmapReader
|
||||
implements VideoFrameProcessorTestRunner.BitmapReader {
|
||||
public static final class Factory
|
||||
implements VideoFrameProcessorTestRunner.BitmapReader.Factory {
|
||||
@Override
|
||||
public SurfaceBitmapReader create(
|
||||
VideoFrameProcessor videoFrameProcessor, int width, int height) {
|
||||
return new SurfaceBitmapReader(width, height);
|
||||
}
|
||||
}
|
||||
|
||||
// ImageReader only supports SDR input.
|
||||
private final ImageReader imageReader;
|
||||
|
||||
@SuppressLint("WrongConstant")
|
||||
private SurfaceBitmapReader(int width, int height) {
|
||||
imageReader =
|
||||
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Surface getSurface() {
|
||||
return imageReader.getSurface();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitmap getBitmap() {
|
||||
Image outputImage = checkNotNull(imageReader).acquireLatestImage();
|
||||
Bitmap outputBitmap = createArgb8888BitmapFromRgba8888Image(outputImage);
|
||||
outputImage.close();
|
||||
return outputBitmap;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.transformer.mh;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE;
|
||||
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
|
||||
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.SurfaceTexture;
|
||||
import android.view.Surface;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.effect.BitmapOverlay;
|
||||
import androidx.media3.effect.DefaultVideoFrameProcessor;
|
||||
import androidx.media3.effect.OverlayEffect;
|
||||
import androidx.media3.test.utils.BitmapPixelTestUtil;
|
||||
import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/**
|
||||
* Pixel test for video frame processing, outputting to a texture, via {@link
|
||||
* DefaultVideoFrameProcessor}.
|
||||
*
|
||||
* <p>Uses a {@link DefaultVideoFrameProcessor} to process one frame, and checks that the actual
|
||||
* output matches expected output, either from a golden file or from another edit.
|
||||
*/
|
||||
// TODO(b/263395272): Move this test to effects/mh tests, and remove @TestOnly dependencies.
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
|
||||
private static final String ORIGINAL_PNG_ASSET_PATH =
|
||||
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
|
||||
private static final String BITMAP_OVERLAY_PNG_ASSET_PATH =
|
||||
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png";
|
||||
private static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png";
|
||||
/** Input video of which we only use the first frame. */
|
||||
private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
|
||||
|
||||
private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner;
|
||||
|
||||
@After
|
||||
public void release() {
|
||||
checkNotNull(videoFrameProcessorTestRunner).release();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void noEffects_matchesGoldenFile() throws Exception {
|
||||
String testId = "noEffects_matchesGoldenFile";
|
||||
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
|
||||
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
|
||||
|
||||
Bitmap actualBitmap = videoFrameProcessorTestRunner.processFirstFrameAndEnd();
|
||||
|
||||
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
|
||||
float averagePixelAbsoluteDifference =
|
||||
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
|
||||
assertThat(averagePixelAbsoluteDifference)
|
||||
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void bitmapOverlay_matchesGoldenFile() throws Exception {
|
||||
String testId = "bitmapOverlay_matchesGoldenFile";
|
||||
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
|
||||
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap);
|
||||
videoFrameProcessorTestRunner =
|
||||
getDefaultFrameProcessorTestRunnerBuilder(testId)
|
||||
.setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay)))
|
||||
.build();
|
||||
Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH);
|
||||
|
||||
Bitmap actualBitmap = videoFrameProcessorTestRunner.processFirstFrameAndEnd();
|
||||
|
||||
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
|
||||
float averagePixelAbsoluteDifference =
|
||||
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
|
||||
assertThat(averagePixelAbsoluteDifference)
|
||||
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
|
||||
}
|
||||
|
||||
// TODO(b/227624622): Add a test for HDR input after BitmapPixelTestUtil can read HDR bitmaps,
|
||||
// using GlEffectWrapper to ensure usage of intermediate textures.
|
||||
|
||||
private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
|
||||
String testId) {
|
||||
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
|
||||
new DefaultVideoFrameProcessor.Factory().setOutputToTexture(true);
|
||||
return new VideoFrameProcessorTestRunner.Builder()
|
||||
.setTestId(testId)
|
||||
.setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
|
||||
.setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING)
|
||||
.setBitmapReaderFactory(new TextureBitmapReader.Factory());
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*
|
||||
* <p>Reads from an OpenGL texture. Only for use on physical devices.
|
||||
*/
|
||||
private static final class TextureBitmapReader
|
||||
implements VideoFrameProcessorTestRunner.BitmapReader {
|
||||
// TODO(b/239172735): This outputs an incorrect black output image on emulators.
|
||||
public static final class Factory
|
||||
implements VideoFrameProcessorTestRunner.BitmapReader.Factory {
|
||||
@Override
|
||||
public TextureBitmapReader create(
|
||||
VideoFrameProcessor videoFrameProcessor, int width, int height) {
|
||||
return new TextureBitmapReader((DefaultVideoFrameProcessor) videoFrameProcessor);
|
||||
}
|
||||
}
|
||||
|
||||
private final DefaultVideoFrameProcessor defaultVideoFrameProcessor;
|
||||
private @MonotonicNonNull Bitmap outputBitmap;
|
||||
|
||||
private TextureBitmapReader(DefaultVideoFrameProcessor defaultVideoFrameProcessor) {
|
||||
this.defaultVideoFrameProcessor = defaultVideoFrameProcessor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Surface getSurface() {
|
||||
int texId;
|
||||
try {
|
||||
texId = GlUtil.createExternalTexture();
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
SurfaceTexture surfaceTexture = new SurfaceTexture(texId);
|
||||
surfaceTexture.setOnFrameAvailableListener(this::onSurfaceTextureFrameAvailable);
|
||||
return new Surface(surfaceTexture);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bitmap getBitmap() {
|
||||
return checkStateNotNull(outputBitmap);
|
||||
}
|
||||
|
||||
private void onSurfaceTextureFrameAvailable(SurfaceTexture surfaceTexture) {
|
||||
defaultVideoFrameProcessor
|
||||
.getTaskExecutor()
|
||||
.submitWithHighPriority(this::getBitmapFromTexture);
|
||||
}
|
||||
|
||||
private void getBitmapFromTexture() throws GlUtil.GlException {
|
||||
GlTextureInfo outputTexture = checkNotNull(defaultVideoFrameProcessor.getOutputTextureInfo());
|
||||
|
||||
GlUtil.focusFramebufferUsingCurrentContext(
|
||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||
outputBitmap =
|
||||
BitmapPixelTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
|
||||
outputTexture.width, outputTexture.height);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue