mirror of
https://github.com/samsonjs/media.git
synced 2026-04-02 10:45:51 +00:00
Support previewing HDR on API33+
FrameProcessor already support using different transfer function for input and output color. This CL has two major changes: - Create an eglSurface that recognizes BT.2020 PQ - This requires a separate extension that works only after 33 - So we current throw, if input is HDR, and this extension doesn't work - Create FrameProcessor with PQ output transfer function PiperOrigin-RevId: 496023758
This commit is contained in:
claincly
Tianyi Feng
parent
5e23b8bfd5
commit
a4bc0959be
5 changed files with 194 additions and 34 deletions
|
|
@ -89,7 +89,16 @@ public final class GlUtil {
|
|||
private static final String EXTENSION_SURFACELESS_CONTEXT = "EGL_KHR_surfaceless_context";
|
||||
// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt
|
||||
private static final String EXTENSION_YUV_TARGET = "GL_EXT_YUV_target";
|
||||
|
||||
// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt
|
||||
private static final String EXTENSION_COLORSPACE_BT2020_PQ = "EGL_EXT_gl_colorspace_bt2020_pq";
|
||||
// https://registry.khronos.org/EGL/extensions/KHR/EGL_KHR_gl_colorspace.txt
|
||||
private static final int EGL_GL_COLORSPACE_KHR = 0x309D;
|
||||
// https://registry.khronos.org/EGL/extensions/EXT/EGL_EXT_gl_colorspace_bt2020_linear.txt
|
||||
private static final int EGL_GL_COLORSPACE_BT2020_PQ_EXT = 0x3340;
|
||||
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ =
|
||||
new int[] {
|
||||
EGL_GL_COLORSPACE_KHR, EGL_GL_COLORSPACE_BT2020_PQ_EXT, EGL14.EGL_NONE, EGL14.EGL_NONE
|
||||
};
|
||||
private static final int[] EGL_WINDOW_SURFACE_ATTRIBUTES_NONE = new int[] {EGL14.EGL_NONE};
|
||||
|
||||
/** Class only contains static methods. */
|
||||
|
|
@ -216,6 +225,13 @@ public final class GlUtil {
|
|||
return glExtensions != null && glExtensions.contains(EXTENSION_YUV_TARGET);
|
||||
}
|
||||
|
||||
/** Returns whether {@value #EXTENSION_COLORSPACE_BT2020_PQ} is supported. */
|
||||
public static boolean isBt2020PqExtensionSupported() {
|
||||
EGLDisplay display = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
|
||||
@Nullable String eglExtensions = EGL14.eglQueryString(display, EGL10.EGL_EXTENSIONS);
|
||||
return eglExtensions != null && eglExtensions.contains(EXTENSION_COLORSPACE_BT2020_PQ);
|
||||
}
|
||||
|
||||
/** Returns an initialized default {@link EGLDisplay}. */
|
||||
@RequiresApi(17)
|
||||
public static EGLDisplay createEglDisplay() throws GlException {
|
||||
|
|
@ -258,30 +274,47 @@ public final class GlUtil {
|
|||
/**
|
||||
* Creates a new {@link EGLSurface} wrapping the specified {@code surface}.
|
||||
*
|
||||
* <p>The {@link EGLSurface} will configure with {@link #EGL_CONFIG_ATTRIBUTES_RGBA_8888} and
|
||||
* OpenGL ES 2.0.
|
||||
* <p>The {@link EGLSurface} will configure with OpenGL ES 2.0.
|
||||
*
|
||||
* @param eglDisplay The {@link EGLDisplay} to attach the surface to.
|
||||
* @param surface The surface to wrap; must be a surface, surface texture or surface holder.
|
||||
*/
|
||||
@RequiresApi(17)
|
||||
public static EGLSurface createEglSurface(EGLDisplay eglDisplay, Object surface)
|
||||
throws GlException {
|
||||
return Api17.createEglSurface(eglDisplay, surface, EGL_CONFIG_ATTRIBUTES_RGBA_8888);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new {@link EGLSurface} wrapping the specified {@code surface}.
|
||||
*
|
||||
* @param eglDisplay The {@link EGLDisplay} to attach the surface to.
|
||||
* @param surface The surface to wrap; must be a surface, surface texture or surface holder.
|
||||
* @param configAttributes The attributes to configure EGL with. Accepts {@link
|
||||
* #EGL_CONFIG_ATTRIBUTES_RGBA_1010102} and {@link #EGL_CONFIG_ATTRIBUTES_RGBA_8888}.
|
||||
* @param colorTransfer The {@linkplain C.ColorTransfer color transfer characteristics} to which
|
||||
* the {@code surface} is configured. The only accepted values are {@link
|
||||
* C#COLOR_TRANSFER_SDR}, {@link C#COLOR_TRANSFER_HLG} and {@link C#COLOR_TRANSFER_ST2084}.
|
||||
* @param isEncoderInputSurface Whether the {@code surface} is the input surface of an encoder.
|
||||
*/
|
||||
@RequiresApi(17)
|
||||
public static EGLSurface createEglSurface(
|
||||
EGLDisplay eglDisplay, Object surface, int[] configAttributes) throws GlException {
|
||||
return Api17.createEglSurface(eglDisplay, surface, configAttributes);
|
||||
EGLDisplay eglDisplay,
|
||||
Object surface,
|
||||
@C.ColorTransfer int colorTransfer,
|
||||
boolean isEncoderInputSurface)
|
||||
throws GlException {
|
||||
int[] configAttributes;
|
||||
int[] windowAttributes;
|
||||
if (colorTransfer == C.COLOR_TRANSFER_SDR) {
|
||||
configAttributes = EGL_CONFIG_ATTRIBUTES_RGBA_8888;
|
||||
windowAttributes = EGL_WINDOW_SURFACE_ATTRIBUTES_NONE;
|
||||
} else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
|
||||
configAttributes = EGL_CONFIG_ATTRIBUTES_RGBA_1010102;
|
||||
if (isEncoderInputSurface) {
|
||||
// Outputting BT2020 PQ with EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ to an encoder causes
|
||||
// the encoder to incorrectly switch to full range color, even if the encoder is configured
|
||||
// with limited range color, because EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ sets full range
|
||||
// color output, and GL windowAttributes overrides encoder settings.
|
||||
windowAttributes = EGL_WINDOW_SURFACE_ATTRIBUTES_NONE;
|
||||
} else {
|
||||
// TODO(b/262259999) HDR10 PQ content looks dark on the screen.
|
||||
windowAttributes = EGL_WINDOW_SURFACE_ATTRIBUTES_BT2020_PQ;
|
||||
}
|
||||
} else if (colorTransfer == C.COLOR_TRANSFER_HLG) {
|
||||
checkArgument(isEncoderInputSurface, "Outputting HLG to the screen is not supported.");
|
||||
configAttributes = EGL_CONFIG_ATTRIBUTES_RGBA_1010102;
|
||||
windowAttributes = EGL_WINDOW_SURFACE_ATTRIBUTES_NONE;
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported color transfer: " + colorTransfer);
|
||||
}
|
||||
return Api17.createEglSurface(eglDisplay, surface, configAttributes, windowAttributes);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -670,13 +703,14 @@ public final class GlUtil {
|
|||
|
||||
@DoNotInline
|
||||
public static EGLSurface createEglSurface(
|
||||
EGLDisplay eglDisplay, Object surface, int[] configAttributes) throws GlException {
|
||||
EGLDisplay eglDisplay, Object surface, int[] configAttributes, int[] windowAttributes)
|
||||
throws GlException {
|
||||
EGLSurface eglSurface =
|
||||
EGL14.eglCreateWindowSurface(
|
||||
eglDisplay,
|
||||
getEglConfig(eglDisplay, configAttributes),
|
||||
surface,
|
||||
EGL_WINDOW_SURFACE_ATTRIBUTES_NONE,
|
||||
windowAttributes,
|
||||
/* offset= */ 0);
|
||||
checkEglException("Error creating surface");
|
||||
return eglSurface;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import androidx.annotation.Nullable;
|
|||
import com.google.android.exoplayer2.Bundleable;
|
||||
import com.google.android.exoplayer2.C;
|
||||
import com.google.android.exoplayer2.Format;
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
import java.lang.annotation.Documented;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
|
|
@ -38,6 +39,96 @@ import org.checkerframework.dataflow.qual.Pure;
|
|||
*/
|
||||
public final class ColorInfo implements Bundleable {
|
||||
|
||||
/**
|
||||
* Builds {@link ColorInfo} instances.
|
||||
*
|
||||
* <p>Use {@link ColorInfo#buildUpon} to obtain a builder representing an existing {@link
|
||||
* ColorInfo}.
|
||||
*/
|
||||
public static final class Builder {
|
||||
private @C.ColorSpace int colorSpace;
|
||||
private @C.ColorRange int colorRange;
|
||||
private @C.ColorTransfer int colorTransfer;
|
||||
@Nullable private byte[] hdrStaticInfo;
|
||||
|
||||
/** Creates a new instance with default values. */
|
||||
public Builder() {
|
||||
colorSpace = Format.NO_VALUE;
|
||||
colorRange = Format.NO_VALUE;
|
||||
colorTransfer = Format.NO_VALUE;
|
||||
}
|
||||
|
||||
/** Creates a new instance to build upon the provided {@link ColorInfo}. */
|
||||
private Builder(ColorInfo colorInfo) {
|
||||
this.colorSpace = colorInfo.colorSpace;
|
||||
this.colorRange = colorInfo.colorRange;
|
||||
this.colorTransfer = colorInfo.colorTransfer;
|
||||
this.hdrStaticInfo = colorInfo.hdrStaticInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the color space.
|
||||
*
|
||||
* <p>Valid values are {@link C#COLOR_SPACE_BT601}, {@link C#COLOR_SPACE_BT709}, {@link
|
||||
* C#COLOR_SPACE_BT2020} or {@link Format#NO_VALUE} if unknown.
|
||||
*
|
||||
* @param colorSpace The color space. The default value is {@link Format#NO_VALUE}.
|
||||
* @return This {@code Builder}.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder setColorSpace(@C.ColorSpace int colorSpace) {
|
||||
this.colorSpace = colorSpace;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the color range.
|
||||
*
|
||||
* <p>Valid values are {@link C#COLOR_RANGE_LIMITED}, {@link C#COLOR_RANGE_FULL} or {@link
|
||||
* Format#NO_VALUE} if unknown.
|
||||
*
|
||||
* @param colorRange The color range. The default value is {@link Format#NO_VALUE}.
|
||||
* @return This {@code Builder}.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder setColorRange(@C.ColorRange int colorRange) {
|
||||
this.colorRange = colorRange;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the color transfer.
|
||||
*
|
||||
* <p>Valid values are {@link C#COLOR_TRANSFER_LINEAR}, {@link C#COLOR_TRANSFER_HLG}, {@link
|
||||
* C#COLOR_TRANSFER_ST2084}, {@link C#COLOR_TRANSFER_SDR} or {@link Format#NO_VALUE} if unknown.
|
||||
*
|
||||
* @param colorTransfer The color transfer. The default value is {@link Format#NO_VALUE}.
|
||||
* @return This {@code Builder}.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder setColorTransfer(@C.ColorTransfer int colorTransfer) {
|
||||
this.colorTransfer = colorTransfer;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the HdrStaticInfo as defined in CTA-861.3.
|
||||
*
|
||||
* @param hdrStaticInfo The HdrStaticInfo. The default value is {@code null}.
|
||||
* @return This {@code Builder}.
|
||||
*/
|
||||
@CanIgnoreReturnValue
|
||||
public Builder setHdrStaticInfo(@Nullable byte[] hdrStaticInfo) {
|
||||
this.hdrStaticInfo = hdrStaticInfo;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Builds a new {@link ColorInfo} instance. */
|
||||
public ColorInfo build() {
|
||||
return new ColorInfo(colorSpace, colorRange, colorTransfer, hdrStaticInfo);
|
||||
}
|
||||
}
|
||||
|
||||
/** Color info representing SDR BT.709 limited range, which is a common SDR video color format. */
|
||||
public static final ColorInfo SDR_BT709_LIMITED =
|
||||
new ColorInfo(
|
||||
|
|
@ -124,7 +215,9 @@ public final class ColorInfo implements Bundleable {
|
|||
* @param colorRange The color range of the video.
|
||||
* @param colorTransfer The color transfer characteristics of the video.
|
||||
* @param hdrStaticInfo HdrStaticInfo as defined in CTA-861.3, or null if none specified.
|
||||
* @deprecated Use {@link Builder}.
|
||||
*/
|
||||
@Deprecated
|
||||
public ColorInfo(
|
||||
@C.ColorSpace int colorSpace,
|
||||
@C.ColorRange int colorRange,
|
||||
|
|
@ -136,6 +229,11 @@ public final class ColorInfo implements Bundleable {
|
|||
this.hdrStaticInfo = hdrStaticInfo;
|
||||
}
|
||||
|
||||
/** Returns a {@link Builder} initialized with the values of this instance. */
|
||||
public Builder buildUpon() {
|
||||
return new Builder(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether this instance is valid.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1885,6 +1885,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
|
|||
return false;
|
||||
}
|
||||
|
||||
ColorInfo inputColorInfo;
|
||||
ColorInfo outputColorInfo;
|
||||
if (inputFormat.colorInfo != null) {
|
||||
inputColorInfo = inputFormat.colorInfo;
|
||||
outputColorInfo =
|
||||
inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG
|
||||
// SurfaceView only supports BT2020 PQ input, converting HLG to PQ.
|
||||
? inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build()
|
||||
: inputColorInfo;
|
||||
} else {
|
||||
inputColorInfo = ColorInfo.SDR_BT709_LIMITED;
|
||||
outputColorInfo = ColorInfo.SDR_BT709_LIMITED;
|
||||
}
|
||||
|
||||
// Playback thread handler.
|
||||
handler = Util.createHandlerForCurrentLooper();
|
||||
try {
|
||||
|
|
@ -1895,10 +1909,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
|
|||
renderer.context,
|
||||
checkNotNull(videoEffects),
|
||||
DebugViewProvider.NONE,
|
||||
inputFormat.colorInfo != null
|
||||
? inputFormat.colorInfo
|
||||
: ColorInfo.SDR_BT709_LIMITED,
|
||||
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
|
||||
inputColorInfo,
|
||||
outputColorInfo,
|
||||
/* releaseFramesAutomatically= */ false,
|
||||
/* executor= */ handler::post,
|
||||
new FrameProcessor.Listener() {
|
||||
|
|
|
|||
|
|
@ -340,15 +340,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
|
||||
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
|
||||
if (outputEglSurface == null) {
|
||||
boolean outputTransferIsHdr = ColorInfo.isTransferHdr(outputColorInfo);
|
||||
|
||||
outputEglSurface =
|
||||
GlUtil.createEglSurface(
|
||||
eglDisplay,
|
||||
outputSurfaceInfo.surface,
|
||||
outputTransferIsHdr
|
||||
? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
|
||||
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888);
|
||||
outputColorInfo.colorTransfer,
|
||||
// Frames are only released automatically when outputting to an encoder.
|
||||
/* isEncoderInputSurface= */ releaseFramesAutomatically);
|
||||
|
||||
@Nullable
|
||||
SurfaceView debugSurfaceView =
|
||||
|
|
@ -356,7 +354,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
outputSurfaceInfo.width, outputSurfaceInfo.height);
|
||||
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
|
||||
debugSurfaceViewWrapper =
|
||||
new SurfaceViewWrapper(eglDisplay, eglContext, outputTransferIsHdr, debugSurfaceView);
|
||||
new SurfaceViewWrapper(
|
||||
eglDisplay, eglContext, ColorInfo.isTransferHdr(outputColorInfo), debugSurfaceView);
|
||||
}
|
||||
this.debugSurfaceView = debugSurfaceView;
|
||||
}
|
||||
|
|
@ -474,13 +473,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||
}
|
||||
|
||||
if (eglSurface == null) {
|
||||
// Screen output supports only BT.2020 PQ (ST2084).
|
||||
eglSurface =
|
||||
GlUtil.createEglSurface(
|
||||
eglDisplay,
|
||||
surface,
|
||||
useHdr
|
||||
? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102
|
||||
: GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888);
|
||||
useHdr ? C.COLOR_TRANSFER_ST2084 : C.COLOR_TRANSFER_SDR,
|
||||
/* isEncoderInputSurface= */ false);
|
||||
}
|
||||
EGLSurface eglSurface = this.eglSurface;
|
||||
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
|
||||
|
|
|
|||
|
|
@ -62,6 +62,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
|
|||
*
|
||||
* <p>Using HDR {@code inputColorInfo} or {@code outputColorInfo} requires OpenGL ES 3.0.
|
||||
*
|
||||
* <p>If outputting HDR content to a display, {@code EGL_GL_COLORSPACE_BT2020_PQ_EXT} is
|
||||
* required, and {@link ColorInfo#colorTransfer outputColorInfo.colorTransfer} must be {@link
|
||||
* C#COLOR_TRANSFER_ST2084}.
|
||||
*
|
||||
* <p>Pass a {@link MoreExecutors#directExecutor() direct listenerExecutor} if invoking the
|
||||
* {@code listener} on {@link GlEffectsFrameProcessor}'s internal thread is desired.
|
||||
*/
|
||||
|
|
@ -161,6 +165,19 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
|
|||
EGLContext eglContext = GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes);
|
||||
GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes);
|
||||
|
||||
// Not releaseFramesAutomatically means outputting to a display surface. HDR display surfaces
|
||||
// require the BT2020 PQ GL extension.
|
||||
if (!releaseFramesAutomatically && ColorInfo.isTransferHdr(outputColorInfo)) {
|
||||
// Display hardware supports PQ only.
|
||||
checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_ST2084);
|
||||
if (Util.SDK_INT < 33 || !GlUtil.isBt2020PqExtensionSupported()) {
|
||||
GlUtil.destroyEglContext(eglDisplay, eglContext);
|
||||
// On API<33, the system cannot display PQ content correctly regardless of whether BT2020 PQ
|
||||
// GL extension is supported.
|
||||
throw new FrameProcessingException("BT.2020 PQ OpenGL output isn't supported.");
|
||||
}
|
||||
}
|
||||
|
||||
ImmutableList<GlTextureProcessor> textureProcessors =
|
||||
getGlTextureProcessorsForGlEffects(
|
||||
context,
|
||||
|
|
|
|||
Loading…
Reference in a new issue