sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-03-25 09:25:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make OverlaySettings dynamic

Browse source 
PiperOrigin-RevId: 687269731
This commit is contained in:
claincly 2024-10-18 05:40:19 -07:00 committed by Copybara-Service
parent 709246ac6a
commit 2a49ffcb23
19 changed files with 363 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
RELEASENOTES.md
View file

@ -154,6 +154,11 @@ This release includes the following changes since the
* Fix `IllegalStateException` from
`DefaultDrmSession.requiresSecureDecoder` after opening a DRM session
failed. This issue was introduced in `1.5.0-alpha01`.
* Effect:
* Moved the functionality of `OverlaySettings` into
`StaticOverlaySettings`. `OverlaySettings` can be subclassed to allow
dynamic overlay settings.
* Muxers:
* IMA extension:
* Fix bug where server-side inserted DAI streams without a preroll can
result in an `ArrayIndexOutOfBoundsException` when playing past the last

3
demos/transformer/src/main/java/androidx/media3/demo/transformer/ClockOverlay.java
View file

@ -25,6 +25,7 @@ import android.graphics.PorterDuff;
import android.graphics.RectF;
import androidx.media3.effect.CanvasOverlay;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.StaticOverlaySettings;
/* package */ final class ClockOverlay extends CanvasOverlay {
private static final int CLOCK_COLOR = Color.WHITE;
@ -96,7 +97,7 @@ import androidx.media3.effect.OverlaySettings;
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return new OverlaySettings.Builder()
return new StaticOverlaySettings.Builder()
.setBackgroundFrameAnchor(
BOTTOM_RIGHT_ANCHOR_X - ANCHOR_INSET_X, BOTTOM_RIGHT_ANCHOR_Y - ANCHOR_INSET_Y)
.setOverlayFrameAnchor(BOTTOM_RIGHT_ANCHOR_X, BOTTOM_RIGHT_ANCHOR_Y)

5
demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java
View file

@ -21,6 +21,7 @@ import android.text.SpannableString;
import android.text.style.ForegroundColorSpan;
import androidx.media3.common.C;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.TextureOverlay;
import java.util.Locale;
@ -31,11 +32,11 @@ import java.util.Locale;
*/
/* package */ final class TimerOverlay extends TextOverlay {
private final OverlaySettings overlaySettings;
private final StaticOverlaySettings overlaySettings;
public TimerOverlay() {
overlaySettings =
new OverlaySettings.Builder()
new StaticOverlaySettings.Builder()
// Place the timer in the bottom left corner of the screen with some padding from the
// edges.
.setOverlayFrameAnchor(/* x= */ -1f, /* y= */ -1f)

14
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View file

@ -71,13 +71,13 @@ import androidx.media3.effect.GlShaderProgram;
import androidx.media3.effect.HslAdjustment;
import androidx.media3.effect.LanczosResample;
import androidx.media3.effect.OverlayEffect;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbAdjustment;
import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.RgbMatrix;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.SingleColorLut;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.TextureOverlay;
import androidx.media3.exoplayer.DefaultLoadControl;
@ -599,8 +599,8 @@ public final class TransformerActivity extends AppCompatActivity {
private OverlayEffect createOverlayEffectFromBundle(Bundle bundle, boolean[] selectedEffects) {
ImmutableList.Builder<TextureOverlay> overlaysBuilder = new ImmutableList.Builder<>();
if (selectedEffects[ConfigurationActivity.OVERLAY_LOGO_AND_TIMER_INDEX]) {
OverlaySettings logoSettings =
new OverlaySettings.Builder()
StaticOverlaySettings logoSettings =
new StaticOverlaySettings.Builder()
// Place the logo in the bottom left corner of the screen with some padding from the
// edges.
.setOverlayFrameAnchor(/* x= */ -1f, /* y= */ -1f)
@ -619,8 +619,8 @@ public final class TransformerActivity extends AppCompatActivity {
overlaysBuilder.add(logoOverlay, timerOverlay);
}
if (selectedEffects[ConfigurationActivity.BITMAP_OVERLAY_INDEX]) {
OverlaySettings overlaySettings =
new OverlaySettings.Builder()
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder()
.setAlphaScale(
bundle.getFloat(
ConfigurationActivity.BITMAP_OVERLAY_ALPHA, /* defaultValue= */ 1))
@ -633,8 +633,8 @@ public final class TransformerActivity extends AppCompatActivity {
overlaysBuilder.add(bitmapOverlay);
}
if (selectedEffects[ConfigurationActivity.TEXT_OVERLAY_INDEX]) {
OverlaySettings overlaySettings =
new OverlaySettings.Builder()
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder()
.setAlphaScale(
bundle.getFloat(ConfigurationActivity.TEXT_OVERLAY_ALPHA, /* defaultValue= */ 1))
.build();

44
libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayShaderProgramPixelTest.java
View file

@ -169,8 +169,8 @@ public class OverlayShaderProgramPixelTest {
public void drawFrame_anchoredAndTranslatedBitmapOverlay_blendsBitmapIntoTopLeftOfFrame()
throws Exception {
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings =
new OverlaySettings.Builder()
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder()
.setOverlayFrameAnchor(/* x= */ -1f, /* y= */ 1f)
.setBackgroundFrameAnchor(/* x= */ -1f, /* y= */ 1f)
.build();
@ -198,8 +198,10 @@ public class OverlayShaderProgramPixelTest {
drawFrame_overlayFrameAnchoredOnlyBitmapOverlay_anchorsOverlayFromTopLeftCornerOfFrame()
throws Exception {
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings =
new OverlaySettings.Builder().setOverlayFrameAnchor(/* x= */ -1f, /* y= */ 1f).build();
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder()
.setOverlayFrameAnchor(/* x= */ -1f, /* y= */ 1f)
.build();
BitmapOverlay staticBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayShaderProgram =
@ -222,7 +224,8 @@ public class OverlayShaderProgramPixelTest {
@Test
public void drawFrame_rotatedBitmapOverlay_blendsBitmapRotated90degrees() throws Exception {
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings = new OverlaySettings.Builder().setRotationDegrees(90f).build();
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder().setRotationDegrees(90f).build();
BitmapOverlay staticBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayShaderProgram =
@ -245,7 +248,8 @@ public class OverlayShaderProgramPixelTest {
@Test
public void drawFrame_translucentBitmapOverlay_blendsBitmapIntoFrame() throws Exception {
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings = new OverlaySettings.Builder().setAlphaScale(0.5f).build();
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder().setAlphaScale(0.5f).build();
BitmapOverlay translucentBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings);
overlayShaderProgram =
@ -268,7 +272,8 @@ public class OverlayShaderProgramPixelTest {
@Test
public void drawFrame_transparentTextOverlay_blendsBitmapIntoFrame() throws Exception {
SpannableString overlayText = new SpannableString(/* source= */ "Text styling");
OverlaySettings overlaySettings = new OverlaySettings.Builder().setAlphaScale(0f).build();
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder().setAlphaScale(0f).build();
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
/* start= */ 0,
@ -354,8 +359,8 @@ public class OverlayShaderProgramPixelTest {
/* start= */ 0,
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings =
new OverlaySettings.Builder().setBackgroundFrameAnchor(0.5f, 0.5f).build();
StaticOverlaySettings overlaySettings =
new StaticOverlaySettings.Builder().setBackgroundFrameAnchor(0.5f, 0.5f).build();
TextOverlay staticTextOverlay =
TextOverlay.createStaticTextOverlay(overlayText, overlaySettings);
overlayShaderProgram =
@ -383,11 +388,12 @@ public class OverlayShaderProgramPixelTest {
/* start= */ 0,
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setBackgroundFrameAnchor(0.5f, 0.5f).build();
StaticOverlaySettings overlaySettings1 =
new StaticOverlaySettings.Builder().setBackgroundFrameAnchor(0.5f, 0.5f).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlphaScale(0.5f).build();
StaticOverlaySettings overlaySettings2 =
new StaticOverlaySettings.Builder().setAlphaScale(0.5f).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayShaderProgram =
new OverlayEffect(ImmutableList.of(textOverlay, bitmapOverlay))
@ -414,12 +420,12 @@ public class OverlayShaderProgramPixelTest {
/* start= */ 0,
/* end= */ overlayText.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setScale(/* x= */ 0.5f, /* y= */ 0.5f).build();
StaticOverlaySettings overlaySettings1 =
new StaticOverlaySettings.Builder().setScale(/* x= */ 0.5f, /* y= */ 0.5f).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings2 =
new OverlaySettings.Builder().setScale(/* x= */ 3, /* y= */ 3).build();
StaticOverlaySettings overlaySettings2 =
new StaticOverlaySettings.Builder().setScale(/* x= */ 3, /* y= */ 3).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayShaderProgram =
@ -461,14 +467,14 @@ public class OverlayShaderProgramPixelTest {
private static final class LetterBoxStretchedBitmapOverlay extends BitmapOverlay {
private final OverlaySettings.Builder overlaySettingsBuilder;
private final StaticOverlaySettings.Builder overlaySettingsBuilder;
private final Bitmap overlayBitmap;
private @MonotonicNonNull OverlaySettings overlaySettings;
private @MonotonicNonNull StaticOverlaySettings overlaySettings;
public LetterBoxStretchedBitmapOverlay(Bitmap overlayBitmap) {
this.overlayBitmap = overlayBitmap;
overlaySettingsBuilder = new OverlaySettings.Builder();
overlaySettingsBuilder = new StaticOverlaySettings.Builder();
}
@Override

10
libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java
View file

@ -79,7 +79,7 @@ public abstract class BitmapOverlay extends TextureOverlay {
* the frames.
*/
public static BitmapOverlay createStaticBitmapOverlay(
Bitmap overlayBitmap, OverlaySettings overlaySettings) {
Bitmap overlayBitmap, StaticOverlaySettings overlaySettings) {
return new BitmapOverlay() {
@Override
public Bitmap getBitmap(long presentationTimeUs) {
@ -95,15 +95,15 @@ public abstract class BitmapOverlay extends TextureOverlay {
/**
* Creates a {@link BitmapOverlay} that shows the input at {@code overlayBitmapUri} with the same
* {@link OverlaySettings} throughout the whole video.
* {@link StaticOverlaySettings} throughout the whole video.
*
* @param context The {@link Context}.
* @param overlayBitmapUri The {@link Uri} pointing to the resource to be converted into a bitmap.
* @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on
* the frames.
* @param overlaySettings The {@link StaticOverlaySettings} configuring how the overlay is
* displayed on the frames.
*/
public static BitmapOverlay createStaticBitmapOverlay(
Context context, Uri overlayBitmapUri, OverlaySettings overlaySettings) {
Context context, Uri overlayBitmapUri, StaticOverlaySettings overlaySettings) {
return new BitmapOverlay() {
private @MonotonicNonNull Bitmap lastBitmap;

2
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java
View file

@ -524,7 +524,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
/* overlaySize= */ new Size(inputTexture.width, inputTexture.height),
inputFrameInfo.overlaySettings);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale);
glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.getAlphaScale());
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.

8
libraries/effect/src/main/java/androidx/media3/effect/DrawableOverlay.java
View file

@ -71,14 +71,14 @@ public abstract class DrawableOverlay extends BitmapOverlay {
/**
* Creates a {@link DrawableOverlay} that shows the {@link Drawable} with the same {@link
* OverlaySettings} throughout the whole video.
* StaticOverlaySettings} throughout the whole video.
*
* @param drawable The {@link Drawable} to be displayed.
* @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on
* the frames.
* @param overlaySettings The {@link StaticOverlaySettings} configuring how the overlay is
* displayed on the frames.
*/
public static DrawableOverlay createStaticDrawableOverlay(
Drawable drawable, OverlaySettings overlaySettings) {
Drawable drawable, StaticOverlaySettings overlaySettings) {
return new DrawableOverlay() {
@Override
public Drawable getDrawable(long presentationTimeUs) {

8
libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java
View file

@ -62,7 +62,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
reset();
// Anchor point of overlay within output frame.
Pair<Float, Float> backgroundFrameAnchor = overlaySettings.backgroundFrameAnchor;
Pair<Float, Float> backgroundFrameAnchor = overlaySettings.getBackgroundFrameAnchor();
Matrix.translateM(
backgroundFrameAnchorMatrix,
MATRIX_OFFSET,
@ -79,13 +79,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* z= */ 1f);
// Scale the image.
Pair<Float, Float> scale = overlaySettings.scale;
Pair<Float, Float> scale = overlaySettings.getScale();
Matrix.scaleM(scaleMatrix, MATRIX_OFFSET, scale.first, scale.second, /* z= */ 1f);
Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET);
// Translate the overlay within its frame. To position the overlay frame's anchor at the correct
// position, it must be translated the opposite direction by the same magnitude.
Pair<Float, Float> overlayFrameAnchor = overlaySettings.overlayFrameAnchor;
Pair<Float, Float> overlayFrameAnchor = overlaySettings.getOverlayFrameAnchor();
Matrix.translateM(
overlayFrameAnchorMatrix,
MATRIX_OFFSET,
@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Matrix.rotateM(
rotateMatrix,
MATRIX_OFFSET,
overlaySettings.rotationDegrees,
overlaySettings.getRotationDegrees(),
/* x= */ 0f,
/* y= */ 0f,
/* z= */ 1f);

278
libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java
View file

@ -1,7 +1,5 @@
package androidx.media3.effect;
/*
* Copyright 2022 The Android Open Source Project
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,199 +13,121 @@ package androidx.media3.effect;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static androidx.media3.common.util.Assertions.checkArgument;
package androidx.media3.effect;
import android.util.Pair;
import androidx.annotation.FloatRange;
import androidx.media3.common.util.UnstableApi;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
/**
* Contains information to control how an input texture (for example, a {@link VideoCompositor} or
* {@link TextureOverlay}) is displayed on a background.
* Provides information of how an input texture (for example, a {@link TextureOverlay} or in {@link
* VideoCompositor}) is presented.
*/
@UnstableApi
public final class OverlaySettings {
public interface OverlaySettings {
/** A builder for {@link OverlaySettings} instances. */
public static final class Builder {
private float alphaScale;
private Pair<Float, Float> backgroundFrameAnchor;
private Pair<Float, Float> overlayFrameAnchor;
private Pair<Float, Float> scale;
private float rotationDegrees;
private float hdrLuminanceMultiplier;
/** The default alpha scale value of the overlay. */
float DEFAULT_ALPHA_SCALE = 1f;
/** Creates a new {@link Builder}. */
public Builder() {
alphaScale = 1f;
backgroundFrameAnchor = Pair.create(0f, 0f);
overlayFrameAnchor = Pair.create(0f, 0f);
scale = Pair.create(1f, 1f);
rotationDegrees = 0f;
hdrLuminanceMultiplier = 1f;
}
/** The default coordinates for the anchor point of the overlay within the background frame. */
Pair<Float, Float> DEFAULT_BACKGROUND_FRAME_ANCHOR = Pair.create(0f, 0f);
private Builder(OverlaySettings overlaySettings) {
this.alphaScale = overlaySettings.alphaScale;
this.backgroundFrameAnchor = overlaySettings.backgroundFrameAnchor;
this.overlayFrameAnchor = overlaySettings.overlayFrameAnchor;
this.scale = overlaySettings.scale;
this.rotationDegrees = overlaySettings.rotationDegrees;
}
/** The default coordinates for the anchor point of the overlay frame. */
Pair<Float, Float> DEFAULT_OVERLAY_FRAME_ANCHOR = Pair.create(0f, 0f);
/**
* Sets the alpha scale value of the overlay, altering its translucency.
*
* <p>An {@code alphaScale} value of {@code 1} means no change is applied. A value below {@code
* 1} increases translucency, and a value above {@code 1} reduces translucency.
*
* <p>Set to always return {@code 1} by default.
*/
@CanIgnoreReturnValue
public Builder setAlphaScale(@FloatRange(from = 0) float alphaScale) {
checkArgument(0 <= alphaScale, "alphaScale needs to be greater than or equal to zero.");
this.alphaScale = alphaScale;
return this;
}
/** The default scaling of the overlay. */
Pair<Float, Float> DEFAULT_SCALE = Pair.create(1f, 1f);
/**
* Sets the coordinates for the anchor point of the overlay within the background frame.
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* background frame. The default value is {@code (0,0)}, the center of the background frame.
*
* <p>The overlay's {@linkplain #setOverlayFrameAnchor(float, float) anchor point} will be
* positioned at the anchor point set in this method. For example, setting a value of {@code
* (+1,+1)} will move the {@linkplain #setOverlayFrameAnchor overlay's anchor} to the top right
* corner. That is, if the overlay's anchor is at {@code (+1,+1)} (the top right corner), the
* overlay's top right corner will be aligned with that of the background frame; whereas if the
* overlay's anchor is at {@code (0,0)} (the center), the overlay's center will be positioned at
* the top right corner of the background frame.
*
* @param x The NDC x-coordinate in the range [-1, 1].
* @param y The NDC y-coordinate in the range [-1, 1].
*/
@CanIgnoreReturnValue
public Builder setBackgroundFrameAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.backgroundFrameAnchor = Pair.create(x, y);
return this;
}
/** The default rotation of the overlay, counter-clockwise. */
float DEFAULT_ROTATION_DEGREES = 0f;
/**
* Sets the coordinates for the anchor point within the overlay.
*
* <p>The anchor point is the point inside the overlay that is placed on the {@linkplain
* #setBackgroundFrameAnchor background frame anchor}
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* overlay. The default value is {@code (0,0)}, the center of the overlay.
*
* <p>See {@link #setBackgroundFrameAnchor} for examples of how to position an overlay.
*
* @param x The NDC x-coordinate in the range [-1, 1].
* @param y The NDC y-coordinate in the range [-1, 1].
*/
@CanIgnoreReturnValue
public Builder setOverlayFrameAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.overlayFrameAnchor = Pair.create(x, y);
return this;
}
/** The default luminance multiplier of an SDR overlay when overlaid on a HDR frame. */
float DEFAULT_HDR_LUMINANCE_MULTIPLIER = 1f;
/**
* Sets the scaling of the overlay.
*
* @param x The desired scaling in the x axis of the overlay.
* @param y The desired scaling in the y axis of the overlay.
*/
@CanIgnoreReturnValue
public Builder setScale(float x, float y) {
this.scale = Pair.create(x, y);
return this;
}
/**
* Sets the rotation of the overlay, counter-clockwise.
*
* <p>The overlay is rotated at the center of its frame.
*
* @param rotationDegree The desired degrees of rotation, counter-clockwise.
*/
@CanIgnoreReturnValue
public Builder setRotationDegrees(float rotationDegree) {
this.rotationDegrees = rotationDegree;
return this;
}
/**
* Set the luminance multiplier of an SDR overlay when overlaid on a HDR frame.
*
* <p>Scales the luminance of the overlay to adjust the output brightness of the overlay on the
* frame. The default value is 1, which scales the overlay colors into the standard HDR
* luminance within the processing pipeline. Use 0.5 to scale the luminance of the overlay to
* SDR range, so that no extra luminance is added.
*
* <p>Currently only supported on text overlays
*/
@CanIgnoreReturnValue
public Builder setHdrLuminanceMultiplier(float hdrLuminanceMultiplier) {
this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
return this;
}
/** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
public OverlaySettings build() {
return new OverlaySettings(
alphaScale,
backgroundFrameAnchor,
overlayFrameAnchor,
scale,
rotationDegrees,
hdrLuminanceMultiplier);
}
/**
* Returns the alpha scale value of the overlay, altering its translucency.
*
* <p>An {@code alphaScale} value of {@code 1} means no change is applied. A value below {@code 1}
* increases translucency, and a value above {@code 1} reduces translucency.
*
* <p>The default value is {@link #DEFAULT_ALPHA_SCALE}.
*/
default float getAlphaScale() {
return DEFAULT_ALPHA_SCALE;
}
/** The alpha scale value of the overlay, altering its translucency. */
public final float alphaScale;
/** The coordinates for the anchor point of the overlay within the background frame. */
public final Pair<Float, Float> backgroundFrameAnchor;
/** The coordinates for the anchor point of the overlay frame. */
public final Pair<Float, Float> overlayFrameAnchor;
/** The scaling of the overlay. */
public final Pair<Float, Float> scale;
/** The rotation of the overlay, counter-clockwise. */
public final float rotationDegrees;
/** The luminance multiplier of an SDR overlay when overlaid on a HDR frame. */
public final float hdrLuminanceMultiplier;
private OverlaySettings(
float alphaScale,
Pair<Float, Float> backgroundFrameAnchor,
Pair<Float, Float> overlayFrameAnchor,
Pair<Float, Float> scale,
float rotationDegrees,
float hdrLuminanceMultiplier) {
this.alphaScale = alphaScale;
this.backgroundFrameAnchor = backgroundFrameAnchor;
this.overlayFrameAnchor = overlayFrameAnchor;
this.scale = scale;
this.rotationDegrees = rotationDegrees;
this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
/**
* Returns the coordinates for the anchor point of the overlay within the background frame.
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* background frame. The ranges for x and y are from {@code -1} to {@code 1}. The default value is
* {@code (0,0)}, the center of the background frame.
*
* <p>The overlay's {@linkplain #getOverlayFrameAnchor anchor point} will be positioned at the
* anchor point returned from this method. For example, a value of {@code (1,1)} will move the
* {@linkplain #getOverlayFrameAnchor overlay's anchor} to the top right corner. That is, if the
* overlay's anchor is at {@code (1,1)} (the top right corner), the overlay's top right corner
* will be aligned with that of the background frame; whereas if the overlay's anchor is at {@code
* (0,0)} (the center), the overlay's center will be positioned at the top right corner of the
* background frame.
*
* <p>The default value is {@link #DEFAULT_BACKGROUND_FRAME_ANCHOR}.
*/
default Pair<Float, Float> getBackgroundFrameAnchor() {
return DEFAULT_BACKGROUND_FRAME_ANCHOR;
}
/** Returns a new {@link Builder} initialized with the values of this instance. */
/* package */ Builder buildUpon() {
return new Builder(this);
/**
* Returns the coordinates for the anchor point within the overlay.
*
* <p>The anchor point is the point inside the overlay that is placed on the {@linkplain
* #getBackgroundFrameAnchor background frame anchor}
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs) relative to the
* overlay. The ranges for x and y are from {@code -1} to {@code 1}. The default value is {@code
* (0,0)}, the center of the overlay.
*
* <p>See {@link #getBackgroundFrameAnchor} for examples of how to position an overlay.
*
* <p>The default value is {@link #DEFAULT_OVERLAY_FRAME_ANCHOR}.
*/
default Pair<Float, Float> getOverlayFrameAnchor() {
return DEFAULT_OVERLAY_FRAME_ANCHOR;
}
/**
* Returns the scaling of the overlay.
*
* <p>The default value is {@link #DEFAULT_SCALE}.
*/
default Pair<Float, Float> getScale() {
return DEFAULT_SCALE;
}
/**
* Returns the rotation of the overlay, counter-clockwise.
*
* <p>The overlay is rotated at the center of its frame.
*
* <p>The default value is {@link #DEFAULT_ROTATION_DEGREES}.
*/
default float getRotationDegrees() {
return DEFAULT_ROTATION_DEGREES;
}
/**
* Returns the luminance multiplier of an SDR overlay when overlaid on a HDR frame.
*
* <p>Scales the luminance of the overlay to adjust the output brightness of the overlay on the
* frame. The default value is 1, which scales the overlay colors into the standard HDR luminance
* within the processing pipeline. Use 0.5 to scale the luminance of the overlay to SDR range, so
* that no extra luminance is added.
*
* <p>Currently only supported on text overlays
*
* <p>The default value is {@link #DEFAULT_HDR_LUMINANCE_MULTIPLIER}.
*/
default float getHdrLuminanceMultiplier() {
return DEFAULT_HDR_LUMINANCE_MULTIPLIER;
}
}

4
libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java
View file

@ -149,7 +149,7 @@ import java.io.IOException;
} else if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_TEXT) {
float[] luminanceMatrix = GlUtil.create4x4IdentityMatrix();
float multiplier =
overlay.getOverlaySettings(presentationTimeUs).hdrLuminanceMultiplier;
overlay.getOverlaySettings(presentationTimeUs).getHdrLuminanceMultiplier();
Matrix.scaleM(luminanceMatrix, /* mOffset= */ 0, multiplier, multiplier, multiplier);
glProgram.setFloatsUniform(
formatInvariant("uLuminanceMatrix%d", texUnitIndex), luminanceMatrix);
@ -169,7 +169,7 @@ import java.io.IOException;
formatInvariant("uTransformationMatrix%d", texUnitIndex),
samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
glProgram.setFloatUniform(
formatInvariant("uOverlayAlphaScale%d", texUnitIndex), overlaySettings.alphaScale);
formatInvariant("uOverlayAlphaScale%d", texUnitIndex), overlaySettings.getAlphaScale());
}
glProgram.setSamplerTexIdUniform("uVideoTexSampler0", inputTexId, /* texUnitIndex= */ 0);

184
libraries/effect/src/main/java/androidx/media3/effect/StaticOverlaySettings.java Normal file
View file

@ -0,0 +1,184 @@
package androidx.media3.effect;
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static androidx.media3.common.util.Assertions.checkArgument;
import android.util.Pair;
import androidx.annotation.FloatRange;
import androidx.media3.common.util.UnstableApi;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
/** A statically valued {@link OverlaySettings}. */
@UnstableApi
public final class StaticOverlaySettings implements OverlaySettings {
/** A builder for {@link StaticOverlaySettings} instances. */
public static final class Builder {
private float alphaScale;
private Pair<Float, Float> backgroundFrameAnchor;
private Pair<Float, Float> overlayFrameAnchor;
private Pair<Float, Float> scale;
private float rotationDegrees;
private float hdrLuminanceMultiplier;
/** Creates a new {@link Builder}. */
public Builder() {
alphaScale = DEFAULT_ALPHA_SCALE;
backgroundFrameAnchor = DEFAULT_BACKGROUND_FRAME_ANCHOR;
overlayFrameAnchor = DEFAULT_OVERLAY_FRAME_ANCHOR;
scale = DEFAULT_SCALE;
rotationDegrees = DEFAULT_ROTATION_DEGREES;
hdrLuminanceMultiplier = DEFAULT_HDR_LUMINANCE_MULTIPLIER;
}
/**
* Sets the alpha scale value of the overlay, altering its translucency.
*
* @see OverlaySettings#getAlphaScale()
*/
@CanIgnoreReturnValue
public Builder setAlphaScale(@FloatRange(from = 0) float alphaScale) {
checkArgument(0 <= alphaScale, "alphaScale needs to be greater than or equal to zero.");
this.alphaScale = alphaScale;
return this;
}
/**
* Sets the coordinates for the anchor point of the overlay within the background frame.
*
* @see OverlaySettings#getBackgroundFrameAnchor()
*/
@CanIgnoreReturnValue
public Builder setBackgroundFrameAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.backgroundFrameAnchor = Pair.create(x, y);
return this;
}
/**
* Sets the coordinates for the anchor point within the overlay.
*
* @see OverlaySettings#getOverlayFrameAnchor()
*/
@CanIgnoreReturnValue
public Builder setOverlayFrameAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(-1 <= x && x <= 1);
checkArgument(-1 <= y && y <= 1);
this.overlayFrameAnchor = Pair.create(x, y);
return this;
}
/**
* Sets the scaling of the overlay.
*
* @see OverlaySettings#getScale()
*/
@CanIgnoreReturnValue
public Builder setScale(float x, float y) {
this.scale = Pair.create(x, y);
return this;
}
/**
* Sets the rotation of the overlay, counter-clockwise.
*
* @see OverlaySettings#getRotationDegrees()
*/
@CanIgnoreReturnValue
public Builder setRotationDegrees(float rotationDegree) {
this.rotationDegrees = rotationDegree;
return this;
}
/**
* Set the luminance multiplier of an SDR overlay when overlaid on a HDR frame.
*
* @see OverlaySettings#getHdrLuminanceMultiplier()
*/
@CanIgnoreReturnValue
public Builder setHdrLuminanceMultiplier(float hdrLuminanceMultiplier) {
this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
return this;
}
/** Creates an instance of {@link StaticOverlaySettings}, using defaults if values are unset. */
public StaticOverlaySettings build() {
return new StaticOverlaySettings(
alphaScale,
backgroundFrameAnchor,
overlayFrameAnchor,
scale,
rotationDegrees,
hdrLuminanceMultiplier);
}
}
private final float alphaScale;
private final Pair<Float, Float> backgroundFrameAnchor;
private final Pair<Float, Float> overlayFrameAnchor;
private final Pair<Float, Float> scale;
private final float rotationDegrees;
private final float hdrLuminanceMultiplier;
private StaticOverlaySettings(
float alphaScale,
Pair<Float, Float> backgroundFrameAnchor,
Pair<Float, Float> overlayFrameAnchor,
Pair<Float, Float> scale,
float rotationDegrees,
float hdrLuminanceMultiplier) {
this.alphaScale = alphaScale;
this.backgroundFrameAnchor = backgroundFrameAnchor;
this.overlayFrameAnchor = overlayFrameAnchor;
this.scale = scale;
this.rotationDegrees = rotationDegrees;
this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
}
@Override
public float getAlphaScale() {
return alphaScale;
}
@Override
public Pair<Float, Float> getBackgroundFrameAnchor() {
return backgroundFrameAnchor;
}
@Override
public Pair<Float, Float> getOverlayFrameAnchor() {
return overlayFrameAnchor;
}
@Override
public Pair<Float, Float> getScale() {
return scale;
}
@Override
public float getRotationDegrees() {
return rotationDegrees;
}
@Override
public float getHdrLuminanceMultiplier() {
return hdrLuminanceMultiplier;
}
}

10
libraries/effect/src/main/java/androidx/media3/effect/TextOverlay.java
View file

@ -43,7 +43,7 @@ public abstract class TextOverlay extends BitmapOverlay {
/**
* Creates a {@link TextOverlay} that shows the {@code overlayText} with the same default settings
* in {@link OverlaySettings} throughout the whole video.
* in {@link StaticOverlaySettings} throughout the whole video.
*/
public static TextOverlay createStaticTextOverlay(SpannableString overlayText) {
return new TextOverlay() {
@ -56,14 +56,14 @@ public abstract class TextOverlay extends BitmapOverlay {
/**
* Creates a {@link TextOverlay} that shows the {@code overlayText} with the same {@link
* OverlaySettings} throughout the whole video.
* StaticOverlaySettings} throughout the whole video.
*
* @param overlayText The text to overlay on the video.
* @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on
* the frames.
* @param overlaySettings The {@link StaticOverlaySettings} configuring how the overlay is
* displayed on the frames.
*/
public static TextOverlay createStaticTextOverlay(
SpannableString overlayText, OverlaySettings overlaySettings) {
SpannableString overlayText, StaticOverlaySettings overlaySettings) {
return new TextOverlay() {
@Override
public SpannableString getText(long presentationTimeUs) {

2
libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java
View file

@ -73,7 +73,7 @@ public abstract class TextureOverlay {
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
*/
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return new OverlaySettings.Builder().build();
return new OverlaySettings() {};
}
/**

2
libraries/effect/src/main/java/androidx/media3/effect/VideoCompositorSettings.java
View file

@ -45,7 +45,7 @@ public interface VideoCompositorSettings {
*/
@Override
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
return new OverlaySettings.Builder().build();
return new OverlaySettings() {};
}
};

11
libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
View file

@ -58,6 +58,7 @@ import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.VideoCompositor;
import androidx.media3.effect.VideoCompositorSettings;
@ -536,7 +537,7 @@ public final class DefaultVideoCompositorPixelTest {
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
if (inputId == 0) {
// This tests all OverlaySettings builder variables.
return new OverlaySettings.Builder()
return new StaticOverlaySettings.Builder()
.setScale(.25f, .5f)
.setOverlayFrameAnchor(1, -1)
.setBackgroundFrameAnchor(.9f, -.7f)
@ -544,7 +545,7 @@ public final class DefaultVideoCompositorPixelTest {
.setAlphaScale(.5f)
.build();
} else {
return new OverlaySettings.Builder().build();
return new StaticOverlaySettings.Builder().build();
}
}
};
@ -575,7 +576,7 @@ public final class DefaultVideoCompositorPixelTest {
@Override
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
return new OverlaySettings.Builder().build();
return new StaticOverlaySettings.Builder().build();
}
};
compositorTestRunner =
@ -617,7 +618,7 @@ public final class DefaultVideoCompositorPixelTest {
@Override
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
return new OverlaySettings.Builder()
return new StaticOverlaySettings.Builder()
.setOverlayFrameAnchor(-1, -1)
.setBackgroundFrameAnchor(-1, -1 + 2f * inputId / NUMBER_OF_INPUT_STREAMS)
.build();
@ -950,7 +951,7 @@ public final class DefaultVideoCompositorPixelTest {
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return new OverlaySettings.Builder()
return new StaticOverlaySettings.Builder()
.setBackgroundFrameAnchor(/* x= */ 0f, /* y= */ 0.5f)
.build();
}

5
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java
View file

@ -42,6 +42,7 @@ import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.test.core.app.ApplicationProvider;
import com.google.common.collect.ImmutableList;
@ -190,13 +191,13 @@ public final class TransformerMultiSequenceCompositionTest {
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
if (inputId == 0) {
// This tests all OverlaySettings builder variables.
return new OverlaySettings.Builder()
return new StaticOverlaySettings.Builder()
.setScale(.25f, .25f)
.setOverlayFrameAnchor(1, -1)
.setBackgroundFrameAnchor(.9f, -.7f)
.build();
} else {
return new OverlaySettings.Builder().build();
return new StaticOverlaySettings.Builder().build();
}
}
};

7
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java
View file

@ -56,7 +56,7 @@ import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.effect.GaussianBlur;
import androidx.media3.effect.GlTextureProducer;
import androidx.media3.effect.OverlayEffect;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.media3.test.utils.TextureBitmapReader;
@ -304,7 +304,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
/* end= */ 7,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
TextOverlay textOverlay =
TextOverlay.createStaticTextOverlay(overlayText, new OverlaySettings.Builder().build());
TextOverlay.createStaticTextOverlay(
overlayText, new StaticOverlaySettings.Builder().build());
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay, textOverlay)))
@ -432,7 +433,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
TextOverlay textOverlay =
TextOverlay.createStaticTextOverlay(
overlayText, new OverlaySettings.Builder().setHdrLuminanceMultiplier(3f).build());
overlayText, new StaticOverlaySettings.Builder().setHdrLuminanceMultiplier(3f).build());
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setEffects(new OverlayEffect(ImmutableList.of(textOverlay)))

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/PlaybackTestUtil.java
View file

@ -24,6 +24,7 @@ import android.text.style.ForegroundColorSpan;
import android.text.style.TypefaceSpan;
import androidx.media3.effect.OverlayEffect;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.StaticOverlaySettings;
import androidx.media3.effect.TextOverlay;
import com.google.common.collect.ImmutableList;
@ -74,7 +75,7 @@ import com.google.common.collect.ImmutableList;
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return new OverlaySettings.Builder().setBackgroundFrameAnchor(x, y).build();
return new StaticOverlaySettings.Builder().setBackgroundFrameAnchor(x, y).build();
}
}
}