From 7bd650b31552ddb4a1030d5b0c32a373c8767fbe Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 6 Apr 2022 16:07:12 +0100 Subject: [PATCH] Add periodic dimming effect to transformer demo. PeriodicDimmingFrameProcessor is an example of how a custom fragment shader can be used to apply color changes that change over time. PiperOrigin-RevId: 439840609 --- .../assets/fragment_shader_vignette_es2.glsl | 31 +++++ .../main/assets/vertex_shader_copy_es2.glsl | 24 ++++ .../ConfigurationActivity.java | 81 +++++++++--- .../PeriodicVignetteFrameProcessor.java | 124 ++++++++++++++++++ .../transformerdemo/TransformerActivity.java | 16 ++- .../res/layout/configuration_activity.xml | 6 +- .../res/layout/periodic_vignette_options.xml | 72 ++++++++++ .../src/main/res/values/strings.xml | 6 +- .../android/exoplayer2/util/GlProgram.java | 79 +++++------ .../android/exoplayer2/util/GlUtil.java | 36 +++-- 10 files changed, 401 insertions(+), 74 deletions(-) create mode 100644 demos/transformer/src/main/assets/fragment_shader_vignette_es2.glsl create mode 100644 demos/transformer/src/main/assets/vertex_shader_copy_es2.glsl create mode 100644 demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteFrameProcessor.java create mode 100644 demos/transformer/src/main/res/layout/periodic_vignette_options.xml diff --git a/demos/transformer/src/main/assets/fragment_shader_vignette_es2.glsl b/demos/transformer/src/main/assets/fragment_shader_vignette_es2.glsl new file mode 100644 index 0000000000..55dea952a5 --- /dev/null +++ b/demos/transformer/src/main/assets/fragment_shader_vignette_es2.glsl @@ -0,0 +1,31 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 fragment shader that samples from a (non-external) texture with uTexSampler, +// copying from this texture to the current output while applying a vignette effect +// by linearly darkening the pixels between uInnerRadius and uOuterRadius. + +precision mediump float; +uniform sampler2D uTexSampler; +uniform vec2 uCenter; +uniform float uInnerRadius; +uniform float uOuterRadius; +varying vec2 vTexSamplingCoord; +void main() { + vec3 src = texture2D(uTexSampler, vTexSamplingCoord).xyz; + float dist = distance(vTexSamplingCoord, uCenter); + float scale = clamp(1.0 - (dist - uInnerRadius) / (uOuterRadius - uInnerRadius), 0.0, 1.0); + gl_FragColor = vec4(src.r * scale, src.g * scale, src.b * scale, 1.0); +} diff --git a/demos/transformer/src/main/assets/vertex_shader_copy_es2.glsl b/demos/transformer/src/main/assets/vertex_shader_copy_es2.glsl new file mode 100644 index 0000000000..b4c1673d25 --- /dev/null +++ b/demos/transformer/src/main/assets/vertex_shader_copy_es2.glsl @@ -0,0 +1,24 @@ +#version 100 +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ES 2 vertex shader that leaves the coordinates unchanged. + +attribute vec4 aFramePosition; +attribute vec4 aTexSamplingCoord; +varying vec2 vTexSamplingCoord; +void main() { + gl_Position = aFramePosition; + vTexSamplingCoord = aTexSamplingCoord.xy; +} diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java index 93ba52c485..b2c6a74f00 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java @@ -34,6 +34,8 @@ import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.android.material.slider.RangeSlider; +import com.google.android.material.slider.Slider; import java.util.Arrays; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -56,7 +58,11 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String ENABLE_FALLBACK = "enable_fallback"; public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; - public static final String FRAME_PROCESSOR_SELECTION = "frame_processor_selection"; + public static final String DEMO_FRAME_PROCESSORS_SELECTIONS = "demo_frame_processors_selections"; + public static final String PERIODIC_VIGNETTE_CENTER_X = "periodic_vignette_center_x"; + public static final String PERIODIC_VIGNETTE_CENTER_Y = "periodic_vignette_center_y"; + public static final String PERIODIC_VIGNETTE_INNER_RADIUS = "periodic_vignette_inner_radius"; + public static final String PERIODIC_VIGNETTE_OUTER_RADIUS = "periodic_vignette_outer_radius"; private static final String[] INPUT_URIS = { "https://html5demos.com/assets/dizzy.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/android-block-1080-hevc.mp4", @@ -81,8 +87,12 @@ public final class ConfigurationActivity extends AppCompatActivity { "SEF slow motion with 240 fps", "MP4 with HDR (HDR10) H265 video (encoding may fail)", }; - private static final String[] FRAME_PROCESSORS = {"Dizzy crop", "3D spin", "Zoom in start"}; + private static final String[] DEMO_FRAME_PROCESSORS = { + "Dizzy crop", "Periodic vignette", "3D spin", "Zoom in start" + }; + private static final int PERIODIC_VIGNETTE_INDEX = 1; private static final String SAME_AS_INPUT_OPTION = "same as input"; + private static final float HALF_DIAGONAL = 1f / (float) Math.sqrt(2); private @MonotonicNonNull Button selectFileButton; private @MonotonicNonNull TextView selectedFileTextView; @@ -97,9 +107,13 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull CheckBox enableFallbackCheckBox; private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; - private @MonotonicNonNull Button selectFrameProcessorsButton; - private boolean @MonotonicNonNull [] selectedFrameProcessors; + private @MonotonicNonNull Button selectDemoFrameProcessorsButton; + private boolean @MonotonicNonNull [] demoFrameProcessorsSelections; private int inputUriPosition; + private float periodicVignetteCenterX; + private float periodicVignetteCenterY; + private float periodicVignetteInnerRadius; + private float periodicVignetteOuterRadius; @Override protected void onCreate(@Nullable Bundle savedInstanceState) { @@ -169,9 +183,9 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.request_sdr_tone_mapping).setEnabled(isRequestSdrToneMappingSupported()); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); - selectedFrameProcessors = new boolean[FRAME_PROCESSORS.length]; - selectFrameProcessorsButton = findViewById(R.id.select_frameprocessors_button); - selectFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors); + demoFrameProcessorsSelections = new boolean[DEMO_FRAME_PROCESSORS.length]; + selectDemoFrameProcessorsButton = findViewById(R.id.select_demo_frameprocessors_button); + selectDemoFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors); } @Override @@ -202,7 +216,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "enableFallbackCheckBox", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectedFrameProcessors" + "demoFrameProcessorsSelections" }) private void startTransformation(View view) { Intent transformerIntent = new Intent(this, TransformerActivity.class); @@ -237,7 +251,11 @@ public final class ConfigurationActivity extends AppCompatActivity { bundle.putBoolean( ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); - bundle.putBooleanArray(FRAME_PROCESSOR_SELECTION, selectedFrameProcessors); + bundle.putBooleanArray(DEMO_FRAME_PROCESSORS_SELECTIONS, demoFrameProcessorsSelections); + bundle.putFloat(PERIODIC_VIGNETTE_CENTER_X, periodicVignetteCenterX); + bundle.putFloat(PERIODIC_VIGNETTE_CENTER_Y, periodicVignetteCenterY); + bundle.putFloat(PERIODIC_VIGNETTE_INNER_RADIUS, periodicVignetteInnerRadius); + bundle.putFloat(PERIODIC_VIGNETTE_OUTER_RADIUS, periodicVignetteOuterRadius); transformerIntent.putExtras(bundle); @Nullable Uri intentUri = getIntent().getData(); @@ -258,9 +276,11 @@ public final class ConfigurationActivity extends AppCompatActivity { private void selectFrameProcessors(View view) { new AlertDialog.Builder(/* context= */ this) - .setTitle(R.string.select_frameprocessors) + .setTitle(R.string.select_demo_frameprocessors) .setMultiChoiceItems( - FRAME_PROCESSORS, checkNotNull(selectedFrameProcessors), this::selectFrameProcessor) + DEMO_FRAME_PROCESSORS, + checkNotNull(demoFrameProcessorsSelections), + this::selectFrameProcessor) .setPositiveButton(android.R.string.ok, /* listener= */ null) .create() .show(); @@ -272,9 +292,36 @@ public final class ConfigurationActivity extends AppCompatActivity { selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); } - @RequiresNonNull("selectedFrameProcessors") + @RequiresNonNull("demoFrameProcessorsSelections") private void selectFrameProcessor(DialogInterface dialog, int which, boolean isChecked) { - selectedFrameProcessors[which] = isChecked; + demoFrameProcessorsSelections[which] = isChecked; + if (!isChecked || which != PERIODIC_VIGNETTE_INDEX) { + return; + } + + View dialogView = + getLayoutInflater().inflate(R.layout.periodic_vignette_options, /* root= */ null); + Slider centerXSlider = + checkNotNull(dialogView.findViewById(R.id.periodic_vignette_center_x_slider)); + Slider centerYSlider = + checkNotNull(dialogView.findViewById(R.id.periodic_vignette_center_y_slider)); + RangeSlider radiusRangeSlider = + checkNotNull(dialogView.findViewById(R.id.periodic_vignette_radius_range_slider)); + radiusRangeSlider.setValues(0f, HALF_DIAGONAL); + new AlertDialog.Builder(/* context= */ this) + .setTitle(R.string.periodic_vignette_options) + .setView(dialogView) + .setPositiveButton( + android.R.string.ok, + (DialogInterface dialogInterface, int i) -> { + periodicVignetteCenterX = centerXSlider.getValue(); + periodicVignetteCenterY = centerYSlider.getValue(); + List radiusRange = radiusRangeSlider.getValues(); + periodicVignetteInnerRadius = radiusRange.get(0); + periodicVignetteOuterRadius = radiusRange.get(1); + }) + .create() + .show(); } @RequiresNonNull({ @@ -286,7 +333,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectFrameProcessorsButton" + "selectDemoFrameProcessorsButton" }) private void onRemoveAudio(View view) { if (((CheckBox) view).isChecked()) { @@ -306,7 +353,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectFrameProcessorsButton" + "selectDemoFrameProcessorsButton" }) private void onRemoveVideo(View view) { if (((CheckBox) view).isChecked()) { @@ -325,7 +372,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectFrameProcessorsButton" + "selectDemoFrameProcessorsButton" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { audioMimeSpinner.setEnabled(isAudioEnabled); @@ -336,7 +383,7 @@ public final class ConfigurationActivity extends AppCompatActivity { enableRequestSdrToneMappingCheckBox.setEnabled( isRequestSdrToneMappingSupported() && isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); - selectFrameProcessorsButton.setEnabled(isVideoEnabled); + selectDemoFrameProcessorsButton.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteFrameProcessor.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteFrameProcessor.java new file mode 100644 index 0000000000..8890298162 --- /dev/null +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteFrameProcessor.java @@ -0,0 +1,124 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformerdemo; + +import static com.google.android.exoplayer2.util.Assertions.checkArgument; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.opengl.GLES20; +import android.util.Size; +import com.google.android.exoplayer2.transformer.GlFrameProcessor; +import com.google.android.exoplayer2.util.GlProgram; +import com.google.android.exoplayer2.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A {@link GlFrameProcessor} that periodically dims the frames such that pixels are darker the + * further they are away from the frame center. + */ +/* package */ final class PeriodicVignetteFrameProcessor implements GlFrameProcessor { + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl"; + private static final float DIMMING_PERIOD_US = 5_600_000f; + + private final Context context; + private float centerX; + private float centerY; + private float minInnerRadius; + private float deltaInnerRadius; + private float outerRadius; + + private @MonotonicNonNull Size outputSize; + private @MonotonicNonNull GlProgram glProgram; + + /** + * Creates a new instance. + * + *

The inner radius of the vignette effect oscillates smoothly between {@code minInnerRadius} + * and {@code maxInnerRadius}. + * + *

The pixels between the inner radius and the {@code outerRadius} are darkened linearly based + * on their distance from {@code innerRadius}. All pixels outside {@code outerRadius} are black. + * + *

The parameters are given in normalized texture coordinates from 0 to 1. + * + * @param context The {@link Context}. + * @param centerX The x-coordinate of the center of the effect. + * @param centerY The y-coordinate of the center of the effect. + * @param minInnerRadius The lower bound of the radius that is unaffected by the effect. + * @param maxInnerRadius The upper bound of the radius that is unaffected by the effect. + * @param outerRadius The radius after which all pixels are black. + */ + public PeriodicVignetteFrameProcessor( + Context context, + float centerX, + float centerY, + float minInnerRadius, + float maxInnerRadius, + float outerRadius) { + checkArgument(minInnerRadius <= maxInnerRadius); + checkArgument(maxInnerRadius <= outerRadius); + this.context = context; + this.centerX = centerX; + this.centerY = centerY; + this.minInnerRadius = minInnerRadius; + this.deltaInnerRadius = maxInnerRadius - minInnerRadius; + this.outerRadius = outerRadius; + } + + @Override + public void initialize(int inputTexId, int inputWidth, int inputHeight) throws IOException { + outputSize = new Size(inputWidth, inputHeight); + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY}); + glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius}); + // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. + glProgram.setBufferAttribute( + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + glProgram.setBufferAttribute( + "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + } + + @Override + public Size getOutputSize() { + return checkStateNotNull(outputSize); + } + + @Override + public void updateProgramAndDraw(long presentationTimeUs) { + checkStateNotNull(glProgram).use(); + double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; + float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); + glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } + + @Override + public void release() { + if (glProgram != null) { + glProgram.delete(); + } + } +} diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index c123174421..4ed70fc4f0 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -243,7 +243,7 @@ public final class TransformerActivity extends AppCompatActivity { ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); @Nullable boolean[] selectedFrameProcessors = - bundle.getBooleanArray(ConfigurationActivity.FRAME_PROCESSOR_SELECTION); + bundle.getBooleanArray(ConfigurationActivity.DEMO_FRAME_PROCESSORS_SELECTIONS); if (selectedFrameProcessors != null) { if (selectedFrameProcessors[0]) { frameProcessors.add( @@ -251,9 +251,21 @@ public final class TransformerActivity extends AppCompatActivity { } if (selectedFrameProcessors[1]) { frameProcessors.add( - AdvancedFrameProcessorFactory.createSpin3dFrameProcessor(/* context= */ this)); + new PeriodicVignetteFrameProcessor( + /* context= */ this, + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), + /* minInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), + /* maxInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } if (selectedFrameProcessors[2]) { + frameProcessors.add( + AdvancedFrameProcessorFactory.createSpin3dFrameProcessor(/* context= */ this)); + } + if (selectedFrameProcessors[3]) { frameProcessors.add( AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor( /* context= */ this)); diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 7464ce9f6b..7d080b7351 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -64,7 +64,7 @@ app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toBottomOf="@+id/selected_file_text_view" - app:layout_constraintBottom_toTopOf="@+id/select_frameprocessors_button"> + app:layout_constraintBottom_toTopOf="@+id/select_demo_frameprocessors_button">