diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 29872f8bb7..2f1ff63ecf 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -17,6 +17,12 @@ * Transformer: * Add support for flattening H.265/HEVC SEF slow motion videos. * Track Selection: + * Add `DefaultTrackSelector.selectImageTrack` to enable image track + selection. + * Add `TrackSelectionParameters.isPrioritizeImageOverVideoEnabled` to + determine whether to select an image track if both an image track and a + video track are available. The default value is `false` which means + selecting a video track is prioritized. * Extractors: * Audio: * Video: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java index 966995854a..b1523f2d7a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java @@ -2615,7 +2615,16 @@ public class DefaultTrackSelector extends MappingTrackSelector rendererFormatSupports, rendererMixedMimeTypeAdaptationSupports, params); - if (selectedVideo != null) { + + @Nullable + Pair selectedImage = + params.isPrioritizeImageOverVideoEnabled || selectedVideo == null + ? selectImageTrack(mappedTrackInfo, rendererFormatSupports, params) + : null; + + if (selectedImage != null) { + definitions[selectedImage.second] = selectedImage.first; + } else if (selectedVideo != null) { definitions[selectedVideo.second] = selectedVideo.first; } @@ -2646,7 +2655,8 @@ public class DefaultTrackSelector extends MappingTrackSelector int trackType = mappedTrackInfo.getRendererType(i); if (trackType != C.TRACK_TYPE_VIDEO && trackType != C.TRACK_TYPE_AUDIO - && trackType != C.TRACK_TYPE_TEXT) { + && trackType != C.TRACK_TYPE_TEXT + && trackType != C.TRACK_TYPE_IMAGE) { definitions[i] = selectOtherTrack( trackType, mappedTrackInfo.getTrackGroups(i), rendererFormatSupports[i], params); @@ -2810,6 +2820,38 @@ public class DefaultTrackSelector extends MappingTrackSelector TextTrackInfo::compareSelections); } + // Image track selection implementation. + + /** + * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a + * {@link ExoTrackSelection.Definition} for an image track selection. + * + * @param mappedTrackInfo Mapped track information. + * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by + * renderer, track group and track (in that order). + * @param params The selector's current constraint parameters. + * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding + * renderer index, or null if no selection was made. + * @throws ExoPlaybackException If an error occurs while selecting the tracks. + */ + @Nullable + protected Pair selectImageTrack( + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] rendererFormatSupports, + Parameters params) + throws ExoPlaybackException { + if (params.audioOffloadPreferences.audioOffloadMode == AUDIO_OFFLOAD_MODE_REQUIRED) { + return null; + } + return selectTracksForType( + C.TRACK_TYPE_IMAGE, + mappedTrackInfo, + rendererFormatSupports, + (int rendererIndex, TrackGroup group, @Capabilities int[] support) -> + ImageTrackInfo.createForTrackGroup(rendererIndex, group, params, support), + ImageTrackInfo::compareSelections); + } + // Generic track selection methods. /** @@ -3975,6 +4017,60 @@ public class DefaultTrackSelector extends MappingTrackSelector } } + private static final class ImageTrackInfo extends TrackInfo + implements Comparable { + + public static ImmutableList createForTrackGroup( + int rendererIndex, + TrackGroup trackGroup, + Parameters params, + @Capabilities int[] formatSupport) { + ImmutableList.Builder imageTracks = ImmutableList.builder(); + for (int i = 0; i < trackGroup.length; i++) { + imageTracks.add( + new ImageTrackInfo( + rendererIndex, trackGroup, /* trackIndex= */ i, params, formatSupport[i])); + } + return imageTracks.build(); + } + + private final @SelectionEligibility int selectionEligibility; + private final int pixelCount; + + public ImageTrackInfo( + int rendererIndex, + TrackGroup trackGroup, + int trackIndex, + Parameters parameters, + @Capabilities int trackFormatSupport) { + super(rendererIndex, trackGroup, trackIndex); + selectionEligibility = + isSupported(trackFormatSupport, parameters.exceedRendererCapabilitiesIfNecessary) + ? SELECTION_ELIGIBILITY_FIXED + : SELECTION_ELIGIBILITY_NO; + pixelCount = format.getPixelCount(); + } + + @Override + public @SelectionEligibility int getSelectionEligibility() { + return selectionEligibility; + } + + @Override + public boolean isCompatibleForAdaptationWith(ImageTrackInfo otherTrack) { + return false; + } + + @Override + public int compareTo(ImageTrackInfo other) { + return Integer.compare(this.pixelCount, other.pixelCount); + } + + public static int compareSelections(List infos1, List infos2) { + return infos1.get(0).compareTo(infos2.get(0)); + } + } + private static final class OtherTrackScore implements Comparable { private final boolean isDefault; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java index 2ea08255bc..d4b8ad02d3 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java @@ -103,11 +103,16 @@ public final class DefaultTrackSelectorTest { private static final RendererCapabilities ALL_AUDIO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES = new FakeRendererCapabilities( C.TRACK_TYPE_AUDIO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES)); + private static final RendererCapabilities ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES = + new FakeRendererCapabilities( + C.TRACK_TYPE_VIDEO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES)); private static final RendererCapabilities VIDEO_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_VIDEO); private static final RendererCapabilities AUDIO_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_AUDIO); + private static final RendererCapabilities IMAGE_CAPABILITIES = + new FakeRendererCapabilities(C.TRACK_TYPE_IMAGE); private static final RendererCapabilities NO_SAMPLE_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_NONE); private static final RendererCapabilities[] RENDERER_CAPABILITIES = @@ -131,6 +136,8 @@ public final class DefaultTrackSelectorTest { .build(); private static final Format TEXT_FORMAT = new Format.Builder().setSampleMimeType(MimeTypes.TEXT_VTT).build(); + private static final Format IMAGE_FORMAT = + new Format.Builder().setSampleMimeType(MimeTypes.IMAGE_PNG).build(); private static final TrackGroup VIDEO_TRACK_GROUP = new TrackGroup(VIDEO_FORMAT); private static final TrackGroup AUDIO_TRACK_GROUP = new TrackGroup(AUDIO_FORMAT); @@ -2906,6 +2913,105 @@ public final class DefaultTrackSelectorTest { verify(invalidationListener).onRendererCapabilitiesChanged(renderer); } + @Test + public void + selectTracks_withImageAndVideoAndPrioritizeImageOverVideoEnabled_selectsOnlyImageTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build()); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertThat(result.selections[ /* video renderer index */0]).isNull(); + assertFixedSelection( + result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT); + } + + @Test + public void selectTracks_withImageAndVideoTracksBothSupported_selectsOnlyVideoTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertFixedSelection( + result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT); + assertThat(result.selections[ /* image renderer index */1]).isNull(); + } + + @Test + public void selectTracks_withVideoAndImageAndOnlyImageSupported_selectsImageTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setExceedRendererCapabilitiesIfNecessary(false)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] { + ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES, IMAGE_CAPABILITIES + }, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertThat(result.selections[ /* video renderer index */0]).isNull(); + assertFixedSelection( + result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT); + } + + @Test + public void selectTracks_withVideoTrackOnlyAndPrioritizeImageOverVideoEnabled_selectsVideoTrack() + throws Exception { + TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build()); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertFixedSelection( + result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT); + assertThat(result.selections[ /* image renderer index */1]).isNull(); + } + + @Test + public void selectTracks_withMultipleImageTracks_selectsHighestResolutionTrack() + throws Exception { + Format image1 = IMAGE_FORMAT.buildUpon().setWidth(320).setHeight(320).build(); + Format image2 = IMAGE_FORMAT.buildUpon().setWidth(480).setHeight(480).build(); + TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(image1, image2)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {IMAGE_CAPABILITIES}, trackGroups, periodId, TIMELINE); + + assertThat(result.length).isEqualTo(1); + assertFixedSelection(result.selections[0], trackGroups, image2); + } + private static void assertSelections(TrackSelectorResult result, TrackSelection[] expected) { assertThat(result.length).isEqualTo(expected.length); for (int i = 0; i < expected.length; i++) {