Add video language to DefaultTrackSelector

PiperOrigin-RevId: 688155680

RELEASENOTES.md

@@ -4,6 +4,13 @@
 
 *   Common Library:
 *   ExoPlayer:
+    *   Consider language when selecting a video track. By default select a
+        'main' video track that matches the language of the selected audio
+        track, if available. Explicit video language preferences can be
+        expressed with
+        `TrackSelectionParameters.Builder.setPreferredVideoLanguage(s)`.
+    *   Add `selectedAudioLanguage` parameter to
+        `DefaultTrackSelector.selectVideoTrack()` method.
 *   Transformer:
 *   Extractors:
     *   Fix media duration parsing in `mdhd` box of MP4 files to handle `-1`

libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java

@@ -90,6 +90,7 @@ public class TrackSelectionParameters {
     private int viewportHeight;
     private boolean viewportOrientationMayChange;
     private ImmutableList<String> preferredVideoMimeTypes;
+    private ImmutableList<String> preferredVideoLanguages;
     private @C.RoleFlags int preferredVideoRoleFlags;
     // Audio
     private ImmutableList<String> preferredAudioLanguages;
@@ -127,6 +128,7 @@ public class TrackSelectionParameters {
       viewportHeight = Integer.MAX_VALUE;
       viewportOrientationMayChange = true;
       preferredVideoMimeTypes = ImmutableList.of();
+      preferredVideoLanguages = ImmutableList.of();
       preferredVideoRoleFlags = 0;
       // Audio
       preferredAudioLanguages = ImmutableList.of();
@@ -194,6 +196,9 @@ public class TrackSelectionParameters {
       preferredVideoMimeTypes =
           ImmutableList.copyOf(
               firstNonNull(bundle.getStringArray(FIELD_PREFERRED_VIDEO_MIMETYPES), new String[0]));
+      preferredVideoLanguages =
+          ImmutableList.copyOf(
+              firstNonNull(bundle.getStringArray(FIELD_PREFERRED_VIDEO_LANGUAGES), new String[0]));
       preferredVideoRoleFlags =
           bundle.getInt(
               FIELD_PREFERRED_VIDEO_ROLE_FLAGS, DEFAULT_WITHOUT_CONTEXT.preferredVideoRoleFlags);
@@ -284,6 +289,7 @@ public class TrackSelectionParameters {
     /** Overrides the value of the builder with the value of {@link TrackSelectionParameters}. */
     @EnsuresNonNull({
       "preferredVideoMimeTypes",
+      "preferredVideoLanguages",
       "preferredAudioLanguages",
       "preferredAudioMimeTypes",
       "audioOffloadPreferences",
@@ -305,6 +311,7 @@ public class TrackSelectionParameters {
       viewportHeight = parameters.viewportHeight;
       viewportOrientationMayChange = parameters.viewportOrientationMayChange;
       preferredVideoMimeTypes = parameters.preferredVideoMimeTypes;
+      preferredVideoLanguages = parameters.preferredVideoLanguages;
       preferredVideoRoleFlags = parameters.preferredVideoRoleFlags;
       // Audio
       preferredAudioLanguages = parameters.preferredAudioLanguages;
@@ -504,6 +511,36 @@ public class TrackSelectionParameters {
       return this;
     }
 
+    /**
+     * Sets the preferred language for video tracks.
+     *
+     * @param preferredVideoLanguage Preferred video language as an IETF BCP 47 conformant tag, or
+     *     {@code null} to express no language preference for video track selection.
+     * @return This builder.
+     */
+    @UnstableApi
+    @CanIgnoreReturnValue
+    public Builder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
+      return preferredVideoLanguage == null
+          ? setPreferredVideoLanguages()
+          : setPreferredVideoLanguages(preferredVideoLanguage);
+    }
+
+    /**
+     * Sets the preferred languages for video tracks.
+     *
+     * @param preferredVideoLanguages Preferred video languages as IETF BCP 47 conformant tags in
+     *     order of preference, or an empty array to express no language preference for video track
+     *     selection.
+     * @return This builder.
+     */
+    @UnstableApi
+    @CanIgnoreReturnValue
+    public Builder setPreferredVideoLanguages(String... preferredVideoLanguages) {
+      this.preferredVideoLanguages = normalizeLanguageCodes(preferredVideoLanguages);
+      return this;
+    }
+
     /**
      * Sets the preferred {@link C.RoleFlags} for video tracks.
      *
@@ -1148,6 +1185,11 @@ public class TrackSelectionParameters {
    */
   public final ImmutableList<String> preferredVideoMimeTypes;
 
+  /**
+   * The preferred languages for video tracks as IETF BCP 47 conformant tags in order of preference.
+   */
+  @UnstableApi public final ImmutableList<String> preferredVideoLanguages;
+
   /**
    * The preferred {@link C.RoleFlags} for video tracks. {@code 0} selects the default track if
    * there is one, or the first track if there's no default. The default value is {@code 0}.
@@ -1267,6 +1309,7 @@ public class TrackSelectionParameters {
     this.viewportHeight = builder.viewportHeight;
     this.viewportOrientationMayChange = builder.viewportOrientationMayChange;
     this.preferredVideoMimeTypes = builder.preferredVideoMimeTypes;
+    this.preferredVideoLanguages = builder.preferredVideoLanguages;
     this.preferredVideoRoleFlags = builder.preferredVideoRoleFlags;
     // Audio
     this.preferredAudioLanguages = builder.preferredAudioLanguages;
@@ -1317,6 +1360,7 @@ public class TrackSelectionParameters {
         && viewportWidth == other.viewportWidth
         && viewportHeight == other.viewportHeight
         && preferredVideoMimeTypes.equals(other.preferredVideoMimeTypes)
+        && preferredVideoLanguages.equals(other.preferredVideoLanguages)
         && preferredVideoRoleFlags == other.preferredVideoRoleFlags
         // Audio
         && preferredAudioLanguages.equals(other.preferredAudioLanguages)
@@ -1355,6 +1399,7 @@ public class TrackSelectionParameters {
     result = 31 * result + viewportWidth;
     result = 31 * result + viewportHeight;
     result = 31 * result + preferredVideoMimeTypes.hashCode();
+    result = 31 * result + preferredVideoLanguages.hashCode();
     result = 31 * result + preferredVideoRoleFlags;
     // Audio
     result = 31 * result + preferredAudioLanguages.hashCode();
@@ -1410,6 +1455,7 @@ public class TrackSelectionParameters {
   private static final String FIELD_AUDIO_OFFLOAD_PREFERENCES = Util.intToStringMaxRadix(30);
   private static final String FIELD_IS_PREFER_IMAGE_OVER_VIDEO_ENABLED =
       Util.intToStringMaxRadix(31);
+  private static final String FIELD_PREFERRED_VIDEO_LANGUAGES = Util.intToStringMaxRadix(32);
 
   /**
    * Defines a minimum field ID value for subclasses to use when implementing {@link #toBundle()}
@@ -1439,6 +1485,8 @@ public class TrackSelectionParameters {
     bundle.putBoolean(FIELD_VIEWPORT_ORIENTATION_MAY_CHANGE, viewportOrientationMayChange);
     bundle.putStringArray(
         FIELD_PREFERRED_VIDEO_MIMETYPES, preferredVideoMimeTypes.toArray(new String[0]));
+    bundle.putStringArray(
+        FIELD_PREFERRED_VIDEO_LANGUAGES, preferredVideoLanguages.toArray(new String[0]));
     bundle.putInt(FIELD_PREFERRED_VIDEO_ROLE_FLAGS, preferredVideoRoleFlags);
     // Audio
     bundle.putStringArray(

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java

@@ -343,6 +343,20 @@ public class DefaultTrackSelector extends MappingTrackSelector
       return this;
     }
 
+    @CanIgnoreReturnValue
+    @Override
+    public ParametersBuilder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
+      super.setPreferredVideoLanguage(preferredVideoLanguage);
+      return this;
+    }
+
+    @CanIgnoreReturnValue
+    @Override
+    public ParametersBuilder setPreferredVideoLanguages(String... preferredVideoLanguages) {
+      super.setPreferredVideoLanguages(preferredVideoLanguages);
+      return this;
+    }
+
     @SuppressWarnings("deprecation") // Intentionally returning deprecated type
     @CanIgnoreReturnValue
     @Override
@@ -1153,6 +1167,20 @@ public class DefaultTrackSelector extends MappingTrackSelector
         return this;
       }
 
+      @CanIgnoreReturnValue
+      @Override
+      public Builder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
+        super.setPreferredVideoLanguage(preferredVideoLanguage);
+        return this;
+      }
+
+      @CanIgnoreReturnValue
+      @Override
+      public Builder setPreferredVideoLanguages(String... preferredVideoLanguages) {
+        super.setPreferredVideoLanguages(preferredVideoLanguages);
+        return this;
+      }
+
       @CanIgnoreReturnValue
       @Override
       public Builder setPreferredVideoRoleFlags(@RoleFlags int preferredVideoRoleFlags) {
@@ -2663,13 +2691,30 @@ public class DefaultTrackSelector extends MappingTrackSelector
     ExoTrackSelection.@NullableType Definition[] definitions =
         new ExoTrackSelection.Definition[rendererCount];
 
+    @Nullable
+    Pair<ExoTrackSelection.Definition, Integer> selectedAudio =
+        selectAudioTrack(
+            mappedTrackInfo,
+            rendererFormatSupports,
+            rendererMixedMimeTypeAdaptationSupports,
+            params);
+    if (selectedAudio != null) {
+      definitions[selectedAudio.second] = selectedAudio.first;
+    }
+    @Nullable
+    String selectedAudioLanguage =
+        selectedAudio == null
+            ? null
+            : selectedAudio.first.group.getFormat(selectedAudio.first.tracks[0]).language;
+
     @Nullable
     Pair<ExoTrackSelection.Definition, Integer> selectedVideo =
         selectVideoTrack(
             mappedTrackInfo,
             rendererFormatSupports,
             rendererMixedMimeTypeAdaptationSupports,
-            params);
+            params,
+            selectedAudioLanguage);
 
     @Nullable
     Pair<ExoTrackSelection.Definition, Integer> selectedImage =
@@ -2683,22 +2728,6 @@ public class DefaultTrackSelector extends MappingTrackSelector
       definitions[selectedVideo.second] = selectedVideo.first;
     }
 
-    @Nullable
-    Pair<ExoTrackSelection.Definition, Integer> selectedAudio =
-        selectAudioTrack(
-            mappedTrackInfo,
-            rendererFormatSupports,
-            rendererMixedMimeTypeAdaptationSupports,
-            params);
-    if (selectedAudio != null) {
-      definitions[selectedAudio.second] = selectedAudio.first;
-    }
-
-    @Nullable
-    String selectedAudioLanguage =
-        selectedAudio == null
-            ? null
-            : selectedAudio.first.group.getFormat(selectedAudio.first.tracks[0]).language;
     @Nullable
     Pair<ExoTrackSelection.Definition, Integer> selectedText =
         selectTextTrack(mappedTrackInfo, rendererFormatSupports, params, selectedAudioLanguage);
@@ -2732,6 +2761,8 @@ public class DefaultTrackSelector extends MappingTrackSelector
    * @param mixedMimeTypeSupports The {@link AdaptiveSupport} for mixed MIME type adaptation for the
    *     renderer.
    * @param params The selector's current constraint parameters.
+   * @param selectedAudioLanguage The language of the selected audio track. May be null if the
+   *     selected audio track declares no language or no audio track was selected.
    * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding
    *     renderer index, or null if no selection was made.
    * @throws ExoPlaybackException If an error occurs while selecting the tracks.
@@ -2741,7 +2772,8 @@ public class DefaultTrackSelector extends MappingTrackSelector
       MappedTrackInfo mappedTrackInfo,
       @Capabilities int[][][] rendererFormatSupports,
       @AdaptiveSupport int[] mixedMimeTypeSupports,
-      Parameters params)
+      Parameters params,
+      @Nullable String selectedAudioLanguage)
       throws ExoPlaybackException {
     if (params.audioOffloadPreferences.audioOffloadMode == AUDIO_OFFLOAD_MODE_REQUIRED) {
       return null;
@@ -2752,7 +2784,12 @@ public class DefaultTrackSelector extends MappingTrackSelector
         rendererFormatSupports,
         (int rendererIndex, TrackGroup group, @Capabilities int[] support) ->
             VideoTrackInfo.createForTrackGroup(
-                rendererIndex, group, params, support, mixedMimeTypeSupports[rendererIndex]),
+                rendererIndex,
+                group,
+                params,
+                support,
+                selectedAudioLanguage,
+                mixedMimeTypeSupports[rendererIndex]),
         VideoTrackInfo::compareSelections);
   }
 
@@ -3486,6 +3523,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
         TrackGroup trackGroup,
         Parameters params,
         @Capabilities int[] formatSupport,
+        @Nullable String selectedAudioLanguage,
         @AdaptiveSupport int mixedMimeTypeAdaptationSupport) {
       int maxPixelsToRetainForViewport =
           getMaxVideoPixelsToRetainForViewport(
@@ -3506,6 +3544,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
                 /* trackIndex= */ i,
                 params,
                 formatSupport[i],
+                selectedAudioLanguage,
                 mixedMimeTypeAdaptationSupport,
                 isSuitableForViewport));
       }
@@ -3525,8 +3564,11 @@ public class DefaultTrackSelector extends MappingTrackSelector
     private final int bitrate;
     private final int pixelCount;
     private final int preferredMimeTypeMatchIndex;
+    private final int preferredLanguageIndex;
+    private final int preferredLanguageScore;
     private final int preferredRoleFlagsScore;
     private final boolean hasMainOrNoRoleFlag;
+    private final int selectedAudioLanguageScore;
     private final boolean allowMixedMimeTypes;
     private final @SelectionEligibility int selectionEligibility;
     private final boolean usesPrimaryDecoder;
@@ -3539,6 +3581,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
         int trackIndex,
         Parameters parameters,
         @Capabilities int formatSupport,
+        @Nullable String selectedAudioLanguage,
         @AdaptiveSupport int mixedMimeTypeAdaptationSupport,
         boolean isSuitableForViewport) {
       super(rendererIndex, trackGroup, trackIndex);
@@ -3574,9 +3617,29 @@ public class DefaultTrackSelector extends MappingTrackSelector
           format.frameRate != Format.NO_VALUE && format.frameRate >= MIN_REASONABLE_FRAME_RATE;
       bitrate = format.bitrate;
       pixelCount = format.getPixelCount();
+      int bestLanguageIndex = Integer.MAX_VALUE;
+      int bestLanguageScore = 0;
+      for (int i = 0; i < parameters.preferredVideoLanguages.size(); i++) {
+        int score =
+            getFormatLanguageScore(
+                format,
+                parameters.preferredVideoLanguages.get(i),
+                /* allowUndeterminedFormatLanguage= */ false);
+        if (score > 0) {
+          bestLanguageIndex = i;
+          bestLanguageScore = score;
+          break;
+        }
+      }
+      preferredLanguageIndex = bestLanguageIndex;
+      preferredLanguageScore = bestLanguageScore;
       preferredRoleFlagsScore =
           getRoleFlagMatchScore(format.roleFlags, parameters.preferredVideoRoleFlags);
       hasMainOrNoRoleFlag = format.roleFlags == 0 || (format.roleFlags & C.ROLE_FLAG_MAIN) != 0;
+      boolean selectedAudioLanguageUndetermined =
+          normalizeUndeterminedLanguageToNull(selectedAudioLanguage) == null;
+      selectedAudioLanguageScore =
+          getFormatLanguageScore(format, selectedAudioLanguage, selectedAudioLanguageUndetermined);
       int bestMimeTypeMatchIndex = Integer.MAX_VALUE;
       for (int i = 0; i < parameters.preferredVideoMimeTypes.size(); i++) {
         if (format.sampleMimeType != null
@@ -3639,9 +3702,15 @@ public class DefaultTrackSelector extends MappingTrackSelector
               .compareFalseFirst(
                   info1.isWithinRendererCapabilities, info2.isWithinRendererCapabilities)
               // 1. Compare match with specific content preferences set by the parameters.
+              .compare(
+                  info1.preferredLanguageIndex,
+                  info2.preferredLanguageIndex,
+                  Ordering.natural().reverse())
+              .compare(info1.preferredLanguageScore, info2.preferredLanguageScore)
               .compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore)
               // 2. Compare match with implicit content preferences set by the media.
               .compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag)
+              .compare(info1.selectedAudioLanguageScore, info2.selectedAudioLanguageScore)
               // 3. Compare match with 'reasonable' frame rate threshold.
               .compareFalseFirst(info1.hasReasonableFrameRate, info2.hasReasonableFrameRate)
               // 4. Compare match with technical preferences set by the parameters.

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java

@@ -743,6 +743,74 @@ public final class DefaultTrackSelectorTest {
     assertFixedSelection(result.selections[0], trackGroups, enNonDefaultFormat);
   }
 
+  /**
+   * Tests that track selector will select a video track with a language that matches the preferred
+   * language given by {@link Parameters}.
+   */
+  @Test
+  public void selectTracksSelectPreferredAudioVideoLanguage() throws Exception {
+    Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon();
+    Format frVideoFormat = formatBuilder.setLanguage("fra").build();
+    Format enVideoFormat = formatBuilder.setLanguage("eng").build();
+    TrackGroupArray trackGroups = wrapFormats(frVideoFormat, enVideoFormat);
+
+    trackSelector.setParameters(defaultParameters.buildUpon().setPreferredVideoLanguage("eng"));
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES},
+            wrapFormats(frVideoFormat, enVideoFormat),
+            periodId,
+            TIMELINE);
+    assertFixedSelection(result.selections[0], trackGroups, enVideoFormat);
+  }
+
+  /**
+   * Tests that the default track selector will select:
+   *
+   * <ul>
+   *   <li>A main video track matching the selected audio language when a main video track in
+   *       another language is present.
+   *   <li>A main video track that doesn't match the selected audio language when a main video track
+   *       in the selected audio language is not present (but alternate video tracks in this
+   *       language are present).
+   * </ul>
+   */
+  @Test
+  public void defaultVideoTracksInteractWithSelectedAudioLanguageAsExpected()
+      throws ExoPlaybackException {
+    Format.Builder mainVideoBuilder = VIDEO_FORMAT.buildUpon().setRoleFlags(C.ROLE_FLAG_MAIN);
+    Format mainEnglish = mainVideoBuilder.setLanguage("eng").build();
+    Format mainGerman = mainVideoBuilder.setLanguage("deu").build();
+    Format mainNoLanguage = mainVideoBuilder.setLanguage(C.LANGUAGE_UNDETERMINED).build();
+    Format alternateGerman =
+        VIDEO_FORMAT.buildUpon().setRoleFlags(C.ROLE_FLAG_ALTERNATE).setLanguage("deu").build();
+
+    Format noLanguageAudio = AUDIO_FORMAT.buildUpon().setLanguage(null).build();
+    Format germanAudio = AUDIO_FORMAT.buildUpon().setLanguage("deu").build();
+
+    RendererCapabilities[] rendererCapabilities =
+        new RendererCapabilities[] {VIDEO_CAPABILITIES, AUDIO_CAPABILITIES};
+
+    // Neither the audio nor the forced text track define a language. We select them both under the
+    // assumption that they have matching language.
+    TrackGroupArray trackGroups = wrapFormats(noLanguageAudio, mainNoLanguage);
+    TrackSelectorResult result =
+        trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
+    assertFixedSelection(result.selections[0], trackGroups, mainNoLanguage);
+
+    // The audio declares german. The main german track should be selected (in favour of the main
+    // english track).
+    trackGroups = wrapFormats(germanAudio, mainGerman, mainEnglish);
+    result = trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
+    assertFixedSelection(result.selections[0], trackGroups, mainGerman);
+
+    // The audio declares german. The main english track should be selected because there's no
+    // main german track.
+    trackGroups = wrapFormats(germanAudio, alternateGerman, mainEnglish);
+    result = trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
+    assertFixedSelection(result.selections[0], trackGroups, mainEnglish);
+  }
+
   /**
    * Tests that track selector will prefer tracks that are within renderer's capabilities over track
    * that exceed renderer's capabilities.

libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java

@@ -1155,13 +1155,18 @@ public final class CompositionPlayer extends SimpleBasePlayer
         MappedTrackInfo mappedTrackInfo,
         @RendererCapabilities.Capabilities int[][][] rendererFormatSupports,
         @RendererCapabilities.AdaptiveSupport int[] mixedMimeTypeSupports,
-        Parameters params)
+        Parameters params,
+        @Nullable String selectedAudioLanguage)
         throws ExoPlaybackException {
       if (disableVideoPlayback) {
         return null;
       }
       return super.selectVideoTrack(
-          mappedTrackInfo, rendererFormatSupports, mixedMimeTypeSupports, params);
+          mappedTrackInfo,
+          rendererFormatSupports,
+          mixedMimeTypeSupports,
+          params,
+          selectedAudioLanguage);
     }
 
     @Nullable