feat: Add audio codec to selectAudioLanguage() (#6723)

- adds optional argument to `selectAudioLanguage()` which gives us a way
to select preferred audio codec
- updates `PreferenceBasedCriteria` to reflect the change
- renders codec in shaka UI to help distinguish tracks with the same
language, role, channels, etc
- can be tested on Chrome with `Tears of Steel (multicodec, TTML)` asset

lib/media/adaptation_set_criteria.js

@@ -115,11 +115,12 @@ shaka.media.PreferenceBasedCriteria = class {
    * @param {string=} videoLabel
    * @param {shaka.config.CodecSwitchingStrategy=} codecSwitchingStrategy
    * @param {boolean=} enableAudioGroups
+   * @param {string=} audioCodec
    */
   constructor(language, role, channelCount, hdrLevel, spatialAudio,
       videoLayout, audioLabel = '', videoLabel = '',
       codecSwitchingStrategy = shaka.config.CodecSwitchingStrategy.RELOAD,
-      enableAudioGroups = false) {
+      enableAudioGroups = false, audioCodec = '') {
     /** @private {string} */
     this.language_ = language;
     /** @private {string} */
@@ -140,6 +141,8 @@ shaka.media.PreferenceBasedCriteria = class {
     this.codecSwitchingStrategy_ = codecSwitchingStrategy;
     /** @private {boolean} */
     this.enableAudioGroups_ = enableAudioGroups;
+    /** @private {string} */
+    this.audioCodec_ = audioCodec;
   }
 
   /** @override */
@@ -229,6 +232,16 @@ shaka.media.PreferenceBasedCriteria = class {
       shaka.log.warning('No exact match for spatial audio could be found.');
     }
 
+    if (this.audioCodec_) {
+      const byAudioCodec = Class.filterVariantsByAudioCodec_(
+          current, this.audioCodec_);
+      if (byAudioCodec.length) {
+        current = byAudioCodec;
+      } else {
+        shaka.log.warning('No exact match for audio codec could be found.');
+      }
+    }
+
     const supportsSmoothCodecTransitions = this.codecSwitchingStrategy_ ==
       shaka.config.CodecSwitchingStrategy.SMOOTH &&
         shaka.media.Capabilities.isChangeTypeSupported();
@@ -403,4 +416,21 @@ shaka.media.PreferenceBasedCriteria = class {
       return true;
     });
   }
+
+
+  /**
+   * Filters variants according to the given audio codec.
+   *
+   * @param {!Array<shaka.extern.Variant>} variants
+   * @param {string} audioCodec
+   * @private
+   */
+  static filterVariantsByAudioCodec_(variants, audioCodec) {
+    return variants.filter((variant) => {
+      if (variant.audio && variant.audio.codecs != audioCodec) {
+        return false;
+      }
+      return true;
+    });
+  }
 };

lib/player.js

@@ -4709,9 +4709,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    * @param {string=} role
    * @param {number=} channelsCount
    * @param {number=} safeMargin
+   * @param {string=} codec
    * @export
    */
-  selectAudioLanguage(language, role, channelsCount = 0, safeMargin = 0) {
+  selectAudioLanguage(language, role, channelsCount = 0, safeMargin = 0,
+      codec = '') {
     if (this.manifest_ && this.playhead_) {
       this.currentAdaptationSetCriteria_ =
           new shaka.media.PreferenceBasedCriteria(
@@ -4724,7 +4726,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
               /* audioLabel= */ '',
               /* videoLabel= */ '',
               this.config_.mediaSource.codecSwitchingStrategy,
-              this.config_.manifest.dash.enableAudioGroups);
+              this.config_.manifest.dash.enableAudioGroups,
+              codec);
 
       const diff = (a, b) => {
         if (!a.video && !b.video) {

test/media/adaptation_set_criteria_unit.js

@@ -909,6 +909,39 @@ describe('AdaptationSetCriteria', () => {
         manifest.variants[2],
       ]);
     });
+
+    it('chooses variants with preferred audio codec', () => {
+      const manifest = shaka.test.ManifestGenerator.generate((manifest) => {
+        manifest.addVariant(1, (variant) => {
+          variant.addAudio(10, (stream) => {
+            stream.codecs = 'mp4a.40.2';
+          });
+        });
+        manifest.addVariant(2, (variant) => {
+          variant.addAudio(20, (stream) => {
+            stream.codecs = 'ec-3';
+          });
+        });
+      });
+
+      const builder = new shaka.media.PreferenceBasedCriteria(
+          /* language= */ 'en',
+          /* role= */ '',
+          /* channelCount= */ 0,
+          /* hdrLevel= */ '',
+          /* spatialAudio= */ false,
+          /* videoLayout= */ '',
+          /* audioLabel= */ '',
+          /* videoLabel= */ '',
+          shaka.config.CodecSwitchingStrategy.RELOAD,
+          /* enableAudioGroups= */ false,
+          /* audioCodec= */ 'ec-3');
+      const set = builder.create(manifest.variants);
+
+      checkSet(set, [
+        manifest.variants[1],
+      ]);
+    });
   });
 
   /**

test/player_unit.js

@@ -1520,6 +1520,25 @@ describe('Player', () => {
           variant.addExistingStream(2);  // video
           variant.addExistingStream(6);  // audio
         });
+        manifest.addVariant(108, (variant) => {  // spanish ec-3, low res
+          variant.language = 'es';
+          variant.bandwidth = 1100;
+          variant.addExistingStream(1);  // video
+          variant.addAudio(7, (stream) => {
+            stream.mime('audio/mp4', 'ec-3');
+            stream.originalId = 'audio-es-ec3';
+            stream.bandwidth = 100;
+            stream.channelsCount = 2;
+            stream.audioSamplingRate = 48000;
+            // stream.codecs = 'ec-3';
+          });
+        });
+        manifest.addVariant(109, (variant) => {  // spanish ec-3, high res
+          variant.language = 'es';
+          variant.bandwidth = 2100;
+          variant.addExistingStream(2);  // video
+          variant.addExistingStream(7);  // audio
+        });
 
         // All text tracks should remain, even with different MIME types.
         manifest.addTextStream(50, (stream) => {
@@ -1884,6 +1903,86 @@ describe('Player', () => {
           originalImageId: null,
           accessibilityPurpose: undefined,
         },
+        {
+          id: 108,
+          active: false,
+          type: 'variant',
+          bandwidth: 1100,
+          language: 'es',
+          originalLanguage: 'es',
+          label: null,
+          kind: null,
+          width: 100,
+          height: 200,
+          frameRate: 1000000 / 42000,
+          pixelAspectRatio: '59:54',
+          hdr: null,
+          colorGamut: null,
+          videoLayout: null,
+          mimeType: 'video/mp4',
+          audioMimeType: 'audio/mp4',
+          videoMimeType: 'video/mp4',
+          codecs: 'avc1.4d401f, ec-3',
+          audioCodec: 'ec-3',
+          videoCodec: 'avc1.4d401f',
+          primary: false,
+          roles: ['main'],
+          audioRoles: [],
+          forced: false,
+          videoId: 1,
+          audioId: 7,
+          channelsCount: 2,
+          audioSamplingRate: 48000,
+          spatialAudio: false,
+          tilesLayout: null,
+          audioBandwidth: 100,
+          videoBandwidth: 1000,
+          originalAudioId: 'audio-es-ec3',
+          originalVideoId: 'video-1kbps',
+          originalTextId: null,
+          originalImageId: null,
+          accessibilityPurpose: undefined,
+        },
+        {
+          id: 109,
+          active: false,
+          type: 'variant',
+          bandwidth: 2100,
+          language: 'es',
+          originalLanguage: 'es',
+          label: null,
+          kind: null,
+          width: 200,
+          height: 400,
+          frameRate: 24,
+          pixelAspectRatio: '59:54',
+          hdr: null,
+          colorGamut: null,
+          videoLayout: null,
+          mimeType: 'video/mp4',
+          audioMimeType: 'audio/mp4',
+          videoMimeType: 'video/mp4',
+          codecs: 'avc1.4d401f, ec-3',
+          audioCodec: 'ec-3',
+          videoCodec: 'avc1.4d401f',
+          primary: false,
+          roles: [],
+          audioRoles: [],
+          forced: false,
+          videoId: 2,
+          audioId: 7,
+          channelsCount: 2,
+          audioSamplingRate: 48000,
+          spatialAudio: false,
+          tilesLayout: null,
+          audioBandwidth: 100,
+          videoBandwidth: 2000,
+          originalAudioId: 'audio-es-ec3',
+          originalVideoId: 'video-2kbps',
+          originalTextId: null,
+          originalImageId: null,
+          accessibilityPurpose: undefined,
+        },
       ];
 
       textTracks = [
@@ -2174,6 +2273,14 @@ describe('Player', () => {
       expect(getActiveVariantTrack().roles).toContain('commentary');
     });
 
+    it('selectAudioLanguage() respects selected audio codec', () => {
+      player.selectAudioLanguage('es', '', 0, 0, 'mp4a.40.2');
+      expect(getActiveVariantTrack().audioCodec).toBe('mp4a.40.2');
+
+      player.selectAudioLanguage('es', '', 0, 0, 'ec-3');
+      expect(getActiveVariantTrack().audioCodec).toBe('ec-3');
+    });
+
     it('selectAudioLanguage() applies role only to audio', () => {
       expect(getActiveVariantTrack().roles).not.toContain('commentary');
       player.selectAudioLanguage('en', 'commentary');

ui/audio_language_selection.js

@@ -97,13 +97,17 @@ shaka.ui.AudioLanguageSelection = class extends shaka.ui.SettingsMenu {
    * @private
    */
   onAudioTrackSelected_(track) {
+    let channelsCount = undefined;
     if (track.channelsCount &&
         this.controls.getConfig().showAudioChannelCountVariants) {
-      this.player.selectAudioLanguage(
-          track.language, track.roles[0], track.channelsCount);
-    } else {
-      this.player.selectAudioLanguage(track.language, track.roles[0]);
+      channelsCount = track.channelsCount;
     }
+    let codec = undefined;
+    if (track.audioCodec) {
+      codec = track.audioCodec;
+    }
+    this.player.selectAudioLanguage(track.language, track.roles[0],
+        channelsCount, /* safeMargin= */ 0, codec);
   }
 
 

ui/language_utils.js

@@ -39,6 +39,20 @@ shaka.ui.LanguageUtils = class {
       return track.active == true;
     });
 
+    /** @type {!Map<string, !Set<string>>} */
+    const codecsByLanguage = new Map();
+    for (const track of tracks) {
+      if (!track.audioCodec) {
+        continue;
+      }
+      if (!codecsByLanguage.has(track.language)) {
+        codecsByLanguage.set(track.language, new Set());
+      }
+      codecsByLanguage.get(track.language).add(track.audioCodec);
+    }
+    const hasDifferentAudioCodecs = (language) =>
+      codecsByLanguage.has(language) && codecsByLanguage.get(language).size > 1;
+
     // Remove old tracks
     // 1. Save the back to menu button
     const backButton = shaka.ui.Utils.getFirstDescendantWithClassName(
@@ -59,14 +73,18 @@ shaka.ui.LanguageUtils = class {
       }
     };
 
-    const getCombination = (language, rolesString, label, channelsCount) => {
+    const getCombination = (language, rolesString, label, channelsCount,
+        audioCodec) => {
       const keys = [
         language,
         rolesString,
       ];
       if (showAudioChannelCountVariants && channelsCount != null) {
         keys.push(channelsCount);
       }
+      if (hasDifferentAudioCodecs(language) && audioCodec) {
+        keys.push(audioCodec);
+      }
       if (label &&
           trackLabelFormat == shaka.ui.Overlay.TrackLabelFormat.LABEL) {
         keys.push(label);
@@ -82,6 +100,21 @@ shaka.ui.LanguageUtils = class {
       return name;
     };
 
+    const getAudioCodecName = (audioCodec) => {
+      let name = '';
+      audioCodec = audioCodec.toLowerCase();
+      if (audioCodec.startsWith('mp4a')) {
+        name = 'AAC';
+      } else if (audioCodec === 'ec-3' || audioCodec === 'ac-3') {
+        name = 'Dolby';
+      } else if (audioCodec === 'opus') {
+        name = 'Opus';
+      } else if (audioCodec === 'flac') {
+        name = 'fLaC';
+      }
+      return name ? ' ' + name : name;
+    };
+
     /** @type {!Map.<string, !Set.<string>>} */
     const rolesByLanguage = new Map();
     for (const track of tracks) {
@@ -96,7 +129,8 @@ shaka.ui.LanguageUtils = class {
     const combinationsMade = new Set();
     const selectedCombination = selectedTrack ? getCombination(
         selectedTrack.language, getRolesString(selectedTrack),
-        selectedTrack.label, selectedTrack.channelsCount) : '';
+        selectedTrack.label, selectedTrack.channelsCount,
+        selectedTrack.audioCodec) : '';
 
     for (const track of tracks) {
       const language = track.language;
@@ -105,8 +139,10 @@ shaka.ui.LanguageUtils = class {
       const rolesString = getRolesString(track);
       const label = track.label;
       const channelsCount = track.channelsCount;
+      const audioCodec = track.audioCodec;
       const combinationName =
-          getCombination(language, rolesString, label, channelsCount);
+          getCombination(language, rolesString, label, channelsCount,
+              audioCodec);
       if (combinationsMade.has(combinationName)) {
         continue;
       }
@@ -124,6 +160,9 @@ shaka.ui.LanguageUtils = class {
           shaka.ui.LanguageUtils.getLanguageName(language, localization);
       switch (trackLabelFormat) {
         case shaka.ui.Overlay.TrackLabelFormat.LANGUAGE:
+          if (hasDifferentAudioCodecs(language)) {
+            span.textContent += getAudioCodecName(audioCodec);
+          }
           if (showAudioChannelCountVariants) {
             span.textContent += getChannelsCountName(channelsCount);
           }
@@ -132,6 +171,9 @@ shaka.ui.LanguageUtils = class {
           }
           break;
         case shaka.ui.Overlay.TrackLabelFormat.ROLE:
+          if (hasDifferentAudioCodecs(language)) {
+            span.textContent += getAudioCodecName(audioCodec);
+          }
           if (showAudioChannelCountVariants) {
             span.textContent += getChannelsCountName(channelsCount);
           }
@@ -148,6 +190,9 @@ shaka.ui.LanguageUtils = class {
           }
           break;
         case shaka.ui.Overlay.TrackLabelFormat.LANGUAGE_ROLE:
+          if (hasDifferentAudioCodecs(language)) {
+            span.textContent += getAudioCodecName(audioCodec);
+          }
           if (showAudioChannelCountVariants) {
             span.textContent += getChannelsCountName(channelsCount);
           }