feat(HLS): Build closed captions metadata on-the-fly

demo/config.js

@@ -263,6 +263,8 @@ shakaDemo.Config = class {
             'manifest.hls.ignoreManifestTimestampsInSegmentsMode')
         .addBoolInput_('Disable codec guessing',
             'manifest.hls.disableCodecGuessing')
+        .addBoolInput_('Disable closed caption detection',
+            'manifest.hls.disableClosedCaptionsDetection')
         .addBoolInput_('Allow LL-HLS byterange optimization',
             'manifest.hls.allowLowLatencyByteRangeOptimization');
   }

externs/shaka/player.js

@@ -1052,6 +1052,7 @@ shaka.extern.DashManifestConfiguration;
  *   sequenceMode: boolean,
  *   ignoreManifestTimestampsInSegmentsMode: boolean,
  *   disableCodecGuessing: boolean,
+ *   disableClosedCaptionsDetection: boolean,
  *   allowLowLatencyByteRangeOptimization: boolean
  * }}
  *
@@ -1120,6 +1121,11 @@ shaka.extern.DashManifestConfiguration;
  *   As a consequence, lazy-loading media playlists won't be possible for this
  *   use case, which may result in longer video startup times.
  *   <i>Defaults to <code>false</code>.</i>
+ * @property {boolean} disableClosedCaptionsDetection
+ *   If set to false, If there is no EXT-X-MEDIA with TYPE="CLOSED-CAPTIONS" we
+ *   will try to detect which closed captions are available.  If your manifest
+ *   has CLOSED-CAPTIONS=NONE we will not try to do any detection.
+ *   <i>Defaults to <code>false</code>.</i>
  * @property {boolean} allowLowLatencyByteRangeOptimization
  *   If set to true, the HLS parser will optimize operation with LL and partial
  *   byte range segments. More info in

lib/hls/hls_parser.js

@@ -244,6 +244,9 @@ shaka.hls.HlsParser = class {
 
     /** @private {?shaka.util.ContentSteeringManager} */
     this.contentSteeringManager_ = null;
+
+    /** @private {boolean} */
+    this.needsClosedCaptionsDetection_ = true;
   }
 
 
@@ -759,10 +762,19 @@ shaka.hls.HlsParser = class {
 
     // Parsing a media playlist results in a single-variant stream.
     if (playlist.type == shaka.hls.PlaylistType.MEDIA) {
+      this.needsClosedCaptionsDetection_ = false;
+
+      /** @type {!Array.<!shaka.hls.Tag>} */
+      const variablesTags = shaka.hls.Utils.filterTagsByName(playlist.tags,
+          'EXT-X-DEFINE');
+
+      const mediaVariables =
+          this.parseMediaVariables_(variablesTags, this.masterPlaylistUri_);
+
       // Get necessary info for this stream. These are things we would normally
       // find from the master playlist (e.g. from values on EXT-X-MEDIA tags).
-      const basicInfo =
-          await this.getMediaPlaylistBasicInfo_(playlist, getUris);
+      const basicInfo = await this.getMediaPlaylistBasicInfo_(
+          playlist, getUris, mediaVariables);
       const type = basicInfo.type;
       const mimeType = basicInfo.mimeType;
       const codecs = basicInfo.codecs;
@@ -787,8 +799,8 @@ shaka.hls.HlsParser = class {
 
       // Make the stream info, with those values.
       const streamInfo = await this.convertParsedPlaylistIntoStreamInfo_(
-          this.globalId_++, playlist, getUris, uri, codecs, type,
-          languageValue, primary, name, channelsCount, closedCaptions,
+          this.globalId_++, mediaVariables, playlist, getUris, uri, codecs,
+          type, languageValue, primary, name, channelsCount, closedCaptions,
           characteristics, forced, sampleRate, spatialAudio, mimeType);
       this.uriToStreamInfosMap_.set(uri, streamInfo);
 
@@ -921,10 +933,11 @@ shaka.hls.HlsParser = class {
   /**
    * @param {shaka.hls.Playlist} playlist
    * @param {function():!Array.<string>} getUris
+   * @param {?Map.<string, string>=} variables
    * @return {!Promise.<shaka.media.SegmentUtils.BasicInfo>}
    * @private
    */
-  async getMediaPlaylistBasicInfo_(playlist, getUris) {
+  async getMediaPlaylistBasicInfo_(playlist, getUris, variables) {
     const HlsParser = shaka.hls.HlsParser;
     const defaultBasicInfo = shaka.media.SegmentUtils.getBasicInfoFromMimeType(
         this.config_.hls.mediaPlaylistFullMimeType);
@@ -934,7 +947,8 @@ shaka.hls.HlsParser = class {
     const firstSegment = playlist.segments[0];
     const firstSegmentUris = shaka.hls.Utils.constructSegmentUris(
         getUris(),
-        firstSegment.verbatimSegmentUri);
+        firstSegment.verbatimSegmentUri,
+        variables);
     const firstSegmentUri = firstSegmentUris[0];
     const parsedUri = new goog.Uri(firstSegmentUri);
     const extension = parsedUri.getPath().split('.').pop();
@@ -948,7 +962,7 @@ shaka.hls.HlsParser = class {
 
     let initData = null;
     const initSegmentRef = this.getInitSegmentReference_(
-        playlist, firstSegment.tags, getUris);
+        playlist, firstSegment.tags, getUris, variables);
     this.mapTagToInitSegmentRefMap_.clear();
     if (initSegmentRef) {
       const initSegmentRequest = shaka.util.Networking.createSegmentRequest(
@@ -1770,10 +1784,12 @@ shaka.hls.HlsParser = class {
     // value of the GROUP-ID attribute of an EXT-X-MEDIA tag elsewhere in the
     // Playlist whose TYPE attribute is CLOSED-CAPTIONS.
     if (type == ContentType.VIDEO) {
-      if (closedCaptionsAttr && closedCaptionsAttr != 'NONE') {
-        return this.groupIdToClosedCaptionsMap_.get(closedCaptionsAttr);
-      } else if (!closedCaptionsAttr &&
-          this.groupIdToClosedCaptionsMap_.size) {
+      if (closedCaptionsAttr) {
+        if (closedCaptionsAttr != 'NONE') {
+          return this.groupIdToClosedCaptionsMap_.get(closedCaptionsAttr);
+        }
+        this.needsClosedCaptionsDetection_ = false;
+      } else if (!closedCaptionsAttr && this.groupIdToClosedCaptionsMap_.size) {
         for (const key of this.groupIdToClosedCaptionsMap_.keys()) {
           return this.groupIdToClosedCaptionsMap_.get(key);
         }
@@ -1923,6 +1939,7 @@ shaka.hls.HlsParser = class {
   parseClosedCaptions_(mediaTags) {
     const closedCaptionsTags =
         shaka.hls.Utils.filterTagsByType(mediaTags, 'CLOSED-CAPTIONS');
+    this.needsClosedCaptionsDetection_ = closedCaptionsTags.length == 0;
     for (const tag of closedCaptionsTags) {
       goog.asserts.assert(tag.name == 'EXT-X-MEDIA',
           'Should only be called on media tags!');
@@ -2264,6 +2281,8 @@ shaka.hls.HlsParser = class {
 
     /** @param {!AbortSignal} abortSignal */
     const downloadSegmentIndex = async (abortSignal) => {
+      const ContentType = shaka.util.ManifestParserUtils.ContentType;
+
       const uris = streamInfo.getUris();
       // Download the actual manifest.
       const response = await this.requestManifest_(
@@ -2283,24 +2302,47 @@ shaka.hls.HlsParser = class {
       /** @type {!shaka.hls.Playlist} */
       const playlist = this.manifestTextParser_.parsePlaylist(response.data);
 
+      /** @type {!Array.<!shaka.hls.Tag>} */
+      const variablesTags = shaka.hls.Utils.filterTagsByName(playlist.tags,
+          'EXT-X-DEFINE');
+
+      const mediaVariables =
+          this.parseMediaVariables_(variablesTags, responseUri);
+
       let mimeType = undefined;
 
+      let closedCaptionsUpdated = false;
+
       // If no codec info was provided in the manifest and codec guessing is
       // disabled we try to get necessary info from the media data.
-      if (!this.codecInfoInManifest_ && this.config_.hls.disableCodecGuessing) {
-        const basicInfo =
-            await this.getMediaPlaylistBasicInfo_(playlist, getUris);
+      if ((!this.codecInfoInManifest_ &&
+          this.config_.hls.disableCodecGuessing) ||
+          (this.needsClosedCaptionsDetection_ && type == ContentType.VIDEO &&
+          !this.config_.hls.disableClosedCaptionsDetection)) {
+        this.needsClosedCaptionsDetection_ = false;
+
+        const basicInfo = await this.getMediaPlaylistBasicInfo_(
+            playlist, getUris, mediaVariables);
 
         goog.asserts.assert(
             type === basicInfo.type, 'Media types should match!');
 
-        mimeType = basicInfo.mimeType;
-        codecs = basicInfo.codecs;
+        if (basicInfo.closedCaptions.size && (!closedCaptions ||
+            closedCaptions.size != basicInfo.closedCaptions.size)) {
+          closedCaptions = basicInfo.closedCaptions;
+          closedCaptionsUpdated = true;
+        }
+
+        if (!this.codecInfoInManifest_ &&
+            this.config_.hls.disableCodecGuessing) {
+          mimeType = basicInfo.mimeType;
+          codecs = basicInfo.codecs;
+        }
       }
 
       const wasLive = this.isLive_();
       const realStreamInfo = await this.convertParsedPlaylistIntoStreamInfo_(
-          streamId, playlist, getUris, responseUri, codecs,
+          streamId, mediaVariables, playlist, getUris, responseUri, codecs,
           type, languageValue, primary, name, channelsCount, closedCaptions,
           characteristics, forced, sampleRate, spatialAudio, mimeType);
       if (abortSignal.aborted) {
@@ -2333,6 +2375,8 @@ shaka.hls.HlsParser = class {
       stream.mimeType = realStream.mimeType;
       stream.bandwidth = stream.bandwidth || realStream.bandwidth;
       stream.codecs = stream.codecs || realStream.codecs;
+      stream.closedCaptions =
+          stream.closedCaptions || realStream.closedCaptions;
       this.setFullTypeForStream_(stream);
 
       // Since we lazy-loaded this content, the player may need to create new
@@ -2341,7 +2385,10 @@ shaka.hls.HlsParser = class {
         this.playerInterface_.newDrmInfo(stream);
       }
 
-      const ContentType = shaka.util.ManifestParserUtils.ContentType;
+      if (this.manifest_ && closedCaptionsUpdated) {
+        this.playerInterface_.makeTextStreamsForClosedCaptions(this.manifest_);
+      }
+
       if (type == ContentType.VIDEO || type == ContentType.AUDIO) {
         for (const otherStreamInfo of this.uriToStreamInfosMap_.values()) {
           if (!otherStreamInfo.loadedOnce && otherStreamInfo.type == type) {
@@ -2572,7 +2619,7 @@ shaka.hls.HlsParser = class {
    * @return {!Promise.<!shaka.hls.HlsParser.StreamInfo>}
    * @private
    */
-  async convertParsedPlaylistIntoStreamInfo_(streamId, playlist,
+  async convertParsedPlaylistIntoStreamInfo_(streamId, variables, playlist,
       getUris, responseUri, codecs, type, languageValue, primary, name,
       channelsCount, closedCaptions, characteristics, forced, sampleRate,
       spatialAudio, mimeType = undefined) {
@@ -2585,13 +2632,6 @@ shaka.hls.HlsParser = class {
           shaka.util.Error.Code.HLS_INVALID_PLAYLIST_HIERARCHY);
     }
 
-    /** @type {!Array.<!shaka.hls.Tag>} */
-    const variablesTags = shaka.hls.Utils.filterTagsByName(playlist.tags,
-        'EXT-X-DEFINE');
-
-    const mediaVariables =
-        this.parseMediaVariables_(variablesTags, responseUri);
-
     goog.asserts.assert(playlist.segments != null,
         'Media playlist should have segments!');
 
@@ -2603,11 +2643,11 @@ shaka.hls.HlsParser = class {
 
     if (!mimeType) {
       mimeType = await this.guessMimeType_(type, codecs, playlist,
-          mediaVariables, getUris);
+          variables, getUris);
     }
 
     const {drmInfos, keyIds, encrypted, aesEncrypted} =
-        await this.parseDrmInfo_(playlist, mimeType, getUris, mediaVariables);
+        await this.parseDrmInfo_(playlist, mimeType, getUris, variables);
 
     if (encrypted && !drmInfos.length && !aesEncrypted) {
       throw new shaka.util.Error(
@@ -2629,8 +2669,7 @@ shaka.hls.HlsParser = class {
         this.mediaSequenceToStartTimeByType_.get(type) : new Map();
 
     const {segments, bandwidth} = this.createSegments_(
-        playlist, stream, mediaSequenceToStartTime, mediaVariables, getUris,
-        type);
+        playlist, stream, mediaSequenceToStartTime, variables, getUris, type);
     if (bandwidth) {
       stream.bandwidth = bandwidth;
     }

lib/util/player_configuration.js

@@ -166,6 +166,7 @@ shaka.util.PlayerConfiguration = class {
         sequenceMode: shaka.util.Platform.supportsSequenceMode(),
         ignoreManifestTimestampsInSegmentsMode: false,
         disableCodecGuessing: false,
+        disableClosedCaptionsDetection: false,
         allowLowLatencyByteRangeOptimization: true,
       },
       mss: {

test/cast/cast_receiver_integration.js

@@ -271,8 +271,8 @@ filterDescribe('CastReceiver', castReceiverIntegrationSupport, () => {
       for (const message of messages) {
         // Check that the update message is of a reasonable size. From previous
         // testing we found that the socket would silently reject data that got
-        // too big. 6KB is safely below the limit.
-        expect(message.length).toBeLessThan(6000);
+        // too big. 7KB is safely below the limit.
+        expect(message.length).toBeLessThan(7000);
       }
     });
 

test/hls/hls_live_unit.js

@@ -32,20 +32,22 @@ describe('HlsParser live', () => {
   beforeEach(() => {
     // TODO: use StreamGenerator?
     initSegmentData = new Uint8Array([
-      0x00, 0x00, 0x00, 0x30, // size (48)
+      0x00, 0x00, 0x00, 0x36, // size (54)
       0x6D, 0x6F, 0x6F, 0x76, // type (moov)
-      0x00, 0x00, 0x00, 0x28, // trak size (40)
+      0x00, 0x00, 0x00, 0x2E, // trak size (46)
       0x74, 0x72, 0x61, 0x6B, // type (trak)
-      0x00, 0x00, 0x00, 0x20, // mdia size (32)
+      0x00, 0x00, 0x00, 0x26, // mdia size (38)
       0x6D, 0x64, 0x69, 0x61, // type (mdia)
 
-      0x00, 0x00, 0x00, 0x18, // mdhd size (24)
+      0x00, 0x00, 0x00, 0x1E, // mdhd size (30)
       0x6D, 0x64, 0x68, 0x64, // type (mdhd)
       0x00, 0x00, 0x00, 0x00, // version and flags
 
       0x00, 0x00, 0x00, 0x00, // creation time (0)
       0x00, 0x00, 0x00, 0x00, // modification time (0)
       0x00, 0x00, 0x03, 0xe8, // timescale (1000)
+      0x00, 0x00, 0x00, 0x00, // duration (0)
+      0x55, 0xC4, // language (und)
     ]);
     segmentData = new Uint8Array([
       0x00, 0x00, 0x00, 0x24, // size (36)
@@ -56,7 +58,7 @@ describe('HlsParser live', () => {
       0x74, 0x66, 0x64, 0x74, // type (tfdt)
       0x01, 0x00, 0x00, 0x00, // version and flags
       0x00, 0x00, 0x00, 0x00, // baseMediaDecodeTime first 4 bytes
-      0x00, 0x00, 0x07, 0xd0,  // baseMediaDecodeTime last 4 bytes (2000)
+      0x00, 0x00, 0x07, 0xd0, // baseMediaDecodeTime last 4 bytes (2000)
     ]);
 
     selfInitializingSegmentData =
@@ -122,11 +124,13 @@ describe('HlsParser live', () => {
         .setResponseText('test:/audio', media1)
         .setResponseValue('test:/init.mp4', initSegmentData)
         .setResponseValue('test:/main.mp4', segmentData)
+        .setResponseValue('test:/main0.mp4', segmentData)
         .setResponseValue('test:/main2.mp4', segmentData)
         .setResponseValue('test:/main3.mp4', segmentData)
         .setResponseValue('test:/main4.mp4', segmentData)
         .setResponseValue('test:/partial.mp4', segmentData)
         .setResponseValue('test:/partial2.mp4', segmentData)
+        .setResponseValue('test:/ref1.mp4', segmentData)
         .setResponseValue('test:/selfInit.mp4', selfInitializingSegmentData);
   }