fix(HLS): Fix external subtitles out of sync in HLS (shaka-project#5491)

Fixes shaka-project#5458 Fixes shaka-project#5443
Robloche · Nov 30, 2023 · 7fc6052 · 7fc6052
1 parent 4911539
commit 7fc6052
Show file tree

Hide file tree

Showing 19 changed files with 43 additions and 9 deletions.
diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js
@@ -388,7 +388,8 @@ shaka.extern.FetchCryptoKeysFunction;
  *   matchedStreams:
  *      (!Array.<shaka.extern.Stream>|!Array.<shaka.extern.StreamDB>|
  *      undefined),
- *   mssPrivateData: (shaka.extern.MssPrivateData|undefined)
+ *   mssPrivateData: (shaka.extern.MssPrivateData|undefined),
+ *   external: boolean
  * }}
  *
  * @description
@@ -510,6 +511,9 @@ shaka.extern.FetchCryptoKeysFunction;
  * @property {(shaka.extern.MssPrivateData|undefined)} mssPrivateData
  *   <i>Microsoft Smooth Streaming only.</i> <br>
  *   Private MSS data that is necessary to be able to do transmuxing.
+ * @property {boolean} external
+ *   Indicate if the stream was added externally.
+ *   Eg: external text tracks.
  *
  * @exportDoc
  */

diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js
@@ -137,7 +137,8 @@ shaka.extern.ManifestDB;
  *   audioSamplingRate: ?number,
  *   spatialAudio: boolean,
  *   closedCaptions: Map.<string, string>,
- *   tilesLayout: (string|undefined)
+ *   tilesLayout: (string|undefined),
+ *   external: boolean
  * }}
  *
  * @property {number} id
@@ -200,6 +201,9 @@ shaka.extern.ManifestDB;
  *   The value is a grid-item-dimension consisting of two positive decimal
  *   integers in the format: column-x-row ('4x3'). It describes the arrangement
  *   of Images in a Grid. The minimum valid LAYOUT is '1x1'.
+ * @property {boolean} external
+ *   Indicate if the stream was added externally.
+ *   Eg: external text tracks.
  */
 shaka.extern.StreamDB;
 

diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js
@@ -1397,6 +1397,7 @@ shaka.dash.DashParser = class {
         tilesLayout,
         matchedStreams: [],
         accessibilityPurpose,
+        external: false,
       };
     }
 

diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
@@ -2487,6 +2487,7 @@ shaka.hls.HlsParser = class {
       hdr: undefined,
       tilesLayout: undefined,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -414,7 +414,7 @@ shaka.media.MediaSourceEngine = class {
     let mimeType = shaka.util.MimeUtils.getFullType(
         stream.mimeType, stream.codecs);
     if (contentType == ContentType.TEXT) {
-      this.reinitText(mimeType, this.sequenceMode_);
+      this.reinitText(mimeType, this.sequenceMode_, stream.external);
     } else {
       let needTransmux = this.config_.forceTransmux;
       if (!shaka.media.Capabilities.isTypeSupported(mimeType) ||
@@ -486,13 +486,14 @@ shaka.media.MediaSourceEngine = class {
    * Reinitialize the TextEngine for a new text type.
    * @param {string} mimeType
    * @param {boolean} sequenceMode
+   * @param {boolean} external
    */
-  reinitText(mimeType, sequenceMode) {
+  reinitText(mimeType, sequenceMode, external) {
     if (!this.textEngine_) {
       this.textEngine_ = new shaka.text.TextEngine(this.textDisplayer_);
     }
     this.textEngine_.initParser(mimeType, sequenceMode,
-        this.segmentRelativeVttTiming_);
+        external || this.segmentRelativeVttTiming_);
   }
 
   /**
@@ -797,7 +798,7 @@ shaka.media.MediaSourceEngine = class {
     if (hasClosedCaptions && contentType == ContentType.VIDEO) {
       if (!this.textEngine_) {
         this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE,
-            this.sequenceMode_);
+            this.sequenceMode_, /* external= */ false);
       }
       if (!this.captionParser_) {
         const basicType = mimeType.split(';', 1)[0];

diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
@@ -260,7 +260,7 @@ shaka.media.StreamingEngine = class {
     const mimeType = shaka.util.MimeUtils.getFullType(
         stream.mimeType, stream.codecs);
     this.playerInterface_.mediaSourceEngine.reinitText(
-        mimeType, this.manifest_.sequenceMode);
+        mimeType, this.manifest_.sequenceMode, stream.external);
 
     const textDisplayer =
         this.playerInterface_.mediaSourceEngine.getTextDisplayer();
@@ -478,7 +478,7 @@ shaka.media.StreamingEngine = class {
       const fullMimeType = shaka.util.MimeUtils.getFullType(
           stream.mimeType, stream.codecs);
       this.playerInterface_.mediaSourceEngine.reinitText(
-          fullMimeType, this.manifest_.sequenceMode);
+          fullMimeType, this.manifest_.sequenceMode, stream.external);
     }
 
     // Releases the segmentIndex of the old stream.

diff --git a/lib/mss/mss_parser.js b/lib/mss/mss_parser.js
@@ -536,6 +536,7 @@ shaka.mss.MssParser = class {
         codecPrivateData: null,
       },
       accessibilityPurpose: null,
+      external: false,
     };
 
     // This is specifically for text tracks.

diff --git a/lib/offline/indexeddb/v1_storage_cell.js b/lib/offline/indexeddb/v1_storage_cell.js
@@ -177,6 +177,7 @@ shaka.offline.indexeddb.V1StorageCell = class
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 

diff --git a/lib/offline/indexeddb/v2_storage_cell.js b/lib/offline/indexeddb/v2_storage_cell.js
@@ -126,6 +126,7 @@ shaka.offline.indexeddb.V2StorageCell = class
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 

diff --git a/lib/offline/manifest_converter.js b/lib/offline/manifest_converter.js
@@ -208,6 +208,7 @@ shaka.offline.ManifestConverter = class {
       closedCaptions: streamDB.closedCaptions,
       tilesLayout: streamDB.tilesLayout,
       accessibilityPurpose: null,
+      external: streamDB.external,
     };
 
     return stream;

diff --git a/lib/offline/storage.js b/lib/offline/storage.js
@@ -1323,6 +1323,7 @@ shaka.offline.Storage = class {
       spatialAudio: stream.spatialAudio,
       closedCaptions: stream.closedCaptions,
       tilesLayout: stream.tilesLayout,
+      external: stream.external,
     };
 
     const startTime =

diff --git a/lib/player.js b/lib/player.js
@@ -2397,6 +2397,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
         spatialAudio: false,
         closedCaptions: null,
         accessibilityPurpose: null,
+        external: false,
       },
       bandwidth: 100,
       allowedByApplication: true,
@@ -4889,6 +4890,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       spatialAudio: false,
       closedCaptions: null,
       accessibilityPurpose: null,
+      external: true,
     };
 
     const fullMimeType = shaka.util.MimeUtils.getFullType(
@@ -5042,6 +5044,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       closedCaptions: null,
       tilesLayout: '1x1',
       accessibilityPurpose: null,
+      external: true,
     };
 
     this.manifest_.imageStreams.push(stream);
@@ -5464,6 +5467,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
               spatialAudio: false,
               closedCaptions: null,
               accessibilityPurpose: null,
+              external: false,
             };
             manifest.textStreams.push(textStream);
             closedCaptionsSet.add(id);

diff --git a/lib/util/periods.js b/lib/util/periods.js
@@ -1514,6 +1514,7 @@ shaka.util.PeriodCombiner = class {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
   }
 
@@ -1552,6 +1553,7 @@ shaka.util.PeriodCombiner = class {
       spatialAudio: false,
       closedCaptions: null,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 

diff --git a/test/media/adaptation_set_unit.js b/test/media/adaptation_set_unit.js
@@ -220,6 +220,7 @@ describe('AdaptationSet', () => {
       trickModeVideo: null,
       type: '',
       accessibilityPurpose: null,
+      external: false,
     };
   }
 });

diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js
@@ -1196,7 +1196,7 @@ describe('MediaSourceEngine', () => {
     });
 
     it('destroys text engines', async () => {
-      mediaSourceEngine.reinitText('text/vtt', false);
+      mediaSourceEngine.reinitText('text/vtt', false, false);
 
       await mediaSourceEngine.destroy();
       expect(mockTextEngine).toBeTruthy();

diff --git a/test/offline/manifest_convert_unit.js b/test/offline/manifest_convert_unit.js
@@ -317,6 +317,7 @@ describe('ManifestConverter', () => {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
 
     return streamDB;
@@ -392,6 +393,7 @@ describe('ManifestConverter', () => {
       closedCaptions: null,
       tilesLayout: undefined,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 
@@ -443,6 +445,7 @@ describe('ManifestConverter', () => {
       closedCaptions: null,
       tilesLayout: undefined,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 
@@ -493,6 +496,7 @@ describe('ManifestConverter', () => {
       closedCaptions: null,
       tilesLayout: undefined,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 
@@ -540,6 +544,7 @@ describe('ManifestConverter', () => {
       closedCaptions: streamDb.closedCaptions,
       tilesLayout: streamDb.tilesLayout,
       accessibilityPurpose: null,
+      external: streamDb.external,
     };
 
     expect(stream).toEqual(expectedStream);

diff --git a/test/test/util/manifest_generator.js b/test/test/util/manifest_generator.js
@@ -555,6 +555,8 @@ shaka.test.ManifestGenerator.Stream = class {
       this.tilesLayout = undefined;
       /** @type {?shaka.media.ManifestParser.AccessibilityPurpose} */
       this.accessibilityPurpose;
+      /** @type {boolean} */
+      this.external = false;
     }
 
     /** @type {shaka.extern.Stream} */

diff --git a/test/test/util/offline_utils.js b/test/test/util/offline_utils.js
@@ -57,6 +57,7 @@ shaka.test.OfflineUtils = class {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
   }
 

diff --git a/test/test/util/streaming_engine_util.js b/test/test/util/streaming_engine_util.js
@@ -415,6 +415,7 @@ shaka.test.StreamingEngineUtil = class {
       forced: false,
       spatialAudio: false,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 
@@ -455,6 +456,7 @@ shaka.test.StreamingEngineUtil = class {
       forced: false,
       spatialAudio: false,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 
@@ -492,6 +494,7 @@ shaka.test.StreamingEngineUtil = class {
       forced: false,
       spatialAudio: false,
       accessibilityPurpose: null,
+      external: false,
     };
   }
 };