fix: embed cc not shown when seeking back (#4643)

For HLS non native, Shaka gets the captions from mux.js transmuxer. This works until you seek back before the buffered range. In mux.js there is a condition that ignores a caption if this belongs to a DTS that is behind the latest it already has parsed. To handle seeks, it adds a reset method that is called by the transmuxer.resetCaptions() method in mp4\transmuxer.js in mux.js. But Shaka Player is not calling it. So I added the resetCaptions() method to the mux.js extern definition and we called it in the appendBuffer method in media_source_engine.js before sending the data to transmux and only if the seeked parameter is True. Closes #4641
shaka-project · Nov 8, 2022 · c514d15 · c514d15
1 parent 687f92c
commit c514d15
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 0 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -32,6 +32,7 @@ Edgeware AB <*@edgeware.tv>
 Enson Choy <[email protected]>
 Esteban Dosztal <[email protected]>
 Fadomire <[email protected]>
+Gerardo Meola <[email protected]>
 Gil Gonen <[email protected]>
 Giorgio Gamberoni <[email protected]>
 Giuseppe Samela <[email protected]>

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
@@ -50,6 +50,7 @@ Enson Choy <[email protected]>
 Esteban Dosztal <[email protected]>
 Fadomire <[email protected]>
 François Beaufort <[email protected]>
+Gerardo Meola <[email protected]>
 Gil Gonen <[email protected]>
 Giorgio Gamberoni <[email protected]>
 Giuseppe Samela <[email protected]>

diff --git a/externs/mux.js b/externs/mux.js
@@ -74,6 +74,9 @@ muxjs.mp4.Transmuxer = class {
 
   /** Remove all handlers and clean up. */
   dispose() {}
+
+  /** Reset captions. */
+  resetCaptions() {}
 };
 
 

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -545,6 +545,12 @@ shaka.media.MediaSourceEngine = class {
     }
 
     if (this.transmuxers_[contentType]) {
+      // When seeked we should reset the transmuxer captionstreams
+      // so it does not ignores the captions from previous segments
+      if (seeked) {
+        this.transmuxers_[contentType].resetCaptions();
+      }
+
       const transmuxedData =
           await this.transmuxers_[contentType].transmux(data);
       // For HLS CEA-608/708 CLOSED-CAPTIONS, text data is embedded in

diff --git a/lib/media/transmuxer.js b/lib/media/transmuxer.js
@@ -182,6 +182,12 @@ shaka.media.Transmuxer = class {
     return this.transmuxPromise_;
   }
 
+  /**
+   * Reset captions from Transport stream to MP4, using the mux.js library.
+  */
+  resetCaptions() {
+    this.muxTransmuxer_.resetCaptions();
+  }
 
   /**
    * Handles the 'data' event of the transmuxer.

diff --git a/test/media/media_source_engine_integration.js b/test/media/media_source_engine_integration.js
@@ -70,6 +70,18 @@ describe('MediaSourceEngine', () => {
         type, segment, reference, /* hasClosedCaptions= */ false);
   }
 
+  function appendWithSeek(type, segmentNumber) {
+    const segment = generators[type]
+        .getSegment(segmentNumber, Date.now() / 1000);
+    const reference = dummyReference(type, segmentNumber);
+    return mediaSourceEngine.appendBuffer(
+        type,
+        segment,
+        reference,
+        /* hasClosedCaptions= */ false,
+        /* seeked= */ true);
+  }
+
   function appendInitWithClosedCaptions(type) {
     const segment = generators[type].getInitSegment(Date.now() / 1000);
     const reference = null;
@@ -380,6 +392,25 @@ describe('MediaSourceEngine', () => {
     expect(textDisplayer.appendSpy).toHaveBeenCalledTimes(3);
   });
 
+  it('extracts CEA-708 captions from previous segment from hls', async () => {
+    // Load TS file with CEA-708 captions.
+    metadata = shaka.test.TestScheme.DATA['cea-708_ts'];
+    generators = shaka.test.TestScheme.GENERATORS['cea-708_ts'];
+
+    const initObject = new Map();
+    initObject.set(ContentType.VIDEO, getFakeStream(metadata.video));
+    initObject.set(ContentType.TEXT, getFakeStream(metadata.text));
+    // Call with forceTransmux = true, so that it will transmux even on
+    // platforms with native TS support.
+    await mediaSourceEngine.init(initObject, /* forceTransmux= */ true);
+    mediaSourceEngine.setSelectedClosedCaptionId('CC1');
+
+    await append(ContentType.VIDEO, 2);
+    await appendWithSeek(ContentType.VIDEO, 0);
+
+    expect(textDisplayer.appendSpy).toHaveBeenCalledTimes(6);
+  });
+
   it('buffers partial TS video segments in sequence mode', async () => {
     metadata = shaka.test.TestScheme.DATA['cea-708_ts'];
     generators = shaka.test.TestScheme.GENERATORS['cea-708_ts'];