fix: Fix captions from MP4s with multiple trun boxes (#5422)

Backported to v4.2.x

Closes #5328

lib/cea/mp4_cea_parser.js

@@ -9,6 +9,7 @@ goog.provide('shaka.cea.Mp4CeaParser');
 goog.require('goog.asserts');
 goog.require('shaka.cea.ICeaParser');
 goog.require('shaka.cea.SeiProcessor');
+goog.require('shaka.log');
 goog.require('shaka.util.DataViewReader');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.Mp4Parser');
@@ -122,12 +123,19 @@ shaka.cea.Mp4CeaParser = class {
     // Fields that are found in MOOF boxes
     let defaultSampleDuration = this.defaultSampleDuration_;
     let defaultSampleSize = this.defaultSampleSize_;
-    let sampleData = [];
+    let moofOffset = 0;
+    /** @type {!Array<shaka.util.ParsedTRUNBox>} */
+    let parsedTRUNs = [];
     let baseMediaDecodeTime = null;
     let timescale = shaka.cea.ICeaParser.DEFAULT_TIMESCALE_VALUE;
 
     new Mp4Parser()
-        .box('moof', Mp4Parser.children)
+        .box('moof', (box) => {
+          moofOffset = box.start;
+          // trun box parsing is reset on each moof.
+          parsedTRUNs = [];
+          Mp4Parser.children(box);
+        })
         .box('traf', Mp4Parser.children)
         .fullBox('trun', (box) => {
           goog.asserts.assert(
@@ -136,10 +144,8 @@ shaka.cea.Mp4CeaParser = class {
 
           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
               box.reader, box.version, box.flags);
-
-          sampleData = parsedTRUN.sampleData;
+          parsedTRUNs.push(parsedTRUN);
         })
-
         .fullBox('tfhd', (box) => {
           goog.asserts.assert(
               box.flags != null,
@@ -163,7 +169,6 @@ shaka.cea.Mp4CeaParser = class {
             timescale = this.trackIdToTimescale_.get(trackId);
           }
         })
-
         .fullBox('tfdt', (box) => {
           goog.asserts.assert(
               box.version != null,
@@ -176,15 +181,18 @@ shaka.cea.Mp4CeaParser = class {
         })
         .box('mdat', (box) => {
           if (baseMediaDecodeTime === null) {
-            // This field should have been populated by
-            // the Base Media Decode time in the TFDT box
+            // This field should have been populated by the Base Media Decode
+            // Time in the tfdt box.
+            shaka.log.alwaysWarn(
+                'Unable to find base media decode time for CEA captions!');
             throw new shaka.util.Error(
                 shaka.util.Error.Severity.CRITICAL,
                 shaka.util.Error.Category.TEXT,
                 shaka.util.Error.Code.INVALID_MP4_CEA);
           }
+          const offset = moofOffset - box.start - 8;
           this.parseMdat_(box.reader, baseMediaDecodeTime, timescale,
-              defaultSampleDuration, defaultSampleSize, sampleData,
+              defaultSampleDuration, defaultSampleSize, offset, parsedTRUNs,
               captionPackets);
         })
         .parse(mediaSegment, /* partialOkay= */ false);
@@ -199,12 +207,13 @@ shaka.cea.Mp4CeaParser = class {
    * @param {number} timescale
    * @param {number} defaultSampleDuration
    * @param {number} defaultSampleSize
-   * @param {!Array<shaka.util.ParsedTRUNSample>} sampleData
+   * @param {number} offset
+   * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
    * @param {!Array<!shaka.cea.ICeaParser.CaptionPacket>} captionPackets
    * @private
    */
   parseMdat_(reader, time, timescale, defaultSampleDuration,
-      defaultSampleSize, sampleData, captionPackets) {
+      defaultSampleSize, offset, parsedTRUNs, captionPackets) {
     let sampleIndex = 0;
 
     // The fields in each ParsedTRUNSample contained in the sampleData
@@ -213,17 +222,24 @@ shaka.cea.Mp4CeaParser = class {
     // composition time offset, we default to 0.
     let sampleSize = defaultSampleSize;
 
+    // Combine all sample data.  This assumes that the samples described across
+    // multiple trun boxes are still continuous in the mdat box.
+    const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
+    const sampleData = sampleDatas.flat();
+
     if (sampleData.length) {
       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
     }
 
+    reader.skip(offset + parsedTRUNs[0].dataOffset);
+
     while (reader.hasMoreData()) {
       const naluSize = reader.readUint32();
       const naluType = reader.readUint8() & 0x1F;
       if (naluType == shaka.cea.ICeaParser.NALU_TYPE_SEI) {
         let timeOffset = 0;
 
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
         }
 
@@ -246,7 +262,7 @@ shaka.cea.Mp4CeaParser = class {
       }
       sampleSize -= (naluSize + 4);
       if (sampleSize == 0) {
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           time += sampleData[sampleIndex].sampleDuration ||
               defaultSampleDuration;
         } else {
@@ -255,7 +271,7 @@ shaka.cea.Mp4CeaParser = class {
 
         sampleIndex++;
 
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
         } else {
           sampleSize = defaultSampleSize;

lib/util/mp4_box_parsers.js

@@ -112,10 +112,11 @@ shaka.util.Mp4BoxParsers = class {
   static parseTRUN(reader, version, flags) {
     const sampleCount = reader.readUint32();
     const sampleData = [];
+    let dataOffset = null;
 
-    // Skip "data_offset" if present.
+    // "data_offset"
     if (flags & 0x000001) {
-      reader.skip(4);
+      dataOffset = reader.readUint32();
     }
 
     // Skip "first_sample_flags" if present.
@@ -159,6 +160,7 @@ shaka.util.Mp4BoxParsers = class {
     return {
       sampleCount,
       sampleData,
+      dataOffset,
     };
   }
 
@@ -252,13 +254,16 @@ shaka.util.ParsedTREXBox;
 /**
  * @typedef {{
  *    sampleCount: number,
- *    sampleData: !Array.<shaka.util.ParsedTRUNSample>
+ *    sampleData: !Array.<shaka.util.ParsedTRUNSample>,
+ *    dataOffset: ?number
  * }}
  *
  * @property {number} sampleCount
  *   As per the spec: the number of samples being added in this run;
  * @property {!Array.<shaka.util.ParsedTRUNSample>} sampleData
  *   An array of size <sampleCount> containing data for each sample
+ * @property {?number} dataOffset
+ *   If specified via flags, this indicate the offset of the sample in bytes.
  *
  * @exportDoc
  */

test/cea/mp4_cea_parser_unit.js

@@ -7,20 +7,33 @@
 describe('Mp4CeaParser', () => {
   const ceaInitSegmentUri = '/base/test/test/assets/cea-init.mp4';
   const ceaSegmentUri = '/base/test/test/assets/cea-segment.mp4';
+  const multipleTrunInitSegmentUri =
+      '/base/test/test/assets/multiple-trun-init.mp4';
+  const multipleTrunSegmentUri =
+      '/base/test/test/assets/multiple-trun-segment.mp4';
   const Util = shaka.test.Util;
 
   /** @type {!ArrayBuffer} */
   let ceaInitSegment;
   /** @type {!ArrayBuffer} */
   let ceaSegment;
+  /** @type {!ArrayBuffer} */
+  let multipleTrunInitSegment;
+  /** @type {!ArrayBuffer} */
+  let multipleTrunSegment;
 
   beforeAll(async () => {
-    const responses = await Promise.all([
+    [
+      ceaInitSegment,
+      ceaSegment,
+      multipleTrunInitSegment,
+      multipleTrunSegment,
+    ] = await Promise.all([
       shaka.test.Util.fetch(ceaInitSegmentUri),
       shaka.test.Util.fetch(ceaSegmentUri),
+      shaka.test.Util.fetch(multipleTrunInitSegmentUri),
+      shaka.test.Util.fetch(multipleTrunSegmentUri),
     ]);
-    ceaInitSegment = responses[0];
-    ceaSegment = responses[1];
   });
 
   /**
@@ -70,6 +83,16 @@ describe('Mp4CeaParser', () => {
         .toEqual(expectedCea708Packet);
   });
 
+  it('parses cea data from a segment with multiple trun boxes', () => {
+    const ceaParser = new shaka.cea.Mp4CeaParser();
+
+    ceaParser.init(multipleTrunInitSegment);
+    const ceaPackets = ceaParser.parse(multipleTrunSegment);
+    // The first trun box references samples with 48 CEA packets.
+    // The second trun box references samples with 132 more, for a total of 180.
+    expect(ceaPackets.length).toBe(180);
+  });
+
   it('parses an invalid init segment', () => {
     const cea708Parser = new shaka.cea.Mp4CeaParser();