[PATCH][v4] cherry pick of needed changes from upstream (shaka-projec…

…t#306) * [PATCH] fix(dash): fix race condition in segment template (shaka-project#5842) Fixes shaka-project#5760 * fix(cea): Fix not rendering CEA-608 Closed Captions (shaka-project#4683) Also added H265 support and a framework for future TS CEA parser support. Fixes shaka-project#4605 Fixes shaka-project#3659 Co-authored-by: Joey Parrish <[email protected]> * fix(cea): Fix not rendering CEA-608 on encrypted mp4 segments (shaka-project#4756) Fixes shaka-project#4605 Co-authored-by: Joey Parrish <[email protected]> * fix: CEA 608 captions not work with H.265 video streams (shaka-project#5252) Fix parsing of CEA 608 captions in H.265 video streams by handling 2 byte nal unit header. Fixes shaka-project#5251 * fix: seeking in segment timeline returns incorrect index (shaka-project#5716) Resolves shaka-project#5664 --------- Co-authored-by: Albin Larsson <[email protected]> Co-authored-by: Álvaro Velad Galván <[email protected]> Co-authored-by: Joey Parrish <[email protected]> Co-authored-by: Aidan Ridley <[email protected]> Co-authored-by: Casey Occhialini <[email protected]>
sky-hugolima · Nov 9, 2023 · ca5942e · ca5942e
1 parent eafef44
commit ca5942e
Show file tree

Hide file tree

Showing 22 changed files with 319 additions and 45 deletions.
diff --git a/build/types/cea b/build/types/cea
@@ -7,6 +7,7 @@
 +../../lib/cea/cea708_service.js
 +../../lib/cea/cea708_window.js
 +../../lib/cea/dtvcc_packet_builder.js
++../../lib/cea/dummy_cea_parser.js
 +../../lib/cea/i_caption_decoder.js
 +../../lib/cea/i_cea_parser.js
 +../../lib/cea/mp4_cea_parser.js

diff --git a/externs/shaka/mp4_parser.js b/externs/shaka/mp4_parser.js
@@ -11,6 +11,7 @@
 
 /**
  * @typedef {{
+ *    name: string,
  *    parser: !shaka.util.Mp4Parser,
  *    partialOkay: boolean,
  *    start: number,
@@ -21,6 +22,8 @@
  *    has64BitSize: boolean
  * }}
  *
+ * @property {string} name
+ *   The box name, a 4-character string (fourcc).
  * @property {!shaka.util.Mp4Parser} parser
  *   The parser that parsed this box. The parser can be used to parse child
  *   boxes where the configuration of the current parser is needed to parsed

diff --git a/lib/cea/dummy_cea_parser.js b/lib/cea/dummy_cea_parser.js
@@ -0,0 +1,28 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.cea.DummyCeaParser');
+
+goog.require('shaka.cea.ICeaParser');
+
+/**
+ * Dummy CEA parser.
+ * @implements {shaka.cea.ICeaParser}
+ */
+shaka.cea.DummyCeaParser = class {
+  /**
+   * @override
+   */
+  init(initSegment) {
+  }
+
+  /**
+   * @override
+   */
+  parse(mediaSegment) {
+    return /* captionPackets= */ [];
+  }
+};
diff --git a/lib/cea/i_cea_parser.js b/lib/cea/i_cea_parser.js
@@ -26,10 +26,22 @@ shaka.cea.ICeaParser = class {
 };
 
 /**
- * NALU type for Supplemental Enhancement Information (SEI).
+ * NALU type for Supplemental Enhancement Information (SEI) for H.264.
  * @const {number}
  */
-shaka.cea.ICeaParser.NALU_TYPE_SEI = 0x06;
+shaka.cea.ICeaParser.H264_NALU_TYPE_SEI = 0x06;
+
+/**
+ * NALU type for Supplemental Enhancement Information (SEI) for H.265.
+ * @const {number}
+ */
+shaka.cea.ICeaParser.H265_PREFIX_NALU_TYPE_SEI = 0x27;
+
+/**
+ * NALU type for Supplemental Enhancement Information (SEI) for H.265.
+ * @const {number}
+ */
+shaka.cea.ICeaParser.H265_SUFFIX_NALU_TYPE_SEI = 0x28;
 
 /**
  * Default timescale value for a track.

diff --git a/lib/cea/mp4_cea_parser.js b/lib/cea/mp4_cea_parser.js
@@ -46,6 +46,11 @@ shaka.cea.Mp4CeaParser = class {
      * @private {number}
      */
     this.defaultSampleSize_ = 0;
+
+    /**
+     * @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
+     */
+    this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
   }
 
   /**
@@ -57,9 +62,12 @@ shaka.cea.Mp4CeaParser = class {
    */
   init(initSegment) {
     const Mp4Parser = shaka.util.Mp4Parser;
+    const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
     const trackIds = [];
     const timescales = [];
 
+    const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
+
     new Mp4Parser()
         .box('moov', Mp4Parser.children)
         .box('mvex', Mp4Parser.children)
@@ -88,6 +96,27 @@ shaka.cea.Mp4CeaParser = class {
               box.reader, box.version);
           timescales.push(parsedMDHDBox.timescale);
         })
+        .box('minf', Mp4Parser.children)
+        .box('stbl', Mp4Parser.children)
+        .fullBox('stsd', Mp4Parser.sampleDescription)
+
+        // These are the various boxes that signal a codec.
+        .box('avc1', codecBoxParser)
+        .box('avc3', codecBoxParser)
+        .box('hev1', codecBoxParser)
+        .box('hvc1', codecBoxParser)
+        .box('dvh1', codecBoxParser)
+        .box('dvhe', codecBoxParser)
+
+        // This signals an encrypted sample, which we can go inside of to find
+        // the codec used.
+        .box('encv', Mp4Parser.visualSampleEntry)
+        .box('sinf', Mp4Parser.children)
+        .box('frma', (box) => {
+          const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
+          this.setBitstreamFormat_(codec);
+        })
+
         .parse(initSegment, /* partialOkay= */ true);
 
     // At least one track should exist, and each track should have a
@@ -100,6 +129,11 @@ shaka.cea.Mp4CeaParser = class {
           shaka.util.Error.Code.INVALID_MP4_CEA);
     }
 
+    if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
+      shaka.log.alwaysWarn(
+          'Unable to determine bitstream format for CEA parsing!');
+    }
+
     // Populate the map from track Id to timescale
     trackIds.forEach((trackId, idx) => {
       this.trackIdToTimescale_.set(trackId, timescales[idx]);
@@ -116,6 +150,12 @@ shaka.cea.Mp4CeaParser = class {
    */
   parse(mediaSegment) {
     const Mp4Parser = shaka.util.Mp4Parser;
+    const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
+
+    if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
+      // We don't know how to extract SEI from this.
+      return [];
+    }
 
     /** @type {!Array<!shaka.cea.ICeaParser.CaptionPacket>} **/
     const captionPackets = [];
@@ -214,6 +254,8 @@ shaka.cea.Mp4CeaParser = class {
    */
   parseMdat_(reader, time, timescale, defaultSampleDuration,
       defaultSampleSize, offset, parsedTRUNs, captionPackets) {
+    const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
+    const ICeaParser = shaka.cea.ICeaParser;
     let sampleIndex = 0;
 
     // The fields in each ParsedTRUNSample contained in the sampleData
@@ -235,8 +277,33 @@ shaka.cea.Mp4CeaParser = class {
 
     while (reader.hasMoreData()) {
       const naluSize = reader.readUint32();
-      const naluType = reader.readUint8() & 0x1F;
-      if (naluType == shaka.cea.ICeaParser.NALU_TYPE_SEI) {
+      const naluHeader = reader.readUint8();
+      let naluType = null;
+      let isSeiMessage = false;
+      let naluHeaderSize = 1;
+
+      goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
+          'Bitstream format should have been checked before now!');
+      switch (this.bitstreamFormat_) {
+        case BitstreamFormat.H264:
+          naluType = naluHeader & 0x1f;
+          isSeiMessage = naluType == ICeaParser.H264_NALU_TYPE_SEI;
+          break;
+
+        case BitstreamFormat.H265:
+          naluHeaderSize = 2;
+          reader.skip(1);
+          naluType = (naluHeader >> 1) & 0x3f;
+          isSeiMessage =
+              naluType == ICeaParser.H265_PREFIX_NALU_TYPE_SEI ||
+              naluType == ICeaParser.H265_SUFFIX_NALU_TYPE_SEI;
+          break;
+
+        default:
+          return;
+      }
+
+      if (isSeiMessage) {
         let timeOffset = 0;
 
         if (sampleIndex < sampleData.length) {
@@ -245,15 +312,15 @@ shaka.cea.Mp4CeaParser = class {
 
         const pts = (time + timeOffset)/timescale;
         for (const packet of this.seiProcessor_
-            .process(reader.readBytes(naluSize - 1))) {
+            .process(reader.readBytes(naluSize - naluHeaderSize))) {
           captionPackets.push({
             packet,
             pts,
           });
         }
       } else {
         try {
-          reader.skip(naluSize - 1);
+          reader.skip(naluSize - naluHeaderSize);
         } catch (e) {
           // It is necessary to ignore this error because it can break the start
           // of playback even if the user does not want to see the subtitles.
@@ -279,4 +346,32 @@ shaka.cea.Mp4CeaParser = class {
       }
     }
   }
+
+  /**
+   * @param {string} codec A fourcc for a codec.
+   * @private
+   */
+  setBitstreamFormat_(codec) {
+    if (codec in shaka.cea.Mp4CeaParser.CodecBitstreamMap_) {
+      this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.CodecBitstreamMap_[codec];
+    }
+  }
+};
+
+/** @enum {number} */
+shaka.cea.Mp4CeaParser.BitstreamFormat = {
+  UNKNOWN: 0,
+  H264: 1,
+  H265: 2,
+};
+
+/** @private {Object.<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
+shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = {
+  'avc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
+  'avc3': shaka.cea.Mp4CeaParser.BitstreamFormat.H264,
+  'hev1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
+  'hvc1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
+  // Dobly vision is also H265.
+  'dvh1': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
+  'dvhe': shaka.cea.Mp4CeaParser.BitstreamFormat.H265,
 };
diff --git a/lib/dash/segment_template.js b/lib/dash/segment_template.js
@@ -907,6 +907,7 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
     let ref = this.references[correctedPosition];
 
     if (!ref) {
+      const mediaTemplate = this.templateInfo_.mediaTemplate;
       const range = this.templateInfo_.timeline[correctedPosition];
       const segmentReplacement = position + this.templateInfo_.startNumber;
       const timeReplacement = this.templateInfo_
@@ -915,7 +916,7 @@ shaka.dash.TimelineSegmentIndex = class extends shaka.media.SegmentIndex {
       const createUrisCb = () => {
         return shaka.dash.TimelineSegmentIndex
             .createUris_(
-                this.templateInfo_.mediaTemplate,
+                mediaTemplate,
                 this.representationId_,
                 segmentReplacement,
                 this.bandwidth_,

diff --git a/lib/media/closed_caption_parser.js b/lib/media/closed_caption_parser.js
@@ -8,6 +8,7 @@ goog.provide('shaka.media.ClosedCaptionParser');
 goog.provide('shaka.media.IClosedCaptionParser');
 
 goog.require('shaka.cea.CeaDecoder');
+goog.require('shaka.cea.DummyCeaParser');
 goog.require('shaka.cea.Mp4CeaParser');
 goog.require('shaka.util.BufferUtils');
 goog.requireType('shaka.cea.ICaptionDecoder');
@@ -54,12 +55,14 @@ shaka.media.IClosedCaptionParser = class {
  */
 shaka.media.ClosedCaptionParser = class {
   /** */
-  constructor() {
-    /**
-     * MP4 Parser to extract closed caption packets from H.264 video.
-     * @private {!shaka.cea.ICeaParser}
-     */
-    this.ceaParser_ = new shaka.cea.Mp4CeaParser();
+  constructor(mimeType) {
+    /** @private {!shaka.cea.ICeaParser} */
+    this.ceaParser_ = new shaka.cea.DummyCeaParser();
+
+    if (mimeType.includes('video/mp4')) {
+      // MP4 Parser to extract closed caption packets from H.264 video.
+      this.ceaParser_ = new shaka.cea.Mp4CeaParser();
+    }
 
     /**
      * Decoder for decoding CEA-X08 data from closed caption packets.

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -11,6 +11,7 @@ goog.require('shaka.config.CodecSwitchingStrategy');
 goog.require('shaka.log');
 goog.require('shaka.media.Capabilities');
 goog.require('shaka.media.ContentWorkarounds');
+goog.require('shaka.media.ClosedCaptionParser');
 goog.require('shaka.media.IClosedCaptionParser');
 goog.require('shaka.media.SegmentReference');
 goog.require('shaka.media.TimeRangesUtils');
@@ -40,18 +41,14 @@ shaka.media.MediaSourceEngine = class {
   /**
    * @param {HTMLMediaElement} video The video element, whose source is tied to
    *   MediaSource during the lifetime of the MediaSourceEngine.
-   * @param {!shaka.media.IClosedCaptionParser} closedCaptionParser
-   *    The closed caption parser that should be used to parser closed captions
-   *    from the video stream. MediaSourceEngine takes ownership of the parser.
-   *    When MediaSourceEngine is destroyed, it will destroy the parser.
    * @param {!shaka.extern.TextDisplayer} textDisplayer
    *    The text displayer that will be used with the text engine.
    *    MediaSourceEngine takes ownership of the displayer. When
    *    MediaSourceEngine is destroyed, it will destroy the displayer.
    * @param {!function(!Array.<shaka.extern.ID3Metadata>, number, ?number)=}
    *    onMetadata
    */
-  constructor(video, closedCaptionParser, textDisplayer, onMetadata) {
+  constructor(video, textDisplayer, onMetadata) {
     /** @private {HTMLMediaElement} */
     this.video_ = video;
 
@@ -98,8 +95,8 @@ shaka.media.MediaSourceEngine = class {
     /** @private {!Object.<string, !shaka.media.Transmuxer>} */
     this.transmuxers_ = {};
 
-    /** @private {shaka.media.IClosedCaptionParser} */
-    this.captionParser_ = closedCaptionParser;
+    /** @private {?shaka.media.IClosedCaptionParser} */
+    this.captionParser_ = null;
 
     /** @private {!shaka.util.PublicPromise} */
     this.mediaSourceOpen_ = new shaka.util.PublicPromise();
@@ -561,6 +558,17 @@ shaka.media.MediaSourceEngine = class {
     }
   }
 
+  /**
+   * Create a new closed caption parser. This will ONLY be replaced by tests as
+   * a way to inject fake closed caption parser instances.
+   *
+   * @param {string} mimeType
+   * @return {!shaka.media.IClosedCaptionParser}
+   */
+  getCaptionParser(mimeType) {
+    return new shaka.media.ClosedCaptionParser(mimeType);
+  }
+
   /**
    * Enqueue an operation to append data to the SourceBuffer.
    * Start and end times are needed for TextEngine, but not for MediaSource.
@@ -637,11 +645,15 @@ shaka.media.MediaSourceEngine = class {
       }
 
       data = transmuxedData.data;
-    } else if (hasClosedCaptions) {
+    } else if (hasClosedCaptions && contentType == ContentType.VIDEO) {
       if (!this.textEngine_) {
         this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE,
             this.sequenceMode_);
       }
+      if (!this.captionParser_) {
+        const mimeType = this.sourceBufferTypes_[contentType];
+        this.captionParser_ = this.getCaptionParser(mimeType);
+      }
       // If it is the init segment for closed captions, initialize the closed
       // caption parser.
       if (!reference) {
@@ -846,7 +858,9 @@ shaka.media.MediaSourceEngine = class {
    * Fully reset the state of the caption parser owned by MediaSourceEngine.
    */
   resetCaptionParser() {
-    this.captionParser_.reset();
+    if (this.captionParser_) {
+      this.captionParser_.reset();
+    }
   }
 
   /**