feat: add event stream support (#1382)

* feat: add event stream support * tests and cue fix * test description * remove value * fix timestamp offset and refactor * refactor and fix tests * logs and spacing * additional test manifest * remove extra line * fix test sourceupdater assignment * update supported feature doc * specify DASH
videojs · Apr 3, 2023 · f6b9498 · f6b9498
1 parent 6bd98d0
commit f6b9498
Show file tree

Hide file tree

Showing 9 changed files with 270 additions and 24 deletions.
diff --git a/docs/supported-features.md b/docs/supported-features.md
@@ -64,6 +64,7 @@ not meant serve as an exhaustive list.
 * In-manifest [WebVTT] subtitles are automatically translated into standard HTML5 subtitle
   tracks
 * [AES-128] segment encryption
+* DASH In-manifest EventStream and Event tags are automatically translated into HTML5 metadata cues
 
 ## Notable Missing Features
 

diff --git a/src/dash-playlist-loader.js b/src/dash-playlist-loader.js
@@ -312,6 +312,7 @@ export default class DashPlaylistLoader extends EventTarget {
 
     this.vhs_ = vhs;
     this.withCredentials = withCredentials;
+    this.addMetadataToTextTrack = options.addMetadataToTextTrack;
 
     if (!srcUrlOrPlaylist) {
       throw new Error('A non-empty playlist URL or object is required');
@@ -773,6 +774,8 @@ export default class DashPlaylistLoader extends EventTarget {
       this.updateMinimumUpdatePeriodTimeout_();
     }
 
+    this.addEventStreamToMetadataTrack_(newMain);
+
     return Boolean(newMain);
   }
 
@@ -901,4 +904,24 @@ export default class DashPlaylistLoader extends EventTarget {
 
     this.trigger('loadedplaylist');
   }
+
+  /**
+   * Takes eventstream data from a parsed DASH manifest and adds it to the metadata text track.
+   *
+   * @param {manifest} newMain the newly parsed manifest
+   */
+  addEventStreamToMetadataTrack_(newMain) {
+    // Only add new event stream metadata if we have a new manifest.
+    if (newMain && this.mainPlaylistLoader_.main.eventStream) {
+      // convert EventStream to ID3-like data.
+      const metadataArray = this.mainPlaylistLoader_.main.eventStream.map((eventStreamNode) => {
+        return {
+          cueTime: eventStreamNode.start,
+          frames: [{ data: eventStreamNode.messageData }]
+        };
+      });
+
+      this.addMetadataToTextTrack('EventStream', metadataArray, this.mainPlaylistLoader_.main.duration);
+    }
+  }
 }
diff --git a/src/playlist-controller.js b/src/playlist-controller.js
@@ -26,6 +26,7 @@ import { codecsForPlaylist, unwrapCodecList, codecCount } from './util/codecs.js
 import { createMediaTypes, setupMediaGroups } from './media-groups';
 import logger from './util/logger';
 import {merge, createTimeRanges} from './util/vjs-compat';
+import { addMetadata, createMetadataTrackIfNotExists } from './util/text-tracks';
 
 const ABORT_EARLY_EXCLUSION_SECONDS = 60 * 2;
 
@@ -254,15 +255,16 @@ export class PlaylistController extends videojs.EventTarget {
       cacheEncryptionKeys,
       sourceUpdater: this.sourceUpdater_,
       timelineChangeController: this.timelineChangeController_,
-      exactManifestTimings: options.exactManifestTimings
+      exactManifestTimings: options.exactManifestTimings,
+      addMetadataToTextTrack: this.addMetadataToTextTrack.bind(this)
     };
 
     // The source type check not only determines whether a special DASH playlist loader
     // should be used, but also covers the case where the provided src is a vhs-json
     // manifest object (instead of a URL). In the case of vhs-json, the default
     // PlaylistLoader should be used.
     this.mainPlaylistLoader_ = this.sourceType_ === 'dash' ?
-      new DashPlaylistLoader(src, this.vhs_, this.requestOptions_) :
+      new DashPlaylistLoader(src, this.vhs_, merge(this.requestOptions_, { addMetadataToTextTrack: this.addMetadataToTextTrack.bind(this) })) :
       new PlaylistLoader(src, this.vhs_, this.requestOptions_);
     this.setupMainPlaylistLoaderListeners_();
 
@@ -2009,4 +2011,19 @@ export class PlaylistController extends videojs.EventTarget {
     return Config.BUFFER_HIGH_WATER_LINE;
   }
 
+  addMetadataToTextTrack(dispatchType, metadataArray, videoDuration) {
+    const timestampOffset = this.sourceUpdater_.videoTimestampOffset() === null ?
+      this.sourceUpdater_.audioTimestampOffset() : this.sourceUpdater_.videoTimestampOffset();
+
+    // There's potentially an issue where we could double add metadata if there's a muxed
+    // audio/video source with a metadata track, and an alt audio with a metadata track.
+    // However, this probably won't happen, and if it does it can be handled then.
+    createMetadataTrackIfNotExists(this.inbandTextTracks_, dispatchType, this.tech_);
+    addMetadata({
+      inbandTextTracks: this.inbandTextTracks_,
+      metadataArray,
+      timestampOffset,
+      videoDuration
+    });
+  }
 }
diff --git a/src/segment-loader.js b/src/segment-loader.js
@@ -14,8 +14,6 @@ import logger from './util/logger';
 import { concatSegments } from './util/segment';
 import {
   createCaptionsTrackIfNotExists,
-  createMetadataTrackIfNotExists,
-  addMetadata,
   addCaptionData,
   removeCuesFromTrack
 } from './util/text-tracks';
@@ -563,6 +561,7 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.useDtsForTimestampOffset_ = settings.useDtsForTimestampOffset;
     this.captionServices_ = settings.captionServices;
     this.exactManifestTimings = settings.exactManifestTimings;
+    this.addMetadataToTextTrack = settings.addMetadataToTextTrack;
 
     // private instance variables
     this.checkBufferTimeout_ = null;
@@ -1872,21 +1871,7 @@ export default class SegmentLoader extends videojs.EventTarget {
       this.metadataQueue_.id3.push(this.handleId3_.bind(this, simpleSegment, id3Frames, dispatchType));
       return;
     }
-
-    const timestampOffset = this.sourceUpdater_.videoTimestampOffset() === null ?
-      this.sourceUpdater_.audioTimestampOffset() :
-      this.sourceUpdater_.videoTimestampOffset();
-
-    // There's potentially an issue where we could double add metadata if there's a muxed
-    // audio/video source with a metadata track, and an alt audio with a metadata track.
-    // However, this probably won't happen, and if it does it can be handled then.
-    createMetadataTrackIfNotExists(this.inbandTextTracks_, dispatchType, this.vhs_.tech_);
-    addMetadata({
-      inbandTextTracks: this.inbandTextTracks_,
-      metadataArray: id3Frames,
-      timestampOffset,
-      videoDuration: this.duration_()
-    });
+    this.addMetadataToTextTrack(dispatchType, id3Frames, this.duration_());
   }
 
   processMetadataQueue_() {

diff --git a/test/loader-common.js b/test/loader-common.js
@@ -52,18 +52,26 @@ export const LoaderCommonHooks = {
         playbackRate: () => this.playbackRate,
         currentTime: () => this.currentTime,
         textTracks: () => {},
-        addRemoteTextTrack: () => {},
+        addRemoteTextTrack: (track) => {
+          return track;
+        },
         trigger: () => {}
       }
     };
     this.tech_ = this.fakeVhs.tech_;
     this.goalBufferLength =
       PlaylistController.prototype.goalBufferLength.bind(this);
     this.mediaSource = new window.MediaSource();
-    this.sourceUpdater = new SourceUpdater(this.mediaSource);
+    this.sourceUpdater_ = new SourceUpdater(this.mediaSource);
+    this.inbandTextTracks_ = {
+      metadataTrack_: {
+        addCue: () => {}
+      }
+    };
     this.syncController = new SyncController();
     this.decrypter = new Decrypter();
     this.timelineChangeController = new TimelineChangeController();
+    this.addMetadataToTextTrack = PlaylistController.prototype.addMetadataToTextTrack.bind(this);
 
     this.video = document.createElement('video');
 
@@ -80,7 +88,7 @@ export const LoaderCommonHooks = {
 
     this.env.restore();
     this.decrypter.terminate();
-    this.sourceUpdater.dispose();
+    this.sourceUpdater_.dispose();
     this.timelineChangeController.dispose();
   }
 };
@@ -105,10 +113,11 @@ export const LoaderCommonSettings = function(settings) {
     duration: () => this.mediaSource.duration,
     goalBufferLength: () => this.goalBufferLength(),
     mediaSource: this.mediaSource,
-    sourceUpdater: this.sourceUpdater,
+    sourceUpdater: this.sourceUpdater_,
     syncController: this.syncController,
     decrypter: this.decrypter,
-    timelineChangeController: this.timelineChangeController
+    timelineChangeController: this.timelineChangeController,
+    addMetadataToTextTrack: this.addMetadataToTextTrack
   }, settings);
 };
 

diff --git a/test/manifests/eventStream.mpd b/test/manifests/eventStream.mpd
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:_XMLSchema-instance="http://www.w3.org/2001/XMLSchema-instance" _XMLSchema-instance:schemaLocation="urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd" id="201" profiles="urn:mpeg:dash:profile:isoff-live:2011" type="dynamic" availabilityStartTime="2021-02-25T22:43:33.112Z" publishTime="2023-03-07T01:01:34.248221397Z" minimumUpdatePeriod="PT5S" minBufferTime="PT30S" timeShiftBufferDepth="PT1M0S" suggestedPresentationDelay="PT25S">
+  <BaseURL>https://547f72e6652371c3.mediapackage.us-east-1.amazonaws.com/out/v1/ef88d1a64fd543f4a4be9a56c913d868/</BaseURL>
+  <Location>https://dai.google.com/linear/dash/pa/event/PSzZMzAkSXCmlJOWDmRj8Q/stream/7e6b742f-0013-412c-b8a9-fd527adafc89:ATL/manifest.mpd</Location>
+  <Period id="dai_pod-0001065804-ad-1" start="PT17738H17M14.156S" duration="PT9.977S">
+    <BaseURL>https://dai.google.com/linear/pods/v1/p/PSzZMzAkSXCmlJOWDmRj8Q/7e6b742f-0013-412c-b8a9-fd527adafc89:ATL/1065804/0/0/</BaseURL>
+    <SegmentTemplate media="$RepresentationID$/$Number$.mp4" initialization="$RepresentationID$/init.mp4"/>
+    <EventStream schemeIdUri="urn:foo:bar:2023" timescale="1000">
+      <Event presentationTime="100" duration="0" id="0">foo</Event>
+      <Event presentationTime="900" duration="0" id="5">bar</Event>
+      <Event presentationTime="1900" duration="0" id="6">foo_bar</Event>
+      <Event presentationTime="2494" duration="0" id="1">bar_foo</Event>
+    </EventStream>
+    <AdaptationSet id="0" contentType="audio">
+      <SegmentTemplate timescale="48000" startNumber="0">
+        <SegmentTimeline>
+          <S d="96240" r="3"/>
+          <S d="93936"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation audioSamplingRate="48000" mimeType="audio/mp4" codecs="mp4a.40.2" id="a87ff679a2f3e71d9181a67b7542122c" bandwidth="98000">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+      </Representation>
+      <Representation audioSamplingRate="48000" mimeType="audio/mp4" codecs="mp4a.40.2" id="eccbc87e4b5ce2fe28308fd9f2a7baf3" bandwidth="130000">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet startWithSAP="1" id="1" contentType="video" minFrameRate="24000/1001" maxFrameRate="30000/1001" segmentAlignment="true">
+      <SegmentTemplate timescale="90000" startNumber="0">
+        <SegmentTimeline>
+          <S d="180180" r="3"/>
+          <S d="177210"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation width="1280" height="720" frameRate="30000/1001" mimeType="video/mp4" codecs="avc1.64001f" id="c4ca4238a0b923820dcc509a6f75849b" bandwidth="331000"/>
+      <Representation width="480" height="270" frameRate="24000/1001" mimeType="video/mp4" codecs="avc1.42c015" id="c81e728d9d4c2f636f067f89cc14862c" bandwidth="137000"/>
+    </AdaptationSet>
+  </Period>
+</MPD>
diff --git a/test/manifests/eventStreamMessageData.mpd b/test/manifests/eventStreamMessageData.mpd
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:_XMLSchema-instance="http://www.w3.org/2001/XMLSchema-instance" _XMLSchema-instance:schemaLocation="urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd" id="201" profiles="urn:mpeg:dash:profile:isoff-live:2011" type="dynamic" availabilityStartTime="2021-02-25T22:43:33.112Z" publishTime="2023-03-07T01:01:34.248221397Z" minimumUpdatePeriod="PT5S" minBufferTime="PT30S" timeShiftBufferDepth="PT1M0S" suggestedPresentationDelay="PT25S">
+  <BaseURL>https://547f72e6652371c3.mediapackage.us-east-1.amazonaws.com/out/v1/ef88d1a64fd543f4a4be9a56c913d868/</BaseURL>
+  <Location>https://dai.google.com/linear/dash/pa/event/PSzZMzAkSXCmlJOWDmRj8Q/stream/7e6b742f-0013-412c-b8a9-fd527adafc89:ATL/manifest.mpd</Location>
+  <Period id="dai_pod-0001065804-ad-1" start="PT17738H17M14.156S" duration="PT9.977S">
+    <BaseURL>https://dai.google.com/linear/pods/v1/p/PSzZMzAkSXCmlJOWDmRj8Q/7e6b742f-0013-412c-b8a9-fd527adafc89:ATL/1065804/0/0/</BaseURL>
+    <SegmentTemplate media="$RepresentationID$/$Number$.mp4" initialization="$RepresentationID$/init.mp4"/>
+    <EventStream schemeIdUri="urn:google:dai:2018" timescale="1000">
+      <Event presentationTime="100" duration="0" id="0" messageData="google_7617584398642699833"/>
+      <Event presentationTime="900" duration="0" id="5" messageData="google_gkmxVFMIdHz413g3pIgZtITUSFFQYDnQ421MGEkVnTA"/>
+      <Event presentationTime="1900" duration="0" id="6" messageData="google_Yl7LFi1Fh-TD39nqQzIiGLDD1lx7tYRjjmYND7tEEjM"/>
+      <Event presentationTime="2494" duration="0" id="1" messageData="google_5437877779805246002"/>
+      <Event presentationTime="2900" duration="0" id="7" messageData="google_8X2eBAFbC2cUJmNNHkrcDKqSJQncj2nrVoB2eIu6lrc"/>
+      <Event presentationTime="3900" duration="0" id="8" messageData="google_Qyxg2ZhKfBUls-J7oj0Re0_-gCQFviaaEMMDvIOTEWE"/>
+      <Event presentationTime="4738" duration="0" id="2" messageData="google_7174574530630198647"/>
+      <Event presentationTime="4900" duration="0" id="9" messageData="google_EFt2jovkcT9PqjuLLC5kH7gIIjWvc0iIhROFED6kqsg"/>
+      <Event presentationTime="5900" duration="0" id="10" messageData="google_eUHx4vMmAikHojJZLOTR2XZdg1A9b9A8TY7F2CVC3cA"/>
+      <Event presentationTime="6900" duration="0" id="11" messageData="google_gkmxVFMIdHz413g3pIgZtITUSFFQYDnQ421MGEkVnTA"/>
+      <Event presentationTime="7482" duration="0" id="3" messageData="google_1443613685977331553"/>
+      <Event presentationTime="7900" duration="0" id="12" messageData="google_Yl7LFi1Fh-TD39nqQzIiGLDD1lx7tYRjjmYND7tEEjM"/>
+      <Event presentationTime="8900" duration="0" id="13" messageData="google_8X2eBAFbC2cUJmNNHkrcDKqSJQncj2nrVoB2eIu6lrc"/>
+      <Event presentationTime="8976" duration="0" id="4" messageData="google_5822903356700578162"/>
+    </EventStream>
+    <AdaptationSet id="0" contentType="audio">
+      <SegmentTemplate timescale="48000" startNumber="0">
+        <SegmentTimeline>
+          <S d="96240" r="3"/>
+          <S d="93936"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation audioSamplingRate="48000" mimeType="audio/mp4" codecs="mp4a.40.2" id="a87ff679a2f3e71d9181a67b7542122c" bandwidth="98000">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+      </Representation>
+      <Representation audioSamplingRate="48000" mimeType="audio/mp4" codecs="mp4a.40.2" id="eccbc87e4b5ce2fe28308fd9f2a7baf3" bandwidth="130000">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet startWithSAP="1" id="1" contentType="video" minFrameRate="24000/1001" maxFrameRate="30000/1001" segmentAlignment="true">
+      <SegmentTemplate timescale="90000" startNumber="0">
+        <SegmentTimeline>
+          <S d="180180" r="3"/>
+          <S d="177210"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation width="1280" height="720" frameRate="30000/1001" mimeType="video/mp4" codecs="avc1.64001f" id="c4ca4238a0b923820dcc509a6f75849b" bandwidth="331000"/>
+      <Representation width="480" height="270" frameRate="24000/1001" mimeType="video/mp4" codecs="avc1.42c015" id="c81e728d9d4c2f636f067f89cc14862c" bandwidth="137000"/>
+    </AdaptationSet>
+  </Period>
+</MPD>