feat: fetch mpd/dash playlists (#35)

fix: decrypt playlists as expected

.gitignore

@@ -24,6 +24,7 @@ npm-debug.log*
 # Dependency directories
 bower_components/
 node_modules/
+hls-fetcher/
 
 # Build-related directories
 dist/

package.json

@@ -33,8 +33,9 @@
     "aes-decrypter": "^3.0.0",
     "bluebird": "^3.4.0",
     "filenamify": "^4.1.0",
-    "m3u8-parser": "^4.3.0",
+    "m3u8-parser": "^4.4.2",
     "mkdirp": "^0.5.1",
+    "mpd-parser": "^0.9.0",
     "pessimist": "^0.3.5",
     "request": "^2.87.0",
     "requestretry": "^2.0.0"

src/walk-manifest.js

@@ -1,5 +1,6 @@
 /* eslint-disable no-console */
 const m3u8 = require('m3u8-parser');
+const mpd = require('mpd-parser');
 const request = require('requestretry');
 const url = require('url');
 const path = require('path');
@@ -55,19 +56,63 @@ const mediaGroupPlaylists = function(mediaGroups) {
   return playlists;
 };
 
-const parseManifest = function(content) {
+const parseM3u8Manifest = function(content) {
   const parser = new m3u8.Parser();
 
   parser.push(content);
   parser.end();
   return parser.manifest;
 };
 
+const collectPlaylists = function(parsed) {
+  return []
+    .concat(parsed.playlists || [])
+    .concat(mediaGroupPlaylists(parsed.mediaGroups || {}) || [])
+    .reduce(function(acc, p) {
+      acc.push(p);
+
+      if (p.playlists) {
+        acc = acc.concat(collectPlaylists(p));
+      }
+      return acc;
+    }, []);
+};
+
+const parseMpdManifest = function(content, srcUrl) {
+  const mpdPlaylists = mpd.toPlaylists(mpd.inheritAttributes(mpd.stringToMpdXml(content), {
+    manifestUri: srcUrl
+  }));
+
+  const m3u8Result = mpd.toM3u8(mpdPlaylists);
+  const m3u8Playlists = collectPlaylists(m3u8Result);
+
+  m3u8Playlists.forEach(function(m) {
+    const mpdPlaylist = m.attributes && mpdPlaylists.find(function(p) {
+      return p.attributes.id === m.attributes.NAME;
+    });
+
+    if (mpdPlaylist) {
+      m.dashattributes = mpdPlaylist.attributes;
+    }
+    // add sidx to segments
+    if (m.sidx) {
+      // fix init segment map if it has one
+      if (m.sidx.map && !m.sidx.map.uri) {
+        m.sidx.map.uri = m.sidx.map.resolvedUri;
+      }
+
+      m.segments.push(m.sidx);
+    }
+  });
+
+  return m3u8Result;
+};
+
 const parseKey = function(requestOptions, basedir, decrypt, resources, manifest, parent) {
   return new Promise(function(resolve, reject) {
 
     if (!manifest.parsed.segments[0] || !manifest.parsed.segments[0].key) {
-      resolve({});
+      return resolve({});
     }
     const key = manifest.parsed.segments[0].key;
 
@@ -92,7 +137,7 @@ const parseKey = function(requestOptions, basedir, decrypt, resources, manifest,
       ));
       key.uri = keyUri;
       resources.push(key);
-      resolve(key);
+      return resolve(key);
     }
 
     requestOptions.url = keyUri;
@@ -155,17 +200,21 @@ const walkPlaylist = function(options) {
       visitedUrls = [],
       requestTimeout = 1500,
       requestRetryMaxAttempts = 5,
+      dashPlaylist = null,
       requestRetryDelay = 5000
     } = options;
 
     let resources = [];
-    const manifest = {};
+    const manifest = {parent};
 
     manifest.uri = uri;
     manifest.file = path.join(basedir, fsSanitize(path.basename(uri)));
 
     // if we are not the master playlist
-    if (parent) {
+    if (dashPlaylist && parent) {
+      manifest.file = parent.file;
+      manifest.uri = parent.uri;
+    } else if (parent) {
       manifest.file = path.join(
         path.dirname(parent.file),
         'manifest' + manifestIndex,
@@ -179,101 +228,124 @@ const walkPlaylist = function(options) {
       parent.content = Buffer.from(parent.content.toString().replace(uri, path.relative(path.dirname(parent.file), manifest.file)));
     }
 
-    if (visitedUrls.includes(manifest.uri)) {
+    if (!dashPlaylist && visitedUrls.includes(manifest.uri)) {
       console.error(`[WARN] Trying to visit the same uri again; skipping to avoid getting stuck in a cycle: ${manifest.uri}`);
       return resolve(resources);
     }
 
-    request({
-      url: manifest.uri,
-      timeout: requestTimeout,
-      maxAttempts: requestRetryMaxAttempts,
-      retryDelay: requestRetryDelay
-    })
-      .then(function(response) {
-        if (response.statusCode !== 200) {
-          const manifestError = new Error(response.statusCode + '|' + manifest.uri);
+    let requestPromise;
 
-          manifestError.reponse = response;
-          return onError(manifestError, manifest.uri, resources, resolve, reject);
-        }
-        // Only push manifest uris that get a non 200 and don't timeout
+    if (dashPlaylist) {
+      requestPromise = Promise.resolve({statusCode: 200});
+    } else {
+      requestPromise = request({
+        url: manifest.uri,
+        timeout: requestTimeout,
+        maxAttempts: requestRetryMaxAttempts,
+        retryDelay: requestRetryDelay
+      });
+    }
+
+    requestPromise.then(function(response) {
+      if (response.statusCode !== 200) {
+        const manifestError = new Error(response.statusCode + '|' + manifest.uri);
+
+        manifestError.reponse = response;
+        return onError(manifestError, manifest.uri, resources, resolve, reject);
+      }
+      // Only push manifest uris that get a non 200 and don't timeout
+      let dash;
+
+      if (!dashPlaylist) {
         resources.push(manifest);
         visitedUrls.push(manifest.uri);
 
         manifest.content = response.body;
+        if ((/^application\/dash\+xml/i).test(response.headers['content-type']) || (/^\<\?xml/i).test(response.body)) {
+          dash = true;
+          manifest.parsed = parseMpdManifest(manifest.content, manifest.uri);
+        } else {
+          manifest.parsed = parseM3u8Manifest(manifest.content);
+        }
+      } else {
+        dash = true;
+        manifest.parsed = dashPlaylist;
+      }
 
-        manifest.parsed = parseManifest(manifest.content);
-        manifest.parsed.segments = manifest.parsed.segments || [];
-        manifest.parsed.playlists = manifest.parsed.playlists || [];
-        manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};
+      manifest.parsed.segments = manifest.parsed.segments || [];
+      manifest.parsed.playlists = manifest.parsed.playlists || [];
+      manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};
 
-        const initSegments = [];
+      const initSegments = [];
 
-        manifest.parsed.segments.forEach(function(s) {
-          if (s.map && s.map.uri && !initSegments.some((m) => s.map.uri === m.uri)) {
-            manifest.parsed.segments.push(s.map);
-            initSegments.push(s.map);
+      manifest.parsed.segments.forEach(function(s) {
+        if (s.map && s.map.uri && !initSegments.some((m) => s.map.uri === m.uri)) {
+          manifest.parsed.segments.push(s.map);
+          initSegments.push(s.map);
+        }
+      });
+
+      const playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
+
+      parseKey({
+        time: requestTimeout,
+        maxAttempts: requestRetryMaxAttempts,
+        retryDelay: requestRetryDelay
+      }, basedir, decrypt, resources, manifest, parent).then(function(key) {
+        // SEGMENTS
+        manifest.parsed.segments.forEach(function(s, i) {
+          if (!s.uri) {
+            return;
           }
-        });
+          // put segments in manifest-name/segment-name.ts
+          s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));
 
-        const playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));
-
-        parseKey({
-          time: requestTimeout,
-          maxAttempts: requestRetryMaxAttempts,
-          retryDelay: requestRetryDelay
-        }, basedir, decrypt, resources, manifest, parent).then(function(key) {
-          // SEGMENTS
-          manifest.parsed.segments.forEach(function(s, i) {
-            if (!s.uri) {
-              return;
-            }
-            // put segments in manifest-name/segment-name.ts
-            s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));
-            if (!isAbsolute(s.uri)) {
-              s.uri = joinURI(path.dirname(manifest.uri), s.uri);
-            }
-            if (key) {
-              s.key = key;
-              s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
-            }
+          if (!isAbsolute(s.uri)) {
+            s.uri = joinURI(path.dirname(manifest.uri), s.uri);
+          }
+          if (key) {
+            s.key = key;
+            s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
+          }
+          if (manifest.content) {
             manifest.content = Buffer.from(manifest.content.toString().replace(
               s.uri,
               path.relative(path.dirname(manifest.file), s.file)
             ));
-            resources.push(s);
-          });
+          }
+          resources.push(s);
+        });
 
-          // SUB Playlists
-          const subs = playlists.map(function(p, z) {
-            if (!p.uri) {
-              return Promise().resolve(resources);
-            }
-            return walkPlaylist({
-              decrypt,
-              basedir,
-              uri: p.uri,
-              parent: manifest,
-              manifestIndex: z,
-              onError,
-              visitedUrls,
-              requestTimeout,
-              requestRetryMaxAttempts,
-              requestRetryDelay
-            });
+        // SUB Playlists
+        const subs = playlists.map(function(p, z) {
+          if (!p.uri && !dash) {
+            return Promise.resolve(resources);
+          }
+          return walkPlaylist({
+            dashPlaylist: dash ? p : null,
+            decrypt,
+            basedir,
+            uri: p.uri,
+            parent: manifest,
+            manifestIndex: z,
+            onError,
+            visitedUrls,
+            requestTimeout,
+            requestRetryMaxAttempts,
+            requestRetryDelay
           });
+        });
 
-          Promise.all(subs).then(function(r) {
-            const flatten = [].concat.apply([], r);
+        Promise.all(subs).then(function(r) {
+          const flatten = [].concat.apply([], r);
 
-            resources = resources.concat(flatten);
-            resolve(resources);
-          }).catch(function(err) {
-            onError(err, manifest.uri, resources, resolve, reject);
-          });
+          resources = resources.concat(flatten);
+          resolve(resources);
+        }).catch(function(err) {
+          onError(err, manifest.uri, resources, resolve, reject);
         });
-      })
+      });
+    })
       .catch(function(err) {
         onError(err, manifest.uri, resources, resolve, reject);
       });

src/write-data.js

@@ -35,25 +35,19 @@ const requestFile = function(uri) {
   });
 };
 
-const toArrayBuffer = function(buffer) {
-  const ab = new ArrayBuffer(buffer.length);
-  const view = new Uint8Array(ab);
-
-  for (let i = 0; i < buffer.length; ++i) {
-    view[i] = buffer[i];
-  }
-  return ab;
+const toUint8Array = function(nodeBuffer) {
+  return new Uint8Array(nodeBuffer.buffer, nodeBuffer.byteOffset, nodeBuffer.byteLength / Uint8Array.BYTES_PER_ELEMENT);
 };
 
 const decryptFile = function(content, encryption) {
   return new Promise(function(resolve, reject) {
     /* eslint-disable no-new */
     // this is how you use it, its kind of bad but :shrug:
-    new AesDecrypter(toArrayBuffer(content), encryption.bytes, encryption.iv, function(err, bytes) {
+    new AesDecrypter(toUint8Array(content), encryption.bytes, encryption.iv, function(err, bytes) {
       if (err) {
         return reject(err);
       }
-      return resolve(new Buffer(bytes));
+      return resolve(Buffer.from(bytes));
     });
     /* eslint-enable no-new */
   });
@@ -68,15 +62,15 @@ const WriteData = function(decrypt, concurrency, resources) {
       operations.push(function() {
         return writeFile(r.file, r.content);
       });
-    } else if (r.key && decrypt) {
+    } else if (r.uri && r.key && decrypt) {
       operations.push(function() {
         return requestFile(r.uri).then(function(content) {
           return decryptFile(content, r.key);
         }).then(function(content) {
           return writeFile(r.file, content);
         });
       });
-    } else if (inProgress.indexOf(r.uri) === -1) {
+    } else if (r.uri && inProgress.indexOf(r.uri) === -1) {
       operations.push(function() {
         return requestFile(r.uri).then(function(content) {
           return writeFile(r.file, content);

test/resources/dash.mpd

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" profiles="urn:mpeg:dash:profile:isoff-live:2011" type="static" mediaPresentationDuration="PT29.611S" minBufferTime="PT7S">
+  <Period>
+    <AdaptationSet id="1" group="5" profiles="ccff" bitstreamSwitching="false" segmentAlignment="true" contentType="audio" mimeType="audio/mp4" codecs="mp4a.40.2" lang="und">
+      <Label>aac_und_2_128_1_1</Label>
+      <SegmentTemplate timescale="10000000" media="QualityLevels($Bandwidth$)/Fragments(aac_und_2_128_1_1=$Time$,format=mpd-time-csf).m4a" initialization="QualityLevels($Bandwidth$)/Fragments(aac_und_2_128_1_1=i,format=mpd-time-csf).mp4">
+        <SegmentTimeline>
+          <S d="60160000" r="3"/>
+          <S d="55466666"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation id="5_A_aac_und_2_128_1_1_1" bandwidth="128000" audioSamplingRate="48000"/>
+    </AdaptationSet>
+    <AdaptationSet id="2" group="1" profiles="ccff" bitstreamSwitching="false" segmentAlignment="true" contentType="video" mimeType="video/mp4" codecs="avc1.64001F" maxWidth="1280" maxHeight="720" startWithSAP="1">
+      <SegmentTemplate timescale="10000000" media="QualityLevels($Bandwidth$)/Fragments(video=$Time$,format=mpd-time-csf).m4v" initialization="QualityLevels($Bandwidth$)/Fragments(video=i,format=mpd-time-csf).mp4">
+        <SegmentTimeline>
+          <S d="60000000" r="3"/>
+          <S d="55600000"/>
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation id="1_V_video_1" bandwidth="1102000" width="1280" height="720"/>
+      <Representation id="1_V_video_2" bandwidth="686000" width="960" height="540"/>
+      <Representation id="1_V_video_3" bandwidth="360000" codecs="avc1.64001E" width="640" height="360"/>
+      <Representation id="1_V_video_4" bandwidth="233000" codecs="avc1.640015" width="480" height="270"/>
+      <Representation id="1_V_video_5" bandwidth="117000" codecs="avc1.64000C" width="320" height="180"/>
+    </AdaptationSet>
+  </Period>
+</MPD>