Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: fetch mpd/dash playlists too #35

Merged
merged 6 commits into from
Sep 5, 2019
Merged

feat: fetch mpd/dash playlists too #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3b136b9
feat: fetch mpd/dash playlists too
brandonocasey Jul 18, 2019
a67a125
do not add dash playlists to resources
brandonocasey Jul 18, 2019
0e3dc2c
prep for find-replace in manifest
brandonocasey Jul 25, 2019
0e3f7e8
use real package versions
brandonocasey Aug 30, 2019
d64d876
less dash specifics, support sidx
brandonocasey Aug 30, 2019
b0c3651
fix: decryption
brandonocasey Aug 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ npm-debug.log*
# Dependency directories
bower_components/
node_modules/
hls-fetcher/

# Build-related directories
dist/
Expand Down
401 changes: 266 additions & 135 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@
"aes-decrypter": "^3.0.0",
"bluebird": "^3.4.0",
"filenamify": "^4.1.0",
"m3u8-parser": "^4.3.0",
"m3u8-parser": "^4.4.2",
"mkdirp": "^0.5.1",
"mpd-parser": "^0.9.0",
"pessimist": "^0.3.5",
"request": "^2.87.0",
"requestretry": "^2.0.0"
Expand Down
226 changes: 149 additions & 77 deletions src/walk-manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/* eslint-disable no-console */
const m3u8 = require('m3u8-parser');
const mpd = require('mpd-parser');
const request = require('requestretry');
const url = require('url');
const path = require('path');
Expand Down Expand Up @@ -55,19 +56,63 @@ const mediaGroupPlaylists = function(mediaGroups) {
return playlists;
};

const parseManifest = function(content) {
const parseM3u8Manifest = function(content) {
const parser = new m3u8.Parser();

parser.push(content);
parser.end();
return parser.manifest;
};

const collectPlaylists = function(parsed) {
return []
.concat(parsed.playlists || [])
.concat(mediaGroupPlaylists(parsed.mediaGroups || {}) || [])
.reduce(function(acc, p) {
acc.push(p);

if (p.playlists) {
acc = acc.concat(collectPlaylists(p));
}
return acc;
}, []);
};

const parseMpdManifest = function(content, srcUrl) {
const mpdPlaylists = mpd.toPlaylists(mpd.inheritAttributes(mpd.stringToMpdXml(content), {
manifestUri: srcUrl
}));

const m3u8Result = mpd.toM3u8(mpdPlaylists);
const m3u8Playlists = collectPlaylists(m3u8Result);

m3u8Playlists.forEach(function(m) {
const mpdPlaylist = m.attributes && mpdPlaylists.find(function(p) {
return p.attributes.id === m.attributes.NAME;
});

if (mpdPlaylist) {
m.dashattributes = mpdPlaylist.attributes;
}
// add sidx to segments
if (m.sidx) {
// fix init segment map if it has one
if (m.sidx.map && !m.sidx.map.uri) {
m.sidx.map.uri = m.sidx.map.resolvedUri;
}

m.segments.push(m.sidx);
}
});

return m3u8Result;
};

const parseKey = function(requestOptions, basedir, decrypt, resources, manifest, parent) {
return new Promise(function(resolve, reject) {

if (!manifest.parsed.segments[0] || !manifest.parsed.segments[0].key) {
resolve({});
return resolve({});
}
const key = manifest.parsed.segments[0].key;

Expand All @@ -92,7 +137,7 @@ const parseKey = function(requestOptions, basedir, decrypt, resources, manifest,
));
key.uri = keyUri;
resources.push(key);
resolve(key);
return resolve(key);
}

requestOptions.url = keyUri;
Expand Down Expand Up @@ -155,17 +200,21 @@ const walkPlaylist = function(options) {
visitedUrls = [],
requestTimeout = 1500,
requestRetryMaxAttempts = 5,
dashPlaylist = null,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In mpd-parser we really only mock child playlists. So if we see dashPlaylist here we know:

  1. That this playlist won't have a uri, or anything to download
  2. That this playlist has already been parsed

requestRetryDelay = 5000
} = options;

let resources = [];
const manifest = {};
const manifest = {parent};
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dash needs access to the parent manifest on child playlists


manifest.uri = uri;
manifest.file = path.join(basedir, fsSanitize(path.basename(uri)));

// if we are not the master playlist
if (parent) {
if (dashPlaylist && parent) {
manifest.file = parent.file;
manifest.uri = parent.uri;
} else if (parent) {
manifest.file = path.join(
path.dirname(parent.file),
'manifest' + manifestIndex,
Expand All @@ -179,101 +228,124 @@ const walkPlaylist = function(options) {
parent.content = Buffer.from(parent.content.toString().replace(uri, path.relative(path.dirname(parent.file), manifest.file)));
}

if (visitedUrls.includes(manifest.uri)) {
if (!dashPlaylist && visitedUrls.includes(manifest.uri)) {
console.error(`[WARN] Trying to visit the same uri again; skipping to avoid getting stuck in a cycle: ${manifest.uri}`);
return resolve(resources);
}

request({
url: manifest.uri,
timeout: requestTimeout,
maxAttempts: requestRetryMaxAttempts,
retryDelay: requestRetryDelay
})
.then(function(response) {
if (response.statusCode !== 200) {
const manifestError = new Error(response.statusCode + '|' + manifest.uri);
let requestPromise;

manifestError.reponse = response;
return onError(manifestError, manifest.uri, resources, resolve, reject);
}
// Only push manifest uris that get a non 200 and don't timeout
if (dashPlaylist) {
requestPromise = Promise.resolve({statusCode: 200});
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only request when needed.

} else {
requestPromise = request({
url: manifest.uri,
timeout: requestTimeout,
maxAttempts: requestRetryMaxAttempts,
retryDelay: requestRetryDelay
});
}

requestPromise.then(function(response) {
if (response.statusCode !== 200) {
const manifestError = new Error(response.statusCode + '|' + manifest.uri);

manifestError.reponse = response;
return onError(manifestError, manifest.uri, resources, resolve, reject);
}
// Only push manifest uris that get a non 200 and don't timeout
let dash;

if (!dashPlaylist) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

only parse when needed.

resources.push(manifest);
visitedUrls.push(manifest.uri);

manifest.content = response.body;
if ((/^application\/dash\+xml/i).test(response.headers['content-type']) || (/^\<\?xml/i).test(response.body)) {
dash = true;
manifest.parsed = parseMpdManifest(manifest.content, manifest.uri);
} else {
manifest.parsed = parseM3u8Manifest(manifest.content);
}
} else {
dash = true;
manifest.parsed = dashPlaylist;
}

manifest.parsed = parseManifest(manifest.content);
manifest.parsed.segments = manifest.parsed.segments || [];
manifest.parsed.playlists = manifest.parsed.playlists || [];
manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};
manifest.parsed.segments = manifest.parsed.segments || [];
manifest.parsed.playlists = manifest.parsed.playlists || [];
manifest.parsed.mediaGroups = manifest.parsed.mediaGroups || {};

const initSegments = [];
const initSegments = [];

manifest.parsed.segments.forEach(function(s) {
if (s.map && s.map.uri && !initSegments.some((m) => s.map.uri === m.uri)) {
manifest.parsed.segments.push(s.map);
initSegments.push(s.map);
manifest.parsed.segments.forEach(function(s) {
if (s.map && s.map.uri && !initSegments.some((m) => s.map.uri === m.uri)) {
manifest.parsed.segments.push(s.map);
initSegments.push(s.map);
}
});

const playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));

parseKey({
time: requestTimeout,
maxAttempts: requestRetryMaxAttempts,
retryDelay: requestRetryDelay
}, basedir, decrypt, resources, manifest, parent).then(function(key) {
// SEGMENTS
manifest.parsed.segments.forEach(function(s, i) {
if (!s.uri) {
return;
}
});
// put segments in manifest-name/segment-name.ts
s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));

const playlists = manifest.parsed.playlists.concat(mediaGroupPlaylists(manifest.parsed.mediaGroups));

parseKey({
time: requestTimeout,
maxAttempts: requestRetryMaxAttempts,
retryDelay: requestRetryDelay
}, basedir, decrypt, resources, manifest, parent).then(function(key) {
// SEGMENTS
manifest.parsed.segments.forEach(function(s, i) {
if (!s.uri) {
return;
}
// put segments in manifest-name/segment-name.ts
s.file = path.join(path.dirname(manifest.file), fsSanitize(path.basename(s.uri)));
if (!isAbsolute(s.uri)) {
s.uri = joinURI(path.dirname(manifest.uri), s.uri);
}
if (key) {
s.key = key;
s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
}
if (!isAbsolute(s.uri)) {
s.uri = joinURI(path.dirname(manifest.uri), s.uri);
}
if (key) {
s.key = key;
s.key.iv = s.key.iv || new Uint32Array([0, 0, 0, manifest.parsed.mediaSequence, i]);
}
if (manifest.content) {
manifest.content = Buffer.from(manifest.content.toString().replace(
s.uri,
path.relative(path.dirname(manifest.file), s.file)
));
resources.push(s);
});
}
resources.push(s);
});

// SUB Playlists
const subs = playlists.map(function(p, z) {
if (!p.uri) {
return Promise().resolve(resources);
}
return walkPlaylist({
decrypt,
basedir,
uri: p.uri,
parent: manifest,
manifestIndex: z,
onError,
visitedUrls,
requestTimeout,
requestRetryMaxAttempts,
requestRetryDelay
});
// SUB Playlists
const subs = playlists.map(function(p, z) {
if (!p.uri && !dash) {
return Promise.resolve(resources);
}
return walkPlaylist({
dashPlaylist: dash ? p : null,
decrypt,
basedir,
uri: p.uri,
parent: manifest,
manifestIndex: z,
onError,
visitedUrls,
requestTimeout,
requestRetryMaxAttempts,
requestRetryDelay
});
});

Promise.all(subs).then(function(r) {
const flatten = [].concat.apply([], r);
Promise.all(subs).then(function(r) {
const flatten = [].concat.apply([], r);

resources = resources.concat(flatten);
resolve(resources);
}).catch(function(err) {
onError(err, manifest.uri, resources, resolve, reject);
});
resources = resources.concat(flatten);
resolve(resources);
}).catch(function(err) {
onError(err, manifest.uri, resources, resolve, reject);
});
})
});
})
.catch(function(err) {
onError(err, manifest.uri, resources, resolve, reject);
});
Expand Down
18 changes: 6 additions & 12 deletions src/write-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,25 +35,19 @@ const requestFile = function(uri) {
});
};

const toArrayBuffer = function(buffer) {
const ab = new ArrayBuffer(buffer.length);
const view = new Uint8Array(ab);

for (let i = 0; i < buffer.length; ++i) {
view[i] = buffer[i];
}
return ab;
const toUint8Array = function(nodeBuffer) {
return new Uint8Array(nodeBuffer.buffer, nodeBuffer.byteOffset, nodeBuffer.byteLength / Uint8Array.BYTES_PER_ELEMENT);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nodes Buffer object to Uint8Array.

};

const decryptFile = function(content, encryption) {
return new Promise(function(resolve, reject) {
/* eslint-disable no-new */
// this is how you use it, its kind of bad but :shrug:
new AesDecrypter(toArrayBuffer(content), encryption.bytes, encryption.iv, function(err, bytes) {
new AesDecrypter(toUint8Array(content), encryption.bytes, encryption.iv, function(err, bytes) {
Copy link
Contributor Author

@brandonocasey brandonocasey Sep 5, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AesDecrypter expects a Uint8Array

if (err) {
return reject(err);
}
return resolve(new Buffer(bytes));
return resolve(Buffer.from(bytes));
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

new Buffer is deprecated in current nodejs and this is now the reccomended usage.

});
/* eslint-enable no-new */
});
Expand All @@ -68,15 +62,15 @@ const WriteData = function(decrypt, concurrency, resources) {
operations.push(function() {
return writeFile(r.file, r.content);
});
} else if (r.key && decrypt) {
} else if (r.uri && r.key && decrypt) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ignore any file that does not have a uri, such as dash playlists.

operations.push(function() {
return requestFile(r.uri).then(function(content) {
return decryptFile(content, r.key);
}).then(function(content) {
return writeFile(r.file, content);
});
});
} else if (inProgress.indexOf(r.uri) === -1) {
} else if (r.uri && inProgress.indexOf(r.uri) === -1) {
operations.push(function() {
return requestFile(r.uri).then(function(content) {
return writeFile(r.file, content);
Expand Down
28 changes: 28 additions & 0 deletions test/resources/dash.mpd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" profiles="urn:mpeg:dash:profile:isoff-live:2011" type="static" mediaPresentationDuration="PT29.611S" minBufferTime="PT7S">
<Period>
<AdaptationSet id="1" group="5" profiles="ccff" bitstreamSwitching="false" segmentAlignment="true" contentType="audio" mimeType="audio/mp4" codecs="mp4a.40.2" lang="und">
<Label>aac_und_2_128_1_1</Label>
<SegmentTemplate timescale="10000000" media="QualityLevels($Bandwidth$)/Fragments(aac_und_2_128_1_1=$Time$,format=mpd-time-csf).m4a" initialization="QualityLevels($Bandwidth$)/Fragments(aac_und_2_128_1_1=i,format=mpd-time-csf).mp4">
<SegmentTimeline>
<S d="60160000" r="3"/>
<S d="55466666"/>
</SegmentTimeline>
</SegmentTemplate>
<Representation id="5_A_aac_und_2_128_1_1_1" bandwidth="128000" audioSamplingRate="48000"/>
</AdaptationSet>
<AdaptationSet id="2" group="1" profiles="ccff" bitstreamSwitching="false" segmentAlignment="true" contentType="video" mimeType="video/mp4" codecs="avc1.64001F" maxWidth="1280" maxHeight="720" startWithSAP="1">
<SegmentTemplate timescale="10000000" media="QualityLevels($Bandwidth$)/Fragments(video=$Time$,format=mpd-time-csf).m4v" initialization="QualityLevels($Bandwidth$)/Fragments(video=i,format=mpd-time-csf).mp4">
<SegmentTimeline>
<S d="60000000" r="3"/>
<S d="55600000"/>
</SegmentTimeline>
</SegmentTemplate>
<Representation id="1_V_video_1" bandwidth="1102000" width="1280" height="720"/>
<Representation id="1_V_video_2" bandwidth="686000" width="960" height="540"/>
<Representation id="1_V_video_3" bandwidth="360000" codecs="avc1.64001E" width="640" height="360"/>
<Representation id="1_V_video_4" bandwidth="233000" codecs="avc1.640015" width="480" height="270"/>
<Representation id="1_V_video_5" bandwidth="117000" codecs="avc1.64000C" width="320" height="180"/>
</AdaptationSet>
</Period>
</MPD>
Loading