From 45c71d1a8e2ba772fa1c56bd3b4cd46a77ee92b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Tue, 16 Nov 2021 15:07:38 +0100 Subject: [PATCH 1/7] Background Fetch API initial implementation. --- rollup.config.js | 2 + src/api/streaming/04-streaming-basics.md | 3 + src/js/classes/BackgroundFetch.js | 84 ++++++++++++++ src/js/classes/DownloadManager.js | 103 +++++++++++++++--- src/js/classes/IDBConnection.js | 13 ++- src/js/classes/ParserMPD.js | 81 +------------- src/js/classes/StorageManager.js | 20 ++-- src/js/sw/sw.js | 52 +++++++++ src/js/typedefs.js | 58 +++++++++- src/js/utils/getFileMetaForDownload.js | 53 +++++++++ src/js/utils/getURLsForDownload.js | 77 +++++-------- src/js/utils/getVideoSources.js | 32 ++++++ src/js/utils/rewriteURL.js | 47 ++++++++ .../video-download/VideoDownloader.js | 58 ++++++---- 14 files changed, 499 insertions(+), 184 deletions(-) create mode 100644 src/js/classes/BackgroundFetch.js create mode 100644 src/js/utils/getFileMetaForDownload.js create mode 100644 src/js/utils/getVideoSources.js create mode 100644 src/js/utils/rewriteURL.js diff --git a/rollup.config.js b/rollup.config.js index 2680b3a..c14a1c7 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -43,7 +43,9 @@ export default [ format: 'cjs', }, plugins: [ + generateApi(), generateCache(), + json(), isWatch ? {} : terser(), ], }, diff --git a/src/api/streaming/04-streaming-basics.md b/src/api/streaming/04-streaming-basics.md index 71b076e..d8ae215 100644 --- a/src/api/streaming/04-streaming-basics.md +++ b/src/api/streaming/04-streaming-basics.md @@ -10,6 +10,9 @@ video-sources: type: application/dash+xml - src: https://storage.googleapis.com/kino-assets/streaming-basics/master.m3u8 type: application/x-mpegURL +url-rewrites: + - online: https://storage.googleapis.com/kino-assets/streaming-basics/manifest.mpd + offline: http://localhost:5000/video/4-manifest-offline.mpd video-subtitles: - default: true kind: captions diff --git a/src/js/classes/BackgroundFetch.js b/src/js/classes/BackgroundFetch.js new file mode 100644 index 0000000..ab2af6d --- /dev/null +++ b/src/js/classes/BackgroundFetch.js @@ -0,0 +1,84 @@ +import getURLsForDownload from '../utils/getURLsForDownload'; + +const BG_FETCH_ID_TEMPLATE = /kino-(?[a-z-_]+)/; + +export default class BackgroundFetch { + /** + * Construct a new Background Fetch wrapper. + */ + constructor() { + this.id = ''; + this.videoId = ''; + this.onprogress = () => {}; + this.ondone = () => {}; + } + + async maybeAbort(swReg) { + const existingBgFetch = await swReg.backgroundFetch.get(this.id); + if (existingBgFetch) await existingBgFetch.abort(); + } + + /** + * + * @param {BackgroundFetchRegistration} registration Background Fetch Registration object. + */ + fromRegistration(registration) { + const matches = registration.id.match(BG_FETCH_ID_TEMPLATE); + + this.id = registration.id; + this.videoId = matches.groups?.videoId || ''; + } + + async start(videoData) { + this.videoId = videoData.id; + this.id = `kino-${this.videoId}`; + + const urls = await getURLsForDownload(this.videoId); + + /** @type {Promise} */ + const requests = urls.map((url) => fetch(url, { method: 'HEAD' })); + + Promise.all(requests).then((responses) => { + const sizes = responses.map((response) => response.headers.get('Content-Length')); + const downloadTotal = sizes.includes(null) + ? null + : sizes.reduce((total, size) => total + parseInt(size, 10), 0); + + // eslint-disable-next-line compat/compat + navigator.serviceWorker.ready.then(async (swReg) => { + this.maybeAbort(swReg); + + /** @type {BackgroundFetchRegistration} */ + const bgFetch = await swReg.backgroundFetch.fetch( + this.id, + urls, + { + title: videoData.title, + icons: videoData['media-session-artwork'] || {}, + downloadTotal, + }, + ); + + bgFetch.addEventListener('progress', () => { + const progress = bgFetch.downloadTotal + ? bgFetch.downloaded / bgFetch.downloadTotal + : 0; + + this.onprogress(progress); + }); + + const messageChannel = new MessageChannel(); + + messageChannel.port1.onmessage = (e) => { + if (e.data === 'done') { + this.ondone(); + } + }; + + navigator.serviceWorker.controller.postMessage({ + type: 'channel-port', + }, [messageChannel.port2]); + }); + }); + } +} diff --git a/src/js/classes/DownloadManager.js b/src/js/classes/DownloadManager.js index dd85f7c..cc471ba 100644 --- a/src/js/classes/DownloadManager.js +++ b/src/js/classes/DownloadManager.js @@ -28,6 +28,8 @@ import FixedBuffer from './FixedBuffer'; * Utils. */ import getMimeByURL from '../utils/getMimeByURL'; +import getFileMetaForDownload from '../utils/getFileMetaForDownload'; +import rewriteURL from '../utils/rewriteURL'; /** * The DownloadManager is responsible for downloading videos from the network. @@ -45,23 +47,56 @@ export default class DownloadManager { /** * Instantiates the download manager. * - * @param {VideoDownloader} videoDownloader The associated video downloader object. + * @param {string} videoId Video ID of the media to be downloaded. */ - constructor(videoDownloader) { - this.files = videoDownloader.internal.files || []; + constructor(videoId) { + this.videoId = videoId; this.paused = false; this.cancelled = false; - this.internal = { - videoDownloader, - }; + /** @type {Response[]} */ + this.responses = []; - this.onflush = () => {}; + /** @type {DownloadFlushHandler[]} */ + this.flushHandlers = []; + + /** @type {DownloadTransformer} */ + this.transformers = []; - this.maybePrepareNextFile(); this.bufferSetup(); } + /** + * Flushes the downloaded data to any handlers. + * + * @param {FileMeta} fileMeta File meta. + * @param {FileChunk} fileChunk File chunk. + * @param {boolean} isDone Is this the last file chunk. + */ + flush(fileMeta, fileChunk, isDone) { + this.flushHandlers.forEach((handler) => { + handler(fileMeta, fileChunk, isDone); + }); + } + + /** + * Attaches a handler to receive downloaded data. + * + * @param {DownloadFlushHandler} flushHandler Flush handler. + */ + attachFlushHandler(flushHandler) { + this.flushHandlers.push(flushHandler); + } + + /** + * Attaches a download data transformer. + * + * @param {DownloadTransformer} transformer Download data transformer. + */ + attachTransformer(transformer) { + this.transformers.push(transformer); + } + /** * Sets the `currentFileMeta` to the first incomplete download. * Also sets the `done` property to indicate if all downloads are completed. @@ -79,7 +114,7 @@ export default class DownloadManager { */ bufferSetup() { /** - * IDB put operations have a lot of overhead, so it's impractical for us to store + * IDB put operations have a lot of overhead, so it's impractical for us to * a data chunk every time our reader has more data, because those chunks * usually are pretty small and generate thousands of IDB data entries. * @@ -115,29 +150,53 @@ export default class DownloadManager { fileMeta.done = true; } this.maybePrepareNextFile(); - this.onflush(fileMeta, fileChunk, this.done); + this.flush(fileMeta, fileChunk, this.done); } /** * Downloads the first file that is not fully downloaded. */ async downloadFile() { - const { bytesDownloaded, url, downloadUrl } = this.currentFileMeta; - const fetchOpts = {}; + const { bytesDownloaded, url } = this.currentFileMeta; + + // Attempts to find an existing response object for the current URL + // before fetching the file from the network. + let response = this.responses.reduce( + (prev, current) => (current.url === url ? current : prev), + null, + ); + + /** + * Some URLs we want to download have their offline versions. + * + * If the current URL is one of those, we want to make sure not to + * use any existing response for the original URL. + */ + const rewrittenUrl = rewriteURL(this.videoId, url, 'online', 'offline'); - if (bytesDownloaded) { - fetchOpts.headers = { - Range: `bytes=${bytesDownloaded}-`, - }; + if (!response || url !== rewrittenUrl) { + const fetchOpts = {}; + + if (bytesDownloaded) { + fetchOpts.headers = { + Range: `bytes=${bytesDownloaded}-`, + }; + } + + response = await fetch(rewrittenUrl, fetchOpts); } - const response = await fetch(downloadUrl, fetchOpts); const reader = response.body.getReader(); const mimeType = response.headers.get('Content-Type') || getMimeByURL(url); const fileLength = response.headers.has('Content-Range') ? Number(response.headers.get('Content-Range').replace(/^[^/]\/(.*)$/, '$1')) : Number(response.headers.get('Content-Length')); + // If this is a full response, throw away any bytes downloaded earlier. + if (!response.headers.has('Content-Range')) { + this.currentFileMeta.bytesDownloaded = 0; + } + this.currentFileMeta.mimeType = mimeType; this.currentFileMeta.bytesTotal = fileLength > 0 ? fileLength : null; @@ -169,9 +228,17 @@ export default class DownloadManager { /** * Starts downloading files. + * + * @param {Response[]} [responses] Already prepared responses for (some of) the donwloaded + * files, e.g. produced by Background Fetch API. + * @param {string[]} [urls] Optional list of URLs to be downloaded. */ - async run() { + async run(responses = [], urls = null) { this.paused = false; + this.responses = responses; + this.files = await getFileMetaForDownload(this.videoId, urls); + + this.maybePrepareNextFile(); while (!this.done && !this.paused && !this.cancelled && this.currentFileMeta) { /* eslint-disable-next-line no-await-in-loop */ await this.downloadFile(); diff --git a/src/js/classes/IDBConnection.js b/src/js/classes/IDBConnection.js index 5637ec7..856e068 100644 --- a/src/js/classes/IDBConnection.js +++ b/src/js/classes/IDBConnection.js @@ -207,8 +207,10 @@ export default () => { * has changed. */ dispatchDataChangedEvent() { - const changeEvent = new Event(IDB_DATA_CHANGED_EVENT); - window.dispatchEvent(changeEvent); + if (typeof window !== 'undefined') { + const changeEvent = new Event(IDB_DATA_CHANGED_EVENT); + window.dispatchEvent(changeEvent); + } } unwrap() { @@ -223,6 +225,8 @@ export default () => { /** * Removes all entries from the database used for video storage. * + * Doesn't clear the Background Fetch API cache. + * * @returns {Promise} Promise that resolves when the DB is deleted. */ abstractedIDB.clearAll = () => new Promise((resolve, reject) => { @@ -375,11 +379,8 @@ export default () => { * - bytesDownloaded (number) How many bytes of data is already downloaded. * - bytesTotal (number) Total size of the file in bytes. * - done (boolean) Whether the file is fully downloaded. - * - downloadUrl (string) The URL used by the application to download file data. * - mimeType (string) File MIME type. - * - url (string) The remote URL representing the file. Can be different from - * `downloadUrl` in some cases, e.g. when we alter the DASH - * manifest to only contain a single video and audio source. + * - url (string) The remote URL. * - videoId (string) Video ID this file is assigned to. */ const fileOS = db.createObjectStore( diff --git a/src/js/classes/ParserMPD.js b/src/js/classes/ParserMPD.js index c91402b..d7e463b 100644 --- a/src/js/classes/ParserMPD.js +++ b/src/js/classes/ParserMPD.js @@ -16,7 +16,6 @@ import '../typedefs'; import iso8601TimeDurationToSeconds from './Duration'; -import selectRepresentations from '../utils/selectRepresentations'; /** * Replaces MPD variables in the chunk URL string with proper values. @@ -205,10 +204,9 @@ export default class { /** * Returns a list of all chunk files referenced in the manifest. * - * @param {Array[]} additionalFileTuples List of tuples in the format [fileId, URL]. - * @returns {string[]} List of all chunk files referenced in the manifest. + * @returns {string[]} List of chunk file URLs referenced in the manifest. */ - listAllChunkURLs(additionalFileTuples = [[]]) { + listAllChunkURLs() { const repObjects = [...this.internal.root.querySelectorAll('Representation')].map(representationElementToObject); const initialSegmentFiles = repObjects.map(getInitialSegment); @@ -222,10 +220,11 @@ export default class { ); const prependBaseURL = (filename) => this.baseURL + filename; - const fileTuples = [...initialSegmentFiles, ...dataChunkFiles].map( - (file) => [prependBaseURL(file), prependBaseURL(file)], + const chunkUrls = [...initialSegmentFiles, ...dataChunkFiles].map( + (file) => prependBaseURL(file), ); - return [...fileTuples, ...additionalFileTuples]; + + return chunkUrls; } /** @@ -272,72 +271,4 @@ export default class { return representationObjects; } - - /** - * Removes all `Representation` elements other than one for video and optionally - * one for audio from the manifest. - * - * @returns {boolean} Whether the operation succeeded. - */ - prepareForOffline() { - const targetResolutionW = 1280; - - /** - * This process is potentially destructive. Clone the root element first. - */ - const RootElementClone = this.internal.root.cloneNode(true); - const videoAdaptationSets = [...RootElementClone.querySelectorAll('AdaptationSet[contentType="video"]')]; - const audioAdaptationSets = [...RootElementClone.querySelectorAll('AdaptationSet[contentType="audio"]')]; - - /** - * Remove all video and audio Adaptation sets apart from the first ones. - */ - const videoAS = videoAdaptationSets.shift(); - const audioAS = audioAdaptationSets.shift(); - [...videoAdaptationSets, ...audioAdaptationSets].forEach( - (as) => as.parentNode.removeChild(as), - ); - - /** - * Remove all but first audio representation from the document. - */ - if (audioAS) { - const audioRepresentations = [...audioAS.querySelectorAll('Representation')]; - audioRepresentations.shift(); - audioRepresentations.forEach( - (rep) => rep.parentNode.removeChild(rep), - ); - } - - /** - * Select the video representation closest to the target resolution and remove the rest. - */ - if (videoAS) { - const videoRepresentations = selectRepresentations(this).video; - if (!videoRepresentations) return false; - let candidate; - - videoRepresentations.forEach( - (videoRep) => { - const satisifiesTarget = Number(videoRep.width) >= targetResolutionW; - const lessData = Number(videoRep.bandwidth) < Number(candidate?.bandwidth || Infinity); - - if (satisifiesTarget && lessData) { - candidate = videoRep; - } - }, - ); - - if (!candidate) return false; - - while (videoAS.firstChild) videoAS.removeChild(videoAS.firstChild); - videoAS.appendChild(candidate.element); - } - - /** - * Everything was OK, assign the altered document as root again. - */ - this.internal.root.innerHTML = RootElementClone.innerHTML; - return true; - } } diff --git a/src/js/classes/StorageManager.js b/src/js/classes/StorageManager.js index faf425b..9d5ce31 100644 --- a/src/js/classes/StorageManager.js +++ b/src/js/classes/StorageManager.js @@ -29,18 +29,18 @@ export default class { /** * Instantiates the storage manager. * - * @param {VideoDownloader} videoDownloader The associated video downloader object. + * @param {string} videoId Video ID to identify stored data. + * @param {object} opts Optional settings. + * @param {VideoDownloader} opts.videoDownloader Video downloader instance. */ - constructor(videoDownloader) { + constructor(videoId, opts = {}) { this.done = false; + this.videoId = videoId; + this.videoDownloader = opts.videoDownloader || null; this.onerror = () => {}; this.onprogress = () => {}; this.ondone = () => {}; - - this.internal = { - videoDownloader, - }; } /** @@ -71,7 +71,7 @@ export default class { const db = await getIDBConnection(); const videoMeta = { done: isDone, - videoId: this.internal.videoDownloader.getId(), + videoId: this.videoId, timestamp: Date.now(), }; const txAbortHandler = (e) => { @@ -125,8 +125,10 @@ export default class { return new Promise((resolve, reject) => { Promise.all([metaWritePromise, dataWritePromise, fileWritePromise]) .then(() => { - const percentage = this.internal.videoDownloader.getProgress(); - this.onprogress(percentage); + if (this.videoDownloader) { + const percentage = this.videoDownloader.getProgress(); + this.onprogress(percentage); + } if (isDone) { this.done = true; diff --git a/src/js/sw/sw.js b/src/js/sw/sw.js index 663df67..688726e 100644 --- a/src/js/sw/sw.js +++ b/src/js/sw/sw.js @@ -26,6 +26,9 @@ import { import getIDBConnection from '../classes/IDBConnection'; import assetsToCache from './cache'; +import BackgroundFetch from '../classes/BackgroundFetch'; +import DownloadManager from '../classes/DownloadManager'; +import StorageManager from '../classes/StorageManager'; /** * Respond to a request to fetch offline video file and construct a response stream. @@ -217,3 +220,52 @@ const fetchHandler = async (event) => { self.addEventListener('install', precacheAssets); self.addEventListener('activate', clearOldCaches); self.addEventListener('fetch', fetchHandler); + +/** @type {MessagePort} */ +let messageChannelPort; + +self.addEventListener( + 'message', + (event) => { + if (event.data.type === 'channel-port') { + [messageChannelPort] = event.ports; + } + }, +); + +const bgFetchHandler = async (e) => { + /** @type {BackgroundFetchRegistration} */ + const bgFetchRegistration = e.registration; + const bgFetch = new BackgroundFetch(); + + bgFetch.fromRegistration(bgFetchRegistration); + + const records = await bgFetchRegistration.matchAll(); + const urls = records.map((record) => record.request.url); + + if (urls.length > 0) { + const responsePromises = records.map((record) => record.responseReady); + const responses = await Promise.all(responsePromises); + + /** + * The `DownloadManager` reads binary data from passed response objects + * and routes the data to the `StorageManager` that saves it along with any + * metadata to IndexedDB. + */ + const downloadManager = new DownloadManager(bgFetch.videoId); + const storageManager = new StorageManager(bgFetch.videoId); + const boundStoreChunkHandler = storageManager.storeChunk.bind(storageManager); + + downloadManager.attachFlushHandler(boundStoreChunkHandler); + downloadManager.attachFlushHandler((fileMeta, fileChunk, isDone) => { + // If we have a message channel open, signal back to the UI when we're done. + if (isDone && messageChannelPort) { + messageChannelPort.postMessage('done'); + } + }); + + // Start the download, i.e. pump binary data out of the response objects. + downloadManager.run(responses, urls); + } +}; +self.addEventListener('backgroundfetchsuccess', bgFetchHandler); diff --git a/src/js/typedefs.js b/src/js/typedefs.js index 07f6cf1..8b7599f 100644 --- a/src/js/typedefs.js +++ b/src/js/typedefs.js @@ -22,8 +22,7 @@ /** * @typedef {object} FileMeta - * @property {string} url The original resource URL. - * @property {string} downloadUrl Rewritten resource URL. + * @property {string} url Resource URL. * @property {string} videoId Identifier for the video this file is associated with. * @property {string} mimeType File MIME type. * @property {number} bytesDownloaded Total bytes downloaded of the resources. @@ -92,3 +91,58 @@ * @property {VideoDownloaderRegistry} VideoDownloaderRegistry Storage for `videoDownload` * instances reuse. */ + +/* eslint-disable max-len */ +/** + * @typedef {object} BackgroundFetchRecord + * @property {Request} request Request. + * @property {Promise} responseReady Returns a promise that resolves with a Response. + */ + +/** + * @callback BgFetchMatch + * @param {Request} request The Request for which you are attempting to find records. + * @param {object} options An object that sets options for the match operation. + * @returns {BackgroundFetchRecord} Background fetch record. + */ + +/** + * @callback BgFetchAbort + * @returns {Promise} Whether the background fetch was successfully aborted. + */ + +/** + * @callback BgFetchMatchAll + * @param {Request} request The Request for which you are attempting to find records. + * @param {object} options An object that sets options for the match operation. + * @returns {Promise} Promise that resolves with an array of BackgroundFetchRecord objects. + */ + +/** + * @typedef {object} BackgroundFetchRegistration + * @property {string} id containing the background fetch's ID. + * @property {number} uploadTotal containing the total number of bytes to be uploaded. + * @property {number} uploaded containing the size in bytes successfully sent, initially 0. + * @property {number} downloadTotal containing the total size in bytes of this download. + * @property {number} downloaded containing the size in bytes that has been downloaded, initially 0. + * @property {""|"aborted"|"bad-status"|"fetch-error"|"quota-exceeded"|"download-total-exceeded"} failureReason Failure reason. + * @property {boolean} recordsAvailable indicating whether the recordsAvailable flag is set. + * @property {BgFetchMatch} match Returns a single BackgroundFetchRecord object which is the first match for the arguments. + * @property {BgFetchMatchAll} matchAll Returns a Promise that resolves with an array of BackgroundFetchRecord objects containing requests and responses. + * @property {BgFetchAbort} abort Aborts the background fetch. Returns a Promise that resolves with true if the fetch was successfully aborted. + */ +/* eslint-enable max-len */ + +/** + * @callback DownloadFlushHandler + * @param {FileMeta} fileMeta File meta. + * @param {FileChunk} fileChunk File chunk. + * @param {boolean} isDone Is this the last downloaded piece? + * @returns {void} + */ + +/** + * @callback DownloadTransformer + * @param {Response} response Response object. + * @returns {Response} + */ diff --git a/src/js/utils/getFileMetaForDownload.js b/src/js/utils/getFileMetaForDownload.js new file mode 100644 index 0000000..80d5ea5 --- /dev/null +++ b/src/js/utils/getFileMetaForDownload.js @@ -0,0 +1,53 @@ +/** + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import '../typedefs'; +import getURLsForDownload from './getURLsForDownload'; +import getIDBConnection from '../classes/IDBConnection'; + +/** + * Returns a list of URLs that need to be downloaded in order to allow + * offline playback of this resource on this device. + * + * @param {string} videoId Video ID. + * @param {string[]} [urls] Optionally a list of URLs to be downloaded. + * @returns {Promise} List of FileMeta objects associated with the given `videoId`. + */ +export default async (videoId, urls = null) => { + const db = await getIDBConnection(); + const dbFiles = await db.file.getByVideoId(videoId); + const dbFilesUrlTuples = dbFiles.map((fileMeta) => [fileMeta.url, fileMeta]); + const dbFilesByUrl = Object.fromEntries(dbFilesUrlTuples); + + if (!urls) { + urls = await getURLsForDownload(videoId); + } + + /** + * If we have an entry for this file in the database, use it. Otherwise + * fall back to the freshly generated FileMeta object. + */ + return urls.map( + (fileUrl) => (dbFilesByUrl[fileUrl] ? dbFilesByUrl[fileUrl] : { + url: fileUrl, + videoId, + mimeType: 'application/octet-stream', // Filled in by `DownloadManager` later. + bytesDownloaded: 0, + bytesTotal: null, // Filled in by `DownloadManager` later. + done: false, + }), + ); +}; diff --git a/src/js/utils/getURLsForDownload.js b/src/js/utils/getURLsForDownload.js index 02dd9c6..9077c11 100644 --- a/src/js/utils/getURLsForDownload.js +++ b/src/js/utils/getURLsForDownload.js @@ -17,73 +17,48 @@ import '../typedefs'; import ParserMPD from '../classes/ParserMPD'; import selectSource from './selectSource'; +import getVideoSources from './getVideoSources'; +import rewriteURL from './rewriteURL'; /** * Returns a list of URLs that need to be downloaded in order to allow * offline playback of this resource on this device. * - * @param {string} videoId Video ID. - * @param {object[]} sources Video sources. - * @returns {Promise} Promise resolving to file meta objects. + * @param {string} videoId Video ID. + * @returns {Promise} List of URLs associated with the given `videoId`. */ -export default async (videoId, sources) => { - let URLTuples = []; - const selectedSource = selectSource(sources); +export default async (videoId) => { + const videoSources = getVideoSources(videoId); + const selectedSource = selectSource(videoSources); + + if (selectedSource === null) { + return []; + } + + let urls = []; /** * If this is a streamed video, we need to read the manifest * first and generate a list of files to be downloaded. */ if (selectedSource?.canPlayTypeMSE) { - const response = await fetch(selectedSource.src); - const responseText = await response.text(); - const parser = new ParserMPD(responseText, selectedSource.src); + const offlineManifestUrl = rewriteURL(videoId, selectedSource.src, 'online', 'offline'); /** - * This removes all but one audio and video representations from the manifest. + * Use the offline version of the manifest to make sure we only line up + * a subset of all available media data for download. + * + * We don't want to download video files in all possible resolutions and formats. + * Instead the offline manifest only contain one manually selected representation. */ - const offlineGenerated = parser.prepareForOffline(); - if (offlineGenerated) { - /** - * The manifest data has been changed in memory. We need to persist the changes. - * Generating a data URI gives us a stable faux-URL that can reliably be used in - * place of a real URL and even stored in the database for later usage. - * - * Note: the offline manifest file is pretty small in size (several kBs max), - * making it a good candidate for a data URI. - */ - const offlineManifestDataURI = parser.toDataURI(); - - /** - * For most files, the `url` and `downloadUrl` are the same thing – see - * `listAllChunkURLs` source code. - * - * The only exception are the manifest files, where the `downloadUrl` is a separate data URI, - * whereas the `url` is the original URL of the manifest. - * - * This allows us to intercept requests for the original manifest file, but serve our - * updated version of the manifest. - */ - const manifestTuple = [selectedSource.src, offlineManifestDataURI]; + const response = await fetch(offlineManifestUrl); + const responseText = await response.text(); + const parser = new ParserMPD(responseText, selectedSource.src); - URLTuples = parser.listAllChunkURLs([manifestTuple]); - } else { - return []; - } + urls.push(selectedSource.src); + urls = [...parser.listAllChunkURLs(), ...urls]; } else { - URLTuples = [[selectedSource.src, selectedSource.src]]; + urls.push(selectedSource.src); } - - const fileMeta = URLTuples.map( - ([url, downloadUrl]) => ({ - url, - downloadUrl, - videoId, - bytesDownloaded: 0, - bytesTotal: null, - done: false, - }), - ); - - return fileMeta; + return urls; }; diff --git a/src/js/utils/getVideoSources.js b/src/js/utils/getVideoSources.js new file mode 100644 index 0000000..fd25637 --- /dev/null +++ b/src/js/utils/getVideoSources.js @@ -0,0 +1,32 @@ +/** + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import api from '../../../public/api.json'; + +/** + * Returns a list of videos sources associated with a video. + * + * @param {string} videoId Video ID. + * @returns {videoSource[]} Video source objects. + */ +export default function getVideoSources(videoId) { + const foundVideo = api.videos.find((video) => video.id === videoId); + + if (foundVideo) { + return foundVideo['video-sources']; + } + return []; +} diff --git a/src/js/utils/rewriteURL.js b/src/js/utils/rewriteURL.js new file mode 100644 index 0000000..86fb631 --- /dev/null +++ b/src/js/utils/rewriteURL.js @@ -0,0 +1,47 @@ +/** + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import '../typedefs'; +import api from '../../../public/api.json'; + +/** + * Rewrite a source URL based on from and to parameters. + * + * Useful for rewriting manifest file URLs, because the offline versions + * of manifests only contain a single video representation. + * + * @param {string} videoId Video ID. + * @param {string} sourceUrl Source URL to find offline alternative for. + * @param {string} from From which key. + * @param {string} to To which key. + * @returns {string} A rewritten URL. + */ +export default function rewriteURL(videoId, sourceUrl, from, to) { + const videoData = api.videos.find((video) => video.id === videoId); + + if (!videoData) { + return sourceUrl; + } + + if (videoData['url-rewrites']) { + videoData['url-rewrites'].forEach((rewrite) => { + if (rewrite[from] === sourceUrl) { + sourceUrl = rewrite[to]; + } + }); + } + return sourceUrl; +} diff --git a/src/js/web-components/video-download/VideoDownloader.js b/src/js/web-components/video-download/VideoDownloader.js index ee04111..0f026e0 100644 --- a/src/js/web-components/video-download/VideoDownloader.js +++ b/src/js/web-components/video-download/VideoDownloader.js @@ -19,9 +19,9 @@ import styles from './VideoDownloader.css'; import getIDBConnection from '../../classes/IDBConnection'; import DownloadManager from '../../classes/DownloadManager'; import StorageManager from '../../classes/StorageManager'; -import getURLsForDownload from '../../utils/getURLsForDownload'; - +import getFileMetaForDownload from '../../utils/getFileMetaForDownload'; import { MEDIA_SESSION_DEFAULT_ARTWORK } from '../../constants'; +import BackgroundFetch from '../../classes/BackgroundFetch'; export default class VideoDownloader extends HTMLElement { static get observedAttributes() { @@ -131,25 +131,12 @@ export default class VideoDownloader extends HTMLElement { }; const videoId = this.getId(); - const sources = this.internal.videoData['video-sources'] || []; - getURLsForDownload(videoId, sources).then(async (files) => { + getFileMetaForDownload(videoId).then(async (fileMeta) => { const db = await getIDBConnection(); - const dbFiles = await db.file.getByVideoId(videoId); - const dbFilesUrlTuples = dbFiles.map((fileMeta) => [fileMeta.url, fileMeta]); - const dbFilesByUrl = Object.fromEntries(dbFilesUrlTuples); - - /** - * If we have an entry for this file in the database, use it. Otherwise - * fall back to the freshly generated FileMeta object. - */ - const filesWithStateUpdatedFromDb = files.map( - (fileMeta) => (dbFilesByUrl[fileMeta.url] ? dbFilesByUrl[fileMeta.url] : fileMeta), - ); - const videoMeta = await db.meta.get(videoId); this.setMeta(videoMeta); - this.internal.files = filesWithStateUpdatedFromDb; + this.internal.files = fileMeta; this.render(); }); @@ -237,10 +224,34 @@ export default class VideoDownloader extends HTMLElement { if (!opts.assetsOnly) { this.downloading = true; - this.runIDBDownloads(); + this.state = 'partial'; + + if ( + 'BackgroundFetchManager' in window + && 'serviceWorker' in navigator + ) { + this.downloadUsingBackgroundFetch(); + } else { + this.downloadSynchronously(); + } } } + downloadUsingBackgroundFetch() { + const bgFetch = new BackgroundFetch(); + + bgFetch.onprogress = (progress) => { + this.progress = progress; + }; + bgFetch.ondone = () => { + this.progress = 100; + this.downloading = false; + this.state = 'done'; + }; + + bgFetch.start(this.internal.videoData); + } + /** * Returns the total download progress for the video. * @@ -271,9 +282,11 @@ export default class VideoDownloader extends HTMLElement { * Takes a list of video URLs, downloads the video using a stream reader * and invokes `storeVideoChunk` to store individual video chunks in IndexedDB. */ - async runIDBDownloads() { - this.downloadManager = new DownloadManager(this); - this.storageManager = new StorageManager(this); + async downloadSynchronously() { + this.downloadManager = new DownloadManager(this.getId()); + this.storageManager = new StorageManager(this.getId(), { + videoDownloader: this, + }); this.storageManager.onprogress = (progress) => { this.progress = progress; @@ -305,9 +318,8 @@ export default class VideoDownloader extends HTMLElement { * to make sure all chunks are sent to the `storeChunk` method of the `StoreManager`. */ const boundStoreChunkHandler = this.storageManager.storeChunk.bind(this.storageManager); - this.downloadManager.onflush = boundStoreChunkHandler; + this.downloadManager.attachFlushHandler(boundStoreChunkHandler); - this.state = 'partial'; this.downloadManager.run(); } From d563412cb9b73e50daa04e01394ffe4ee4588569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Tue, 16 Nov 2021 19:36:20 +0100 Subject: [PATCH 2/7] Add offline URLs rewrites for MPD manifests. --- src/api/streaming/04-streaming-basics.md | 2 +- src/api/streaming/05-efficient-formats.md | 3 +++ src/api/streaming/06-adaptive-streaming.md | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/api/streaming/04-streaming-basics.md b/src/api/streaming/04-streaming-basics.md index d8ae215..7571730 100644 --- a/src/api/streaming/04-streaming-basics.md +++ b/src/api/streaming/04-streaming-basics.md @@ -12,7 +12,7 @@ video-sources: type: application/x-mpegURL url-rewrites: - online: https://storage.googleapis.com/kino-assets/streaming-basics/manifest.mpd - offline: http://localhost:5000/video/4-manifest-offline.mpd + offline: https://storage.googleapis.com/kino-assets/streaming-basics/manifest-offline.mpd video-subtitles: - default: true kind: captions diff --git a/src/api/streaming/05-efficient-formats.md b/src/api/streaming/05-efficient-formats.md index 23465eb..aa472bc 100644 --- a/src/api/streaming/05-efficient-formats.md +++ b/src/api/streaming/05-efficient-formats.md @@ -10,6 +10,9 @@ video-sources: type: application/dash+xml - src: https://storage.googleapis.com/kino-assets/efficient-formats/master.m3u8 type: application/x-mpegURL +url-rewrites: + - online: https://storage.googleapis.com/kino-assets/efficient-formats/manifest.mpd + offline: https://storage.googleapis.com/kino-assets/efficient-formats/manifest-offline.mpd video-subtitles: - default: true kind: captions diff --git a/src/api/streaming/06-adaptive-streaming.md b/src/api/streaming/06-adaptive-streaming.md index 2555101..1511565 100644 --- a/src/api/streaming/06-adaptive-streaming.md +++ b/src/api/streaming/06-adaptive-streaming.md @@ -10,6 +10,9 @@ video-sources: type: application/dash+xml - src: https://storage.googleapis.com/kino-assets/adaptive-streaming/master.m3u8 type: application/x-mpegURL +url-rewrites: + - online: https://storage.googleapis.com/kino-assets/adaptive-streaming/manifest.mpd + offline: https://storage.googleapis.com/kino-assets/adaptive-streaming/manifest-offline.mpd video-subtitles: - default: true kind: captions From 9d6d36d6834a2022636ab081c6bb6cb51318ffd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Tue, 16 Nov 2021 19:37:20 +0100 Subject: [PATCH 3/7] Remove the file transformers concept. --- src/js/classes/DownloadManager.js | 12 ------------ src/js/typedefs.js | 6 ------ 2 files changed, 18 deletions(-) diff --git a/src/js/classes/DownloadManager.js b/src/js/classes/DownloadManager.js index cc471ba..9553c6f 100644 --- a/src/js/classes/DownloadManager.js +++ b/src/js/classes/DownloadManager.js @@ -60,9 +60,6 @@ export default class DownloadManager { /** @type {DownloadFlushHandler[]} */ this.flushHandlers = []; - /** @type {DownloadTransformer} */ - this.transformers = []; - this.bufferSetup(); } @@ -88,15 +85,6 @@ export default class DownloadManager { this.flushHandlers.push(flushHandler); } - /** - * Attaches a download data transformer. - * - * @param {DownloadTransformer} transformer Download data transformer. - */ - attachTransformer(transformer) { - this.transformers.push(transformer); - } - /** * Sets the `currentFileMeta` to the first incomplete download. * Also sets the `done` property to indicate if all downloads are completed. diff --git a/src/js/typedefs.js b/src/js/typedefs.js index 8b7599f..b6a3ebe 100644 --- a/src/js/typedefs.js +++ b/src/js/typedefs.js @@ -140,9 +140,3 @@ * @param {boolean} isDone Is this the last downloaded piece? * @returns {void} */ - -/** - * @callback DownloadTransformer - * @param {Response} response Response object. - * @returns {Response} - */ From d8b2757f80e25362b1bec905b830757ecc68b1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Wed, 17 Nov 2021 16:52:28 +0100 Subject: [PATCH 4/7] Abstract getProgress away from the VideoDownloader class. --- src/js/classes/DownloadManager.js | 20 +++++- src/js/classes/StorageManager.js | 14 ++-- src/js/constants.js | 1 + src/js/pages/Settings.js | 12 +++- src/js/sw/sw.js | 3 +- src/js/utils/getProgress.js | 26 +++++++ src/js/utils/getURLsForDownload.js | 15 ++-- .../video-download/VideoDownloader.js | 68 +++++++------------ 8 files changed, 99 insertions(+), 60 deletions(-) create mode 100644 src/js/utils/getProgress.js diff --git a/src/js/classes/DownloadManager.js b/src/js/classes/DownloadManager.js index c6f9df1..aa835e1 100644 --- a/src/js/classes/DownloadManager.js +++ b/src/js/classes/DownloadManager.js @@ -60,6 +60,8 @@ export default class DownloadManager { /** @type {DownloadFlushHandler[]} */ this.flushHandlers = []; + this.onfilemeta = () => {}; + this.bufferSetup(); } @@ -245,19 +247,31 @@ export default class DownloadManager { this.cancelled = true; } + /** + * Generates a list of URLs to be downloaded and turns them into + * a list of FileMeta objects that track download properties + * for each of the files. + * + * @param {string[]} [urls] Optional list of URLs to be downloaded. + * @returns {Promise} Promise resolving with FileMeta objects prepared for download. + */ + async prepareFileMeta(urls = null) { + this.files = await getFileMetaForDownload(this.videoId, urls); + return this.files; + } + /** * Starts downloading files. * * @param {Response[]} [responses] Already prepared responses for (some of) the donwloaded * files, e.g. produced by Background Fetch API. - * @param {string[]} [urls] Optional list of URLs to be downloaded. */ - async run(responses = [], urls = null) { + async run(responses = []) { this.paused = false; this.responses = responses; - this.files = await getFileMetaForDownload(this.videoId, urls); this.maybePrepareNextFile(); + while (!this.done && !this.paused && !this.cancelled && this.currentFileMeta) { /* eslint-disable-next-line no-await-in-loop */ await this.downloadFile(); diff --git a/src/js/classes/StorageManager.js b/src/js/classes/StorageManager.js index 9d5ce31..2a5fc3e 100644 --- a/src/js/classes/StorageManager.js +++ b/src/js/classes/StorageManager.js @@ -15,6 +15,7 @@ */ import '../typedefs'; +import getProgress from '../utils/getProgress'; import getIDBConnection from './IDBConnection'; /** @@ -29,14 +30,14 @@ export default class { /** * Instantiates the storage manager. * - * @param {string} videoId Video ID to identify stored data. - * @param {object} opts Optional settings. - * @param {VideoDownloader} opts.videoDownloader Video downloader instance. + * @param {string} videoId Video ID to identify stored data. + * @param {object} opts Optional settings. + * @param {FileMeta[]} opts.fileMeta File meta objects to observe progress for. */ constructor(videoId, opts = {}) { this.done = false; this.videoId = videoId; - this.videoDownloader = opts.videoDownloader || null; + this.fileMeta = opts.fileMeta || []; this.onerror = () => {}; this.onprogress = () => {}; @@ -125,9 +126,8 @@ export default class { return new Promise((resolve, reject) => { Promise.all([metaWritePromise, dataWritePromise, fileWritePromise]) .then(() => { - if (this.videoDownloader) { - const percentage = this.videoDownloader.getProgress(); - this.onprogress(percentage); + if (this.fileMeta.length > 0) { + this.onprogress(getProgress(this.fileMeta)); } if (isDone) { diff --git a/src/js/constants.js b/src/js/constants.js index 90610ac..483a383 100644 --- a/src/js/constants.js +++ b/src/js/constants.js @@ -119,6 +119,7 @@ export const ALL_STREAM_TYPES = ['audio', 'video']; */ export const SETTING_KEY_TOGGLE_OFFLINE = 'toggle-offline'; export const SETTING_KEY_DARK_MODE = 'dark-mode'; +export const SETTING_KEY_BG_FETCH_API = 'allow-bg-fetch-api'; /** * Event name signalling that data in IDB has changes. diff --git a/src/js/pages/Settings.js b/src/js/pages/Settings.js index 1e4415e..bd1408d 100644 --- a/src/js/pages/Settings.js +++ b/src/js/pages/Settings.js @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SETTING_KEY_DARK_MODE } from '../constants'; +import { SETTING_KEY_DARK_MODE, SETTING_KEY_BG_FETCH_API } from '../constants'; /** * @param {RouterContext} routerContext Context object passed by the Router. @@ -55,6 +55,16 @@ export default (routerContext) => { +
+
+ +
+
+

Download videos in the background

+

Use Background Fetch API in browsers that support it.

+
+ +
diff --git a/src/js/sw/sw.js b/src/js/sw/sw.js index 688726e..77f0113 100644 --- a/src/js/sw/sw.js +++ b/src/js/sw/sw.js @@ -256,6 +256,7 @@ const bgFetchHandler = async (e) => { const storageManager = new StorageManager(bgFetch.videoId); const boundStoreChunkHandler = storageManager.storeChunk.bind(storageManager); + await downloadManager.prepareFileMeta(urls); downloadManager.attachFlushHandler(boundStoreChunkHandler); downloadManager.attachFlushHandler((fileMeta, fileChunk, isDone) => { // If we have a message channel open, signal back to the UI when we're done. @@ -265,7 +266,7 @@ const bgFetchHandler = async (e) => { }); // Start the download, i.e. pump binary data out of the response objects. - downloadManager.run(responses, urls); + downloadManager.run(responses); } }; self.addEventListener('backgroundfetchsuccess', bgFetchHandler); diff --git a/src/js/utils/getProgress.js b/src/js/utils/getProgress.js new file mode 100644 index 0000000..bf8b45a --- /dev/null +++ b/src/js/utils/getProgress.js @@ -0,0 +1,26 @@ +/** + * Returns the total download progress for the video. + * + * @param {FileMeta[]} fileMetas File meta objects to calculate progress for. + * @returns {number} Percentage progress for the video in the range 0–100. + */ +export default function getProgress(fileMetas) { + const pieceValue = 1 / fileMetas.length; + const percentageProgress = fileMetas.reduce( + (percentage, fileMeta) => { + if (fileMeta.done) { + percentage += pieceValue; + } else if (fileMeta.bytesDownloaded === 0 || !fileMeta.bytesTotal) { + percentage += 0; + } else { + const percentageOfCurrent = fileMeta.bytesDownloaded / fileMeta.bytesTotal; + percentage += percentageOfCurrent * pieceValue; + } + return percentage; + }, + 0, + ); + const clampedPercents = Math.max(0, Math.min(percentageProgress, 1)); + + return clampedPercents; +} diff --git a/src/js/utils/getURLsForDownload.js b/src/js/utils/getURLsForDownload.js index 9077c11..6558a52 100644 --- a/src/js/utils/getURLsForDownload.js +++ b/src/js/utils/getURLsForDownload.js @@ -51,12 +51,17 @@ export default async (videoId) => { * We don't want to download video files in all possible resolutions and formats. * Instead the offline manifest only contain one manually selected representation. */ - const response = await fetch(offlineManifestUrl); - const responseText = await response.text(); - const parser = new ParserMPD(responseText, selectedSource.src); + try { + const response = await fetch(offlineManifestUrl); + const responseText = await response.text(); + const parser = new ParserMPD(responseText, selectedSource.src); - urls.push(selectedSource.src); - urls = [...parser.listAllChunkURLs(), ...urls]; + urls.push(selectedSource.src); + urls = [...parser.listAllChunkURLs(), ...urls]; + } catch (e) { + /* eslint-disable-next-line no-console */ + console.error('Error fetching and parsing the offline MPD manifest.', e); + } } else { urls.push(selectedSource.src); } diff --git a/src/js/web-components/video-download/VideoDownloader.js b/src/js/web-components/video-download/VideoDownloader.js index 57fee17..19ec8e3 100644 --- a/src/js/web-components/video-download/VideoDownloader.js +++ b/src/js/web-components/video-download/VideoDownloader.js @@ -19,9 +19,10 @@ import styles from './VideoDownloader.css'; import getIDBConnection from '../../classes/IDBConnection'; import DownloadManager from '../../classes/DownloadManager'; import StorageManager from '../../classes/StorageManager'; -import getFileMetaForDownload from '../../utils/getFileMetaForDownload'; -import { MEDIA_SESSION_DEFAULT_ARTWORK } from '../../constants'; +import { MEDIA_SESSION_DEFAULT_ARTWORK, SETTING_KEY_BG_FETCH_API } from '../../constants'; import BackgroundFetch from '../../classes/BackgroundFetch'; +import { loadSetting } from '../../utils/settings'; +import getProgress from '../../utils/getProgress'; export default class VideoDownloader extends HTMLElement { /** @@ -38,7 +39,14 @@ export default class VideoDownloader extends HTMLElement { connectionStatus, changeCallbacks: [], root: this.attachShadow({ mode: 'open' }), + files: [], }; + + /** @type {DownloadManager} */ + this.downloadManager = null; + + /** @type {StorageManager} */ + this.storageManager = null; } /** @@ -138,7 +146,7 @@ export default class VideoDownloader extends HTMLElement { * @param {object} videoData Video data coming from the API. * @param {string} cacheName Cache name. */ - init(videoData, cacheName = 'v1') { + async init(videoData, cacheName = 'v1') { this.internal = { ...this.internal, videoData, @@ -147,15 +155,18 @@ export default class VideoDownloader extends HTMLElement { }; const videoId = this.getId(); + const db = await getIDBConnection(); + const videoMeta = await db.meta.get(videoId); - getFileMetaForDownload(videoId).then(async (fileMeta) => { - const db = await getIDBConnection(); - const videoMeta = await db.meta.get(videoId); - this.setMeta(videoMeta); - this.internal.files = fileMeta; + this.downloadManager = new DownloadManager(this.getId()); + this.internal.files = await this.downloadManager.prepareFileMeta(); - this.render(); + this.storageManager = new StorageManager(this.getId(), { + fileMeta: this.internal.files, }); + + this.setMeta(videoMeta); + this.render(); } /** @@ -243,8 +254,9 @@ export default class VideoDownloader extends HTMLElement { this.state = 'partial'; if ( - 'BackgroundFetchManager' in window - && 'serviceWorker' in navigator + loadSetting(SETTING_KEY_BG_FETCH_API) + && 'BackgroundFetchManager' in window + && 'serviceWorker' in navigator ) { this.downloadUsingBackgroundFetch(); } else { @@ -268,45 +280,15 @@ export default class VideoDownloader extends HTMLElement { bgFetch.start(this.internal.videoData); } - /** - * Returns the total download progress for the video. - * - * @returns {number} Percentage progress for the video in the range 0–100. - */ - getProgress() { - const pieceValue = 1 / this.internal.files.length; - const percentageProgress = this.internal.files.reduce( - (percentage, fileMeta) => { - if (fileMeta.done) { - percentage += pieceValue; - } else if (fileMeta.bytesDownloaded === 0 || !fileMeta.bytesTotal) { - percentage += 0; - } else { - const percentageOfCurrent = fileMeta.bytesDownloaded / fileMeta.bytesTotal; - percentage += percentageOfCurrent * pieceValue; - } - return percentage; - }, - 0, - ); - const clampedPercents = Math.max(0, Math.min(percentageProgress, 1)); - - return clampedPercents; - } - /** * Takes a list of video URLs, downloads the video using a stream reader * and invokes `storeVideoChunk` to store individual video chunks in IndexedDB. */ async downloadSynchronously() { - this.downloadManager = new DownloadManager(this.getId()); - this.storageManager = new StorageManager(this.getId(), { - videoDownloader: this, - }); - this.storageManager.onprogress = (progress) => { this.progress = progress; }; + this.storageManager.onerror = (error) => { if (this.downloading && error.name === 'QuotaExceededError') { /** @@ -499,7 +481,7 @@ export default class VideoDownloader extends HTMLElement { */ async setDownloadState() { const videoMeta = this.getMeta(); - const downloadProgress = this.getProgress(); + const downloadProgress = getProgress(this.internal.files); if (videoMeta.done) { this.state = 'done'; From 5be1b5094c41430913845c9312edb9fc56bf0a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Fri, 19 Nov 2021 13:06:15 +0100 Subject: [PATCH 5/7] Background Fetch has to be controlled using browser UI. --- .../video-download/VideoDownloader.css | 9 +++++++++ .../web-components/video-download/VideoDownloader.js | 12 +++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/js/web-components/video-download/VideoDownloader.css b/src/js/web-components/video-download/VideoDownloader.css index a7dc9d9..f80d0d7 100644 --- a/src/js/web-components/video-download/VideoDownloader.css +++ b/src/js/web-components/video-download/VideoDownloader.css @@ -164,6 +164,15 @@ stroke: var(--accent); } +/* Removes button interactivity when controls are disabled */ +:host( [state="partial"][downloading="true"] ) button.downloading { + pointer-events: none; +} +:host( [state="partial"][downloading="true"][nocontrols="true"] ) button.downloading svg path[fill] { + display: none; +} + + /* Progress icon (paused state) */ :host( [state="partial"][downloading="false"] ) button.paused { display: flex; diff --git a/src/js/web-components/video-download/VideoDownloader.js b/src/js/web-components/video-download/VideoDownloader.js index 19ec8e3..277a102 100644 --- a/src/js/web-components/video-download/VideoDownloader.js +++ b/src/js/web-components/video-download/VideoDownloader.js @@ -29,7 +29,7 @@ export default class VideoDownloader extends HTMLElement { * @type {string[]} */ static get observedAttributes() { - return ['state', 'progress', 'downloading', 'willremove']; + return ['state', 'progress', 'downloading', 'willremove', 'nocontrols']; } constructor({ connectionStatus }) { @@ -108,6 +108,14 @@ export default class VideoDownloader extends HTMLElement { this.setAttribute('willremove', willremove); } + get nocontrols() { + return this.getAttribute('nocontrols') === 'true'; + } + + set nocontrols(nocontrols) { + this.setAttribute('nocontrols', nocontrols); + } + /** * Observed attributes callbacks. * @@ -125,6 +133,7 @@ export default class VideoDownloader extends HTMLElement { const currentState = { state: this.state, willremove: this.willremove, + nocontrols: this.nocontrols, }; if (Object.keys(currentState).includes(name)) { @@ -258,6 +267,7 @@ export default class VideoDownloader extends HTMLElement { && 'BackgroundFetchManager' in window && 'serviceWorker' in navigator ) { + this.nocontrols = true; // Browser will handle the download UI. this.downloadUsingBackgroundFetch(); } else { this.downloadSynchronously(); From 7979e3282d469de3d8166bd4d7ea889fa62760c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Thu, 9 Dec 2021 14:55:07 +0100 Subject: [PATCH 6/7] Address PR feedback related to Background Fetch API. --- src/js/classes/BackgroundFetch.js | 21 ++++++++--- src/js/sw/sw.js | 59 +++++++++++++++++++------------ 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/js/classes/BackgroundFetch.js b/src/js/classes/BackgroundFetch.js index ab2af6d..61bf531 100644 --- a/src/js/classes/BackgroundFetch.js +++ b/src/js/classes/BackgroundFetch.js @@ -46,14 +46,14 @@ export default class BackgroundFetch { // eslint-disable-next-line compat/compat navigator.serviceWorker.ready.then(async (swReg) => { - this.maybeAbort(swReg); + await this.maybeAbort(swReg); /** @type {BackgroundFetchRegistration} */ const bgFetch = await swReg.backgroundFetch.fetch( this.id, urls, { - title: videoData.title, + title: `Downloading "${videoData.title}" video`, icons: videoData['media-session-artwork'] || {}, downloadTotal, }, @@ -75,9 +75,20 @@ export default class BackgroundFetch { } }; - navigator.serviceWorker.controller.postMessage({ - type: 'channel-port', - }, [messageChannel.port2]); + const swController = navigator.serviceWorker.controller; + + /** + * Need to guard the `postMessage` logic below, because + * `navigator.serviceWorker.controller` will be `null` + * when a force-refresh request is made. + * + * @see https://developer.mozilla.org/en-US/docs/Web/API/ServiceWorkerContainer/controller + */ + if (swController) { + navigator.serviceWorker.controller.postMessage({ + type: 'channel-port', + }, [messageChannel.port2]); + } }); }); } diff --git a/src/js/sw/sw.js b/src/js/sw/sw.js index 77f0113..cfd2ac6 100644 --- a/src/js/sw/sw.js +++ b/src/js/sw/sw.js @@ -221,31 +221,46 @@ self.addEventListener('install', precacheAssets); self.addEventListener('activate', clearOldCaches); self.addEventListener('fetch', fetchHandler); -/** @type {MessagePort} */ -let messageChannelPort; - -self.addEventListener( - 'message', - (event) => { - if (event.data.type === 'channel-port') { - [messageChannelPort] = event.ports; - } - }, -); +if ('BackgroundFetchManager' in self) { + /** + * When Background Fetch API is used to download a file + * for offline viewing, Channel Messaging API is used + * to signal that a video is fully saved to IDB back + * to the UI. + * + * Because the operation is initiated from the UI, the + * `MessageChannel` instance is created there and one of + * the newly created channel ports is then sent to the + * service worker and stored in this variable. + * + * @type {MessagePort} + */ + let messageChannelPort; -const bgFetchHandler = async (e) => { - /** @type {BackgroundFetchRegistration} */ - const bgFetchRegistration = e.registration; - const bgFetch = new BackgroundFetch(); + self.addEventListener( + 'message', + (event) => { + if (event.data.type === 'channel-port') { + [messageChannelPort] = event.ports; + } + }, + ); - bgFetch.fromRegistration(bgFetchRegistration); + const bgFetchHandler = async (e) => { + /** @type {BackgroundFetchRegistration} */ + const bgFetchRegistration = e.registration; + const records = await bgFetchRegistration.matchAll(); + const urls = records.map((record) => record.request.url); - const records = await bgFetchRegistration.matchAll(); - const urls = records.map((record) => record.request.url); + if (urls.length === 0) { + return; + } - if (urls.length > 0) { const responsePromises = records.map((record) => record.responseReady); const responses = await Promise.all(responsePromises); + const bgFetch = new BackgroundFetch(); + + bgFetch.fromRegistration(bgFetchRegistration); /** * The `DownloadManager` reads binary data from passed response objects @@ -267,6 +282,6 @@ const bgFetchHandler = async (e) => { // Start the download, i.e. pump binary data out of the response objects. downloadManager.run(responses); - } -}; -self.addEventListener('backgroundfetchsuccess', bgFetchHandler); + }; + self.addEventListener('backgroundfetchsuccess', bgFetchHandler); +} From fb53402c04d583be0cee03238a90589f9cad82bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Polakovi=C4=8D?= Date: Thu, 9 Dec 2021 14:55:39 +0100 Subject: [PATCH 7/7] Only display the BG Fetch API settings in supporting browsers. --- src/js/pages/Settings.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/js/pages/Settings.js b/src/js/pages/Settings.js index bd1408d..19603f9 100644 --- a/src/js/pages/Settings.js +++ b/src/js/pages/Settings.js @@ -24,6 +24,20 @@ export default (routerContext) => { mainContent, connectionStatus, } = routerContext; + + const backgroundFetchSettingMarkup = !('BackgroundFetchManager' in window) + ? '' + : `
+
+ +
+
+

Download videos in the background

+

Use Background Fetch API to download videos in the background.

+
+ +
`; + mainContent.innerHTML = `

Settings

@@ -55,16 +69,7 @@ export default (routerContext) => {
-
-
- -
-
-

Download videos in the background

-

Use Background Fetch API in browsers that support it.

-
- -
+ ${backgroundFetchSettingMarkup}