Downloading videos using Background Fetch API

rollup.config.js

@@ -45,7 +45,9 @@ export default [
       format: 'cjs',
     },
     plugins: [
+      generateApi(),
       generateCache(),
+      json(),
       isWatch ? {} : terser(),
     ],
   },

src/api/02-streaming/04-streaming-basics.md

@@ -10,6 +10,9 @@ video-sources:
     type: application/dash+xml
   - src: https://storage.googleapis.com/kino-assets/streaming-basics/master.m3u8
     type: application/x-mpegURL
+url-rewrites:
+  - online: https://storage.googleapis.com/kino-assets/streaming-basics/manifest.mpd
+    offline: https://storage.googleapis.com/kino-assets/streaming-basics/manifest-offline.mpd
 video-subtitles:
   - default: true
     kind: captions

src/api/02-streaming/05-efficient-formats.md

@@ -10,6 +10,9 @@ video-sources:
     type: application/dash+xml
   - src: https://storage.googleapis.com/kino-assets/efficient-formats/master.m3u8
     type: application/x-mpegURL
+url-rewrites:
+  - online: https://storage.googleapis.com/kino-assets/efficient-formats/manifest.mpd
+    offline: https://storage.googleapis.com/kino-assets/efficient-formats/manifest-offline.mpd
 video-subtitles:
   - default: true
     kind: captions

src/api/02-streaming/06-adaptive-streaming.md

@@ -10,6 +10,9 @@ video-sources:
     type: application/dash+xml
   - src: https://storage.googleapis.com/kino-assets/adaptive-streaming/master.m3u8
     type: application/x-mpegURL
+url-rewrites:
+  - online: https://storage.googleapis.com/kino-assets/adaptive-streaming/manifest.mpd
+    offline: https://storage.googleapis.com/kino-assets/adaptive-streaming/manifest-offline.mpd
 video-subtitles:
   - default: true
     kind: captions

src/js/classes/BackgroundFetch.js

@@ -0,0 +1,95 @@
+import getURLsForDownload from '../utils/getURLsForDownload';
+
+const BG_FETCH_ID_TEMPLATE = /kino-(?<videoId>[a-z-_]+)/;
+
+export default class BackgroundFetch {
+  /**
+   * Construct a new Background Fetch wrapper.
+   */
+  constructor() {
+    this.id = '';
+    this.videoId = '';
+    this.onprogress = () => {};
+    this.ondone = () => {};
+  }
+
+  async maybeAbort(swReg) {
+    const existingBgFetch = await swReg.backgroundFetch.get(this.id);
+    if (existingBgFetch) await existingBgFetch.abort();
+  }
+
+  /**
+   *
+   * @param {BackgroundFetchRegistration} registration Background Fetch Registration object.
+   */
+  fromRegistration(registration) {
+    const matches = registration.id.match(BG_FETCH_ID_TEMPLATE);
+
+    this.id = registration.id;
+    this.videoId = matches.groups?.videoId || '';
+  }
+
+  async start(videoData) {
+    this.videoId = videoData.id;
+    this.id = `kino-${this.videoId}`;
+
+    const urls = await getURLsForDownload(this.videoId);
+
+    /** @type {Promise<Response[]>} */
+    const requests = urls.map((url) => fetch(url, { method: 'HEAD' }));
+
+    Promise.all(requests).then((responses) => {
+      const sizes = responses.map((response) => response.headers.get('Content-Length'));
+      const downloadTotal = sizes.includes(null)
+        ? null
+        : sizes.reduce((total, size) => total + parseInt(size, 10), 0);
+
+      // eslint-disable-next-line compat/compat
+      navigator.serviceWorker.ready.then(async (swReg) => {
+        await this.maybeAbort(swReg);
+
+        /** @type {BackgroundFetchRegistration} */
+        const bgFetch = await swReg.backgroundFetch.fetch(
+          this.id,
+          urls,
+          {
+            title: `Downloading "${videoData.title}" video`,
+            icons: videoData['media-session-artwork'] || {},
+            downloadTotal,
+          },
+        );
+
+        bgFetch.addEventListener('progress', () => {
+          const progress = bgFetch.downloadTotal
+            ? bgFetch.downloaded / bgFetch.downloadTotal
+            : 0;
+
+          this.onprogress(progress);
+        });
+
+        const messageChannel = new MessageChannel();
+
+        messageChannel.port1.onmessage = (e) => {
+          if (e.data === 'done') {
+            this.ondone();
+          }
+        };
+
+        const swController = navigator.serviceWorker.controller;
+
+        /**
+         * Need to guard the `postMessage` logic below, because
+         * `navigator.serviceWorker.controller` will be `null`
+         * when a force-refresh request is made.
+         *
+         * @see https://developer.mozilla.org/en-US/docs/Web/API/ServiceWorkerContainer/controller
+         */
+        if (swController) {
+          navigator.serviceWorker.controller.postMessage({
+            type: 'channel-port',
+          }, [messageChannel.port2]);
+        }
+      });
+    });
+  }
+}

src/js/classes/DownloadManager.js

@@ -28,6 +28,8 @@ import FixedBuffer from './FixedBuffer';
  * Utils.
  */
 import getMimeByURL from '../utils/getMimeByURL';
+import getFileMetaForDownload from '../utils/getFileMetaForDownload';
+import rewriteURL from '../utils/rewriteURL';
 
 /**
  * The DownloadManager is responsible for downloading videos from the network.
@@ -45,23 +47,46 @@ export default class DownloadManager {
   /**
    * Instantiates the download manager.
    *
-   * @param {VideoDownloader} videoDownloader The associated video downloader object.
+   * @param {string} videoId Video ID of the media to be downloaded.
    */
-  constructor(videoDownloader) {
-    this.files = videoDownloader.internal.files || [];
+  constructor(videoId) {
+    this.videoId = videoId;
     this.paused = false;
     this.cancelled = false;
 
-    this.internal = {
-      videoDownloader,
-    };
+    /** @type {Response[]} */
+    this.responses = [];
 
-    this.onflush = () => {};
+    /** @type {DownloadFlushHandler[]} */
+    this.flushHandlers = [];
+
+    this.onfilemeta = () => {};
 
-    this.maybePrepareNextFile();
     this.bufferSetup();
   }
 
+  /**
+   * Flushes the downloaded data to any handlers.
+   *
+   * @param {FileMeta}  fileMeta  File meta.
+   * @param {FileChunk} fileChunk File chunk.
+   * @param {boolean}   isDone    Is this the last file chunk.
+   */
+  flush(fileMeta, fileChunk, isDone) {
+    this.flushHandlers.forEach((handler) => {
+      handler(fileMeta, fileChunk, isDone);
+    });
+  }
+
+  /**
+   * Attaches a handler to receive downloaded data.
+   *
+   * @param {DownloadFlushHandler} flushHandler Flush handler.
+   */
+  attachFlushHandler(flushHandler) {
+    this.flushHandlers.push(flushHandler);
+  }
+
   /**
    * Sets the `currentFileMeta` to the first incomplete download.
    * Also sets the `done` property to indicate if all downloads are completed.
@@ -79,7 +104,7 @@ export default class DownloadManager {
    */
   bufferSetup() {
     /**
-     * IDB put operations have a lot of overhead, so it's impractical for us to store
+     * IDB put operations have a lot of overhead, so it's impractical for us to
      * a data chunk every time our reader has more data, because those chunks
      * usually are pretty small and generate thousands of IDB data entries.
      *
@@ -115,29 +140,46 @@ export default class DownloadManager {
       fileMeta.done = true;
     }
     this.maybePrepareNextFile();
-    this.onflush(fileMeta, fileChunk, this.done);
+    this.flush(fileMeta, fileChunk, this.done);
   }
 
   /**
    * Downloads the first file that is not fully downloaded.
    */
   async downloadFile() {
-    const { bytesDownloaded, url, downloadUrl } = this.currentFileMeta;
-    const fetchOpts = {};
+    const { bytesDownloaded, url } = this.currentFileMeta;
 
-    if (bytesDownloaded) {
-      fetchOpts.headers = {
-        Range: `bytes=${bytesDownloaded}-`,
-      };
-    }
+    // Attempts to find an existing response object for the current URL
+    // before fetching the file from the network.
+    let response = this.responses.reduce(
+      (prev, current) => (current.url === url ? current : prev),
+      null,
+    );
+
+    /**
+     * Some URLs we want to download have their offline versions.
+     *
+     * If the current URL is one of those, we want to make sure not to
+     * use any existing response for the original URL.
+     */
+    const rewrittenUrl = rewriteURL(this.videoId, url, 'online', 'offline');
 
-    let response;
-    try {
-      response = await fetch(downloadUrl, fetchOpts);
-    } catch (e) {
-      this.warning(`Pausing the download of ${downloadUrl} due to network error.`);
-      this.forcePause();
-      return;
+    if (!response || url !== rewrittenUrl) {
+      const fetchOpts = {};
+
+      if (bytesDownloaded) {
+        fetchOpts.headers = {
+          Range: `bytes=${bytesDownloaded}-`,
+        };
+      }
+
+      try {
+        response = await fetch(rewrittenUrl, fetchOpts);
+      } catch (e) {
+        this.warning(`Pausing the download of ${rewrittenUrl} due to network error.`);
+        this.forcePause();
+        return;
+      }
     }
 
     const reader = response.body.getReader();
@@ -146,6 +188,11 @@ export default class DownloadManager {
       ? Number(response.headers.get('Content-Range').replace(/^[^/]\/(.*)$/, '$1'))
       : Number(response.headers.get('Content-Length'));
 
+    // If this is a full response, throw away any bytes downloaded earlier.
+    if (!response.headers.has('Content-Range')) {
+      this.currentFileMeta.bytesDownloaded = 0;
+    }
+
     this.currentFileMeta.mimeType = mimeType;
     this.currentFileMeta.bytesTotal = fileLength > 0 ? fileLength : null;
 
@@ -155,7 +202,7 @@ export default class DownloadManager {
         /* eslint-disable-next-line no-await-in-loop */
         dataChunk = await reader.read();
       } catch (e) {
-        this.warning(`Pausing the download of ${downloadUrl} due to network error.`);
+        this.warning(`Pausing the download of ${rewrittenUrl} due to network error.`);
         this.forcePause();
       }
 
@@ -200,11 +247,31 @@ export default class DownloadManager {
     this.cancelled = true;
   }
 
+  /**
+   * Generates a list of URLs to be downloaded and turns them into
+   * a list of FileMeta objects that track download properties
+   * for each of the files.
+   *
+   * @param {string[]} [urls]       Optional list of URLs to be downloaded.
+   * @returns {Promise<FileMeta[]>} Promise resolving with FileMeta objects prepared for download.
+   */
+  async prepareFileMeta(urls = null) {
+    this.files = await getFileMetaForDownload(this.videoId, urls);
+    return this.files;
+  }
+
   /**
    * Starts downloading files.
+   *
+   * @param {Response[]} [responses] Already prepared responses for (some of) the donwloaded
+   *                                 files, e.g. produced by Background Fetch API.
    */
-  async run() {
+  async run(responses = []) {
     this.paused = false;
+    this.responses = responses;
+
+    this.maybePrepareNextFile();
+
     while (!this.done && !this.paused && !this.cancelled && this.currentFileMeta) {
       /* eslint-disable-next-line no-await-in-loop */
       await this.downloadFile();

src/js/classes/IDBConnection.js

@@ -207,8 +207,10 @@ export default () => {
        * has changed.
        */
       dispatchDataChangedEvent() {
-        const changeEvent = new Event(IDB_DATA_CHANGED_EVENT);
-        window.dispatchEvent(changeEvent);
+        if (typeof window !== 'undefined') {
+          const changeEvent = new Event(IDB_DATA_CHANGED_EVENT);
+          window.dispatchEvent(changeEvent);
+        }
       }
 
       unwrap() {
@@ -223,6 +225,8 @@ export default () => {
     /**
      * Removes all entries from the database used for video storage.
      *
+     * Doesn't clear the Background Fetch API cache.
+     *
      * @returns {Promise} Promise that resolves when the DB is deleted.
      */
     abstractedIDB.clearAll = () => new Promise((resolve, reject) => {
@@ -375,11 +379,8 @@ export default () => {
        * - bytesDownloaded (number)  How many bytes of data is already downloaded.
        * - bytesTotal      (number)  Total size of the file in bytes.
        * - done            (boolean) Whether the file is fully downloaded.
-       * - downloadUrl     (string)  The URL used by the application to download file data.
        * - mimeType        (string)  File MIME type.
-       * - url             (string)  The remote URL representing the file. Can be different from
-       *                             `downloadUrl` in some cases, e.g. when we alter the DASH
-       *                             manifest to only contain a single video and audio source.
+       * - url             (string)  The remote URL.
        * - videoId         (string)  Video ID this file is assigned to.
        */
       const fileOS = db.createObjectStore(