feat(gatsby-plugin-offline): replace no-cache detection with dynamic …

…path whitelist (#9907)

* Remove all no-cache code

* Remove references to no-cache in offline plugin

* Initial work on hybrid navigation handler

* Refactor whitelist code to allow it to support onPostPrefetchPathname

* Fix service worker detection

* Fix IndexedDB race condition

* Prevent race conditions + reset whitelist on SW update

* Remove unnecessary API handler (onPostPrefetchPathname is called anyway)

* Add debugging statements + fix some minor problems

* Fix back/forward not working after 404

* Remove unneeded debugging statements

* Bundle idb-keyval instead of using an external CDN

* Update README

* Backport fixes from #9907

* minor fixes for things I copy-pasted wrong

* Refactor prefetching so we can detect success

packages/gatsby-plugin-offline/README.md

@@ -24,8 +24,8 @@ plugins: [`gatsby-plugin-offline`]
 When adding this plugin to your `gatsby-config.js`, you can pass in options to
 override the default [Workbox](https://developers.google.com/web/tools/workbox/modules/workbox-build) config.
 
-The default config is as follows. Warning, you can break the offline support
-and AppCache setup by changing these options so tread carefully.
+The default config is as follows. Warning: you can break the offline support by
+changing these options, so tread carefully.
 
 ```javascript
 const options = {
@@ -37,17 +37,6 @@ const options = {
     // the default prefix with `pathPrefix`.
     "/": `${pathPrefix}/`,
   },
-  navigateFallback: `${pathPrefix}/offline-plugin-app-shell-fallback/index.html`,
-  // Only match URLs without extensions or the query `no-cache=1`.
-  // So example.com/about/ will pass but
-  // example.com/about/?no-cache=1 and
-  // example.com/cheeseburger.jpg will not.
-  // We only want the service worker to handle our "clean"
-  // URLs and not any files hosted on the site.
-  //
-  // Regex based on http://stackoverflow.com/a/18017805
-  navigateFallbackWhitelist: [/^([^.?]*|[^?]*\.([^.?]{5,}|html))(\?.*)?$/],
-  navigateFallbackBlacklist: [/\?(.+&)?no-cache=1$/],
   cacheId: `gatsby-plugin-offline`,
   // Don't cache-bust JS or CSS files, and anything in the static directory,
   // since these files have unique URLs and their contents will never change

packages/gatsby-plugin-offline/package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "@babel/runtime": "^7.0.0",
     "cheerio": "^1.0.0-rc.2",
+    "idb-keyval": "^3.1.0",
     "lodash": "^4.17.10",
     "workbox-build": "^3.6.3"
   },

packages/gatsby-plugin-offline/src/gatsby-browser.js

@@ -1,23 +1,12 @@
 exports.registerServiceWorker = () => true
 
-let swNotInstalled = true
 const prefetchedPathnames = []
-
-exports.onPostPrefetchPathname = ({ pathname }) => {
-  // if SW is not installed, we need to record any prefetches
-  // that happen so we can then add them to SW cache once installed
-  if (swNotInstalled && `serviceWorker` in navigator) {
-    prefetchedPathnames.push(pathname)
-  }
-}
+const whitelistedPathnames = []
 
 exports.onServiceWorkerActive = ({
   getResourceURLsForPathname,
   serviceWorker,
 }) => {
-  // stop recording prefetch events
-  swNotInstalled = false
-
   // grab nodes from head of document
   const nodes = document.querySelectorAll(`
     head > script[src],
@@ -51,4 +40,40 @@ exports.onServiceWorkerActive = ({
 
     document.head.appendChild(link)
   })
+
+  serviceWorker.active.postMessage({
+    gatsbyApi: `whitelistPathnames`,
+    pathnames: whitelistedPathnames,
+  })
+}
+
+function whitelistPathname(pathname, includesPrefix) {
+  if (`serviceWorker` in navigator) {
+    const { serviceWorker } = navigator
+
+    if (serviceWorker.controller !== null) {
+      serviceWorker.controller.postMessage({
+        gatsbyApi: `whitelistPathnames`,
+        pathnames: [{ pathname, includesPrefix }],
+      })
+    } else {
+      whitelistedPathnames.push({ pathname, includesPrefix })
+    }
+  }
+}
+
+exports.onPostPrefetchPathname = ({ pathname }) => {
+  whitelistPathname(pathname, false)
+
+  // if SW is not installed, we need to record any prefetches
+  // that happen so we can then add them to SW cache once installed
+  if (
+    `serviceWorker` in navigator &&
+    !(
+      navigator.serviceWorker.controller !== null &&
+      navigator.serviceWorker.controller.state === `activated`
+    )
+  ) {
+    prefetchedPathnames.push(pathname)
+  }
 }

packages/gatsby-plugin-offline/src/gatsby-node.js

@@ -79,17 +79,6 @@ exports.onPostBuild = (args, pluginOptions) => {
       // the default prefix with `pathPrefix`.
       "/": `${pathPrefix}/`,
     },
-    navigateFallback: `${pathPrefix}/offline-plugin-app-shell-fallback/index.html`,
-    // Only match URLs without extensions or the query `no-cache=1`.
-    // So example.com/about/ will pass but
-    // example.com/about/?no-cache=1 and
-    // example.com/cheeseburger.jpg will not.
-    // We only want the service worker to handle our "clean"
-    // URLs and not any files hosted on the site.
-    //
-    // Regex based on http://stackoverflow.com/a/18017805
-    navigateFallbackWhitelist: [/^([^.?]*|[^?]*\.([^.?]{5,}|html))(\?.*)?$/],
-    navigateFallbackBlacklist: [/\?(.+&)?no-cache=1$/],
     cacheId: `gatsby-plugin-offline`,
     // Don't cache-bust JS or CSS files, and anything in the static directory,
     // since these files have unique URLs and their contents will never change
@@ -122,15 +111,22 @@ exports.onPostBuild = (args, pluginOptions) => {
   delete pluginOptions.plugins
   const combinedOptions = _.defaults(pluginOptions, options)
 
+  const idbKeyvalFile = `idb-keyval-iife.min.js`
+  const idbKeyvalSource = require.resolve(`idb-keyval/dist/${idbKeyvalFile}`)
+  const idbKeyvalDest = `public/${idbKeyvalFile}`
+  fs.createReadStream(idbKeyvalSource).pipe(fs.createWriteStream(idbKeyvalDest))
+
   const swDest = `public/sw.js`
   return workboxBuild
     .generateSW({ swDest, ...combinedOptions })
     .then(({ count, size, warnings }) => {
       if (warnings) warnings.forEach(warning => console.warn(warning))
 
-      const swAppend = fs.readFileSync(`${__dirname}/sw-append.js`)
-      fs.appendFileSync(`public/sw.js`, swAppend)
+      const swAppend = fs
+        .readFileSync(`${__dirname}/sw-append.js`, `utf8`)
+        .replace(/%pathPrefix%/g, pathPrefix)
 
+      fs.appendFileSync(`public/sw.js`, swAppend)
       console.log(
         `Generated ${swDest}, which will precache ${count} files, totaling ${size} bytes.`
       )

packages/gatsby-plugin-offline/src/sw-append.js

@@ -1 +1,83 @@
-// noop
+/* global importScripts, workbox, idbKeyval */
+
+importScripts(`idb-keyval-iife.min.js`)
+const WHITELIST_KEY = `custom-navigation-whitelist`
+
+const navigationRoute = new workbox.routing.NavigationRoute(({ event }) => {
+  const { pathname } = new URL(event.request.url)
+
+  return idbKeyval.get(WHITELIST_KEY).then((customWhitelist = []) => {
+    // Respond with the offline shell if we match the custom whitelist
+    if (customWhitelist.includes(pathname)) {
+      const offlineShell = `%pathPrefix%/offline-plugin-app-shell-fallback/index.html`
+      const cacheName = workbox.core.cacheNames.precache
+
+      return caches.match(offlineShell, { cacheName })
+    }
+
+    return fetch(event.request)
+  })
+})
+
+workbox.routing.registerRoute(navigationRoute)
+
+let updatingWhitelist = null
+
+function rawWhitelistPathnames(pathnames) {
+  if (updatingWhitelist !== null) {
+    // Prevent the whitelist from being updated twice at the same time
+    return updatingWhitelist.then(() => rawWhitelistPathnames(pathnames))
+  }
+
+  updatingWhitelist = idbKeyval
+    .get(WHITELIST_KEY)
+    .then((customWhitelist = []) => {
+      pathnames.forEach(pathname => {
+        if (!customWhitelist.includes(pathname)) customWhitelist.push(pathname)
+      })
+
+      return idbKeyval.set(WHITELIST_KEY, customWhitelist)
+    })
+    .then(() => {
+      updatingWhitelist = null
+    })
+
+  return updatingWhitelist
+}
+
+function rawResetWhitelist() {
+  if (updatingWhitelist !== null) {
+    return updatingWhitelist.then(() => rawResetWhitelist())
+  }
+
+  updatingWhitelist = idbKeyval.set(WHITELIST_KEY, []).then(() => {
+    updatingWhitelist = null
+  })
+
+  return updatingWhitelist
+}
+
+const messageApi = {
+  whitelistPathnames(event) {
+    let { pathnames } = event.data
+
+    pathnames = pathnames.map(({ pathname, includesPrefix }) => {
+      if (!includesPrefix) {
+        return `%pathPrefix%${pathname}`
+      } else {
+        return pathname
+      }
+    })
+
+    event.waitUntil(rawWhitelistPathnames(pathnames))
+  },
+
+  resetWhitelist(event) {
+    event.waitUntil(rawResetWhitelist())
+  },
+}
+
+self.addEventListener(`message`, event => {
+  const { gatsbyApi } = event.data
+  if (gatsbyApi) messageApi[gatsbyApi](event)
+})

packages/gatsby/cache-dir/ensure-resources.js

@@ -2,7 +2,6 @@ import React from "react"
 import PropTypes from "prop-types"
 import loader from "./loader"
 import shallowCompare from "shallow-compare"
-import { getRedirectUrl } from "./load-directly-or-404"
 
 // Pass pathname in as prop.
 // component will try fetching resources. If they exist,
@@ -94,15 +93,19 @@ class EnsureResources extends React.Component {
   }
 
   render() {
+    // This should only occur if the network is offline, or if the
+    // path is nonexistent and there's no custom 404 page.
     if (
       process.env.NODE_ENV === `production` &&
       !(this.state.pageResources && this.state.pageResources.json)
     ) {
-      // This should only occur if there's no custom 404 page
-      const url = getRedirectUrl(this.state.location.href)
-      if (url) {
-        window.location.replace(url)
-      }
+      // Do this, rather than simply `window.location.reload()`, so that
+      // pressing the back/forward buttons work - otherwise Reach Router will
+      // try to handle back/forward navigation, causing the URL to change but
+      // the page displayed to stay the same.
+      const originalUrl = new URL(location.href)
+      window.history.replaceState({}, `404`, `${location.pathname}?gatsby-404`)
+      window.location.replace(originalUrl)
 
       return null
     }