Improve release caching and prevent download loops

README.md

@@ -88,7 +88,6 @@ configuration files, set `CONTAINER_PRESERVE_CONFIG` to `true`.
         image: felddy/foundryvtt:release
         hostname: my_foundry_host
         init: true
-        restart: "unless-stopped"
         volumes:
           - type: bind
             source: <your_data_dir>
@@ -150,7 +149,6 @@ uses `secrets.json`.  Regardless of the name you choose it must be targeted to
         image: felddy/foundryvtt:release
         hostname: my_foundry_host
         init: true
-        restart: "unless-stopped"
         volumes:
           - type: bind
             source: <your_data_dir>
@@ -210,7 +208,7 @@ It is recommended that most users use the `:release` tag.
 | Image:tag | Description |
 |-----------|-------------|
 |`felddy/foundryvtt:release` | The most recent image from the `stable` channel.  These images are **considered stable**, and well-tested.  Most users will use this tag.  The `latest` tag always points to the same version as `release`.|
-|`felddy/foundryvtt:prerelease` | The most recent image from the `testing`, `development`, or `prototype` channels.  Pre-releases are **VERY LIKELY to introduce breaking bugs** that will be disruptive to play. Do not install this version unless you are using for the specific purposes of testing. The intention of pre-release builds are to allow for previewing new features and to help developers to begin updating modules which are impacted by the changes. If you choose to update to this version for a live game you do so entirely at your own risk of having a bad experience. _Please back up your critical user data before installing this version._ |
+|`felddy/foundryvtt:prerelease` | The most recent image from the `testing`, `development`, or `prototype` channels.  Pre-releases are **VERY LIKELY to introduce breaking bugs** that will be disruptive to play. Do not install this version unless you are using for the specific purposes of testing. The intention of pre-release builds are to allow for previewing new features and to help developers to begin updating modules which are impacted by the changes. If you choose to update to this version for a live game you do so entirely at your own risk of having a bad experience. *Please back up your critical user data before installing this version.* |
 |`felddy/foundryvtt:9.255.0`| An exact image version. |
 |`felddy/foundryvtt:9.255`| The most recent image matching the major and minor version numbers. |
 |`felddy/foundryvtt:9`| The most recent image matching the major version number. |
@@ -262,23 +260,15 @@ secrets](#using-secrets) instead of environment variables.
 |------------------|----------|
 | `FOUNDRY_RELEASE_URL` | S3 pre-signed URL generate from the user's profile.  Required for downloading an application distribution. |
 
-#### Pre-cached distribution variable ####
-
-A distribution can be downloaded and placed into a cache directory.  The
-distribution's name must be of the form: `foundryvtt-9.255.zip`
-
-| Name             | Purpose  |
-|------------------|----------|
-| `CONTAINER_CACHE` | Set a path to cache downloads of the Foundry distribution archive and speed up subsequent container startups.  The path should be in `/data` or another persistent mount point in the container. e.g.; `/data/container_cache`| |
-
 ### Optional ###
 
 | Name  | Purpose | Default |
 |-------|---------|---------|
-| `CONTAINER_PATCHES` | Set a path to a directory of shell scripts to be sourced after Foundry is installed but before it is started.  The path should be in `/data` or another persistent mount point in the container. e.g.; `/data/container_patches`  Patch files are sourced in lexicographic order.  `CONTAINER_PATCHES` are processed after `CONTAINER_PATCH_URLS`.|  |
+| `CONTAINER_CACHE` | Set a path to cache downloads of the Foundry distribution archive and speed up subsequent container startups.  The path should be in `/data` or another persistent mount point in the container.  Set to `""` to disable.  ***Note***: When the cache is disabled the container may sleep instead of exiting, in certian circumstances, to prevent a download loop.  A distribution can be pre-downloaded and placed into a cache directory.  The distribution's name must be of the form: `foundryvtt-9.255.zip`| `/data/container_cache` |
+| `CONTAINER_PATCHES` | Set a path to a directory of shell scripts to be sourced after Foundry is installed but before it is started.  The path should be in `/data` or another persistent mount point in the container. e.g.; `/data/container_patches`  Patch files are sourced in lexicographic order.  `CONTAINER_PATCHES` are processed after `CONTAINER_PATCH_URLS`.| |
 | `CONTAINER_PATCH_URLS` | Set to a space-delimited list of URLs to be sourced after Foundry is installed but before it is started.  Patch URLs are sourced in the order specified.  `CONTAINER_PATCH_URLS` are processed before `CONTAINER_PATCHES`.  ⚠️ **Only use patch URLs from trusted sources!** | |
 | `CONTAINER_PRESERVE_CONFIG` | Normally new `options.json` and `admin.txt` files are generated by the container at each startup.  Setting this to `true` prevents the container from modifying these files when they exist.  If they do not exist, they will be created as normal. | `false` |
-| `CONTAINER_PRESERVE_OWNER` | Normally the ownership of the `/data` directory and its contents are changed to match that of the server at startup.  Setting this to a regular expression will exclude any matching paths and preserve their ownership.   _Note: This is a match on the whole path, not a search._  This is useful if you want mount a volume as read-only inside `/data` (e.g.; a volume that contains assets mounted at `/data/Data/assets`).  | |
+| `CONTAINER_PRESERVE_OWNER` | Normally the ownership of the `/data` directory and its contents are changed to match that of the server at startup.  Setting this to a regular expression will exclude any matching paths and preserve their ownership.   *Note: This is a match on the whole path, not a search.*  This is useful if you want mount a volume as read-only inside `/data` (e.g.; a volume that contains assets mounted at `/data/Data/assets`).  | |
 | `CONTAINER_VERBOSE` | Set to `true` to enable verbose logging for the container utility scripts. | `false` |
 | `FOUNDRY_ADMIN_KEY` | Admin password to be applied at startup.  If omitted the admin password will be cleared.  May be set [using secrets](#using-secrets). | |
 | `FOUNDRY_AWS_CONFIG` | An absolute or relative path that points to the [awsConfig.json](https://foundryvtt.com/article/aws-s3/) or `true` for AWS environment variable [credentials evaluation](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html) usage. | `null` |

src/authenticate.ts

@@ -9,14 +9,16 @@ EXIT STATUS
     >0  An error occurred.
 
 Usage:
-  authenticate.js [--log-level=LEVEL] <username> <password> <cookiejar>
+  authenticate.js [options] <username> <password> <cookiejar>
   authenticate.js (-h | --help)
 
 Options:
   -h --help              Show this message.
   --log-level=LEVEL      If specified, then the log level will be set to
                          the specified value.  Valid values are "debug", "info",
                          "warn", and "error". [default: info]
+  --user-agent=USERAGENT If specified, then the user-agent header will be set to
+                         the specified value. [default: node-fetch]
 `;
 
 // Imports
@@ -27,7 +29,7 @@ import createLogger from "./logging.js";
 import winston from "winston";
 import docopt from "docopt";
 import fetchCookie from "fetch-cookie";
-import nodeFetch from "node-fetch";
+import nodeFetch, { Headers } from "node-fetch";
 import process from "process";
 
 // Setup globals, to be configured in main()
@@ -41,12 +43,12 @@ const LOCAL_DOMAIN = "felddy.com";
 const LOGIN_URL = BASE_URL + "/auth/login/";
 const USERNAME_RE = /\/community\/(?<username>.+)/;
 
-const HEADERS = {
+const HEADERS: Headers = new Headers({
   DNT: "1",
   Referer: BASE_URL,
   "Upgrade-Insecure-Requests": "1",
-  "User-Agent": "Mozilla/5.0",
-};
+  "User-Agent": "node-fetch",
+});
 
 /**
  * fetchTokens - Fetch the CSRF form and cookie tokens.
@@ -154,6 +156,7 @@ async function main(): Promise<number> {
   // Extract values from CLI options.
   const cookiejar_filename = options["<cookiejar>"];
   const log_level = options["--log-level"].toLowerCase();
+  HEADERS.set("User-Agent", options["--user-agent"]);
   const password = options["<password>"];
   const username = options["<username>"].toLowerCase();
 

src/entrypoint.sh

@@ -41,6 +41,11 @@ cookiejar_file="cookiejar.json"
 license_min_length=24
 secret_file="/run/secrets/config.json"
 
+# Calculate a user-agent comment to use in for curl and node-fetch requests
+CONTAINER_USER_AGENT_COMMENT="(felddy/foundryvtt:${image_version})"
+curl_user_agent=$(curl --version | awk 'NR==1 {print $1 "/" $2}')" ${CONTAINER_USER_AGENT_COMMENT}"
+node_user_agent="node-fetch ${CONTAINER_USER_AGENT_COMMENT}"
+
 # Warn user if the container version does not start with the FOUNDRY_VERSION.
 # The FOUNDRY_VERSION looks like x.yyy
 # The container version is a semver x.y.z
@@ -68,6 +73,9 @@ fi
 
 # Check to see if an install is required
 install_required=false
+# Track whether an S3 URL request is made.
+# We use this information to protect from a download loop.
+requested_s3_url=false
 if [ -f "resources/app/package.json" ]; then
   # FoundryVTT no longer supports the "version" field in package.json
   # We need to build up a pseudo-version using the generation and build values
@@ -97,19 +105,30 @@ if [ $install_required = true ]; then
     # The resulting cookiejar is used to get a release URL or license.
     # CONTAINER_VERBOSE default value should not be quoted.
     # shellcheck disable=SC2086
-    ./authenticate.js ${CONTAINER_VERBOSE+--log-level=debug} "${FOUNDRY_USERNAME}" "${FOUNDRY_PASSWORD}" "${cookiejar_file}"
+    ./authenticate.js ${CONTAINER_VERBOSE+--log-level=debug} \
+      --user-agent="${node_user_agent}" \
+      "${FOUNDRY_USERNAME}" "${FOUNDRY_PASSWORD}" "${cookiejar_file}"
     if [[ ! "${s3_url:-}" ]]; then
       # If the s3_url wasn't set by FOUNDRY_RELEASE_URL generate one now.
       log "Using authenticated credentials to download release."
       # CONTAINER_VERBOSE default value should not be quoted.
       # shellcheck disable=SC2086
-      s3_url=$(./get_release_url.js ${CONTAINER_VERBOSE+--log-level=debug} "${cookiejar_file}" "${FOUNDRY_VERSION}")
+      s3_url=$(./get_release_url.js ${CONTAINER_VERBOSE+--log-level=debug} \
+        --user-agent="${node_user_agent}" \
+        "${cookiejar_file}" "${FOUNDRY_VERSION}")
+      requested_s3_url=true
     fi
   fi
 
+  # If CONTAINER_CACHE is null, set it to a default.
+  # If it set to an empty string, disable the caching.
+  CONTAINER_CACHE="${CONTAINER_CACHE-/data/container_cache}"
+
   if [[ "${CONTAINER_CACHE:-}" ]]; then
     log "Using CONTAINER_CACHE: ${CONTAINER_CACHE}"
     mkdir -p "${CONTAINER_CACHE}"
+  else
+    log_warn "CONTAINER_CACHE has been unset.  Release caching is disabled."
   fi
 
   set +o nounset
@@ -121,7 +140,9 @@ if [ $install_required = true ]; then
     log "Downloading Foundry Virtual Tabletop release."
     # Download release if newer than cached version.
     # Filter out warnings about bad date formats if the file is missing.
-    curl --fail --location --time-cond "${release_filename}" \
+    curl ${CONTAINER_VERBOSE+--verbose} --fail --location \
+      --user-agent "${curl_user_agent}" \
+      --time-cond "${release_filename}" \
       --output "${downloading_filename}" "${s3_url}" 2>&1 \
       | tr "\r" "\n" \
       | sed --unbuffered '/^Warning: .* date/d'
@@ -156,7 +177,9 @@ if [ $install_required = true ]; then
     for url in ${CONTAINER_PATCH_URLS}; do
       log "Downloading patch from URL: $url"
       patch_file=$(mktemp -t patch_url.sh.XXXXXX)
-      curl --silent --output "${patch_file}" "${url}"
+      curl ${CONTAINER_VERBOSE+--verbose} --silent \
+        --user-agent "${curl_user_agent}" \
+        --output "${patch_file}" "${url}"
       log_debug "Sourcing patch file: ${patch_file}"
       # shellcheck disable=SC1090
       source "${patch_file}"
@@ -202,10 +225,15 @@ if [ ! -f "${LICENSE_FILE}" ]; then
       # FOUNDRY_LICENSE_KEY can be an index, try passing it.
       # CONTAINER_VERBOSE default value should not be quoted.
       # shellcheck disable=SC2086
-      fetched_license_key=$(./get_license.js ${CONTAINER_VERBOSE+--log-level=debug} --select="${FOUNDRY_LICENSE_KEY}" "${cookiejar_file}")
+      fetched_license_key=$(./get_license.js ${CONTAINER_VERBOSE+--log-level=debug} \
+        --user-agent="${node_user_agent}" \
+        --select="${FOUNDRY_LICENSE_KEY}" \
+        "${cookiejar_file}")
     else
       # shellcheck disable=SC2086
-      fetched_license_key=$(./get_license.js ${CONTAINER_VERBOSE+--log-level=debug} "${cookiejar_file}")
+      fetched_license_key=$(./get_license.js ${CONTAINER_VERBOSE+--log-level=debug} \
+        --user-agent="${node_user_agent}" \
+        "${cookiejar_file}")
     fi
     echo "{ \"license\": \"${fetched_license_key}\" }" > "${LICENSE_FILE}"
   else
@@ -234,5 +262,17 @@ export CONTAINER_PRESERVE_CONFIG FOUNDRY_ADMIN_KEY FOUNDRY_AWS_CONFIG \
   FOUNDRY_LOCAL_HOSTNAME FOUNDRY_MINIFY_STATIC_FILES FOUNDRY_PASSWORD_SALT \
   FOUNDRY_PROXY_PORT FOUNDRY_PROXY_SSL FOUNDRY_ROUTE_PREFIX FOUNDRY_SSL_CERT \
   FOUNDRY_SSL_KEY FOUNDRY_UPNP FOUNDRY_UPNP_LEASE_DURATION FOUNDRY_WORLD
-su-exec "${FOUNDRY_UID:-foundry}:${FOUNDRY_GID:-foundry}" ./launcher.sh "$@"
+su-exec "${FOUNDRY_UID:-foundry}:${FOUNDRY_GID:-foundry}" ./launcher.sh "$@" \
+  || log_error "Launcher exited with error code: $?"
+
+# If the container requested a new S3 URL but disabled the cache
+# we are going to sleep forever to prevent a downlaod loop.
+if [[ "${requested_s3_url}" == "true" && "${CONTAINER_CACHE:-}" == "" ]]; then
+  log_warn "Server exited after downloading a release while the CONTAINER_CACHE was disabled."
+  log_warn "This configuration could lead to a restart loop putting excessive load on the release server."
+  log_warn "Please re-enable the CONTAINER_CACHE to allow the container to safely exit."
+  log_warn "Sleeping..."
+  while true; do sleep 60; done
+fi
+
 exit 0

src/get_license.ts

@@ -12,7 +12,7 @@ EXIT STATUS
     >0  An error occurred.
 
 Usage:
-  get_license.js [--log-level=LEVEL] [--select=MODE] <cookiejar>
+  get_license.js [options] <cookiejar>
   get_license.js (-h | --help)
 
 Options:
@@ -24,6 +24,9 @@ Options:
                          account return the one specified by index.  In
                          unspecified, a random license will be returned.  Index
                          starts at 1.
+  --user-agent=USERAGENT If specified, then the user-agent header will be set to
+                         the specified value. [default: node-fetch]
+
 `;
 
 // Imports
@@ -33,7 +36,7 @@ import cheerio from "cheerio";
 import createLogger from "./logging.js";
 import docopt from "docopt";
 import fetchCookie from "fetch-cookie";
-import nodeFetch, { HeadersInit } from "node-fetch";
+import nodeFetch, { Headers } from "node-fetch";
 import process from "process";
 import winston from "winston";
 
@@ -46,12 +49,12 @@ var logger: winston.Logger;
 const BASE_URL: string = "https://foundryvtt.com";
 const LOCAL_DOMAIN: string = "felddy.com";
 
-const HEADERS: HeadersInit = {
+const HEADERS: Headers = new Headers({
   DNT: "1",
   Referer: BASE_URL,
   "Upgrade-Insecure-Requests": "1",
-  "User-Agent": "Mozilla/5.0",
-};
+  "User-Agent": "node-fetch",
+});
 
 /**
  * fetchLicense - Fetch a license key for a user.
@@ -94,6 +97,7 @@ async function main(): Promise<number> {
   const cookiejar_filename: string = options["<cookiejar>"];
   const log_level: string = options["--log-level"].toLowerCase();
   const select_mode: string = options["--select"];
+  HEADERS.set("User-Agent", options["--user-agent"]);
 
   // Setup logging.
   logger = createLogger("License", log_level);

src/get_release_url.ts

@@ -12,14 +12,17 @@ EXIT STATUS
     >0  An error occurred.
 
 Usage:
-  get_release_url.js [--log-level=LEVEL] <cookiejar> <version>
+  get_release_url.js [options] <cookiejar> <version>
   get_release_url.js (-h | --help)
 
 Options:
   -h --help              Show this message.
   --log-level=LEVEL      If specified, then the log level will be set to
                          the specified value.  Valid values are "debug", "info",
                          "warn", and "error". [default: info]
+  --user-agent=USERAGENT If specified, then the user-agent header will be set to
+                         the specified value. [default: node-fetch]
+
 `;
 
 // Imports
@@ -28,7 +31,7 @@ import { FileCookieStore } from "tough-cookie-file-store";
 import createLogger from "./logging.js";
 import docopt from "docopt";
 import fetchCookie from "fetch-cookie";
-import nodeFetch, { Response } from "node-fetch";
+import nodeFetch, { Headers, Response } from "node-fetch";
 import process from "process";
 import winston from "winston";
 
@@ -40,12 +43,12 @@ var logger: winston.Logger;
 // Constants
 const BASE_URL = "https://foundryvtt.com";
 
-const HEADERS = {
+const HEADERS: Headers = new Headers({
   DNT: "1",
   Referer: BASE_URL,
   "Upgrade-Insecure-Requests": "1",
-  "User-Agent": "Mozilla/5.0",
-};
+  "User-Agent": "node-fetch",
+});
 
 /**
  * fetchReleaseURL - Fetch the pre-signed S3 URL.
@@ -85,6 +88,7 @@ async function main(): Promise<number> {
   const cookiejar_filename: string = options["<cookiejar>"];
   const foundry_version: string = options["<version>"];
   const log_level: string = options["--log-level"].toLowerCase();
+  HEADERS.set("User-Agent", options["--user-agent"]);
 
   // Setup logging.
   logger = createLogger("ReleaseURL", log_level);

src/launcher.sh

@@ -60,4 +60,4 @@ fi
 
 # Spawn node with clean environment to prevent credential leaks
 log "Starting Foundry Virtual Tabletop."
-env -i HOME="$HOME" node "$@"
+env -i HOME="$HOME" node "$@" || log_error "Node process exited with code $?"