diff --git a/.github/workflows/test-template-changes.yml b/.github/workflows/test-template-changes.yml
new file mode 100644
index 00000000000..e2f30ae35b0
--- /dev/null
+++ b/.github/workflows/test-template-changes.yml
@@ -0,0 +1,51 @@
+######################################
+## Custom Web Almanac GitHub action ##
+######################################
+#
+# This Action generates the dynamic templates. It also then takes a copy
+# of the templates folder, and then checks out main branch and regenerates the
+# templates and compares the two. If run from a pull request it adds a comment
+# showing the diffs.
+#
+# Note it does not run a full test of the website as that will be completed by
+# the test_website.yml GitHub Action.
+#
+# This Action is useful to check impact of dependency upgrades (e.g. if
+# prettier version change results in different HTML being generated), since we
+# do not store generated templates in Git and so cannot otherwise see these
+# changes. Changes can then be accepted or investigated further before merging.
+#
+name: Test Template Changes
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '**/package.json'
+      - '**/package-lock.json'
+jobs:
+  build:
+    name: Build and Test Template Changes
+    runs-on: ubuntu-latest
+    if: github.repository == 'HTTPArchive/almanac.httparchive.org'
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2.3.2
+      with:
+        fetch-depth: 0
+    - name: Setup Node.js for use with actions
+      uses: actions/setup-node@v1.4.2
+      with:
+        node-version: 12.x
+    - name: Test Template Changes
+      id: test-template-changes
+      run: ./src/tools/scripts/test_template_changes.sh
+    - name: 'Comment PR'
+      uses: actions/github-script@0.3.0
+      if: github.event_name == 'pull_request' && env.PR_COMMENT != ''
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const { issue: { number: issue_number }, repo: { owner, repo }  } = context;
+          // Unescape any escaped comments so code blocks appear correctly
+          const pr_comment = process.env.PR_COMMENT.replace(/%0A/g,'\n').replace(/%0D/g,'\r').replace(/%25/g,'%');
+          github.issues.createComment({ issue_number, owner, repo, body: pr_comment });
diff --git a/src/package.json b/src/package.json
index b0a0e215c99..da6f95c39e6 100644
--- a/src/package.json
+++ b/src/package.json
@@ -20,14 +20,14 @@
     "deploy": "./tools/scripts/deploy.sh"
   },
   "devDependencies": {
-    "ejs": "^3.1.5",
-    "fs-extra": "^9.0.1",
-    "jsdom": "^16.4.0",
-    "prettier": "^2.1.1",
-    "recursive-readdir": "^2.2.2",
-    "showdown": "^1.9.1",
-    "web-vitals": "^0.2.4",
-    "node-fetch": "^2.6.0",
-    "xml-js": "^1.6.11"
+    "ejs": "3.1.5",
+    "fs-extra": "9.0.1",
+    "jsdom": "16.4.0",
+    "prettier": "2.1.1",
+    "recursive-readdir": "2.2.2",
+    "showdown": "1.9.1",
+    "web-vitals": "0.2.4",
+    "node-fetch": "2.6.0",
+    "xml-js": "1.6.11"
   }
 }
diff --git a/src/tools/scripts/test_template_changes.sh b/src/tools/scripts/test_template_changes.sh
new file mode 100755
index 00000000000..a8c37151748
--- /dev/null
+++ b/src/tools/scripts/test_template_changes.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+######################################
+## Custom Web Almanac script        ##
+######################################
+#
+# This script installs all the required dependencies needed to run the
+# Web Almanac website providing you have python and node installed.
+#
+# It also runs our tests to ensure the website is working for all pages.
+#
+# It is used by various GitHub actions to build and test the site.
+#
+
+# exit when any command fails instead of trying to continue on
+set -e
+
+TEMP_TEMPLATES_DIRECTORY=templates_new
+DIFF_FILENAME=/tmp/template_differences.txt
+
+# This script must be run from src directory
+if [ -d "src" ]; then
+  cd src
+fi
+
+if [ -d "${TEMP_TEMPLATES_DIRECTORY}" ]; then
+  echo "${TEMP_TEMPLATES_DIRECTORY} already exists. Exiting"
+  exit 1
+fi
+
+echo "Installing node modules"
+npm install
+
+echo "Building website"
+npm run generate
+
+echo "Backing up templates"
+cp -r templates "${TEMP_TEMPLATES_DIRECTORY}"
+
+echo "Checkout main branch"
+git checkout main
+
+echo "Reinstalling node modules"
+rm -rf node_modules
+npm install
+
+echo "Building website"
+npm run generate
+
+echo "Diff the two folders"
+#Don't fail if there are differences so turn that check off that temporarily
+set +e
+diff -r templates "${TEMP_TEMPLATES_DIRECTORY}" > "${DIFF_FILENAME}"
+set -e
+
+echo "Differences:"
+cat "${DIFF_FILENAME}"
+
+NUM_DIFFS=$(wc -l < "${DIFF_FILENAME}")
+if [ "${NUM_DIFFS}" -ne "0" ]; then
+  # Weird syntax is to make is Mac compatible: https://stackoverflow.com/a/1252191/2144578
+  ESCAPED_OUTPUT=$(sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g' -e 's/\r/%0D/g' -e 's/\%/%25/g' "${DIFF_FILENAME}")
+  PR_COMMENT="Please note, that the following diffs happen in the templates on this branch compared to main:%0A\`\`\`%0A${ESCAPED_OUTPUT}%0A\`\`\`%0A"
+  echo "::set-env name=PR_COMMENT::${PR_COMMENT}"
+fi
+
+echo "Removing templates backup"
+rm -rf "${TEMP_TEMPLATES_DIRECTORY}"
+rm -f "${DIFF_FILENAME}"
+
+echo "Comparison complete"