Run e2e tests inside a docker container

.dockerignore

@@ -4,7 +4,6 @@
 # directories
 node_modules/
 dist/
-test/
 .storybook/
 .husky/
 .github/

.github/workflows/ci.yml

@@ -70,11 +70,8 @@ jobs:
 
       - uses: ./.github/actions/setup-node-env
 
-      - name: Install E2E browsers
-        run: pnpx playwright install
-
       - name: Run E2E
-        run: pnpm ci:e2e-test
+        run: PW_USER=root ./bin/playwright.sh e2e
 
   storybook:
     name: Check Storybook build

.gitignore

@@ -31,3 +31,7 @@ storybook-static
 /src/locales/scripts/untranslated-locales.json
 /src/locales/scripts/wp-locales.json
 .zshrc
+
+# Playwright creates these two folders
+test-results
+test/Default

Dockerfile

@@ -7,6 +7,8 @@ FROM node:16 AS builder
 
 WORKDIR /usr/app
 
+ARG API_URL
+
 # Install pnpm
 RUN npm install -g pnpm
 
@@ -16,7 +18,7 @@ COPY pnpm-lock.yaml .
 COPY .npmrc .
 
 # install dependencies including local development tools
-RUN pnpm install
+RUN pnpm install --store=pnpm-store
 
 # copy the rest of the content
 COPY . /usr/app
@@ -44,11 +46,13 @@ ENV CYPRESS_INSTALL_BINARY=0
 # copy files from local machine
 COPY . /usr/app
 
+COPY --from=builder /usr/app/pnpm-store /usr/app/pnpm-store
+
 # disable telemetry when building the app
 ENV NUXT_TELEMETRY_DISABLED=1
 
 # install dependencies (development dependencies included)
-RUN pnpm install
+RUN pnpm install --store=pnpm-store
 
 # expose port 8443
 EXPOSE 8443
@@ -89,8 +93,9 @@ COPY --from=builder /usr/app/src/locales /usr/app/src/locales
 COPY --from=builder /usr/app/src/utils  /usr/app/src/utils
 COPY --from=builder /usr/app/src/constants  /usr/app/src/constants
 COPY --from=builder /usr/app/src/server-middleware  /usr/app/src/server-middleware
+COPY --from=builder /usr/app/pnpm-store /usr/app/pnpm-store
 
-RUN pnpm install --frozen-lockfile
+RUN pnpm install --frozen-lockfile --store=pnpm-store
 
 # set app serving to permissive / assigned
 ENV NUXT_HOST=0.0.0.0

Dockerfile.playwright

@@ -0,0 +1,5 @@
+ARG PLAYWRIGHT_VERSION
+
+FROM mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-focal
+
+RUN npm install -g pnpm

TESTING_GUIDELINES.md

@@ -92,36 +92,46 @@ Openverse uses [Vue Testing Library](https://testing-library.com/docs/vue-testin
 
 There are also legacy unit tests written in [Vue Test Utils](https://vue-test-utils.vuejs.org/) but those are slated to be re-written using testing library.
 
-### E2e tests
+### End-to-end tests
 
-Before running the e2e tests, install the browsers that Playwright needs:
+Our end-to-end test suite runs inside a Playwright docker container in order to prevent cross-platform browser differences from creating flaky test behavior.
 
-```
-pnpx install playwright
-```
+Having docker and docker-compose is a pre-requisite to running the end-to-end tests locally. Please follow [the relevant instructions for your operating system for how to install docker and docker-compose](https://docs.docker.com/get-docker/). If you're on Windows 10 Home Edition, please note that you'll need to [install and run docker inside WSL2](https://www.freecodecamp.org/news/how-to-run-docker-on-windows-10-home-edition/).
 
-If you don't have the app running, start by running it in the dev mode:
+If it's not possible for you to run docker locally, don't fret! Our CI will run it on every pull request and another contributor who is able to run the tests locally can help you develop new or update existing tests for your changes.
 
-```
-pnpm run dev
+The Playwright docker container runs everything needed for the end-to-end tests, including the Nuxt server and a [Talkback proxy](https://github.com/ijpiantanida/talkback) for the API.
+
+To run the end-to-end tests, after having installed docker, run the following:
+
+```bash
+pnpm test:e2e
 ```
 
-After the dev server has finished building, run the e2e tests:
+If you've added new tests or updated existing ones, you may get errors about API responses not being found. To remedy this, you'll need to update the tapes:
 
+```bash
+pnpm test:e2e:update-tapes
 ```
-pnpm run test:e2e
+
+Additional debugging may be accomplished in two ways. You may inspect the trace output of failed tests by finding the `trace.zip` for the relevant test in the `test-results` folder. Traces are only saved for failed tests. You can use the [Playwright Trace Viewer](https://playwright.dev/docs/trace-viewer) to inspect these (or open the zip yourself and poke around the files on your own).
+
+Additionally, you can run run the tests in debug mode. This will run the tests with a headed browser as opposed to a headless (invisible) one and allow you to watch the test happen in real time. It's not possible for a headed browser to run inside the docker container, however, so be aware that when debugging the environment will be slightly different. For example, if you're on any OS other than Linux, the browser you're running will have small differences in how it renders the page compared to the docker container.
+
+To run the debug tests:
+
+```bash
+pnpm test:e2e:debug
 ```
 
+Note that this still runs the talkback proxy and the Nuxt server for you. If you'd like to avoid this, simply run the Nuxt server before you run the `test:e2e:debug` script and Playwright will automatically prefer your previously running Nuxt server.
+
 When writing e2e tests, it can be helpful to use Playwright [codegen](https://playwright.dev/docs/cli#generate-code) to generate the tests by performing actions in the browser:
 
 ```
-pnpm run generate-e2e-tests
+pnpm run generate-playwright-tests
 ```
 
 This will open the app in a new browser window, and record any actions you take in a format that can be used in e2e tests.
 
-The CI uses [talkback](https://github.com/ijpiantanida/talkback) to ensure that the e2e tests are independent of the network status by recording the network responses in the `/test/tapes` folder. If you add new e2e tests, you may need to update the tapes by running
-
-```
-pnpm run update-tapes
-```
+Note that this does _not_ run the server for you; you must run the Nuxt server using `pnpm start` or `pnpm dev` separately before running the codegen script.

bin/playwright.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+# todo: find a way to make this portable for Windows users so it can be used for the package.json script
+
+export PLAYWRIGHT_VERSION=$(pnpm ls --dev --depth=0 | grep playwright | awk '{print $2}')
+export TEST_TYPE=$1
+docker-compose -f docker-compose.playwright.yml run playwright

docker-compose.playwright.yml

@@ -0,0 +1,15 @@
+version: "3"
+services:
+  playwright:
+    build:
+      context: .
+      dockerfile: Dockerfile.playwright
+      args:
+        - PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION}
+    volumes:
+      - .:/app
+    user: ${PW_USER:-pwuser}
+    working_dir: /app
+    command: pnpm test:${TEST_TYPE}:local
+    environment:
+      - UPDATE_TAPES=${UPDATE_TAPES:-false}

package.json

@@ -14,20 +14,25 @@
   "scripts": {
     "dev": "pnpm install && pnpm i18n:create-locales-list && pnpm i18n:update-locales && cross-env nuxt --hostname 0.0.0.0",
     "build": "pnpm i18n:get-translations && nuxt build",
+    "build-only": "nuxt build",
     "generate": "nuxt generate",
     "start": "nuxt start",
+    "prod": "pnpm build-only && pnpm start",
     "storybook": "nuxt storybook",
     "test": "jest",
-    "test:all": "pnpm e2e:ci & jest",
+    "test:all": "pnpm test:e2e & jest",
     "test:watch": "jest --collectCoverage=false --watch",
-    "test:e2e": "playwright test",
-    "test:e2e-debug": "PWDEBUG=1 playwright test",
-    "build-and-e2e": "start-server-and-test 'cross-env API_URL=http://localhost:49152/ nuxt build && pnpm start' http://localhost:8443 test:e2e",
+    "talkback": "node ./test/proxy.js",
+    "pretest:e2e": "pnpm install",
+    "test:e2e": "cross-env PLAYWRIGHT_VERSION=1.17.1 TEST_TYPE=e2e docker-compose -f docker-compose.playwright.yml up --abort-on-container-exit --exit-code-from playwright",
+    "test:e2e:local": "playwright test -c test/e2e",
+    "test:e2e:debug": "cross-env PWDEBUG=1 pnpm test:e2e:local",
+    "test:e2e:update-tapes": "npm-run-all delete-tapes update-tapes fix-tape-permissions --continue-on-error",
     "ci:unit-test": "jest --collectCoverage=false",
-    "ci:e2e-test": "sh ./e2e.sh",
-    "recreate-tapes": "rimraf test/tapes && sh ./e2e.sh -u",
-    "update-tapes": "sh ./e2e.sh -u",
-    "generate-e2e-tests": "playwright codegen localhost:8443/",
+    "delete-tapes": "rimraf test/tapes",
+    "update-tapes": "cross-env UPDATE_TAPES=true pnpm test:e2e",
+    "fix-tape-permissions": "docker run -it --rm --mount type=bind,source=$(pwd),target=/app node:alpine chmod -R 0777 /app/test/tapes",
+    "generate-playwright-test": "playwright codegen localhost:8443/",
     "types": "tsc --noEmit",
     "lint": "eslint --ext .js,.vue,.ts --ignore-path .gitignore --ignore-path .eslintignore .",
     "lintfix": "pnpm run lint -- --fix",
@@ -123,13 +128,13 @@
     "jest": "^26.6.3",
     "jest-transform-stub": "^2.0.0",
     "lint-staged": "^11.1.1",
+    "npm-run-all": "^4.1.5",
     "pinia": "^2.0.11",
     "postcss": "^8.4.5",
     "prettier": "^2.2.1",
     "rimraf": "^3.0.2",
     "sass": "^1.34.0",
     "sass-loader": "^10.1.1",
-    "start-server-and-test": "^1.14.0",
     "tailwindcss": "^3.0.7",
     "tailwindcss-rtl": "^0.8.0",
     "talkback": "^2.5.0",