Run storybook tests with playwright (#249)

* resolve conflicts

* test cleanup

* add post deployment action for testing

* update lockfile to fix failing action

* update action name

* correct config dir path

* add workflow dispatch for testing

* add push action for wip branch

* fix failing test for radio group pattern edit

* update documentation and add script for storybook testing with playwright

* update lockfile

* update path in action

* set endpoint in action

* update workflow endpoint and run mode

* install playwright in gh action

* create a self-contained image that runs through the tests on docker build

* use docker in post-deploy action

* clean up logging for docker build

* simplify end to end tests

* debugging tests

* rename test

* install unfiltered deps in docker--no-verify

* change end-to-end back to imported convention

* test deliberate failure in pipeline

* remove deliberately failing test

* update adr for test strategy

* add drag and drop test

* dry up test code

* add mouse interaction test for drag-and-drop

* fix flaky test

* remove flaky test

* remove commented code

* ignore pnpm cache dir

* use netcat instead of sleep to start server. pare down the copied files in the base image.

* use netcat instead of sleep to start server. pare down the copied files in the base image.

* add script to provide config for docker functions

* update documentation and add provisions for running stories locally

* grammar correction

.github/workflows/_end-to-end.yml

@@ -0,0 +1,12 @@
+name: End-to-end tests
+on:
+  workflow_call:
+jobs:
+  end-to-end:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Docker to run tests
+        run: |
+          docker build --platform linux/amd64 --tag 'playwright' . -f ./e2e/Dockerfile --target test

.github/workflows/run-tests.yml

@@ -10,6 +10,5 @@ jobs:
   run-tests:
     uses: ./.github/workflows/_validate.yml
   e2e:
-    needs: [run-tests]
-    uses: ./.github/workflows/_playwright.yml
+    uses: ./.github/workflows/_end-to-end.yml
     secrets: inherit

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .env
+.pnpm-store/
 *.code-workspace
 _site
 .turbo/

.husky/pre-commit

@@ -1,3 +1,4 @@
+#!/bin/sh
 pnpm lint
 pnpm format
-pnpm test
+pnpm test

documents/adr/0010-end-to-end-testing.md

@@ -1,6 +1,6 @@
-# 10. End to end testing
+# 10. End-to-end and interaction testing
 
-Date: 2024-07-01
+Date: 2024-07-08
 
 ## Status
 
@@ -11,8 +11,8 @@ Pending
 Certain tests are not able to be performed with Storybook and JSDOM (e.g. drag-and-drop). The ability to replicate more complex user interactions in the test suite through an actual browser can provide this feature.
 
 ## Decision
-The end-to-end tests should be used sparingly since they are slower to run than the ones through JSDOM. Storybook still should be the primary mechanism for testing, and the Playwright tests will round out what isn't possible there. 
+The end-to-end tests should be used sparingly since they are slower to run than the ones through JSDOM. We will use Playwright in CI/CD for the comprehensive tests and JSDOM during development for speed. Storybook still should be the primary mechanism for UI testing, and during CI/CD, the interaction tests will be run in Playwright using a docker container against the build. 
 
 ## Consequences
 
-The deployed application will include Playwright tests in the e2e package.
+There are some tests that will be end-to-end that run against the built application, while the interaction tests will run against the built Storybook. The end-to-end tests will be in the e2e directory, and docker will be used to make the test environment consistent.

e2e/Dockerfile

@@ -3,20 +3,27 @@ FROM mcr.microsoft.com/playwright:v1.43.1-jammy as base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 ENV NODE_ENV=test
+RUN apt-get update && apt-get install -y netcat
 WORKDIR /srv/apps/atj-platform
-COPY . .
+COPY ./pnpm-lock.yaml ./pnpm-lock.yaml
+COPY ./package.json ./package.json
 RUN corepack enable
-RUN pnpm install --filter=@atj/spotlight --frozen-lockfile
 
 FROM base as test
 ENV E2E_ENDPOINT=http://localhost:9090
+ENV CI=true
+COPY . .
 RUN npm install -g serve
 EXPOSE 9090
-RUN pnpm build --filter=@atj/spotlight
+EXPOSE 9191
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm build --filter=@atj/spotlight --filter=@atj/design
 WORKDIR ./e2e
-RUN serve ../apps/spotlight/dist -p 9090 & sleep 5; pnpm playwright test;
+RUN serve ../apps/spotlight/dist -p 9090 -L & while ! nc -z localhost 9090; do sleep 1; done; pnpm playwright test;
+RUN serve ../packages/design/storybook-static -p 9191 -L & while ! nc -z localhost 9191; do sleep 1; done; pnpm --filter=end-to-end-tests test:storybook --url http://localhost:9191 --config-dir ../packages/design/.storybook/ --browsers firefox chromium
 
 FROM base as serve
 ENV E2E_ENDPOINT=http://localhost:4321
-EXPOSE 4321 9292 9323
-CMD ["pnpm", "dev", "--filter=@atj/spotlight", "--", "--host"]
+EXPOSE 4321 9292 9323 9009 8080
+RUN git config --global --add safe.directory /srv/apps/atj-platform
+CMD ["pnpm", "dev"]

e2e/README.md

@@ -1,34 +1,45 @@
-# End-to-end testing
-E2E testing runs in a docker container.
+# End-to-end and interaction testing
+E2E testing runs in a docker container. There is a shell script (`./scripts/end-to-end.sh`) that provides configuration to automate several Docker-related commands. 
 
-```bash
-# run from project root
-docker build --tag 'playwright' . -f ./e2e/Dockerfile
-```
+Parameters:
+-p : Configure the port on which Storybook will be served. Defaults to 9009 if no value is specified.
+-c : Configure the name of the Docker container. Defaults to e2e if no value is specified.
+-f : Specify the function(s) you'd like to run. You can add multiple -f parameters followed by function name, like `-f build_container -f run_container`
+-t : This parameter lets you specify the Docker build target. This should be either `serve` or `test`.
+
+Functions:
+`build_container` : Builds a Docker container, where the build target is specified using a -t flag (the default target is `test`).
+`run_container` : Runs the built Docker container.
+`end_to_end` : Performs playwright test command inside the Docker container for end-to-end testing. Requires the container to be running.
+`interaction` : Performs the interaction tests inside the Docker container against the storybook instance. Requires the container to be running.
+
+If Docker is not installed on your machine, running the script will prompt you to install Docker. If no function(s) are defined using -f flag, it runs `end_to_end` and `interaction` function by default.
+
+## Build targets
+The `test` target is self-contained and meant to run mostly in the build pipeline. The `serve` target is most useful during local development. When the `serve` container is run, it will mount a volume from your local machine and start a dev server, so you get persistent storage without having to rebuild the image. 
+
+## Getting Started
+
+First, make sure the script is executable: 
 
-To see the output of the tests and run everything when the docker container is built, run the command below:
 ```bash
-# run from project root
-docker build --tag 'playwright' . -f ./e2e/Dockerfile --progress=plain --target test
+# from the project root
+chmod +x ./e2e/scripts/end-to-end.sh
 ```
-You can add the `--no-cache` flag to build from scratch.
 
-To run the container (best for development):
+Examples:
 
 ```bash
-# run from project root
-docker run -p 4321:4321 -it --name e2e --rm playwright
+# builds the test container (also will run the tests)
+./e2e/scripts/end-to-end.sh -f build_container -t test
 ```
 
 ```bash
-# run from project root
-docker exec -it e2e pnpm playwright test
+# builds the serve container and start it
+./e2e/scripts/end-to-end.sh -f build_container -t serve -f run_container
 ```
 
-### Debugging
-To debug and follow the flow of a test in a browser, you can run:
-
 ```bash
-# run from this directory
-export E2E_ENDPOINT=http://localhost:4321; pnpm playwright test --ui-port=8080 --ui-host=0.0.0.0
-```
+# Runs the default tasks `end_to_end` and `interaction`
+./e2e/scripts/end-to-end.sh
+```

e2e/package.json

@@ -3,10 +3,15 @@
   "version": "1.0.0",
   "scripts": {
     "dev": "tsc -w",
+    "test:storybook": "test-storybook",
     "test": "export E2E_ENDPOINT=http://localhost:4321; pnpm playwright test --ui-port=8080 --ui-host=0.0.0.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.43.1",
+    "@storybook/test-runner": "^0.16.0",
     "path-to-regexp": "^7.0.0"
+  },
+  "dependencies": {
+    "@atj/common": "workspace:*"
   }
 }

e2e/script/end-to-end.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# default values
+STORYBOOK_PORT=9009
+CONTAINER_NAME="e2e"
+BUILD_TARGET="test"
+BASE_PATH=$(dirname $0)
+FUNCS=""
+
+# Parse flag parameters
+while getopts "p:c:f:t:" flag
+do
+    case "${flag}" in
+        p) STORYBOOK_PORT=${OPTARG};;
+        c) CONTAINER_NAME=${OPTARG};;
+        f) FUNCS="$FUNCS ${OPTARG}";;
+        t) BUILD_TARGET="${OPTARG}";;
+    esac
+done
+
+build_container() {
+    if [[ $BUILD_TARGET != "serve" ]] && [[ $BUILD_TARGET != "test" ]]; then
+        echo "Invalid BUILD_TARGET! It should be either 'serve' or 'test.' You need to pass these values in with the -t flag (e.g. -t serve)."
+        exit 1
+    fi
+
+    docker build --tag 'playwright' . -f $BASE_PATH/../Dockerfile --progress=plain --target $BUILD_TARGET
+}
+
+run_container() {
+    docker run -p 4321:4321 -p $STORYBOOK_PORT:$STORYBOOK_PORT --name $CONTAINER_NAME -v $(dirname $0)/../../:/srv/apps/atj-platform -it --rm playwright
+}
+
+end_to_end() {
+    docker exec -w /srv/apps/atj-platform/e2e -it $CONTAINER_NAME pnpm playwright test
+}
+
+interaction() {
+    docker exec -it $CONTAINER_NAME pnpm --filter=end-to-end-tests test:storybook --url http://localhost:$STORYBOOK_PORT --config-dir ../packages/design/.storybook/ --browsers firefox chromium
+}
+
+if ! command -v docker &> /dev/null; then
+    echo "Docker is not installed. Please install Docker to run this script."
+    exit 1
+fi
+
+
+if [ -z "$FUNCS" ]; then
+    end_to_end
+    interaction
+else
+    for FUNC in $FUNCS; do
+        case $FUNC in
+            "build_container") build_container ;;
+            "run_container") run_container ;;
+            "interaction") interaction ;;
+            "end_to_end") end_to_end ;;
+            *) echo "Invalid flag: ${FUNC}. Ignored." ;;
+        esac
+    done
+fi

e2e/src/constants.ts

@@ -1 +1,2 @@
 export const BASE_URL = process.env.E2E_ENDPOINT;
+export const STORYBOOK_PATH = 'design/iframe.html?id=';

e2e/src/create.spec.ts

@@ -5,11 +5,18 @@ import { pathToRegexp } from 'path-to-regexp';
 
 
 const createNewForm = async (page: Page) => {
-  console.log(`${BASE_URL}/${GuidedFormCreation.getUrl()}`);
   await page.goto(`${BASE_URL}/${GuidedFormCreation.getUrl()}`);
   await page.getByRole('button', { name: 'Create New' }).click();
 }
 
+const addQuestions = async (page: Page) => {
+  const menuButton = page.getByRole('button', { name: 'Question' });
+  await menuButton.click();
+  await page.getByRole('button', { name: 'Short Answer' }).click();
+  await menuButton.click();
+  await page.getByRole('button', { name: 'Radio Buttons' }).click();
+}
+
 test('Create form from scratch', async ({ page }) => {
   const regexp = pathToRegexp(Create.path);
   await createNewForm(page);
@@ -30,16 +37,11 @@ test('Create form from scratch', async ({ page }) => {
 
 test('Add questions', async ({ page }) => {
   await createNewForm(page);
-
-  const menuButton = page.getByRole('button', { name: 'Question' });
-  await menuButton.click();
-  await page.getByRole('button', { name: 'Short Answer' }).click();
-  await menuButton.click();
-  await page.getByRole('button', { name: 'Radio Buttons' }).click();
+  await addQuestions(page);
 
   // Create locators for both elements
   const fields = page.locator('.usa-label');
-  const element1 = fields.filter({ hasText: 'Field Label' })
+  const element1 = fields.filter({ hasText: 'Field label' });
   const element2 = fields.filter({ hasText: 'Radio group label' });
   expect(element1.first()).toBeTruthy();
   expect(element2.first()).toBeTruthy();
@@ -50,4 +52,25 @@ test('Add questions', async ({ page }) => {
   const element2Index = htmlContent.indexOf((await element2.textContent() as string));
   expect(element1Index).toBeLessThan(element2Index);
 
+});
+
+test('Drag-and-drop via mouse interaction', async ({ page }) => {
+  await createNewForm(page);
+  await addQuestions(page);
+
+  const handle = page.locator('[aria-describedby="DndDescribedBy-0"]').first();
+  await handle.hover();
+  await page.mouse.down();
+  const nextElement = page.locator('.draggable-list-item-wrapper').nth(1);
+  await nextElement.hover();
+  await page.mouse.up();
+
+  // Initiating a reorder clones the element. We want to await the count to go back to 1 to
+  // signify the invocation of the drag event has completed and the update has rendered.
+  await expect(page.getByText('Field label')).toHaveCount(1, {
+    timeout: 10000
+  });
+
+  await expect(page.locator('.draggable-list-item-wrapper').nth(1)).toContainText('Field label');
+
 });

e2e/src/my-forms.spec.ts

@@ -5,4 +5,4 @@ import { BASE_URL } from './constants';
 test('Go to MyForms', async ({ page }) => {
   const response = await page.goto(`${BASE_URL}/${MyForms.getUrl()}`);
   expect(response?.ok()).toBe(true);
-});
+});