Setting up End-To-End Testing (#7)

* Setup Playwright with common browsers

* Update gitignore to add trace files

* chore: update deps [next 13.5]

* e2e testing on push/pull playwright

- Testing via containers to run Playwright Tests

* adding pnpm to install dependencies

- Changed install dependencies step to use pnpm actions

* fix missing dependencies and change to pnpm

- Attempt to fix missing msedge
- Change all npm commands to execute via pnpm

* fix: nightly cannot be run from root

- attempting solution from https://github.com/mxschmitt/firefox-github-actions-container/pull/3/files
- thread: microsoft/playwright#6500

* feat: On Deploy Playwright Testing & Modify Push/pull for Main Only

- On Deploy (Preview and Production), Run Playwright E2E Tests
- Modify Push/Pull to run Playwright E2E Tests only on main/master

* fix: Improve Job Names in Worflow Runs

- Changed Job Names under each playwright testing environment

* fix: Deployment URL is not being hit when Playwright E2E Tests are run

- added Deployment URL via env variable in URL attribute of playwright config

* improve: Removed Redundant Testing Stage & Switched TO E2E Tests on Preview Deployments

- Due to the nature of Vercel Preview, it creates preview deployments for all
branches associated to the project. As such, the E2E tests run on all branches for which
Vercel releases a deployment for (including preview) which applies to every branch.

- We will get stability coverage across all branches on all pushes across the project.
As such, it is not necessary to build and execute a web server on a container to E2E
test again. As the build and execute process is automatically tested and reported by
Vercel.

* Removed WebServer from Playwright

- Running into an issue where WebServer causes issues on workflow run
after deployment with "reuseExistingServer" & URL / Port Already active

- Decided to change local E2E Testing policy to run `pnpm run dev` on a
separate terminal & run pnpm exec playwright test on a separate terminal

* attempt: determining if pnpm commands can execute playwright tests on deploy

* attempt: Playwright Package Missing Fix

* attempt: fix yaml syntax

* fix: Update test to use baseURL

* improve: Added Type Documentation for PLAYWRIGHT_TEST_BASE_URL

.github/workflows/playwright-deploy.yml

@@ -0,0 +1,33 @@
+name: Playwright Tests (Deploy)
+on:
+  deployment_status:
+jobs:
+  playwright_deploy:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    if: github.event.deployment_status.state == 'success'
+    container:
+      image: mcr.microsoft.com/playwright:v1.38.0-jammy
+    steps:
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'pnpm'
+      - name: Install project dependencies
+        run: pnpm install
+      - name: Install msedge - playwright dependency
+        run: pnpm exec playwright install msedge
+      - name: Run end-to-end tests
+        run: HOME=/root pnpm exec playwright test
+        env:
+          PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: playwright-test-results
+          path: test-results/
+          retention-days: 7

.gitignore

@@ -25,6 +25,7 @@ yarn-debug.log*
 yarn-error.log*
 
 # local env files
+.env
 .env*.local
 
 # vercel
@@ -33,3 +34,6 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# playwright e2e test-results
+/test-results/

.vscode/extensions.json

@@ -3,6 +3,7 @@
         "eamodio.gitlens",
         "bradlc.vscode-tailwindcss",
         "dbaeumer.vscode-eslint",
-        "rvest.vs-code-prettier-eslint"
+        "rvest.vs-code-prettier-eslint",
+        "ms-playwright.playwright"
     ]
 }

e2e/example.spec.ts

@@ -0,0 +1,12 @@
+import { test, expect } from '@playwright/test'
+
+test('should navigate to home page', async ({ page }) => {
+    // Start from the index page (the baseURL is set via playwright.config.ts)
+    await page.goto('/')
+
+    // Find an element with the text 'Shad CN Test' and click on it
+    await page.click('text=ShadCN Test')
+
+    // The current page in view should contain a paragraph text with  "sample change"
+    await expect(page.locator('p')).toContainText('sample change')
+})

environment.d.ts

@@ -0,0 +1,15 @@
+namespace NodeJS {
+  interface ProcessEnv {
+
+    /**
+     *
+     * URL generated by Vercel Preview Deployment
+     * Triggered by any code push on any branch within repo.
+     *
+     * Used in baseURL attribute of `playwright.config.ts` 
+     * to determine endpoint for running E2E Tests.
+     * @type {string}
+     */
+    PLAYWRIGHT_TEST_BASE_URL: string
+  }
+}

package.json

@@ -6,30 +6,32 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
-    "lint": "next lint"
+    "lint": "next lint",
+    "test:e2e": "playwright test"
   },
   "dependencies": {
     "@radix-ui/react-slot": "^1.0.2",
-    "@types/node": "20.5.9",
-    "@types/react": "18.2.21",
-    "@types/react-dom": "18.2.7",
     "autoprefixer": "10.4.15",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.0.0",
     "eslint": "8.48.0",
-    "eslint-config-next": "13.4.19",
+    "eslint-config-next": "^13.5.2",
     "lucide-react": "^0.274.0",
-    "next": "13.4.19",
+    "next": "^13.5.2",
     "postcss": "8.4.29",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "tailwind-merge": "^1.14.0",
-    "tailwindcss": "3.3.3",
+    "tailwindcss": "^3.3.3",
     "tailwindcss-animate": "^1.0.7",
-    "typescript": "5.2.2"
+    "typescript": "^5.2.2"
   },
   "devDependencies": {
+    "@types/node": "^20.6.3",
+    "@types/react": "^18.2.22",
+    "@types/react-dom": "^18.2.7",
+    "@playwright/test": "^1.38.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-tailwindcss": "^3.13.0"
   }
-}
+}

playwright.config.ts

@@ -0,0 +1,54 @@
+import { defineConfig, devices } from '@playwright/test'
+import path from 'path'
+
+export default defineConfig({
+    // Timeout per test
+    timeout: 60 * 1000,
+    // Test Directory
+    testDir: path.join(__dirname, 'e2e'),
+    // If a test fails, retry it additional 2 times
+    retries: 2,
+    // Artifacts folder where screenshots, videos, and traces are stored.
+    outputDir: 'test-results/',
+
+    use: {
+        // Retry a test if its failing with enabled tracing.
+        // Allows for us to analyze DOM, console logs, network traffic, etc.
+        trace: 'retry-with-trace',
+        baseURL: process.env.PLAYWRIGHT_TEST_BASE_URL || 'http://localhost:3000'
+    },
+    projects: [
+        {
+            name: 'chromium',
+            use: { ...devices['Desktop Chrome'] }
+        },
+        {
+            name: 'firefox',
+            use: { ...devices['Desktop Firefox'] }
+        },
+        {
+            name: 'webkit',
+            use: { ...devices['Desktop Safari'] }
+        },
+
+        /* Test against mobile view ports. */
+        {
+            name: 'Mobile Chrome',
+            use: { ...devices['Pixel 5'] }
+        },
+        {
+            name: 'Mobile Safari',
+            use: { ...devices['iPhone 12'] }
+        },
+
+        /* Test against branded browsers. */
+        {
+            name: 'edge',
+            use: {
+                ...devices['Desktop Edge'],
+                channel: 'msedge'
+            },
+        },
+    ],
+    reporter: process.env.CI ? 'github' : 'list'
+})