test(quay): add first playwright tests (#1201)

* test(quay): add first playwright tests * Fix typo Co-authored-by: Karthik Jeeyar <[email protected]> --------- Co-authored-by: Karthik Jeeyar <[email protected]>
janus-idp · Feb 16, 2024 · a7e936d · a7e936d
1 parent 352c609
commit a7e936d
Show file tree

Hide file tree

Showing 8 changed files with 316 additions and 2 deletions.
diff --git a/.github/workflows/pr-playwright.yaml b/.github/workflows/pr-playwright.yaml
@@ -0,0 +1,134 @@
+name: Playwright tests
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  changes:
+    name: Scan for changes
+    runs-on: ubuntu-latest
+    outputs:
+      plugins: ${{ steps.scan.outputs.plugins }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Determine changes
+        id: scan
+        env:
+          HEAD: ${{ github.event.pull_request.head.sha }}
+          BASE: ${{ github.event.pull_request.base.sha }}
+        run: |
+          root=$(pwd)
+          cd plugins
+
+          changed=()
+          for f in */; do
+            if git diff --name-only $BASE $HEAD | grep $f -q; then
+              if [[ ! -L "$f" && -f "$f/package.json" ]]; then
+                cd $f
+
+                if npm run | grep ui-test -q; then
+                  changed+=($f)
+                fi
+                cd $root/plugins
+              fi
+            fi
+          done
+
+          JSON="[$(echo ${changed[@]} | sed 's/ /,/g')]"
+          echo "plugins=$(echo $JSON)" >> $GITHUB_OUTPUT
+  playwright:
+    name: 'Run Playwright Tests'
+    needs: changes
+    if: needs.changes.outputs.plugins != '[]'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Install dependencies
+        run: yarn
+      - name: Install playwright
+        run: yarn playwright install --with-deps chromium
+      - name: Run tests
+        env:
+          PLUGINS: ${{ needs.changes.outputs.plugins }}
+        run: |
+          root=$(pwd)
+          cd packages/backend
+          readarray folders < <(echo $PLUGINS | sed 's/[][]//g' | sed 's/,/ /g')
+
+          # Start backend
+          echo "Starting backend"
+          logfile=$(mktemp)
+          yarn start >$logfile 2>&1 &
+
+          for attempt in $(seq 1 45); do
+            sleep 1
+            if grep -q "Error:" $logfile; then
+              cat $logfile
+              exit 1
+            fi
+            if grep -q "Listening on" $logfile; then
+              echo "Backend started"
+              break
+            fi
+            if [[ attempt -eq 45 ]]; then
+              echo "Failed to launch backend"
+              cat $logfile
+              exit 1
+            fi
+          done
+
+          cd $root/plugins
+
+          # Launch suitable plugins with changes
+          for f in $folders; do
+            cd $f
+
+            echo "Starting $f plugin"
+            tmpfile=$(mktemp)
+
+            # Start the plugin
+            yarn start >$tmpfile 2>&1 &
+            for attempt in $(seq 1 45); do
+              sleep 1
+              if grep -q "Error:" $tmpfile; then
+              cat $tmpfile
+              exit 1
+              fi
+              if grep -q "webpack compiled" $tmpfile; then
+                echo "$f started"
+                break
+              fi
+              if [[ attempt -eq 45 ]]; then
+                echo "Failed to launch $f"
+                cat $tmpfile
+                exit 1
+              fi
+            done
+
+            # Run UI tests
+            yarn run ui-test
+            
+            # Kill the plugin
+            pid=$(lsof -i :3000 -Fp | grep p | sed s/p//)
+            kill -9 $pid && echo "$f shut down"
+            cd $root/plugins
+          done
+
+          # Kill backend
+          pid=$(lsof -i :7007 -Fp | grep p | sed s/p//)
+          kill -9 $pid && echo "Backend shut down"
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: plugins/*/playwright-report/
+          retention-days: 1
diff --git a/.gitignore b/.gitignore
@@ -65,3 +65,9 @@ site
 .webpack-cache
 
 .tmp
+
+# playwright
+test-results/
+playwright-report/
+blob-report/
+playwright/.cache/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -312,6 +312,45 @@ Before pushing your code changes make sure all **tests pass** and the **coverage
 $ yarn test
 ```
 
+### UI Tests
+
+Some plugins (e.g. [quay](https://github.com/janus-idp/backstage-plugins/tree/main/plugins/quay)) also have playwright-based UI tests. When making changes to such plugin, make sure these tests pass.
+
+To run the UI tests locally, take the following steps:
+
+First, install playwright dependencies:
+
+```bash
+$ yarn install --with-deps chromium
+```
+
+The remaining steps need to be run in parallel.
+Launch the backend package and wait for it to start:
+
+```bash
+$ cd packages/backend && yarn start
+```
+
+Launch the plugin:
+
+```bash
+$ cd plugins/${plugin} && yarn start
+```
+
+Finally, launch the UI tests (headless):
+
+```bash
+$ cd plugins/${plugin} && yarn run ui-test
+```
+
+If you wish to see the test runner UI, instead of headless:
+
+```bash
+$ cd plugins/${plugin} && yarn playwright test --ui
+```
+
+Test results from the headless run will be available in `plugins/${plugin}/playwright-report` folder.
+
 ## Releasing changes
 
 This repository defaults to a rapid release scheme where we would rather release on every PR merge than restrict ourselves by a strict release cadence and policy. This brings contributors the opportunity to see the direct impact of their contributions since they are released immediately after the merge. The release process itself is done via the [semantic-release](https://semantic-release.gitbook.io/semantic-release/) tool. In order for it to work properly, it requires contributors to follow a simple set of rules:

diff --git a/plugins/quay/dev/index.tsx b/plugins/quay/dev/index.tsx
@@ -13,7 +13,7 @@ const mockEntity: Entity = {
     name: 'backstage',
     description: 'backstage.io',
     annotations: {
-      'quay.io/repository-slug': 'janus-idp/redhat-backstage-build',
+      'quay.io/repository-slug': 'backstage-test/test-images',
     },
   },
   spec: {

diff --git a/plugins/quay/package.json b/plugins/quay/package.json
@@ -23,7 +23,8 @@
     "prepare": "",
     "start": "backstage-cli package start",
     "test": "backstage-cli package test --passWithNoTests --coverage",
-    "tsc": "tsc"
+    "tsc": "tsc",
+    "ui-test": "yarn playwright test"
   },
   "dependencies": {
     "@backstage/catalog-model": "^1.4.3",
@@ -47,6 +48,7 @@
     "@backstage/dev-utils": "1.0.22",
     "@backstage/test-utils": "1.4.4",
     "@janus-idp/cli": "1.7.1",
+    "@playwright/test": "^1.41.0",
     "@testing-library/jest-dom": "5.17.0",
     "@testing-library/react": "12.1.5",
     "@testing-library/react-hooks": "8.0.1",

diff --git a/plugins/quay/playwright.config.ts b/plugins/quay/playwright.config.ts
@@ -0,0 +1,33 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Run tests in sequence. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});
diff --git a/plugins/quay/tests/quay.spec.ts b/plugins/quay/tests/quay.spec.ts
@@ -0,0 +1,74 @@
+import { expect, Page, test } from '@playwright/test';
+
+test.describe('Quay plugin', () => {
+  let page: Page;
+
+  test.beforeAll(async ({ browser }) => {
+    const context = await browser.newContext();
+    page = await context.newPage();
+    await page.goto('http://localhost:3000/quay');
+    await expect(
+      page.getByRole('link', { name: 'backstage-test/test-images' }),
+    ).toBeEnabled({ timeout: 20000 });
+  });
+
+  test.afterAll(async ({ browser }) => {
+    await browser.close();
+  });
+
+  test('All columns are shown', async () => {
+    const columns = [
+      'Tag',
+      'Last Modified',
+      'Security Scan',
+      'Size',
+      'Expires',
+      'Manifest',
+    ];
+    const thead = page.locator('thead');
+
+    for (const col of columns) {
+      await expect(thead.getByText(col)).toBeVisible();
+    }
+  });
+
+  test('Vulnerabilities are listed', async () => {
+    const severity = ['High:', 'Medium:', 'Low:'];
+    for (const lvl of severity) {
+      await expect(page.getByRole('link', { name: lvl })).toBeVisible();
+    }
+  });
+
+  test('Vulnerability details are accessible', async () => {
+    await page.getByRole('link', { name: 'High' }).first().click();
+    await expect(page.getByText('Vulnerabilities for')).toBeVisible({
+      timeout: 15000,
+    });
+  });
+
+  test('Vulnerability columns are shown', async () => {
+    const columns = [
+      'Advisory',
+      'Severity',
+      'Package Name',
+      'Current Version',
+      'Fixed By',
+    ];
+
+    for (const col of columns) {
+      await expect(page.getByText(col)).toBeVisible();
+    }
+  });
+
+  test('Vulnerability rows are shown', async () => {
+    const tbody = page.locator('tbody');
+    await expect(tbody.locator('tr')).toHaveCount(5);
+  });
+
+  test('Link back to repository works', async () => {
+    await page.getByRole('link', { name: 'Back to repository' }).click();
+    await expect(
+      page.getByRole('link', { name: 'backstage-test/test-images' }),
+    ).toBeEnabled();
+  });
+});
diff --git a/yarn.lock b/yarn.lock
@@ -9543,6 +9543,13 @@
   resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
   integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
 
+"@playwright/test@^1.41.0":
+  version "1.41.2"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.2.tgz#bd9db40177f8fd442e16e14e0389d23751cdfc54"
+  integrity sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg==
+  dependencies:
+    playwright "1.41.2"
+
 "@pmmmwh/react-refresh-webpack-plugin@^0.5.5", "@pmmmwh/react-refresh-webpack-plugin@^0.5.7":
   version "0.5.11"
   resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz#7c2268cedaa0644d677e8c4f377bc8fb304f714a"
@@ -21535,6 +21542,11 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
+[email protected]:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
 fsevents@^2.3.2, fsevents@~2.3.2:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -29400,6 +29412,20 @@ [email protected], pkginfo@^0.4.1:
   resolved "https://registry.yarnpkg.com/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff"
   integrity sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ==
 
+[email protected]:
+  version "1.41.2"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.2.tgz#db22372c708926c697acc261f0ef8406606802d9"
+  integrity sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA==
+
+[email protected]:
+  version "1.41.2"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.2.tgz#4e760b1c79f33d9129a8c65cc27953be6dd35042"
+  integrity sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A==
+  dependencies:
+    playwright-core "1.41.2"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 pluralize@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"