From a7e936d88e0c21f2948cb4f46da491ec878f4c9b Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Fri, 16 Feb 2024 13:41:15 +0100 Subject: [PATCH] test(quay): add first playwright tests (#1201) * test(quay): add first playwright tests * Fix typo Co-authored-by: Karthik Jeeyar --------- Co-authored-by: Karthik Jeeyar --- .github/workflows/pr-playwright.yaml | 134 +++++++++++++++++++++++++++ .gitignore | 6 ++ CONTRIBUTING.md | 39 ++++++++ plugins/quay/dev/index.tsx | 2 +- plugins/quay/package.json | 4 +- plugins/quay/playwright.config.ts | 33 +++++++ plugins/quay/tests/quay.spec.ts | 74 +++++++++++++++ yarn.lock | 26 ++++++ 8 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pr-playwright.yaml create mode 100644 plugins/quay/playwright.config.ts create mode 100644 plugins/quay/tests/quay.spec.ts diff --git a/.github/workflows/pr-playwright.yaml b/.github/workflows/pr-playwright.yaml new file mode 100644 index 0000000000..990a6e5d46 --- /dev/null +++ b/.github/workflows/pr-playwright.yaml @@ -0,0 +1,134 @@ +name: Playwright tests + +on: + pull_request: + branches: [main] + +jobs: + changes: + name: Scan for changes + runs-on: ubuntu-latest + outputs: + plugins: ${{ steps.scan.outputs.plugins }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Determine changes + id: scan + env: + HEAD: ${{ github.event.pull_request.head.sha }} + BASE: ${{ github.event.pull_request.base.sha }} + run: | + root=$(pwd) + cd plugins + + changed=() + for f in */; do + if git diff --name-only $BASE $HEAD | grep $f -q; then + if [[ ! -L "$f" && -f "$f/package.json" ]]; then + cd $f + + if npm run | grep ui-test -q; then + changed+=($f) + fi + cd $root/plugins + fi + fi + done + + JSON="[$(echo ${changed[@]} | sed 's/ /,/g')]" + echo "plugins=$(echo $JSON)" >> $GITHUB_OUTPUT + playwright: + name: 'Run Playwright Tests' + needs: changes + if: needs.changes.outputs.plugins != '[]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - name: Install dependencies + run: yarn + - name: Install playwright + run: yarn playwright install --with-deps chromium + - name: Run tests + env: + PLUGINS: ${{ needs.changes.outputs.plugins }} + run: | + root=$(pwd) + cd packages/backend + readarray folders < <(echo $PLUGINS | sed 's/[][]//g' | sed 's/,/ /g') + + # Start backend + echo "Starting backend" + logfile=$(mktemp) + yarn start >$logfile 2>&1 & + + for attempt in $(seq 1 45); do + sleep 1 + if grep -q "Error:" $logfile; then + cat $logfile + exit 1 + fi + if grep -q "Listening on" $logfile; then + echo "Backend started" + break + fi + if [[ attempt -eq 45 ]]; then + echo "Failed to launch backend" + cat $logfile + exit 1 + fi + done + + cd $root/plugins + + # Launch suitable plugins with changes + for f in $folders; do + cd $f + + echo "Starting $f plugin" + tmpfile=$(mktemp) + + # Start the plugin + yarn start >$tmpfile 2>&1 & + for attempt in $(seq 1 45); do + sleep 1 + if grep -q "Error:" $tmpfile; then + cat $tmpfile + exit 1 + fi + if grep -q "webpack compiled" $tmpfile; then + echo "$f started" + break + fi + if [[ attempt -eq 45 ]]; then + echo "Failed to launch $f" + cat $tmpfile + exit 1 + fi + done + + # Run UI tests + yarn run ui-test + + # Kill the plugin + pid=$(lsof -i :3000 -Fp | grep p | sed s/p//) + kill -9 $pid && echo "$f shut down" + cd $root/plugins + done + + # Kill backend + pid=$(lsof -i :7007 -Fp | grep p | sed s/p//) + kill -9 $pid && echo "Backend shut down" + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: plugins/*/playwright-report/ + retention-days: 1 diff --git a/.gitignore b/.gitignore index d11efe63ab..df28a93880 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,9 @@ site .webpack-cache .tmp + +# playwright +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ccdec1069..d130cf4a7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -312,6 +312,45 @@ Before pushing your code changes make sure all **tests pass** and the **coverage $ yarn test ``` +### UI Tests + +Some plugins (e.g. [quay](https://github.com/janus-idp/backstage-plugins/tree/main/plugins/quay)) also have playwright-based UI tests. When making changes to such plugin, make sure these tests pass. + +To run the UI tests locally, take the following steps: + +First, install playwright dependencies: + +```bash +$ yarn install --with-deps chromium +``` + +The remaining steps need to be run in parallel. +Launch the backend package and wait for it to start: + +```bash +$ cd packages/backend && yarn start +``` + +Launch the plugin: + +```bash +$ cd plugins/${plugin} && yarn start +``` + +Finally, launch the UI tests (headless): + +```bash +$ cd plugins/${plugin} && yarn run ui-test +``` + +If you wish to see the test runner UI, instead of headless: + +```bash +$ cd plugins/${plugin} && yarn playwright test --ui +``` + +Test results from the headless run will be available in `plugins/${plugin}/playwright-report` folder. + ## Releasing changes This repository defaults to a rapid release scheme where we would rather release on every PR merge than restrict ourselves by a strict release cadence and policy. This brings contributors the opportunity to see the direct impact of their contributions since they are released immediately after the merge. The release process itself is done via the [semantic-release](https://semantic-release.gitbook.io/semantic-release/) tool. In order for it to work properly, it requires contributors to follow a simple set of rules: diff --git a/plugins/quay/dev/index.tsx b/plugins/quay/dev/index.tsx index 896db7ecb7..60a3c3d038 100644 --- a/plugins/quay/dev/index.tsx +++ b/plugins/quay/dev/index.tsx @@ -13,7 +13,7 @@ const mockEntity: Entity = { name: 'backstage', description: 'backstage.io', annotations: { - 'quay.io/repository-slug': 'janus-idp/redhat-backstage-build', + 'quay.io/repository-slug': 'backstage-test/test-images', }, }, spec: { diff --git a/plugins/quay/package.json b/plugins/quay/package.json index 92877c0604..58a9691262 100644 --- a/plugins/quay/package.json +++ b/plugins/quay/package.json @@ -23,7 +23,8 @@ "prepare": "", "start": "backstage-cli package start", "test": "backstage-cli package test --passWithNoTests --coverage", - "tsc": "tsc" + "tsc": "tsc", + "ui-test": "yarn playwright test" }, "dependencies": { "@backstage/catalog-model": "^1.4.3", @@ -47,6 +48,7 @@ "@backstage/dev-utils": "1.0.22", "@backstage/test-utils": "1.4.4", "@janus-idp/cli": "1.7.1", + "@playwright/test": "^1.41.0", "@testing-library/jest-dom": "5.17.0", "@testing-library/react": "12.1.5", "@testing-library/react-hooks": "8.0.1", diff --git a/plugins/quay/playwright.config.ts b/plugins/quay/playwright.config.ts new file mode 100644 index 0000000000..fb9e51b5af --- /dev/null +++ b/plugins/quay/playwright.config.ts @@ -0,0 +1,33 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Run tests in sequence. */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/plugins/quay/tests/quay.spec.ts b/plugins/quay/tests/quay.spec.ts new file mode 100644 index 0000000000..fec2f01d72 --- /dev/null +++ b/plugins/quay/tests/quay.spec.ts @@ -0,0 +1,74 @@ +import { expect, Page, test } from '@playwright/test'; + +test.describe('Quay plugin', () => { + let page: Page; + + test.beforeAll(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('http://localhost:3000/quay'); + await expect( + page.getByRole('link', { name: 'backstage-test/test-images' }), + ).toBeEnabled({ timeout: 20000 }); + }); + + test.afterAll(async ({ browser }) => { + await browser.close(); + }); + + test('All columns are shown', async () => { + const columns = [ + 'Tag', + 'Last Modified', + 'Security Scan', + 'Size', + 'Expires', + 'Manifest', + ]; + const thead = page.locator('thead'); + + for (const col of columns) { + await expect(thead.getByText(col)).toBeVisible(); + } + }); + + test('Vulnerabilities are listed', async () => { + const severity = ['High:', 'Medium:', 'Low:']; + for (const lvl of severity) { + await expect(page.getByRole('link', { name: lvl })).toBeVisible(); + } + }); + + test('Vulnerability details are accessible', async () => { + await page.getByRole('link', { name: 'High' }).first().click(); + await expect(page.getByText('Vulnerabilities for')).toBeVisible({ + timeout: 15000, + }); + }); + + test('Vulnerability columns are shown', async () => { + const columns = [ + 'Advisory', + 'Severity', + 'Package Name', + 'Current Version', + 'Fixed By', + ]; + + for (const col of columns) { + await expect(page.getByText(col)).toBeVisible(); + } + }); + + test('Vulnerability rows are shown', async () => { + const tbody = page.locator('tbody'); + await expect(tbody.locator('tr')).toHaveCount(5); + }); + + test('Link back to repository works', async () => { + await page.getByRole('link', { name: 'Back to repository' }).click(); + await expect( + page.getByRole('link', { name: 'backstage-test/test-images' }), + ).toBeEnabled(); + }); +}); diff --git a/yarn.lock b/yarn.lock index 9b5ddbd591..5c25d8db6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9543,6 +9543,13 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@playwright/test@^1.41.0": + version "1.41.2" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.2.tgz#bd9db40177f8fd442e16e14e0389d23751cdfc54" + integrity sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg== + dependencies: + playwright "1.41.2" + "@pmmmwh/react-refresh-webpack-plugin@^0.5.5", "@pmmmwh/react-refresh-webpack-plugin@^0.5.7": version "0.5.11" resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.11.tgz#7c2268cedaa0644d677e8c4f377bc8fb304f714a" @@ -21535,6 +21542,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + fsevents@^2.3.2, fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -29400,6 +29412,20 @@ pkginfo@0.4.x, pkginfo@^0.4.1: resolved "https://registry.yarnpkg.com/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff" integrity sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ== +playwright-core@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.2.tgz#db22372c708926c697acc261f0ef8406606802d9" + integrity sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA== + +playwright@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.2.tgz#4e760b1c79f33d9129a8c65cc27953be6dd35042" + integrity sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A== + dependencies: + playwright-core "1.41.2" + optionalDependencies: + fsevents "2.3.2" + pluralize@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"