From 020a3c942e00b27b03302fd5aa14ebee9ee618be Mon Sep 17 00:00:00 2001 From: Jan Richter Date: Tue, 20 Feb 2024 11:28:21 +0100 Subject: [PATCH] test(tekton): add playwright tests for the plugin --- .github/workflows/pr-playwright.yaml | 2 +- plugins/tekton/package.json | 4 +- plugins/tekton/playwright.config.ts | 33 +++ plugins/tekton/tests/tekton.spec.ts | 328 +++++++++++++++++++++++++++ 4 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 plugins/tekton/playwright.config.ts create mode 100644 plugins/tekton/tests/tekton.spec.ts diff --git a/.github/workflows/pr-playwright.yaml b/.github/workflows/pr-playwright.yaml index 990a6e5d46b..28cf041c514 100644 --- a/.github/workflows/pr-playwright.yaml +++ b/.github/workflows/pr-playwright.yaml @@ -20,7 +20,7 @@ jobs: - name: Determine changes id: scan env: - HEAD: ${{ github.event.pull_request.head.sha }} + HEAD: ${{ github.sha }} BASE: ${{ github.event.pull_request.base.sha }} run: | root=$(pwd) diff --git a/plugins/tekton/package.json b/plugins/tekton/package.json index 7cb27be4b77..5fd38294898 100644 --- a/plugins/tekton/package.json +++ b/plugins/tekton/package.json @@ -22,7 +22,8 @@ "prepack": "backstage-cli package prepack", "start": "backstage-cli package start", "test": "backstage-cli package test --passWithNoTests --coverage", - "tsc": "tsc" + "tsc": "tsc", + "ui-test": "yarn playwright test" }, "dependencies": { "@aonic-ui/pipelines": "^1.1.0", @@ -60,6 +61,7 @@ "@backstage/dev-utils": "1.0.22", "@backstage/test-utils": "1.4.4", "@janus-idp/cli": "1.7.1", + "@playwright/test": "^1.41.0", "@testing-library/jest-dom": "5.17.0", "@testing-library/react": "12.1.5", "@testing-library/react-hooks": "8.0.1", diff --git a/plugins/tekton/playwright.config.ts b/plugins/tekton/playwright.config.ts new file mode 100644 index 00000000000..fb9e51b5af8 --- /dev/null +++ b/plugins/tekton/playwright.config.ts @@ -0,0 +1,33 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Run tests in sequence. */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/plugins/tekton/tests/tekton.spec.ts b/plugins/tekton/tests/tekton.spec.ts new file mode 100644 index 00000000000..345142ef7d9 --- /dev/null +++ b/plugins/tekton/tests/tekton.spec.ts @@ -0,0 +1,328 @@ +import { expect, Locator, Page, test } from '@playwright/test'; + +test.describe('Tekton plugin', () => { + let page: Page; + + test.beforeAll(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('http://localhost:3000/tekton'); + await expect( + page.getByRole('heading', { name: 'Pipeline Runs' }), + ).toBeVisible({ timeout: 20000 }); + }); + + test.afterAll(async ({ browser }) => { + await browser.close(); + }); + + test('Control elements are shown', async () => { + const clusterSelect = page.locator('.bs-tkn-cluster-selector'); + await expect( + clusterSelect.getByText('Cluster', { exact: true }), + ).toBeVisible(); + await expect(clusterSelect.getByText('mock-cluster')).toBeVisible(); + + const statusSelect = page.locator('.bs-tkn-status-selector'); + await expect(statusSelect.getByText('Status')).toBeVisible(); + await expect(statusSelect.getByText('All')).toBeVisible(); + + const columns = [ + 'NAME', + 'VULNERABILITIES', + 'STATUS', + 'TASK STATUS', + 'STARTED', + 'DURATION', + 'ACTIONS', + ]; + const thead = page.locator('thead'); + for (const col of columns) { + await expect( + thead.getByRole('columnheader', { name: col, exact: true }), + ).toBeVisible(); + } + }); + + test('Pipelines are shown', async () => { + const plrLabel = page.locator('.bs-tkn-pipeline-visualization__label'); + expect(await plrLabel.all()).toHaveLength(5); + for (const plr of await plrLabel.all()) { + expect(plr).toBeVisible(); + } + }); + + test('Pipeline without scan or sbom only shows logs', async () => { + const row = page.getByRole('row', { name: 'pipeline-test-wbvtlk' }); + await expect(row.getByRole('cell').nth(2)).toHaveText('-'); + + const actions = row.getByRole('cell').last(); + await expect(actions.getByRole('button').first()).toBeEnabled(); + await expect(row.getByTestId('view-sbom-icon')).toBeDisabled(); + await expect(row.getByTestId('view-output-icon')).toBeDisabled(); + }); + + test.describe('Pipeline with scanner', () => { + const output = { + vulnerabilities: { + critical: 13, + high: 29, + medium: 32, + low: 3, + unknown: 0, + }, + unpatched_vulnerabilities: { + critical: 0, + high: 1, + medium: 0, + low: 1, + }, + }; + let row: Locator; + + test.beforeAll(() => { + row = page.getByRole('row', { name: 'pipelinerun-with-scanner-task' }); + }); + + test.afterAll(async () => { + await page.getByLabel('close').click(); + }); + + test('Vulnerabilities are shown in the run', async () => { + const vuln = row.locator('.makeStyles-severityContainer-122'); + await expect(vuln.first()).toContainText( + new RegExp(`Critical\s*${output.vulnerabilities.critical}`), + ); + await expect(vuln.nth(1)).toContainText( + new RegExp(`High\s*${output.vulnerabilities.high}`), + ); + await expect(vuln.nth(2)).toContainText( + new RegExp(`Medium\s*${output.vulnerabilities.medium}`), + ); + await expect(vuln.last()).toContainText( + new RegExp(`Low\s*${output.vulnerabilities.low}`), + ); + }); + + test('Output action is available', async () => { + const btn = row.getByTestId('view-output-icon'); + await expect(btn).toBeEnabled(); + await btn.click(); + await expect(page.getByTestId('pipelinerun-output-dialog')).toBeVisible(); + }); + + test('Enterprise contract output is shown', async () => { + const card = page.getByTestId('enterprise-contract'); + const title = page.locator('[id="{enterprise contract-title}"]'); + // check the title and the badge + await expect(title.getByTestId('card-title')).toBeVisible(); + await expect(title.getByTestId('card-title')).toHaveText( + 'Enterprise Contract', + ); + await expect(title.getByTestId('card-badge')).toHaveText('Failed'); + + // check the description + await expect(card).toContainText('Enterprise Contract is a set of tools'); + + // check the summary + const summary = card.locator('.pf-v5-c-card'); + await expect(summary.getByText('Summary')).toBeVisible(); + await expect(summary.getByText('Failed')).toBeVisible(); + await expect(summary.getByText('Success')).toBeVisible(); + await expect(summary.getByText('Warning')).toBeVisible(); + + // check the rules + const rules = page.getByTestId('ec-policy-table'); + const statuses = rules.getByTestId('rule-status'); + await expect(statuses).toHaveCount(4); + await expect(statuses.filter({ hasText: 'Failed' })).toHaveCount(2); + await expect(statuses.filter({ hasText: 'Warning' })).toHaveCount(1); + await expect(statuses.filter({ hasText: 'Success' })).toHaveCount(1); + }); + + test('ACS Image Scan is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card + .locator(`[id='advanced cluster security-toggle-button']`) + .click(); + await card.scrollIntoViewIfNeeded(); + + // check the title and the badge + await expect(card.getByTestId('card-title')).toBeVisible(); + await expect(card.getByTestId('card-title')).toHaveText( + 'Advanced Cluster Security', + ); + await expect(card.getByTestId('card-badge')).toHaveText('Issues found'); + + const sections = [ + 'CVEs by severity', + 'CVEs by status', + 'Total scan results', + ]; + const columns = [ + 'CVE ID', + 'Severity', + 'Component', + 'Component version', + 'Fixed in version', + ]; + + await checkCards(card, sections, 'image-scan-table', columns); + }); + + test('ACS Image Check is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card.getByRole('tab', { name: 'Image Check' }).click(); + const cards = ['CVEs by severity', 'Failing policy checks']; + const columns = [ + 'Name', + 'Severity', + 'Breaks build', + 'Description', + 'Violation', + 'Remediation', + ]; + + await checkCards(card, cards, 'image-check-table', columns); + }); + + test('ACS Deployment Check is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card.getByRole('tab', { name: 'Deployment Check' }).click(); + const cards = ['Violations by severity', 'Failing policy checks']; + const columns = [ + 'Name', + 'Severity', + 'Breaks build', + 'Description', + 'Violation', + 'Remediation', + ]; + + await checkCards(card, cards, 'deployment-check-table', columns); + }); + + test('Check other output', async () => { + const card = page.locator('[id="others"]'); + await card.locator(`[id='others-toggle-button']`).click(); + await card.scrollIntoViewIfNeeded(); + + await expect(card.getByRole('gridcell').first()).toContainText( + 'SCAN_OUTPUT', + ); + + const text = (await card + .getByRole('gridcell') + .last() + .textContent()) as string; + expect(JSON.parse(text)).toEqual(output); + }); + }); + + test('Pipeline with sbom has the show sbom action', async () => { + const row = page.getByRole('row', { name: 'pipelinerun-with-sbom-task' }); + await expect(row.getByRole('cell').nth(2)).toHaveText('-'); + + const showSbom = row.getByTestId('view-sbom-icon'); + await expect(showSbom).toBeEnabled(); + await expect(row.getByTestId('view-output-icon')).toBeDisabled(); + await showSbom.click(); + + const dialog = page.getByTitle('PipelineRun Logs'); + await expect(dialog.getByText('sbom-task')).toBeVisible(); + + await page.getByLabel('close').click(); + }); + + test.describe('Pipeline with external sbom', () => { + let row: Locator; + const output = { + vulnerabilities: { + critical: 1, + high: 9, + medium: 20, + low: 1, + unknown: 0, + }, + unpatched_vulnerabilities: { + critical: 0, + high: 1, + medium: 0, + low: 1, + }, + }; + + test.beforeAll(() => { + row = page.getByRole('row', { + name: 'pipelinerun-with-external-sbom-task', + }); + }); + + test('Vulnerability scan is shown', async () => { + const vuln = row.locator('.makeStyles-severityContainer-122'); + await expect(vuln.first()).toContainText( + new RegExp(`Critical\s*${output.vulnerabilities.critical}`), + ); + await expect(vuln.nth(1)).toContainText( + new RegExp(`High\s*${output.vulnerabilities.high}`), + ); + await expect(vuln.nth(2)).toContainText( + new RegExp(`Medium\s*${output.vulnerabilities.medium}`), + ); + await expect(vuln.last()).toContainText( + new RegExp(`Low\s*${output.vulnerabilities.low}`), + ); + }); + + test('Show sbom action points to quay.io', async () => { + const showSbom = row.getByTestId('view-sbom-icon'); + await expect(showSbom).toBeEnabled(); + expect(await showSbom.locator('a').getAttribute('href')).toContain( + 'https://quay.io', + ); + }); + + test('View output action is enabled', async () => { + const viewOutput = row.getByTestId('view-output-icon'); + await expect(viewOutput).toBeEnabled(); + + await viewOutput.click(); + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + + await expect(dialog.locator('tbody')).toContainText('MY_SCAN_OUTPUT'); + const text = (await dialog.locator('td').last().textContent()) as string; + expect(JSON.parse(text)).toEqual(output); + await page.getByLabel('close').click(); + }); + }); + + test('Signed pipeline shows the signed indicator', async () => { + const row = page.getByRole('row', { name: 'ruby-ex-git-xf45fo' }); + await expect(row.locator('.makeStyles-signedIndicator-120')).toBeVisible(); + }); +}); + +async function checkCards( + base: Locator, + sectionTitles: string[], + tableName: string, + columns: string[], +) { + // check the violations summary + const sections = base.locator('.pf-v5-c-card:visible'); + for (const item of sectionTitles) { + await expect(sections.filter({ hasText: item })).toBeVisible(); + } + + // check the violations table + const table = base.getByTestId(tableName); + await expect(table).toBeVisible(); + + for (const col of columns) { + await expect( + table.locator('thead').getByText(col, { exact: true }), + ).toBeVisible(); + } + expect(await table.getByRole('row').count()).toBeGreaterThan(1); +}