diff --git a/.github/workflows/pr-playwright.yaml b/.github/workflows/pr-playwright.yaml index 990a6e5d46..28cf041c51 100644 --- a/.github/workflows/pr-playwright.yaml +++ b/.github/workflows/pr-playwright.yaml @@ -20,7 +20,7 @@ jobs: - name: Determine changes id: scan env: - HEAD: ${{ github.event.pull_request.head.sha }} + HEAD: ${{ github.sha }} BASE: ${{ github.event.pull_request.base.sha }} run: | root=$(pwd) diff --git a/plugins/quay/package.json b/plugins/quay/package.json index a886831bc9..274360b01a 100644 --- a/plugins/quay/package.json +++ b/plugins/quay/package.json @@ -48,7 +48,7 @@ "@backstage/dev-utils": "1.0.27", "@backstage/test-utils": "1.5.0", "@janus-idp/cli": "1.7.1", - "@playwright/test": "1.41.0", + "@playwright/test": "1.41.2", "@testing-library/jest-dom": "6.4.2", "@testing-library/react": "14.2.1", "@testing-library/react-hooks": "8.0.1", diff --git a/plugins/quay/playwright.config.ts b/plugins/quay/playwright.config.ts index fb9e51b5af..fed138d0ce 100644 --- a/plugins/quay/playwright.config.ts +++ b/plugins/quay/playwright.config.ts @@ -17,6 +17,7 @@ export default defineConfig({ reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { + baseURL: process.env.PLUGIN_BASE_URL || 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', screenshot: 'only-on-failure', diff --git a/plugins/quay/tests/quay.spec.ts b/plugins/quay/tests/quay.spec.ts index fec2f01d72..a2be3be5a0 100644 --- a/plugins/quay/tests/quay.spec.ts +++ b/plugins/quay/tests/quay.spec.ts @@ -6,7 +6,7 @@ test.describe('Quay plugin', () => { test.beforeAll(async ({ browser }) => { const context = await browser.newContext(); page = await context.newPage(); - await page.goto('http://localhost:3000/quay'); + await page.goto('/'); await expect( page.getByRole('link', { name: 'backstage-test/test-images' }), ).toBeEnabled({ timeout: 20000 }); diff --git a/plugins/tekton/package.json b/plugins/tekton/package.json index 7e70c5c020..459c636df8 100644 --- a/plugins/tekton/package.json +++ b/plugins/tekton/package.json @@ -22,7 +22,8 @@ "prepack": "backstage-cli package prepack", "start": "backstage-cli package start", "test": "backstage-cli package test --passWithNoTests --coverage", - "tsc": "tsc" + "tsc": "tsc", + "ui-test": "yarn playwright test" }, "dependencies": { "@aonic-ui/pipelines": "^1.1.0", @@ -62,6 +63,7 @@ "@janus-idp/cli": "1.7.1", "@testing-library/jest-dom": "6.4.2", "@testing-library/react": "14.2.1", + "@playwright/test": "1.41.2", "@testing-library/react-hooks": "8.0.1", "@testing-library/user-event": "14.5.1", "@types/node": "18.18.5", diff --git a/plugins/tekton/playwright.config.ts b/plugins/tekton/playwright.config.ts new file mode 100644 index 0000000000..fed138d0ce --- /dev/null +++ b/plugins/tekton/playwright.config.ts @@ -0,0 +1,34 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Run tests in sequence. */ + workers: 1, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + baseURL: process.env.PLUGIN_BASE_URL || 'http://localhost:3000', + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/plugins/tekton/tests/tekton.spec.ts b/plugins/tekton/tests/tekton.spec.ts new file mode 100644 index 0000000000..024851ba13 --- /dev/null +++ b/plugins/tekton/tests/tekton.spec.ts @@ -0,0 +1,319 @@ +import { expect, Locator, Page, test } from '@playwright/test'; + +test.describe('Tekton plugin', () => { + let page: Page; + + test.beforeAll(async ({ browser }) => { + const context = await browser.newContext(); + page = await context.newPage(); + await page.goto('/'); + await expect( + page.getByRole('heading', { name: 'Pipeline Runs' }), + ).toBeVisible({ timeout: 20000 }); + }); + + test.afterAll(async ({ browser }) => { + await browser.close(); + }); + + test('Control elements are shown', async () => { + const clusterSelect = page.locator('.bs-tkn-cluster-selector'); + await expect( + clusterSelect.getByText('Cluster', { exact: true }), + ).toBeVisible(); + await expect(clusterSelect.getByText('mock-cluster')).toBeVisible(); + + const statusSelect = page.locator('.bs-tkn-status-selector'); + await expect(statusSelect.getByText('Status')).toBeVisible(); + await expect(statusSelect.getByText('All')).toBeVisible(); + + const columns = [ + 'NAME', + 'VULNERABILITIES', + 'STATUS', + 'TASK STATUS', + 'STARTED', + 'DURATION', + 'ACTIONS', + ]; + const thead = page.locator('thead'); + for (const col of columns) { + await expect( + thead.getByRole('columnheader', { name: col, exact: true }), + ).toBeVisible(); + } + }); + + test('Pipelines are shown', async () => { + const plrLabel = page.locator('.bs-tkn-pipeline-visualization__label'); + expect(await plrLabel.all()).toHaveLength(5); + for (const plr of await plrLabel.all()) { + expect(plr).toBeVisible(); + } + }); + + test('Pipeline without scan or sbom only shows logs', async () => { + const row = page.getByRole('row', { name: 'pipeline-test-wbvtlk' }); + await expect(row.getByRole('cell').nth(2)).toHaveText('-'); + + const actions = row.getByRole('cell').last(); + await expect(actions.getByRole('button').first()).toBeEnabled(); + await expect(row.getByTestId('view-sbom-icon')).toBeDisabled(); + await expect(row.getByTestId('view-output-icon')).toBeDisabled(); + }); + + test.describe('Pipeline with scanner', () => { + const output = { + vulnerabilities: { + critical: 13, + high: 29, + medium: 32, + low: 3, + unknown: 0, + }, + unpatched_vulnerabilities: { + critical: 0, + high: 1, + medium: 0, + low: 1, + }, + }; + const policyColumns = [ + 'Name', + 'Severity', + 'Breaks build', + 'Description', + 'Violation', + 'Remediation', + ]; + let row: Locator; + + test.beforeAll(() => { + row = page.getByRole('row', { name: 'pipelinerun-with-scanner-task' }); + }); + + test.afterAll(async () => { + await page.getByLabel('close').click(); + }); + + test('Vulnerabilities are shown in the run', async () => { + await checkVulnerabilities(row, output.vulnerabilities); + }); + + test('Output action is available', async () => { + const btn = row.getByTestId('view-output-icon'); + await expect(btn).toBeEnabled(); + await btn.click(); + await expect(page.getByTestId('pipelinerun-output-dialog')).toBeVisible(); + }); + + test('Enterprise contract output is shown', async () => { + const card = page.getByTestId('enterprise-contract'); + const title = page.locator('[id="{enterprise contract-title}"]'); + // check the title and the badge + await expect(title.getByTestId('card-title')).toBeVisible(); + await expect(title.getByTestId('card-title')).toHaveText( + 'Enterprise Contract', + ); + await expect(title.getByTestId('card-badge')).toHaveText('Failed'); + + // check the description + await expect(card).toContainText('Enterprise Contract is a set of tools'); + + // check the summary + const summary = card.locator('.pf-v5-c-card'); + await expect(summary.getByText('Summary')).toBeVisible(); + await expect(summary.getByText('Failed')).toBeVisible(); + await expect(summary.getByText('Success')).toBeVisible(); + await expect(summary.getByText('Warning')).toBeVisible(); + + // check the rules + const rules = page.getByTestId('ec-policy-table'); + const statuses = rules.getByTestId('rule-status'); + await expect(statuses).toHaveCount(4); + await expect(statuses.filter({ hasText: 'Failed' })).toHaveCount(2); + await expect(statuses.filter({ hasText: 'Warning' })).toHaveCount(1); + await expect(statuses.filter({ hasText: 'Success' })).toHaveCount(1); + }); + + test('ACS Image Scan is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card + .locator(`[id='advanced cluster security-toggle-button']`) + .click(); + await card.scrollIntoViewIfNeeded(); + + // check the title and the badge + await expect(card.getByTestId('card-title')).toBeVisible(); + await expect(card.getByTestId('card-title')).toHaveText( + 'Advanced Cluster Security', + ); + await expect(card.getByTestId('card-badge')).toHaveText('Issues found'); + + const sections = [ + 'CVEs by severity', + 'CVEs by status', + 'Total scan results', + ]; + const columns = [ + 'CVE ID', + 'Severity', + 'Component', + 'Component version', + 'Fixed in version', + ]; + + await checkCards(card, sections, 'image-scan-table', columns); + }); + + test('ACS Image Check is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card.getByRole('tab', { name: 'Image Check' }).click(); + const cards = ['CVEs by severity', 'Failing policy checks']; + + await checkCards(card, cards, 'image-check-table', policyColumns); + }); + + test('ACS Deployment Check is shown', async () => { + const card = page.locator(`[id='advanced cluster security']`); + await card.getByRole('tab', { name: 'Deployment Check' }).click(); + const cards = ['Violations by severity', 'Failing policy checks']; + + await checkCards(card, cards, 'deployment-check-table', policyColumns); + }); + + test('Check other output', async () => { + const card = page.locator('[id="others"]'); + await card.locator(`[id='others-toggle-button']`).click(); + await card.scrollIntoViewIfNeeded(); + + await expect(card.getByRole('gridcell').first()).toContainText( + 'SCAN_OUTPUT', + ); + + const text = (await card + .getByRole('gridcell') + .last() + .textContent()) as string; + expect(JSON.parse(text)).toEqual(output); + }); + }); + + test('Pipeline with sbom has the show sbom action', async () => { + const row = page.getByRole('row', { name: 'pipelinerun-with-sbom-task' }); + await expect(row.getByRole('cell').nth(2)).toHaveText('-'); + + const showSbom = row.getByTestId('view-sbom-icon'); + await expect(showSbom).toBeEnabled(); + await expect(row.getByTestId('view-output-icon')).toBeDisabled(); + await showSbom.click(); + + const dialog = page.getByTitle('PipelineRun Logs'); + await expect(dialog.getByText('sbom-task')).toBeVisible(); + + await page.getByLabel('close').click(); + }); + + test.describe('Pipeline with external sbom', () => { + let row: Locator; + const output = { + vulnerabilities: { + critical: 1, + high: 9, + medium: 20, + low: 1, + unknown: 0, + }, + unpatched_vulnerabilities: { + critical: 0, + high: 1, + medium: 0, + low: 1, + }, + }; + + test.beforeAll(() => { + row = page.getByRole('row', { + name: 'pipelinerun-with-external-sbom-task', + }); + }); + + test('Vulnerability scan is shown', async () => { + await checkVulnerabilities(row, output.vulnerabilities); + }); + + test('Show sbom action points to quay.io', async () => { + const showSbom = row.getByTestId('view-sbom-icon'); + await expect(showSbom).toBeEnabled(); + expect(await showSbom.locator('a').getAttribute('href')).toContain( + 'https://quay.io', + ); + }); + + test('View output action is enabled', async () => { + const viewOutput = row.getByTestId('view-output-icon'); + await expect(viewOutput).toBeEnabled(); + + await viewOutput.click(); + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + + await expect(dialog.locator('tbody')).toContainText('MY_SCAN_OUTPUT'); + const text = (await dialog.locator('td').last().textContent()) as string; + expect(JSON.parse(text)).toEqual(output); + await page.getByLabel('close').click(); + }); + }); + + test('Signed pipeline shows the signed indicator', async () => { + const row = page.getByRole('row', { name: 'ruby-ex-git-xf45fo' }); + await expect(row.locator('.makeStyles-signedIndicator-120')).toBeVisible(); + }); +}); + +async function checkCards( + base: Locator, + sectionTitles: string[], + tableName: string, + columns: string[], +) { + // check the violations summary + const sections = base.locator('.pf-v5-c-card:visible'); + for (const item of sectionTitles) { + await expect(sections.filter({ hasText: item })).toBeVisible(); + } + + // check the violations table + const table = base.getByTestId(tableName); + await expect(table).toBeVisible(); + + for (const col of columns) { + await expect( + table.locator('thead').getByText(col, { exact: true }), + ).toBeVisible(); + } + expect(await table.getByRole('row').count()).toBeGreaterThan(1); +} + +async function checkVulnerabilities( + row: Locator, + vulnerabilities: { + critical: number; + high: number; + medium: number; + low: number; + }, +) { + const vuln = row.locator('.makeStyles-severityContainer-122'); + let i = 0; + + for (const prop in vulnerabilities) { + if (vulnerabilities[prop] > 0) { + await expect(vuln.nth(i)).toContainText( + new RegExp(`${prop}\s*${vulnerabilities[prop]}`), + { ignoreCase: true }, + ); + i++; + } + } +} diff --git a/yarn.lock b/yarn.lock index 0945244d5e..2d061ac792 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9268,12 +9268,12 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@playwright/test@1.41.0": - version "1.41.0" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.0.tgz#b083e976295f1fe039c15d451c66554d7f37278c" - integrity sha512-Grvzj841THwtpBOrfiHOeYTJQxDRnKofMSzCiV8XeyLWu3o89qftQ4BCKfkziJhSUQRd0utKhrddtIsiraIwmw== +"@playwright/test@1.41.2": + version "1.41.2" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.41.2.tgz#bd9db40177f8fd442e16e14e0389d23751cdfc54" + integrity sha512-qQB9h7KbibJzrDpkXkYvsmiDJK14FULCCZgEcoe2AvFAS64oCirWTwzTlAYEbKaRxWs5TFesE1Na6izMv3HfGg== dependencies: - playwright "1.41.0" + playwright "1.41.2" "@pmmmwh/react-refresh-webpack-plugin@^0.5.5", "@pmmmwh/react-refresh-webpack-plugin@^0.5.7": version "0.5.11" @@ -29975,17 +29975,17 @@ pkginfo@0.4.x, pkginfo@^0.4.1: resolved "https://registry.yarnpkg.com/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff" integrity sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ== -playwright-core@1.41.0: - version "1.41.0" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.0.tgz#dbda9c3948df28a8deae76cc90b424e47174f9d7" - integrity sha512-UGKASUhXmvqm2Lxa1fNr8sFwAtqjpgBRr9jQ7XBI8Rn5uFiEowGUGwrruUQsVPIom4bk7Lt+oLGpXobnXzrBIw== +playwright-core@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.41.2.tgz#db22372c708926c697acc261f0ef8406606802d9" + integrity sha512-VaTvwCA4Y8kxEe+kfm2+uUUw5Lubf38RxF7FpBxLPmGe5sdNkSg5e3ChEigaGrX7qdqT3pt2m/98LiyvU2x6CA== -playwright@1.41.0: - version "1.41.0" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.0.tgz#77ab5f3a5fde479522167f74a5070e72ce884bff" - integrity sha512-XOsfl5ZtAik/T9oek4V0jAypNlaCNzuKOwVhqhgYT3os6kH34PzbRb74F0VWcLYa5WFdnmxl7qyAHBXvPv7lqQ== +playwright@1.41.2: + version "1.41.2" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.41.2.tgz#4e760b1c79f33d9129a8c65cc27953be6dd35042" + integrity sha512-v0bOa6H2GJChDL8pAeLa/LZC4feoAMbSQm1/jF/ySsWWoaNItvrMP7GEkvEEFyCTUYKMxjQKaTSg5up7nR6/8A== dependencies: - playwright-core "1.41.0" + playwright-core "1.41.2" optionalDependencies: fsevents "2.3.2"