test(tekton): add playwright tests for the plugin

.github/workflows/pr-playwright.yaml

@@ -20,7 +20,7 @@ jobs:
       - name: Determine changes
         id: scan
         env:
-          HEAD: ${{ github.event.pull_request.head.sha }}
+          HEAD: ${{ github.sha }}
           BASE: ${{ github.event.pull_request.base.sha }}
         run: |
           root=$(pwd)

plugins/tekton/package.json

@@ -22,7 +22,8 @@
     "prepack": "backstage-cli package prepack",
     "start": "backstage-cli package start",
     "test": "backstage-cli package test --passWithNoTests --coverage",
-    "tsc": "tsc"
+    "tsc": "tsc",
+    "ui-test": "yarn playwright test"
   },
   "dependencies": {
     "@aonic-ui/pipelines": "^1.1.0",
@@ -60,6 +61,7 @@
     "@backstage/dev-utils": "1.0.22",
     "@backstage/test-utils": "1.4.4",
     "@janus-idp/cli": "1.7.1",
+    "@playwright/test": "^1.41.0",
     "@testing-library/jest-dom": "5.17.0",
     "@testing-library/react": "12.1.5",
     "@testing-library/react-hooks": "8.0.1",

plugins/tekton/playwright.config.ts

@@ -0,0 +1,33 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Run tests in sequence. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});

plugins/tekton/tests/tekton.spec.ts

@@ -0,0 +1,328 @@
+import { expect, Locator, Page, test } from '@playwright/test';
+
+test.describe('Tekton plugin', () => {
+  let page: Page;
+
+  test.beforeAll(async ({ browser }) => {
+    const context = await browser.newContext();
+    page = await context.newPage();
+    await page.goto('http://localhost:3000/tekton');
+    await expect(
+      page.getByRole('heading', { name: 'Pipeline Runs' }),
+    ).toBeVisible({ timeout: 20000 });
+  });
+
+  test.afterAll(async ({ browser }) => {
+    await browser.close();
+  });
+
+  test('Control elements are shown', async () => {
+    const clusterSelect = page.locator('.bs-tkn-cluster-selector');
+    await expect(
+      clusterSelect.getByText('Cluster', { exact: true }),
+    ).toBeVisible();
+    await expect(clusterSelect.getByText('mock-cluster')).toBeVisible();
+
+    const statusSelect = page.locator('.bs-tkn-status-selector');
+    await expect(statusSelect.getByText('Status')).toBeVisible();
+    await expect(statusSelect.getByText('All')).toBeVisible();
+
+    const columns = [
+      'NAME',
+      'VULNERABILITIES',
+      'STATUS',
+      'TASK STATUS',
+      'STARTED',
+      'DURATION',
+      'ACTIONS',
+    ];
+    const thead = page.locator('thead');
+    for (const col of columns) {
+      await expect(
+        thead.getByRole('columnheader', { name: col, exact: true }),
+      ).toBeVisible();
+    }
+  });
+
+  test('Pipelines are shown', async () => {
+    const plrLabel = page.locator('.bs-tkn-pipeline-visualization__label');
+    expect(await plrLabel.all()).toHaveLength(5);
+    for (const plr of await plrLabel.all()) {
+      expect(plr).toBeVisible();
+    }
+  });
+
+  test('Pipeline without scan or sbom only shows logs', async () => {
+    const row = page.getByRole('row', { name: 'pipeline-test-wbvtlk' });
+    await expect(row.getByRole('cell').nth(2)).toHaveText('-');
+
+    const actions = row.getByRole('cell').last();
+    await expect(actions.getByRole('button').first()).toBeEnabled();
+    await expect(row.getByTestId('view-sbom-icon')).toBeDisabled();
+    await expect(row.getByTestId('view-output-icon')).toBeDisabled();
+  });
+
+  test.describe('Pipeline with scanner', () => {
+    const output = {
+      vulnerabilities: {
+        critical: 13,
+        high: 29,
+        medium: 32,
+        low: 3,
+        unknown: 0,
+      },
+      unpatched_vulnerabilities: {
+        critical: 0,
+        high: 1,
+        medium: 0,
+        low: 1,
+      },
+    };
+    let row: Locator;
+
+    test.beforeAll(() => {
+      row = page.getByRole('row', { name: 'pipelinerun-with-scanner-task' });
+    });
+
+    test.afterAll(async () => {
+      await page.getByLabel('close').click();
+    });
+
+    test('Vulnerabilities are shown in the run', async () => {
+      const vuln = row.locator('.makeStyles-severityContainer-122');
+      await expect(vuln.first()).toContainText(
+        new RegExp(`Critical\s*${output.vulnerabilities.critical}`),
+      );
+      await expect(vuln.nth(1)).toContainText(
+        new RegExp(`High\s*${output.vulnerabilities.high}`),
+      );
+      await expect(vuln.nth(2)).toContainText(
+        new RegExp(`Medium\s*${output.vulnerabilities.medium}`),
+      );
+      await expect(vuln.last()).toContainText(
+        new RegExp(`Low\s*${output.vulnerabilities.low}`),
+      );
+    });
+
+    test('Output action is available', async () => {
+      const btn = row.getByTestId('view-output-icon');
+      await expect(btn).toBeEnabled();
+      await btn.click();
+      await expect(page.getByTestId('pipelinerun-output-dialog')).toBeVisible();
+    });
+
+    test('Enterprise contract output is shown', async () => {
+      const card = page.getByTestId('enterprise-contract');
+      const title = page.locator('[id="{enterprise contract-title}"]');
+      // check the title and the badge
+      await expect(title.getByTestId('card-title')).toBeVisible();
+      await expect(title.getByTestId('card-title')).toHaveText(
+        'Enterprise Contract',
+      );
+      await expect(title.getByTestId('card-badge')).toHaveText('Failed');
+
+      // check the description
+      await expect(card).toContainText('Enterprise Contract is a set of tools');
+
+      // check the summary
+      const summary = card.locator('.pf-v5-c-card');
+      await expect(summary.getByText('Summary')).toBeVisible();
+      await expect(summary.getByText('Failed')).toBeVisible();
+      await expect(summary.getByText('Success')).toBeVisible();
+      await expect(summary.getByText('Warning')).toBeVisible();
+
+      // check the rules
+      const rules = page.getByTestId('ec-policy-table');
+      const statuses = rules.getByTestId('rule-status');
+      await expect(statuses).toHaveCount(4);
+      await expect(statuses.filter({ hasText: 'Failed' })).toHaveCount(2);
+      await expect(statuses.filter({ hasText: 'Warning' })).toHaveCount(1);
+      await expect(statuses.filter({ hasText: 'Success' })).toHaveCount(1);
+    });
+
+    test('ACS Image Scan is shown', async () => {
+      const card = page.locator(`[id='advanced cluster security']`);
+      await card
+        .locator(`[id='advanced cluster security-toggle-button']`)
+        .click();
+      await card.scrollIntoViewIfNeeded();
+
+      // check the title and the badge
+      await expect(card.getByTestId('card-title')).toBeVisible();
+      await expect(card.getByTestId('card-title')).toHaveText(
+        'Advanced Cluster Security',
+      );
+      await expect(card.getByTestId('card-badge')).toHaveText('Issues found');
+
+      const sections = [
+        'CVEs by severity',
+        'CVEs by status',
+        'Total scan results',
+      ];
+      const columns = [
+        'CVE ID',
+        'Severity',
+        'Component',
+        'Component version',
+        'Fixed in version',
+      ];
+
+      await checkCards(card, sections, 'image-scan-table', columns);
+    });
+
+    test('ACS Image Check is shown', async () => {
+      const card = page.locator(`[id='advanced cluster security']`);
+      await card.getByRole('tab', { name: 'Image Check' }).click();
+      const cards = ['CVEs by severity', 'Failing policy checks'];
+      const columns = [
+        'Name',
+        'Severity',
+        'Breaks build',
+        'Description',
+        'Violation',
+        'Remediation',
+      ];
+
+      await checkCards(card, cards, 'image-check-table', columns);
+    });
+
+    test('ACS Deployment Check is shown', async () => {
+      const card = page.locator(`[id='advanced cluster security']`);
+      await card.getByRole('tab', { name: 'Deployment Check' }).click();
+      const cards = ['Violations by severity', 'Failing policy checks'];
+      const columns = [
+        'Name',
+        'Severity',
+        'Breaks build',
+        'Description',
+        'Violation',
+        'Remediation',
+      ];
+
+      await checkCards(card, cards, 'deployment-check-table', columns);
+    });
+
+    test('Check other output', async () => {
+      const card = page.locator('[id="others"]');
+      await card.locator(`[id='others-toggle-button']`).click();
+      await card.scrollIntoViewIfNeeded();
+
+      await expect(card.getByRole('gridcell').first()).toContainText(
+        'SCAN_OUTPUT',
+      );
+
+      const text = (await card
+        .getByRole('gridcell')
+        .last()
+        .textContent()) as string;
+      expect(JSON.parse(text)).toEqual(output);
+    });
+  });
+
+  test('Pipeline with sbom has the show sbom action', async () => {
+    const row = page.getByRole('row', { name: 'pipelinerun-with-sbom-task' });
+    await expect(row.getByRole('cell').nth(2)).toHaveText('-');
+
+    const showSbom = row.getByTestId('view-sbom-icon');
+    await expect(showSbom).toBeEnabled();
+    await expect(row.getByTestId('view-output-icon')).toBeDisabled();
+    await showSbom.click();
+
+    const dialog = page.getByTitle('PipelineRun Logs');
+    await expect(dialog.getByText('sbom-task')).toBeVisible();
+
+    await page.getByLabel('close').click();
+  });
+
+  test.describe('Pipeline with external sbom', () => {
+    let row: Locator;
+    const output = {
+      vulnerabilities: {
+        critical: 1,
+        high: 9,
+        medium: 20,
+        low: 1,
+        unknown: 0,
+      },
+      unpatched_vulnerabilities: {
+        critical: 0,
+        high: 1,
+        medium: 0,
+        low: 1,
+      },
+    };
+
+    test.beforeAll(() => {
+      row = page.getByRole('row', {
+        name: 'pipelinerun-with-external-sbom-task',
+      });
+    });
+
+    test('Vulnerability scan is shown', async () => {
+      const vuln = row.locator('.makeStyles-severityContainer-122');
+      await expect(vuln.first()).toContainText(
+        new RegExp(`Critical\s*${output.vulnerabilities.critical}`),
+      );
+      await expect(vuln.nth(1)).toContainText(
+        new RegExp(`High\s*${output.vulnerabilities.high}`),
+      );
+      await expect(vuln.nth(2)).toContainText(
+        new RegExp(`Medium\s*${output.vulnerabilities.medium}`),
+      );
+      await expect(vuln.last()).toContainText(
+        new RegExp(`Low\s*${output.vulnerabilities.low}`),
+      );
+    });
+
+    test('Show sbom action points to quay.io', async () => {
+      const showSbom = row.getByTestId('view-sbom-icon');
+      await expect(showSbom).toBeEnabled();
+      expect(await showSbom.locator('a').getAttribute('href')).toContain(
+        'https://quay.io',
+      );
+    });
+
+    test('View output action is enabled', async () => {
+      const viewOutput = row.getByTestId('view-output-icon');
+      await expect(viewOutput).toBeEnabled();
+
+      await viewOutput.click();
+      const dialog = page.getByRole('dialog');
+      await expect(dialog).toBeVisible();
+
+      await expect(dialog.locator('tbody')).toContainText('MY_SCAN_OUTPUT');
+      const text = (await dialog.locator('td').last().textContent()) as string;
+      expect(JSON.parse(text)).toEqual(output);
+      await page.getByLabel('close').click();
+    });
+  });
+
+  test('Signed pipeline shows the signed indicator', async () => {
+    const row = page.getByRole('row', { name: 'ruby-ex-git-xf45fo' });
+    await expect(row.locator('.makeStyles-signedIndicator-120')).toBeVisible();
+  });
+});
+
+async function checkCards(
+  base: Locator,
+  sectionTitles: string[],
+  tableName: string,
+  columns: string[],
+) {
+  // check the violations summary
+  const sections = base.locator('.pf-v5-c-card:visible');
+  for (const item of sectionTitles) {
+    await expect(sections.filter({ hasText: item })).toBeVisible();
+  }
+
+  // check the violations table
+  const table = base.getByTestId(tableName);
+  await expect(table).toBeVisible();
+
+  for (const col of columns) {
+    await expect(
+      table.locator('thead').getByText(col, { exact: true }),
+    ).toBeVisible();
+  }
+  expect(await table.getByRole('row').count()).toBeGreaterThan(1);
+}