wp-scripts: add support for Playwright (#53108)

* wp-scripts: add support for Playwright

* Fix lock file

* Lint fixes

* Downgrade Playwright

* Fix config arg & browser install

* Use relative path for testDir

* Fix config inheritance

* Update npm scripts

* Fix e2e config too

* Suppress eslint errors

* Fix lock file

* Move `ASSETS_PATH` definition

* Fix typo

* Update readme & changelogs

* Move changelog entries

* Add missing dependency

.eslintrc.js

@@ -294,7 +294,7 @@ module.exports = {
 				'packages/e2e-test-utils-playwright/**/*.[tj]s',
 			],
 			extends: [
-				'plugin:eslint-plugin-playwright/playwright-test',
+				'plugin:@wordpress/eslint-plugin/test-playwright',
 				'plugin:@typescript-eslint/base',
 			],
 			parserOptions: {
@@ -308,7 +308,6 @@ module.exports = {
 			rules: {
 				'@wordpress/no-global-active-element': 'off',
 				'@wordpress/no-global-get-selection': 'off',
-				'playwright/no-page-pause': 'error',
 				'no-restricted-syntax': [
 					'error',
 					{

package.json

@@ -194,7 +194,6 @@
 		"eslint-plugin-import": "2.25.2",
 		"eslint-plugin-jest": "27.2.1",
 		"eslint-plugin-jest-dom": "5.0.2",
-		"eslint-plugin-playwright": "0.8.0",
 		"eslint-plugin-ssr-friendly": "1.0.6",
 		"eslint-plugin-storybook": "0.6.13",
 		"eslint-plugin-testing-library": "5.7.2",
@@ -314,7 +313,7 @@
 		"test:create-block": "bash ./bin/test-create-block.sh",
 		"test:e2e": "wp-scripts test-e2e --config packages/e2e-tests/jest.config.js",
 		"test:e2e:debug": "wp-scripts --inspect-brk test-e2e  --runInBand --no-cache --verbose --config packages/e2e-tests/jest.config.js --puppeteer-devtools",
-		"test:e2e:playwright": "playwright test --config test/e2e/playwright.config.ts",
+		"test:e2e:playwright": "wp-scripts test-playwright --config test/e2e/playwright.config.ts",
 		"test:e2e:storybook": "playwright test --config test/storybook-playwright/playwright.config.ts",
 		"test:e2e:watch": "npm run test:e2e -- --watch",
 		"test:native": "cross-env NODE_ENV=test jest --config test/native/jest.config.js",
@@ -323,8 +322,8 @@
 		"test:native:perf": "cross-env TEST_RUNNER_ARGS='--runInBand --config test/native/jest.config.js --testMatch \"**/performance/*.native.[jt]s?(x)\"' reassure",
 		"test:native:perf:baseline": "cross-env TEST_RUNNER_ARGS='--runInBand --config test/native/jest.config.js --testMatch \"**/performance/*.native.[jt]s?(x)\"' reassure --baseline",
 		"test:native:update": "npm run test:native -- --updateSnapshot",
-		"test:performance": "playwright test --config test/performance/playwright.config.ts",
-		"test:performance:debug": "playwright test --config test/performance/playwright.config.ts --debug",
+		"test:performance": "wp-scripts test-playwright --config test/performance/playwright.config.ts",
+		"test:performance:debug": "wp-scripts test-playwright --config test/performance/playwright.config.ts --debug",
 		"test:php": "npm-run-all lint:php test:unit:php",
 		"test:php:watch": "wp-env run --env-cwd='wp-content/plugins/gutenberg' tests-cli composer run-script test:watch",
 		"test:unit": "wp-scripts test-unit-js --config test/unit/jest.config.js",

packages/eslint-plugin/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Enhancement
+
+-   Added a new `test-playwright` ruleset using [`eslint-plugin-playwright`](https://www.npmjs.com/package/eslint-plugin-playwright).
+
 ## 15.1.0 (2023-08-31)
 
 ## 15.0.0 (2023-08-16)

packages/eslint-plugin/README.md

@@ -53,6 +53,7 @@ Alternatively, you can opt-in to only the more granular rulesets offered by the
 -   `react` – rules for React components.
 -   `test-e2e` – rules for end-to-end tests written in Puppeteer.
 -   `test-unit`– rules for unit tests written in Jest.
+-   `test-playwright` – rules for end-to-end tests written in Playwright.
 
 For example, if your project does not use React, you could consider extending including only the ESNext rules in your project using the following `extends` definition:
 

packages/eslint-plugin/configs/test-playwright.js

@@ -0,0 +1,3 @@
+module.exports = {
+	extends: [ 'plugin:playwright/recommended' ],
+};

packages/eslint-plugin/package.json

@@ -42,6 +42,7 @@
 		"eslint-plugin-jest": "^27.2.1",
 		"eslint-plugin-jsdoc": "^46.4.6",
 		"eslint-plugin-jsx-a11y": "^6.5.1",
+		"eslint-plugin-playwright": "^0.15.3",
 		"eslint-plugin-prettier": "^3.3.0",
 		"eslint-plugin-react": "^7.27.0",
 		"eslint-plugin-react-hooks": "^4.3.0",

packages/scripts/CHANGELOG.md

@@ -2,13 +2,17 @@
 
 ## Unreleased
 
+### Enhancement
+
+-   Added support for `test-playwright` script ([#53108](https://github.com/WordPress/gutenberg/pull/53108)).
+
 ## 26.12.0 (2023-08-31)
 
 ## 26.11.0 (2023-08-16)
 
 ### Enhancement
 
--   Updated `npm-package-json-lint` peer dependency to require v6.0.0 [#53636](https://github.com/WordPress/gutenberg/pull/53636)
+-   Updated `npm-package-json-lint` peer dependency to require v6.0.0 [#53636](https://github.com/WordPress/gutenberg/pull/53636).
 -   The bundled `@svgr/webpack` dependency has been updated from requiring ^6.2.1 to requiring ^8.0.1 ([#53630](https://github.com/WordPress/gutenberg/pull/53630)).
 -   The bundled `cssnano` dependency has been updated from requiring ^5.07 to requiring ^6.0.1 ([#53630](https://github.com/WordPress/gutenberg/pull/53630)).
 

packages/scripts/README.md

@@ -491,6 +491,47 @@ Should there be any situation where you want to provide your own Jest config, yo
 -   there is a file called `jest-unit.config.js`, `jest-unit.config.json`, `jest.config.js`, or `jest.config.json` in the top-level directory of your package (at the same level than your `package.json`).
 -   a `jest` object can be provided in the `package.json` file with the test configuration.
 
+### `test-plyawright`
+
+Launches the Playwright End-To-End (E2E) test runner. Similar to Puppeteer, it provides a high-level API to control a headless browser.
+
+Refer to the [Getting Started guide](https://playwright.dev/docs/writing-tests) to learn how to write tests.
+
+_Example:_
+
+```json
+{
+	"scripts": {
+		"test:playwright": "wp-scripts test-playwright",
+		"test:playwright:help": "wp-scripts test-playwright --help",
+		"test:playwright:debug": "wp-scripts test-playwright --debug"
+	}
+}
+```
+
+This is how you execute those scripts using the presented setup:
+
+-   `npm run test:playwright` - runs all tests.
+-   `npm run test:playwright:help` - prints all available options to configure the test runner.
+-   `npm run test:playwright:debug` - runs all tests interactively with the Playwright inspector.
+-   `npm run test:playwright FILE_NAME` - runs a specific test file.
+-   `npm run test:playwright -- --watch` - runs all tests interactively with watch mode and enhanced debugging.
+
+By default, Playwright looks for JavaScript or TypeScript files with `.test` or `.spec` suffix in the project root-level `/specs` folder, for example `/specs/login-screen.wrong-credentials.spec.ts`.
+
+This script automatically detects the best config to start Playwright, but sometimes you may need to specify custom options.
+To do so, you can add a file called `playwright.config.ts` or `playwright.config.js` in the top-level directory of your package (at the same level as your `package.json`).
+
+#### Failed Test Artifacts
+
+When tests fail, snapshots will be taken of the page and stored in the `artifacts/` directory at the root of your project. These snapshots may help debug failed tests during development or when running tests in a CI environment.
+
+The `artifacts/` directory can be customized by setting the `WP_ARTIFACTS_PATH` environment variable to the relative path of the desired directory within your project's root. For example: to change the default directory from `artifacts/` to `my/custom/artifacts`, you could use `WP_ARTIFACTS_PATH=my/custom/artifacts npm run test:playwright`.
+
+#### Advanced information
+
+You are able to use all of Playwright's [CLI options](https://playwright.dev/docs/test-cli#reference). You can also run `./node_modules/.bin/wp-scripts test-playwright --help` or `npm run test:playwright:help` (as mentioned above) to view all the available options. Learn more in the [Advanced Usage](#advanced-usage) section.
+
 ## Passing Node.js options
 
 `wp-scripts` supports the full array of [Node.js CLI options](https://nodejs.org/api/cli.html). They can be passed after the `wp-scripts` command and before the script name.

packages/scripts/config/playwright.config.ts

@@ -0,0 +1,63 @@
+/**
+ * External dependencies
+ */
+const path = require( 'path' );
+const { fileURLToPath } = require( 'url' );
+const { defineConfig, devices } = require( '@playwright/test' );
+
+process.env.WP_ARTIFACTS_PATH ??= path.join( process.cwd(), 'artifacts' );
+process.env.STORAGE_STATE_PATH ??= path.join(
+	process.env.WP_ARTIFACTS_PATH,
+	'storage-states/admin.json'
+);
+
+const config = defineConfig( {
+	reporter: process.env.CI ? [ [ 'github' ] ] : [ [ 'list' ] ],
+	forbidOnly: !! process.env.CI,
+	// fullyParallel: false,
+	workers: 1,
+	retries: process.env.CI ? 2 : 0,
+	timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 100_000, // Defaults to 100 seconds.
+	// Don't report slow test "files", as we will be running our tests in serial.
+	reportSlowTests: null,
+	testDir: './specs',
+	outputDir: path.join( process.env.WP_ARTIFACTS_PATH, 'test-results' ),
+	snapshotPathTemplate:
+		'{testDir}/{testFileDir}/__snapshots__/{arg}-{projectName}{ext}',
+	globalSetup: fileURLToPath(
+		new URL( './playwright/global-setup.ts', 'file:' + __filename ).href
+	),
+	use: {
+		baseURL: process.env.WP_BASE_URL || 'http://localhost:8889',
+		headless: true,
+		viewport: {
+			width: 960,
+			height: 700,
+		},
+		ignoreHTTPSErrors: true,
+		locale: 'en-US',
+		contextOptions: {
+			reducedMotion: 'reduce',
+			strictSelectors: true,
+		},
+		storageState: process.env.STORAGE_STATE_PATH,
+		actionTimeout: 10_000, // 10 seconds.
+		trace: 'retain-on-failure',
+		screenshot: 'only-on-failure',
+		video: 'on-first-retry',
+	},
+	webServer: {
+		command: 'npm run wp-env start',
+		port: 8889,
+		timeout: 120_000, // 120 seconds.
+		reuseExistingServer: true,
+	},
+	projects: [
+		{
+			name: 'chromium',
+			use: { ...devices[ 'Desktop Chrome' ] },
+		},
+	],
+} );
+
+export default config;

packages/scripts/config/playwright/global-setup.ts

@@ -0,0 +1,31 @@
+/**
+ * External dependencies
+ */
+import { request } from '@playwright/test';
+import type { FullConfig } from '@playwright/test';
+
+/**
+ * WordPress dependencies
+ */
+import { RequestUtils } from '@wordpress/e2e-test-utils-playwright';
+
+async function globalSetup( config: FullConfig ) {
+	const { storageState, baseURL } = config.projects[ 0 ].use;
+	const storageStatePath =
+		typeof storageState === 'string' ? storageState : undefined;
+
+	const requestContext = await request.newContext( {
+		baseURL,
+	} );
+
+	const requestUtils = new RequestUtils( requestContext, {
+		storageStatePath,
+	} );
+
+	// Authenticate and save the storageState to disk.
+	await requestUtils.setupRest();
+
+	await requestContext.dispose();
+}
+
+export default globalSetup;