Replace TT1-Blocks with Empty Theme in the wp-env of gutenberg and CI (…

…#37446)

Co-authored-by: Bernie Reiter <[email protected]>

.gitignore

@@ -12,6 +12,7 @@ coverage
 *.log
 yarn.lock
 /artifacts
+/perf-envs
 
 .cache
 *.tsbuildinfo

.wp-env.json

@@ -1,7 +1,7 @@
 {
 	"core": "WordPress/WordPress",
 	"plugins": [ "." ],
-	"themes": [ "WordPress/theme-experiments/tt1-blocks#[email protected]" ],
+	"themes": [ "./test/emptytheme" ],
 	"env": {
 		"tests": {
 			"mappings": {

bin/plugin/commands/performance.js

@@ -3,7 +3,7 @@
  */
 const fs = require( 'fs' );
 const path = require( 'path' );
-const { mapValues } = require( 'lodash' );
+const { mapValues, kebabCase } = require( 'lodash' );
 
 /**
  * Internal dependencies
@@ -213,7 +213,7 @@ async function runPerformanceTests( branches, options ) {
 	}
 
 	// 1- Preparing the tests directory.
-	log( '\n>> Preparing the tests directory' );
+	log( '\n>> Preparing the tests directories' );
 	log( '    >> Cloning the repository' );
 	const baseDirectory = await git.clone( config.gitRepositoryURL );
 	const performanceTestDirectory = getRandomTemporaryPath();
@@ -236,19 +236,33 @@ async function runPerformanceTests( branches, options ) {
 		'npm install && npm run build:packages',
 		performanceTestDirectory
 	);
+	log( '    >> Creating the environment folders' );
+	await runShellScript( 'mkdir perf-envs', performanceTestDirectory );
 
 	// 2- Preparing the environment directories per branch.
 	log( '\n>> Preparing an environment directory per branch' );
 	const branchDirectories = {};
 	for ( const branch of branches ) {
 		log( '    >> Branch: ' + branch );
-		const environmentDirectory = getRandomTemporaryPath();
+		const environmentDirectory =
+			performanceTestDirectory + '/perf-envs/' + kebabCase( branch );
 		// @ts-ignore
 		branchDirectories[ branch ] = environmentDirectory;
+		await runShellScript( 'mkdir ' + environmentDirectory );
 		await runShellScript(
-			'cp -R ' + baseDirectory + ' ' + environmentDirectory
+			'cp -R ' + baseDirectory + ' ' + environmentDirectory + '/plugin'
+		);
+		await setUpGitBranch( branch, environmentDirectory + '/plugin' );
+		await runShellScript(
+			'cp ' +
+				path.resolve(
+					performanceTestDirectory,
+					'bin/plugin/utils/.wp-env.performance.json'
+				) +
+				' ' +
+				environmentDirectory +
+				'/.wp-env.json'
 		);
-		await setUpGitBranch( branch, environmentDirectory );
 
 		if ( options.wpVersion ) {
 			// In order to match the topology of ZIP files at wp.org, remap .0
@@ -316,7 +330,7 @@ async function runPerformanceTests( branches, options ) {
 				log( '    >> Branch: ' + branch + ', Suite: ' + testSuite );
 				log( '        >> Starting the environment.' );
 				await runShellScript(
-					'npm run wp-env start',
+					'../../node_modules/.bin/wp-env start',
 					environmentDirectory
 				);
 				log( '        >> Running the test.' );
@@ -326,7 +340,7 @@ async function runPerformanceTests( branches, options ) {
 				);
 				log( '        >> Stopping the environment' );
 				await runShellScript(
-					'npm run wp-env stop',
+					'../../node_modules/.bin/wp-env stop',
 					environmentDirectory
 				);
 			}

bin/plugin/utils/.wp-env.performance.json

@@ -0,0 +1,13 @@
+{
+	"core": "WordPress/WordPress",
+	"plugins": [ "./plugin" ],
+	"themes": [ "../../test/emptytheme" ],
+	"env": {
+		"tests": {
+			"mappings": {
+				"wp-content/mu-plugins": "../../packages/e2e-tests/mu-plugins",
+				"wp-content/plugins/gutenberg-test-plugins": "../../packages/e2e-tests/plugins"
+			}
+		}
+	}
+}

docs/explanations/architecture/performance.md

@@ -4,7 +4,7 @@ Performance is a key feature for editor applications and the Block editor is not
 
 ## Metrics
 
-To ensure the block editor stays performant across releases and development, we monitor some key metrics using [performance testing](/docs/contributors/code/testing-overview.md#performance-testing).
+To ensure the block editor stays performant across releases and development, we monitor some key metrics using [performance benchmark job](#the-performance-benchmark-job).
 
 **Loading Time:** The time it takes to load an editor page. This includes time the server takes to respond, times to first paint, first contentful paint, DOM content load complete, load complete and first block render.
 **Typing Time:** The time it takes for the browser to respond while typing on the editor.
@@ -24,6 +24,47 @@ Rendering asynchronously in this context means that if a change is triggered in
 
 Based on the idea that **when editing a given block, it is very rare that an update to that block affects other parts of the content**, the block editor canvas only renders the selected block is synchronous mode, all the remaining blocks are rendered asynchronously. This ensures that the editor stays responsive as the post grows.
 
+## The performance benchmark job
+
+A tool to compare performance accross multiple branches/tags/commits is provided. You can run it locally like so: `./bin/plugin/cli.js perf [branches]`, example:
+
+```
+./bin/plugin/cli.js perf trunk v8.1.0 v8.0.0
+```
+
+To get the most accurate results, it's is important to use the exact same version of the tests and environment (theme...) when running the tests, the only thing that need to be different between the branches is the Gutenberg plugin version (or branch used to build the plugin).
+
+To achieve that the command first prepares the following folder structure:
+
+    │
+    ├── packages/e2e-tests/specs/performance/*
+    |   The actual performance tests to run
+    │
+    ├── test/emptytheme
+    |   The theme used for the tests environment. (site editor)
+    │
+    │── perf-envs/branch1/.wp-env.json
+    │   The wp-env config file for branch1 (similar to all other branches except the plugin folder).
+    │── perf-envs/branch1/plugin
+    │   A built clone of the Gutenberg plugin for branch1 (git checkout branch1)
+    │
+    ├── perf-envs/branchX
+    │   The structure of perf-envs/branch1 is duplicated for all other branches.
+    │
+    └── ...
+        The remaining files of the Gutenberg repository (packages..., all what's necessary to actually run the performance tests job)
+
+Once the directory above is in place, the performance command loop over the performance test suites (post editor and site editor) and does the following:
+
+ 1- Start the environment for branch1
+ 2- Run the performance test for the current suite
+ 3- Stop the environment for branch1
+ 4- Repeat the first 3 steps for all other branches
+ 5- Repeat the previous 4 steps 3 times.
+ 6- Compute medians for all the performance metrics of the current suite.
+
+Once all the test suites are executed, a summary report is printed.
+
 ## Going further
 
 -   [Journey towards a performant editor](https://riad.blog/2020/02/14/a-journey-towards-a-performant-web-editor/)

packages/e2e-tests/specs/editor/various/font-size-picker.test.js

@@ -7,7 +7,6 @@ import {
 	createNewPost,
 	pressKeyWithModifier,
 	pressKeyTimes,
-	activateTheme,
 	openTypographyToolsPanelMenu,
 } from '@wordpress/e2e-test-utils';
 
@@ -100,21 +99,68 @@ describe( 'Font Size Picker', () => {
 		` );
 		} );
 	} );
+
 	// A different control is rendered based on the available font sizes number.
 	describe( 'More font sizes', () => {
-		beforeAll( async () => {
-			await activateTheme( 'tt1-blocks' );
-		} );
-		afterAll( async () => {
-			await activateTheme( 'twentytwentyone' );
+		beforeEach( async () => {
+			await page.evaluate( () => {
+				wp.data.dispatch( 'core/block-editor' ).updateSettings(
+					// eslint-disable-next-line no-undef
+					lodash.merge(
+						wp.data.select( 'core/block-editor' ).getSettings(),
+						{
+							__experimentalFeatures: {
+								typography: {
+									fontSizes: {
+										default: [
+											{
+												name: 'Tiny',
+												slug: 'tiny',
+												size: '11px',
+											},
+											,
+											{
+												name: 'Small',
+												slug: 'small',
+												size: '13px',
+											},
+											{
+												name: 'Medium',
+												slug: 'medium',
+												size: '20px',
+											},
+											{
+												name: 'Large',
+												slug: 'large',
+												size: '36px',
+											},
+											{
+												name: 'Extra Large',
+												slug: 'x-large',
+												size: '42px',
+											},
+											{
+												name: 'Huge',
+												slug: 'huge',
+												size: '48px',
+											},
+										],
+									},
+								},
+							},
+						}
+					)
+				);
+			} );
 		} );
+
 		it( 'should apply a named font size using the font size buttons', async () => {
 			// Create a paragraph block with some content.
 			await clickBlockAppender();
 			await page.keyboard.type( 'Paragraph to be made "large"' );
 
 			await openFontSizeSelectControl();
-			await pressKeyTimes( 'ArrowDown', 4 );
+			await pressKeyTimes( 'ArrowDown', 5 );
 			await page.keyboard.press( 'Enter' );
 
 			expect( await getEditedPostContent() ).toMatchInlineSnapshot( `
@@ -131,11 +177,11 @@ describe( 'Font Size Picker', () => {
 			);
 
 			await openFontSizeSelectControl();
-			await pressKeyTimes( 'ArrowDown', 3 );
+			await pressKeyTimes( 'ArrowDown', 4 );
 			await page.keyboard.press( 'Enter' );
 			expect( await getEditedPostContent() ).toMatchInlineSnapshot( `
-			"<!-- wp:paragraph {\\"fontSize\\":\\"normal\\"} -->
-			<p class=\\"has-normal-font-size\\">Paragraph with font size reset using tools panel menu</p>
+			"<!-- wp:paragraph {\\"fontSize\\":\\"medium\\"} -->
+			<p class=\\"has-medium-font-size\\">Paragraph with font size reset using tools panel menu</p>
 			<!-- /wp:paragraph -->"
 		` );
 
@@ -158,7 +204,7 @@ describe( 'Font Size Picker', () => {
 			);
 
 			await openFontSizeSelectControl();
-			await pressKeyTimes( 'ArrowDown', 2 );
+			await pressKeyTimes( 'ArrowDown', 3 );
 			await page.keyboard.press( 'Enter' );
 			expect( await getEditedPostContent() ).toMatchInlineSnapshot( `
 			"<!-- wp:paragraph {\\"fontSize\\":\\"small\\"} -->

packages/e2e-tests/specs/editor/various/post-editor-template-mode.test.js

@@ -102,7 +102,7 @@ describe( 'Post Editor Template mode', () => {
 	} );
 
 	it( 'Allow to switch to template mode, edit the template and check the result', async () => {
-		await activateTheme( 'tt1-blocks' );
+		await activateTheme( 'emptytheme' );
 		await createNewPost();
 		// Create a random post.
 		await page.type( '.editor-post-title__input', 'Just an FSE Post' );

packages/e2e-tests/specs/performance/site-editor.test.js

@@ -31,7 +31,7 @@ jest.setTimeout( 1000000 );
 
 describe( 'Site Editor Performance', () => {
 	beforeAll( async () => {
-		await activateTheme( 'tt1-blocks' );
+		await activateTheme( 'emptytheme' );
 		await trashAllPosts( 'wp_template' );
 		await trashAllPosts( 'wp_template', 'auto-draft' );
 		await trashAllPosts( 'wp_template_part' );

packages/e2e-tests/specs/site-editor/document-settings.test.js

@@ -26,7 +26,7 @@ async function getDocumentSettingsSecondaryTitle() {
 
 describe( 'Document Settings', () => {
 	beforeAll( async () => {
-		await activateTheme( 'tt1-blocks' );
+		await activateTheme( 'emptytheme' );
 		await trashAllPosts( 'wp_template' );
 		await trashAllPosts( 'wp_template_part' );
 	} );
@@ -43,7 +43,7 @@ describe( 'Document Settings', () => {
 		it( 'should display the selected templates name in the document header', async () => {
 			// Navigate to a template
 			await siteEditor.visit( {
-				postId: 'tt1-blocks//index',
+				postId: 'emptytheme//index',
 				postType: 'wp_template',
 			} );
 
@@ -79,7 +79,7 @@ describe( 'Document Settings', () => {
 		it( "should display the selected template part's name in the document header", async () => {
 			// Navigate to a template part
 			await siteEditor.visit( {
-				postId: 'tt1-blocks//header',
+				postId: 'emptytheme//header',
 				postType: 'wp_template_part',
 			} );
 

packages/e2e-tests/specs/site-editor/multi-entity-editing.test.js

@@ -134,7 +134,7 @@ const removeErrorMocks = () => {
 
 describe( 'Multi-entity editor states', () => {
 	beforeAll( async () => {
-		await activateTheme( 'tt1-blocks' );
+		await activateTheme( 'emptytheme' );
 		await trashAllPosts( 'wp_template' );
 		await trashAllPosts( 'wp_template_part' );
 	} );
@@ -152,7 +152,7 @@ describe( 'Multi-entity editor states', () => {
 	// Skip reason: This should be rewritten to use other methods to switching to different templates.
 	it.skip( 'should not dirty an entity by switching to it in the template dropdown', async () => {
 		await siteEditor.visit( {
-			postId: 'tt1-blocks//header',
+			postId: 'emptytheme//header',
 			postType: 'wp_template_part',
 		} );
 		await page.waitForFunction( () =>