From 647ea574918f5784cad2086b17be96ec7ec5e5f6 Mon Sep 17 00:00:00 2001 From: Bart Kalisz Date: Mon, 10 Jul 2023 16:30:15 +0200 Subject: [PATCH] Make performance.js comparison runner results-agnostic --- bin/plugin/commands/performance.js | 142 +------- .../config/performance-reporter.ts | 340 +++++++++--------- .../specs/front-end-block-theme.spec.js | 2 +- .../specs/front-end-classic-theme.spec.js | 2 +- test/performance/specs/post-editor.spec.js | 2 +- test/performance/specs/site-editor.spec.js | 2 +- test/performance/utils.js | 26 +- 7 files changed, 201 insertions(+), 315 deletions(-) diff --git a/bin/plugin/commands/performance.js b/bin/plugin/commands/performance.js index 31fdcfa332bde..032c4d28141ab 100644 --- a/bin/plugin/commands/performance.js +++ b/bin/plugin/commands/performance.js @@ -31,62 +31,6 @@ const RESULTS_FILE_SUFFIX = '.performance-results.json'; * @property {string=} wpVersion The WordPress version to be used as the base install for testing. */ -/** - * @typedef WPRawPerformanceResults - * - * @property {number[]} timeToFirstByte Represents the time since the browser started the request until it received a response. - * @property {number[]} largestContentfulPaint Represents the time when the main content of the page has likely loaded. - * @property {number[]} lcpMinusTtfb Represents the difference between LCP and TTFB. - * @property {number[]} serverResponse Represents the time the server takes to respond. - * @property {number[]} firstPaint Represents the time when the user agent first rendered after navigation. - * @property {number[]} domContentLoaded Represents the time immediately after the document's DOMContentLoaded event completes. - * @property {number[]} loaded Represents the time when the load event of the current document is completed. - * @property {number[]} firstContentfulPaint Represents the time when the browser first renders any text or media. - * @property {number[]} firstBlock Represents the time when Puppeteer first sees a block selector in the DOM. - * @property {number[]} type Average type time. - * @property {number[]} typeContainer Average type time within a container. - * @property {number[]} focus Average block selection time. - * @property {number[]} inserterOpen Average time to open global inserter. - * @property {number[]} inserterSearch Average time to search the inserter. - * @property {number[]} inserterHover Average time to move mouse between two block item in the inserter. - * @property {number[]} listViewOpen Average time to open listView - */ - -/** - * @typedef WPPerformanceResults - * - * @property {number=} timeToFirstByte Represents the time since the browser started the request until it received a response. - * @property {number=} largestContentfulPaint Represents the time when the main content of the page has likely loaded. - * @property {number=} lcpMinusTtfb Represents the difference between LCP and TTFB. - * @property {number=} serverResponse Represents the time the server takes to respond. - * @property {number=} firstPaint Represents the time when the user agent first rendered after navigation. - * @property {number=} domContentLoaded Represents the time immediately after the document's DOMContentLoaded event completes. - * @property {number=} loaded Represents the time when the load event of the current document is completed. - * @property {number=} firstContentfulPaint Represents the time when the browser first renders any text or media. - * @property {number=} firstBlock Represents the time when Puppeteer first sees a block selector in the DOM. - * @property {number=} type Average type time. - * @property {number=} minType Minimum type time. - * @property {number=} maxType Maximum type time. - * @property {number=} typeContainer Average type time within a container. - * @property {number=} minTypeContainer Minimum type time within a container. - * @property {number=} maxTypeContainer Maximum type time within a container. - * @property {number=} focus Average block selection time. - * @property {number=} minFocus Min block selection time. - * @property {number=} maxFocus Max block selection time. - * @property {number=} inserterOpen Average time to open global inserter. - * @property {number=} minInserterOpen Min time to open global inserter. - * @property {number=} maxInserterOpen Max time to open global inserter. - * @property {number=} inserterSearch Average time to open global inserter. - * @property {number=} minInserterSearch Min time to open global inserter. - * @property {number=} maxInserterSearch Max time to open global inserter. - * @property {number=} inserterHover Average time to move mouse between two block item in the inserter. - * @property {number=} minInserterHover Min time to move mouse between two block item in the inserter. - * @property {number=} maxInserterHover Max time to move mouse between two block item in the inserter. - * @property {number=} listViewOpen Average time to open list view. - * @property {number=} minListViewOpen Min time to open list view. - * @property {number=} maxListViewOpen Max time to open list view. - */ - /** * Sanitizes branch name to be used in a path or a filename. * @@ -98,17 +42,6 @@ function sanitizeBranchName( branch ) { return branch.replace( /[^a-zA-Z0-9-]/g, '-' ); } -/** - * Computes the average number from an array numbers. - * - * @param {number[]} array - * - * @return {number} Average. - */ -function average( array ) { - return array.reduce( ( a, b ) => a + b, 0 ) / array.length; -} - /** * Computes the median number from an array numbers. * @@ -124,69 +57,6 @@ function median( array ) { : ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2; } -/** - * Rounds and format a time passed in milliseconds. - * - * @param {number} number - * - * @return {number} Formatted time. - */ -function formatTime( number ) { - const factor = Math.pow( 10, 2 ); - return Math.round( number * factor ) / factor; -} - -/** - * Curate the raw performance results. - * - * @param {string} testSuite - * @param {WPRawPerformanceResults} results - * - * @return {WPPerformanceResults} Curated Performance results. - */ -function curateResults( testSuite, results ) { - if ( - testSuite === 'front-end-classic-theme' || - testSuite === 'front-end-block-theme' - ) { - return { - timeToFirstByte: median( results.timeToFirstByte ), - largestContentfulPaint: median( results.largestContentfulPaint ), - lcpMinusTtfb: median( results.lcpMinusTtfb ), - }; - } - - return { - serverResponse: average( results.serverResponse ), - firstPaint: average( results.firstPaint ), - domContentLoaded: average( results.domContentLoaded ), - loaded: average( results.loaded ), - firstContentfulPaint: average( results.firstContentfulPaint ), - firstBlock: average( results.firstBlock ), - type: average( results.type ), - minType: Math.min( ...results.type ), - maxType: Math.max( ...results.type ), - typeContainer: average( results.typeContainer ), - minTypeContainer: Math.min( ...results.typeContainer ), - maxTypeContainer: Math.max( ...results.typeContainer ), - focus: average( results.focus ), - minFocus: Math.min( ...results.focus ), - maxFocus: Math.max( ...results.focus ), - inserterOpen: average( results.inserterOpen ), - minInserterOpen: Math.min( ...results.inserterOpen ), - maxInserterOpen: Math.max( ...results.inserterOpen ), - inserterSearch: average( results.inserterSearch ), - minInserterSearch: Math.min( ...results.inserterSearch ), - maxInserterSearch: Math.max( ...results.inserterSearch ), - inserterHover: average( results.inserterHover ), - minInserterHover: Math.min( ...results.inserterHover ), - maxInserterHover: Math.max( ...results.inserterHover ), - listViewOpen: average( results.listViewOpen ), - minListViewOpen: Math.min( ...results.listViewOpen ), - maxListViewOpen: Math.max( ...results.listViewOpen ), - }; -} - /** * Runs the performance tests on the current branch. * @@ -466,7 +336,7 @@ async function runPerformanceTests( branches, options ) { const resultFiles = getFilesFromDir( ARTIFACTS_PATH ).filter( ( file ) => file.endsWith( RESULTS_FILE_SUFFIX ) ); - /** @type {Record>} */ + /** @type {Record>>} */ const results = {}; for ( const testSuite of testSuites ) { @@ -477,9 +347,7 @@ async function runPerformanceTests( branches, options ) { .filter( ( file ) => file.includes( `${ testSuite }_${ branch }_round-` ) ) - .map( ( file ) => - curateResults( testSuite, readJSONFile( file ) ) - ); + .map( ( file ) => readJSONFile( file ) ); const metrics = Object.keys( resultsRounds[ 0 ] ); results[ testSuite ][ branch ] = {}; @@ -488,9 +356,7 @@ async function runPerformanceTests( branches, options ) { // @ts-ignore const values = resultsRounds.map( ( item ) => item[ metric ] ); // @ts-ignore - results[ testSuite ][ branch ][ metric ] = formatTime( - median( values ) - ); + results[ testSuite ][ branch ][ metric ] = median( values ); } } @@ -527,7 +393,7 @@ async function runPerformanceTests( branches, options ) { } } - // Printing the results. + // Print the results. console.table( invertedResult ); } } diff --git a/test/performance/config/performance-reporter.ts b/test/performance/config/performance-reporter.ts index fcc47113b8ae3..98e815621208f 100644 --- a/test/performance/config/performance-reporter.ts +++ b/test/performance/config/performance-reporter.ts @@ -2,190 +2,188 @@ * External dependencies */ import path from 'path'; -import chalk from 'chalk'; -import { readFileSync, existsSync } from 'fs'; -import type { Reporter, TestCase } from '@playwright/test/reporter'; +import type { + Reporter, + FullConfig, + Suite, + FullResult, +} from '@playwright/test/reporter'; /** * Internal dependencies */ -import { average, round } from '../utils'; - -const title = chalk.bold; -const success = chalk.bold.green; - -class PerformanceReporter implements Reporter { - onTestEnd( test: TestCase ) { - const basename = path.basename( test.location.file, '.js' ); - const filepath = path.join( - process.env.WP_ARTIFACTS_PATH as string, - basename + '.performance-results.json' - ); - - if ( ! existsSync( filepath ) ) { - return; - } - - const results = readFileSync( filepath, 'utf8' ); - const { - serverResponse, - firstPaint, - domContentLoaded, - loaded, - firstContentfulPaint, - firstBlock, - type, - typeContainer, - focus, - listViewOpen, - inserterOpen, - inserterHover, - inserterSearch, - timeToFirstByte, - largestContentfulPaint, - lcpMinusTtfb, - } = JSON.parse( results ); - - if ( serverResponse && serverResponse.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Loading Time:' ) } -Average time to server response (subtracted from client side metrics): ${ success( - round( average( serverResponse ) ) + 'ms' - ) } -Average time to first paint: ${ success( - round( average( firstPaint ) ) + 'ms' - ) } -Average time to DOM content load: ${ success( - round( average( domContentLoaded ) ) + 'ms' - ) } -Average time to load: ${ success( round( average( loaded ) ) + 'ms' ) } -Average time to first contentful paint: ${ success( - round( average( firstContentfulPaint ) ) + 'ms' - ) } -Average time to first block: ${ success( - round( average( firstBlock ) ) + 'ms' - ) }` ); - } - - if ( type && type.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Typing:' ) } -Average time to type character: ${ success( round( average( type ) ) + 'ms' ) } -Slowest time to type character: ${ success( - round( Math.max( ...type ) ) + 'ms' - ) } -Fastest time to type character: ${ success( - round( Math.min( ...type ) ) + 'ms' - ) }` ); - } +import { + readFile, + average, + median, + formatTime, + saveResultsFile, +} from '../utils'; + +export interface WPRawPerformanceResults { + timeToFirstByte: number[]; + largestContentfulPaint: number[]; + lcpMinusTtfb: number[]; + serverResponse: number[]; + firstPaint: number[]; + domContentLoaded: number[]; + loaded: number[]; + firstContentfulPaint: number[]; + firstBlock: number[]; + type: number[]; + typeContainer: number[]; + focus: number[]; + inserterOpen: number[]; + inserterSearch: number[]; + inserterHover: number[]; + listViewOpen: number[]; +} - if ( typeContainer && typeContainer.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Typing within a container:' ) } -Average time to type within a container: ${ success( - round( average( typeContainer ) ) + 'ms' - ) } -Slowest time to type within a container: ${ success( - round( Math.max( ...typeContainer ) ) + 'ms' - ) } -Fastest time to type within a container: ${ success( - round( Math.min( ...typeContainer ) ) + 'ms' - ) }` ); - } +export interface WPPerformanceResults { + timeToFirstByte?: number; + largestContentfulPaint?: number; + lcpMinusTtfb?: number; + serverResponse?: number; + firstPaint?: number; + domContentLoaded?: number; + loaded?: number; + firstContentfulPaint?: number; + firstBlock?: number; + type?: number; + minType?: number; + maxType?: number; + typeContainer?: number; + minTypeContainer?: number; + maxTypeContainer?: number; + focus?: number; + minFocus?: number; + maxFocus?: number; + inserterOpen?: number; + minInserterOpen?: number; + maxInserterOpen?: number; + inserterSearch?: number; + minInserterSearch?: number; + maxInserterSearch?: number; + inserterHover?: number; + minInserterHover?: number; + maxInserterHover?: number; + listViewOpen?: number; + minListViewOpen?: number; + maxListViewOpen?: number; +} - if ( focus && focus.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Block Selection:' ) } -Average time to select a block: ${ success( round( average( focus ) ) + 'ms' ) } -Slowest time to select a block: ${ success( - round( Math.max( ...focus ) ) + 'ms' - ) } -Fastest time to select a block: ${ success( - round( Math.min( ...focus ) ) + 'ms' - ) }` ); - } +/** + * Curate the raw performance results. + * + * @param {string} testSuite + * @param {WPRawPerformanceResults} results + * + * @return {WPPerformanceResults} Curated Performance results. + */ +export function curateResults( + testSuite: string, + results: WPRawPerformanceResults +): WPPerformanceResults { + let output: WPPerformanceResults; + + if ( testSuite.includes( 'front-end' ) ) { + output = { + timeToFirstByte: median( results.timeToFirstByte ), + largestContentfulPaint: median( results.largestContentfulPaint ), + lcpMinusTtfb: median( results.lcpMinusTtfb ), + }; + } else { + output = { + serverResponse: average( results.serverResponse ), + firstPaint: average( results.firstPaint ), + domContentLoaded: average( results.domContentLoaded ), + loaded: average( results.loaded ), + firstContentfulPaint: average( results.firstContentfulPaint ), + firstBlock: average( results.firstBlock ), + type: average( results.type ), + minType: Math.min( ...results.type ), + maxType: Math.max( ...results.type ), + typeContainer: average( results.typeContainer ), + minTypeContainer: Math.min( ...results.typeContainer ), + maxTypeContainer: Math.max( ...results.typeContainer ), + focus: average( results.focus ), + minFocus: Math.min( ...results.focus ), + maxFocus: Math.max( ...results.focus ), + inserterOpen: average( results.inserterOpen ), + minInserterOpen: Math.min( ...results.inserterOpen ), + maxInserterOpen: Math.max( ...results.inserterOpen ), + inserterSearch: average( results.inserterSearch ), + minInserterSearch: Math.min( ...results.inserterSearch ), + maxInserterSearch: Math.max( ...results.inserterSearch ), + inserterHover: average( results.inserterHover ), + minInserterHover: Math.min( ...results.inserterHover ), + maxInserterHover: Math.max( ...results.inserterHover ), + listViewOpen: average( results.listViewOpen ), + minListViewOpen: Math.min( ...results.listViewOpen ), + maxListViewOpen: Math.max( ...results.listViewOpen ), + }; + } - if ( listViewOpen && listViewOpen.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Opening List View:' ) } -Average time to open list view: ${ success( - round( average( listViewOpen ) ) + 'ms' - ) } -Slowest time to open list view: ${ success( - round( Math.max( ...listViewOpen ) ) + 'ms' - ) } -Fastest time to open list view: ${ success( - round( Math.min( ...listViewOpen ) ) + 'ms' - ) }` ); - } + return Object.fromEntries( + Object.entries( output ).map( ( [ key, value ] ) => { + return [ key, formatTime( value ) ]; + } ) + ); +} +class PerformanceReporter implements Reporter { + private testSuites: string[]; - if ( inserterOpen && inserterOpen.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Opening Global Inserter:' ) } -Average time to open global inserter: ${ success( - round( average( inserterOpen ) ) + 'ms' - ) } -Slowest time to open global inserter: ${ success( - round( Math.max( ...inserterOpen ) ) + 'ms' - ) } -Fastest time to open global inserter: ${ success( - round( Math.min( ...inserterOpen ) ) + 'ms' - ) }` ); - } + constructor() { + this.testSuites = []; + } - if ( inserterSearch && inserterSearch.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Inserter Search:' ) } -Average time to type the inserter search input: ${ success( - round( average( inserterSearch ) ) + 'ms' - ) } -Slowest time to type the inserter search input: ${ success( - round( Math.max( ...inserterSearch ) ) + 'ms' - ) } -Fastest time to type the inserter search input: ${ success( - round( Math.min( ...inserterSearch ) ) + 'ms' - ) }` ); - } + onBegin( _: FullConfig, suite: Suite ) { + // Map suites from the reporter state instead of reading from the disk + // to avoid including existing result files that are irrelevant to the + // current run. + const suites = suite.suites[ 0 ].suites; // It's where the suites are at. + this.testSuites = suites.map( ( s ) => + path.basename( s.location?.file as string, '.spec.js' ) + ); + } - if ( inserterHover && inserterHover.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Inserter Block Item Hover:' ) } -Average time to move mouse between two block item in the inserter: ${ success( - round( average( inserterHover ) ) + 'ms' - ) } -Slowest time to move mouse between two block item in the inserter: ${ success( - round( Math.max( ...inserterHover ) ) + 'ms' - ) } -Fastest time to move mouse between two block item in the inserter: ${ success( - round( Math.min( ...inserterHover ) ) + 'ms' - ) }` ); + onEnd( result: FullResult ) { + if ( result.status !== 'passed' ) { + return; } - if ( timeToFirstByte && timeToFirstByte.length ) { - // eslint-disable-next-line no-console - console.log( ` -${ title( 'Front End Performance:' ) } -Average time to first byte (TTFB): ${ success( - round( average( timeToFirstByte ) ) + 'ms' - ) } -Average time to largest contentful paint (LCP): ${ success( - round( average( largestContentfulPaint ) ) + 'ms' - ) } -Average primary content load time (LCP - TTFB): ${ success( - round( average( lcpMinusTtfb ) ) + 'ms' - ) }` ); + for ( const testSuite of this.testSuites ) { + // Get raw results filepaths. + const rawResultsPath = path.join( + process.env.WP_ARTIFACTS_PATH as string, + testSuite + '.performance-results.raw.json' + ); + + // Read the raw metrics. + const rawResults = JSON.parse( + readFile( rawResultsPath ) + ) as WPRawPerformanceResults; + + // Curate the results. + const results = curateResults( testSuite, rawResults ); + + // Save curated results to file. + saveResultsFile( testSuite, results ); + + if ( ! process.env.CI ) { + // Print the results. + const printableResults = Object.fromEntries( + Object.entries( results ).map( ( [ key, value ] ) => { + return [ key, { value: `${ value } ms` } ]; + } ) + ); + + // eslint-disable-next-line no-console + console.log( `\n${ testSuite }\n` ); + // eslint-disable-next-line no-console + console.table( printableResults ); + } } - - // eslint-disable-next-line no-console - console.log( '' ); } } diff --git a/test/performance/specs/front-end-block-theme.spec.js b/test/performance/specs/front-end-block-theme.spec.js index 93edaa15bdeab..094673d32728d 100644 --- a/test/performance/specs/front-end-block-theme.spec.js +++ b/test/performance/specs/front-end-block-theme.spec.js @@ -20,7 +20,7 @@ test.describe( 'Front End Performance', () => { } ); test.afterAll( async ( { requestUtils } ) => { - saveResultsFile( __filename, results ); + saveResultsFile( __filename, results, true ); await requestUtils.activateTheme( 'twentytwentyone' ); } ); diff --git a/test/performance/specs/front-end-classic-theme.spec.js b/test/performance/specs/front-end-classic-theme.spec.js index 2f22f579c1143..4f33edbdeeb0e 100644 --- a/test/performance/specs/front-end-classic-theme.spec.js +++ b/test/performance/specs/front-end-classic-theme.spec.js @@ -20,7 +20,7 @@ test.describe( 'Front End Performance', () => { } ); test.afterAll( async () => { - saveResultsFile( __filename, results ); + saveResultsFile( __filename, results, true ); } ); test( 'Measure TTFB, LCP, and LCP-TTFB', async ( { page } ) => { diff --git a/test/performance/specs/post-editor.spec.js b/test/performance/specs/post-editor.spec.js index de1582161ea80..30f573c13f619 100644 --- a/test/performance/specs/post-editor.spec.js +++ b/test/performance/specs/post-editor.spec.js @@ -48,7 +48,7 @@ test.describe( 'Post Editor Performance', () => { const traceFilePath = getTraceFilePath(); test.afterAll( async () => { - saveResultsFile( __filename, results ); + saveResultsFile( __filename, results, true ); // Delete the trace file. deleteFile( traceFilePath ); } ); diff --git a/test/performance/specs/site-editor.spec.js b/test/performance/specs/site-editor.spec.js index 59b50e1f1ce08..52c507805ae61 100644 --- a/test/performance/specs/site-editor.spec.js +++ b/test/performance/specs/site-editor.spec.js @@ -49,7 +49,7 @@ test.describe( 'Site Editor Performance', () => { } ); test.afterAll( async ( { requestUtils } ) => { - saveResultsFile( __filename, results ); + saveResultsFile( __filename, results, true ); await requestUtils.deleteAllTemplates( 'wp_template' ); await requestUtils.deleteAllTemplates( 'wp_template_part' ); diff --git a/test/performance/utils.js b/test/performance/utils.js index e7bbb4206ee8d..1110d1fcae143 100644 --- a/test/performance/utils.js +++ b/test/performance/utils.js @@ -20,10 +20,12 @@ export function getTraceFilePath() { return path.join( process.env.WP_ARTIFACTS_PATH, '/trace.json' ); } -export function saveResultsFile( testFilename, results ) { +export function saveResultsFile( testFilename, results, isRaw = false ) { const resultsFilename = process.env.RESULTS_FILENAME || - path.basename( testFilename, '.js' ) + '.performance-results.json'; + `${ path.basename( testFilename, '.spec.js' ) }.performance-results${ + isRaw ? '.raw' : '' + }.json`; return writeFileSync( path.join( process.env.WP_ARTIFACTS_PATH, resultsFilename ), @@ -145,11 +147,31 @@ export function average( array ) { return array.reduce( ( a, b ) => a + b ) / array.length; } +export function median( array ) { + const mid = Math.floor( array.length / 2 ), + numbers = [ ...array ].sort( ( a, b ) => a - b ); + return array.length % 2 !== 0 + ? numbers[ mid ] + : ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2; +} + export function round( number, decimalPlaces = 2 ) { const factor = Math.pow( 10, decimalPlaces ); return Math.round( number * factor ) / factor; } +/** + * Rounds and format a time passed in milliseconds. + * + * @param {number} number + * + * @return {number} Formatted time. + */ +export function formatTime( number ) { + const factor = Math.pow( 10, 2 ); + return Math.round( number * factor ) / factor; +} + export async function loadBlocksFromHtml( page, filepath ) { if ( ! existsSync( filepath ) ) { throw new Error( `File not found (${ filepath })` );