Update messaging on tests and add more comments to messaging system.

actions/ui-check/package.json

@@ -9,10 +9,10 @@
 		"test:e2e:interactive": "PUPPETEER_HEADLESS=false wp-scripts test-e2e --config tests/e2e/jest.config.js",
 		"test:e2e": "wp-scripts test-e2e --config tests/e2e/jest.config.js",
 		"test:e2e:dev": "DEV_MODE=true wp-scripts test-e2e --config tests/e2e/jest.config.js",
-		"test:e2e:sanity": "SANITY_MODE=true DEV_MODE=true wp-scripts test-e2e --config tests/e2e/jest.sanity.js",
+		"test:e2e:sanity": "SANITY_MODE=true wp-scripts test-e2e --config tests/e2e/jest.sanity.js",
 		"test:unit": "wp-scripts test-unit-js --config tests/unit/jest.config.js --passWithNoTests",
 		"test:unit:dev": "DEV_MODE=true wp-scripts test-unit-js --config tests/unit/jest.config.js",
-		"test:unit:sanity": "SANITY_MODE=true DEV_MODE=true wp-scripts test-e2e --config tests/unit/jest.sanity.js",
+		"test:unit:sanity": "SANITY_MODE=true wp-scripts test-e2e --config tests/unit/jest.sanity.js",
 		"start": "npm run install:environment && npm run test:unit && npm run test:e2e"
 	},
 	"author": "The WordPress Contributors",

actions/ui-check/tests/e2e/jest.config.js

@@ -22,6 +22,10 @@ let configs = {
 	reporters: [ '<rootDir>/../reporters/index.js' ],
 };
 
+if ( process.env.DEV_MODE ) {
+	configs.reporters = [ 'default' ];
+}
+
 // When run using NPX there are issues related to running in a `node_modules` folder.
 if (
 	process.env.INIT_CWD &&

actions/ui-check/tests/e2e/specs/page/body-class/index.js

@@ -12,7 +12,7 @@ export default async ( url, bodyClass ) => {
 	const bodyClassName = await getElementPropertyAsync( body, 'className' );
 
 	return errorWithMessageOnFail(
-		`${ url } does not contain a body class`,
+		`Unable to find body class when loading a page using ${ url }`,
 		() => {
 			expect( bodyClassName.split( ' ' ) ).toContain( bodyClass );
 		}

actions/ui-check/tests/e2e/specs/page/complete-output/index.js

@@ -5,7 +5,7 @@ import { errorWithMessageOnFail } from '../../../../utils';
 
 export default async ( url, responseText ) => {
 	return errorWithMessageOnFail(
-		`${ url } contains incomplete output. Make sure the page contains valid html.`,
+		`Loading a page using ${ url } contains incomplete output. Make sure the page contains valid html.`,
 		() => {
 			expect( responseText ).toMatch( /<\/(html|rss)>\s*$/ );
 		}

actions/ui-check/tests/e2e/specs/page/frontpage-template/index.js

@@ -4,11 +4,15 @@
 import { errorWithMessageOnFail, getFileNameFromPath } from '../../../../utils';
 
 export default async ( url ) => {
-	const template = await page.$eval( '#template', ( el ) => el.value );
-	const filename = getFileNameFromPath( template );
+	let template, filename;
+
+	try {
+		template = await page.$eval( '#template', ( el ) => el.value );
+		filename = getFileNameFromPath( template );
+	} catch ( ex ) {}
 
 	return errorWithMessageOnFail(
-		`${ url } Frontpage template file must not be page.php`,
+		`Frontpage template file must not be page.php`,
 		() => {
 			expect( filename ).not.toBe( 'page.php' );
 		}

actions/ui-check/tests/e2e/specs/page/index.test.js

@@ -28,7 +28,7 @@ let urls = [ [ '/', '?feed=rss2', '' ], ...getTestUrls() ];
 
 const getUrlPathWithTemplate = async ( urlPath ) => {
 	const template = await page.$eval( '#template', ( el ) => el.value );
-	return `${ urlPath } (via: ${ getFileNameFromPath( template ) })`;
+	return getFileNameFromPath( template );
 };
 
 // Some basic tests that apply to every page

actions/ui-check/tests/e2e/specs/page/js-errors/index.js

@@ -14,7 +14,7 @@ page.on( 'pageerror', ( error ) => {
 
 export default async ( url ) => {
 	return errorWithMessageOnFail(
-		`${ url } contains javascript errors. Found ${ removeLocalPathRefs(
+		`Loading the page using ${ url } contains javascript errors. Found ${ removeLocalPathRefs(
 			jsError
 		) }`,
 		() => {

actions/ui-check/tests/e2e/specs/page/php-errors/index.js

@@ -11,7 +11,9 @@ export default async ( url ) => {
 	const pageError = await getPageError();
 
 	return errorWithMessageOnFail(
-		`${ url } contains PHP errors: ${ removeLocalPathRefs( pageError ) }`,
+		`Loading the page using ${ url } contains PHP errors: ${ removeLocalPathRefs(
+			pageError
+		) }`,
 		() => {
 			expect( pageError ).toBe( null );
 		}

actions/ui-check/tests/e2e/specs/ui/sub-menu/index.js

@@ -104,7 +104,6 @@ export default async () => {
 				expect( false ).toEqual( true );
 			} );
 		} else {
-			console.log( ex );
 		}
 
 		return false;

actions/ui-check/tests/reporters/index.js

@@ -14,9 +14,6 @@ const WARNING_DOCS_URL =
  */
 const getTemplate = ( key, obj ) => `Test Name: ${ obj.title }
 
-Found on: 
-${ obj.pages.join( ',' ) }
-
 Details: 
 ${ obj.details.join( '\n' ) }
 
@@ -74,17 +71,26 @@ class MyCustomReporter {
 
 				// Get the error type
 				const errorTypeRegex = /\[\[\[(.*)]]]/g;
-				const [ , errorTypeMatch ] = errorTypeRegex.exec(
+				const typeMatches = errorTypeRegex.exec(
 					result.failureMessages[ 0 ]
 				);
 
+				if ( ! typeMatches || ! typeMatches[ 1 ] ) {
+					return;
+				}
+
 				// // Get the error message
-				const regex = /{{{(.*)}}}/g;
-				const [ , match ] = regex.exec( result.failureMessages[ 0 ] );
+				const regex = /{{{([\s\S]*?)}}}/g;
+				const matches = regex.exec( result.failureMessages[ 0 ] );
 
-				this.errors[ id ].pages.push( `"${ result.ancestorTitles }"` );
-				this.errors[ id ].severity = errorTypeMatch;
-				this.errors[ id ].details.push( match.trim() );
+				if ( matches && matches[ 1 ] ) {
+					this.errors[ id ].pages.push(
+						`"${ result.ancestorTitles }"`
+					);
+
+					this.errors[ id ].severity = typeMatches[ 1 ];
+					this.errors[ id ].details.push( matches[ 1 ].trim() );
+				}
 			}
 		}
 	}
@@ -109,6 +115,10 @@ class MyCustomReporter {
 		if ( errors.length ) {
 			printToLog( 'errors', errors );
 		}
+
+		console.log(
+			'\n\nCompleted test run. See logs for test result details.\n'
+		);
 	}
 }
 

actions/ui-check/tests/unit/jest.config.js

@@ -10,4 +10,9 @@ module.exports = {
 		// Don't run sanity tests
 		'!**/unit/**/sanity/**',
 	],
+	reporters: [ '<rootDir>/../reporters/index.js' ],
 };
+
+if ( process.env.DEV_MODE ) {
+	configs.reporters = [ 'default' ];
+}

actions/ui-check/tests/utils/messaging.js

@@ -6,22 +6,27 @@
  * @returns {bool} Whether the test has passed(true) or not(false).
  */
 const expectWithMessage = ( type, message, testToRun ) => {
+	// Dev mode is considered the default jest environment
+	if ( process.env.DEV_MODE ) {
+		testToRun();
+		return;
+	}
+
+	// Custom handling of error messages
+	// This is interpreted by a custom reporter (../reporters)
 	try {
 		testToRun();
+
+		// We return true because the tests are also used for sanity checks
+		// which assume the test either pass or fail
 		return true;
 	} catch ( e ) {
-		const output = Array.isArray( message ) ? message : [ message ];
-
-		if ( ! process.env.DEV_MODE ) {
-			throw new Error( `[[[${ type }]]] {{{ ${ output } }}} ` );
-		} else if ( process.env.SANITY_MODE ) {
-			// No need to do anything
-			// We expect tests to fail and don't want that to appear as test failures.
-		} else {
-			throw new Error( e );
+		// Sanity tests only care about whether the test failed or not
+		if ( process.env.SANITY_MODE ) {
+			return false;
 		}
 
-		return false;
+		throw new Error( `[[[${ type }]]] {{{ ${ message } }}} ` );
 	}
 };