Skip to content

Commit

Permalink
test: add WPTRunner support for variants and generating WPT reports
Browse files Browse the repository at this point in the history 
PR-URL: #46498
Backport-PR-URL: #46768
Reviewed-By: Yagiz Nizipli <[email protected]>
Reviewed-By: Richard Lau <[email protected]>
  • Loading branch information
@panva @BethGriggs
panva authored and BethGriggs committed Mar 23, 2023
1 parent 7af9bdb commit 088e9cd
Show file tree
Hide file tree
Showing 2 changed files with 207 additions and 57 deletions.
6 changes: 6 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,12 @@ test-message: test-build
test-wpt: all
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt

.PHONY: test-wpt-report
test-wpt-report:
$(RM) -r out/wpt
mkdir -p out/wpt
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt

.PHONY: test-simple
test-simple: | cctest # Depends on 'all'.
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
Expand Down
258 changes: 201 additions & 57 deletions test/common/wpt.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,119 @@ const fs = require('fs');
const fsPromises = fs.promises;
const path = require('path');
const events = require('events');
const os = require('os');
const { inspect } = require('util');
const { Worker } = require('worker_threads');

function getBrowserProperties() {
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
const release = /^\d+\.\d+\.\d+$/.test(version);
const browser = {
browser_channel: release ? 'stable' : 'experimental',
browser_version: version,
};

return browser;
}

/**
* Return one of three expected values
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
*/
function getOs() {
switch (os.type()) {
case 'Linux':
return 'linux';
case 'Darwin':
return 'mac';
case 'Windows_NT':
return 'win';
default:
throw new Error('Unsupported os.type()');
}
}

// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
function sanitizeUnpairedSurrogates(str) {
return str.replace(
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
function(_, low, prefix, high) {
let output = prefix || ''; // Prefix may be undefined
const string = low || high; // Only one of these alternates can match
for (let i = 0; i < string.length; i++) {
output += codeUnitStr(string[i]);
}
return output;
});
}

function codeUnitStr(char) {
return 'U+' + char.charCodeAt(0).toString(16);
}

class WPTReport {
constructor() {
this.results = [];
this.time_start = Date.now();
}

addResult(name, status) {
const result = {
test: name,
status,
subtests: [],
addSubtest(name, status, message) {
const subtest = {
status,
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
name: sanitizeUnpairedSurrogates(name),
};
if (message) {
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
subtest.message = sanitizeUnpairedSurrogates(message);
}
this.subtests.push(subtest);
return subtest;
},
};
this.results.push(result);
return result;
}

write() {
this.time_end = Date.now();
this.results = this.results.filter((result) => {
return result.status === 'SKIP' || result.subtests.length !== 0;
}).map((result) => {
const url = new URL(result.test, 'http://wpt');
url.pathname = url.pathname.replace(/\.js$/, '.html');
result.test = url.href.slice(url.origin.length);
return result;
});

if (fs.existsSync('out/wpt/wptreport.json')) {
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
this.results = [...prev.results, ...this.results];
this.time_start = prev.time_start;
this.time_end = Math.max(this.time_end, prev.time_end);
this.run_info = prev.run_info;
} else {
/**
* Return required and some optional properties
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
*/
this.run_info = {
product: 'node.js',
...getBrowserProperties(),
revision: process.env.WPT_REVISION || 'unknown',
os: getOs(),
};
}

fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
}
}

// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
// TODO: get rid of this half-baked harness in favor of the one
// pulled from WPT
Expand Down Expand Up @@ -313,6 +423,10 @@ class WPTRunner {
this.unexpectedFailures = [];

this.scriptsModifier = null;

if (process.env.WPT_REPORT != null) {
this.report = new WPTReport();
}
}

/**
Expand All @@ -339,18 +453,27 @@ class WPTRunner {
this.scriptsModifier = modifier;
}

get fullInitScript() {
if (this.initScript === null && this.dummyGlobalThisScript === null) {
fullInitScript(hasSubsetScript, locationSearchString) {
let { initScript } = this;
if (hasSubsetScript || locationSearchString) {
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
}

if (locationSearchString) {
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
}

if (initScript === null && this.dummyGlobalThisScript === null) {
return null;
}

if (this.initScript === null) {
if (initScript === null) {
return this.dummyGlobalThisScript;
} else if (this.dummyGlobalThisScript === null) {
return this.initScript;
return initScript;
}

return `${this.dummyGlobalThisScript}\n\n//===\n${this.initScript}`;
return `${this.dummyGlobalThisScript}\n\n//===\n${initScript}`;
}

/**
Expand Down Expand Up @@ -398,15 +521,20 @@ class WPTRunner {
for (const spec of queue) {
const testFileName = spec.filename;
const content = spec.getContent();
const meta = spec.title = this.getMeta(content);
const meta = spec.meta = this.getMeta(content);

const absolutePath = spec.getAbsolutePath();
const relativePath = spec.getRelativePath();
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
const scriptsToRun = [];
let hasSubsetScript = false;

// Scripts specified with the `// META: script=` header
if (meta.script) {
for (const script of meta.script) {
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
hasSubsetScript = true;
}
const obj = {
filename: this.resource.toRealFilePath(relativePath, script),
code: this.resource.read(relativePath, script, false)
Expand All @@ -423,54 +551,65 @@ class WPTRunner {
this.scriptsModifier?.(obj);
scriptsToRun.push(obj);

const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript,
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
/**
* Example test with no META variant
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
*
* Example test with multiple META variants
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
*/
for (const variant of meta.variant || ['']) {
const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript(hasSubsetScript, variant),
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
},
scriptsToRun,
},
scriptsToRun,
},
});
this.workers.set(testFileName, worker);

worker.on('message', (message) => {
switch (message.type) {
case 'result':
return this.resultCallback(testFileName, message.result);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});
});
this.workers.set(testFileName, worker);

let reportResult;
worker.on('message', (message) => {
switch (message.type) {
case 'result':
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
return this.resultCallback(testFileName, message.result, reportResult);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});

worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
// The test is already finished. Ignore errors that occur after it.
// This can happen normally, for example in timers tests.
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err)
},
kUncaught
);
this.inProgress.delete(testFileName);
});
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err)
},
kUncaught
);
this.inProgress.delete(testFileName);
});

await events.once(worker, 'exit').catch(() => {});
await events.once(worker, 'exit').catch(() => {});
}
}

process.on('exit', () => {
Expand Down Expand Up @@ -529,6 +668,8 @@ class WPTRunner {
}
}

this.report?.write();

const ran = total - skipped;
const passed = ran - expectedFailures - failures.length;
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
Expand All @@ -552,8 +693,7 @@ class WPTRunner {

getTestTitle(filename) {
const spec = this.specMap.get(filename);
const title = spec.meta && spec.meta.title;
return title ? `${filename} : ${title}` : filename;
return spec.meta?.title || filename;
}

// Map WPT test status to strings
Expand All @@ -579,14 +719,14 @@ class WPTRunner {
* @param {string} filename
* @param {Test} test The Test object returned by WPT harness
*/
resultCallback(filename, test) {
resultCallback(filename, test, reportResult) {
const status = this.getTestStatus(test.status);
const title = this.getTestTitle(filename);
console.log(`---- ${title} ----`);
if (status !== kPass) {
this.fail(filename, test, status);
this.fail(filename, test, status, reportResult);
} else {
this.succeed(filename, test, status);
this.succeed(filename, test, status, reportResult);
}
}

Expand Down Expand Up @@ -634,11 +774,12 @@ class WPTRunner {
}
}

succeed(filename, test, status) {
succeed(filename, test, status, reportResult) {
console.log(`[${status.toUpperCase()}] ${test.name}`);
reportResult?.addSubtest(test.name, 'PASS');
}

fail(filename, test, status) {
fail(filename, test, status, reportResult) {
const spec = this.specMap.get(filename);
const expected = spec.failedTests.includes(test.name);
if (expected) {
Expand All @@ -654,6 +795,9 @@ class WPTRunner {
const command = `${process.execPath} ${process.execArgv}` +
` ${require.main.filename} ${filename}`;
console.log(`Command: ${command}\n`);

reportResult?.addSubtest(test.name, 'FAIL', test.message);

this.addTestResult(filename, {
name: test.name,
expected,
Expand Down Expand Up @@ -683,7 +827,7 @@ class WPTRunner {
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
const key = parts[1];
const value = parts[2];
if (key === 'script') {
if (key === 'script' || key === 'variant') {
if (result[key]) {
result[key].push(value);
} else {
Expand Down

0 comments on commit 088e9cd

Please sign in to comment.