From 4ada7ce43d6c1de5d2565db86bfc972c15e51d56 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 3 Feb 2023 15:19:55 +0100 Subject: [PATCH 1/2] test: add WPTRunner support for variants and generating WPT reports --- Makefile | 6 + test/common/wpt.js | 260 ++++++++++++++++++++++++++++--------- test/wpt/test-encoding.js | 1 - test/wpt/test-webcrypto.js | 4 - 4 files changed, 208 insertions(+), 63 deletions(-) diff --git a/Makefile b/Makefile index 7648063958a199..830413fdba81da 100644 --- a/Makefile +++ b/Makefile @@ -595,6 +595,12 @@ test-message: test-build test-wpt: all $(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt +.PHONY: test-wpt-report +test-wpt-report: + $(RM) -r out/wpt + mkdir -p out/wpt + WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt + .PHONY: test-simple test-simple: | cctest # Depends on 'all'. $(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential diff --git a/test/common/wpt.js b/test/common/wpt.js index dcf5d943416950..8a4482de7142af 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -6,9 +6,119 @@ const fs = require('fs'); const fsPromises = fs.promises; const path = require('path'); const events = require('events'); +const os = require('os'); const { inspect } = require('util'); const { Worker } = require('worker_threads'); +function getBrowserProperties() { + const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481 + const release = /^\d+\.\d+\.\d+$/.test(version); + const browser = { + browser_channel: release ? 'stable' : 'experimental', + browser_version: version, + }; + + return browser; +} + +/** + * Return one of three expected values + * https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958 + */ +function getOs() { + switch (os.type()) { + case 'Linux': + return 'linux'; + case 'Darwin': + return 'mac'; + case 'Windows_NT': + return 'win'; + default: + throw new Error('Unsupported os.type()'); + } +} + +// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705 +function sanitizeUnpairedSurrogates(str) { + return str.replace( + /([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g, + function(_, low, prefix, high) { + let output = prefix || ''; // Prefix may be undefined + const string = low || high; // Only one of these alternates can match + for (let i = 0; i < string.length; i++) { + output += codeUnitStr(string[i]); + } + return output; + }); +} + +function codeUnitStr(char) { + return 'U+' + char.charCodeAt(0).toString(16); +} + +class WPTReport { + constructor() { + this.results = []; + this.time_start = Date.now(); + } + + addResult(name, status) { + const result = { + test: name, + status, + subtests: [], + addSubtest(name, status, message) { + const subtest = { + status, + // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722 + name: sanitizeUnpairedSurrogates(name), + }; + if (message) { + // https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506 + subtest.message = sanitizeUnpairedSurrogates(message); + } + this.subtests.push(subtest); + return subtest; + }, + }; + this.results.push(result); + return result; + } + + write() { + this.time_end = Date.now(); + this.results = this.results.filter((result) => { + return result.status === 'SKIP' || result.subtests.length !== 0; + }).map((result) => { + const url = new URL(result.test, 'http://wpt'); + url.pathname = url.pathname.replace(/\.js$/, '.html'); + result.test = url.href.slice(url.origin.length); + return result; + }); + + if (fs.existsSync('out/wpt/wptreport.json')) { + const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json')); + this.results = [...prev.results, ...this.results]; + this.time_start = prev.time_start; + this.time_end = Math.max(this.time_end, prev.time_end); + this.run_info = prev.run_info; + } else { + /** + * Return required and some optional properties + * https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335 + */ + this.run_info = { + product: 'node.js', + ...getBrowserProperties(), + revision: process.env.WPT_REVISION || 'unknown', + os: getOs(), + }; + } + + fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this)); + } +} + // https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js // TODO: get rid of this half-baked harness in favor of the one // pulled from WPT @@ -313,6 +423,10 @@ class WPTRunner { this.unexpectedFailures = []; this.scriptsModifier = null; + + if (process.env.WPT_REPORT != null) { + this.report = new WPTReport(); + } } /** @@ -339,18 +453,27 @@ class WPTRunner { this.scriptsModifier = modifier; } - get fullInitScript() { + fullInitScript(hasSubsetScript, locationSearchString) { + let { initScript } = this; + if (hasSubsetScript || locationSearchString) { + initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`; + } + + if (locationSearchString) { + initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`; + } + if (this.globalThisInitScripts.length === null) { - return this.initScript; + return initScript; } const globalThisInitScript = this.globalThisInitScripts.join('\n\n//===\n'); - if (this.initScript === null) { + if (initScript === null) { return globalThisInitScript; } - return `${globalThisInitScript}\n\n//===\n${this.initScript}`; + return `${globalThisInitScript}\n\n//===\n${initScript}`; } /** @@ -455,15 +578,20 @@ class WPTRunner { for (const spec of queue) { const testFileName = spec.filename; const content = spec.getContent(); - const meta = spec.title = this.getMeta(content); + const meta = spec.meta = this.getMeta(content); const absolutePath = spec.getAbsolutePath(); const relativePath = spec.getRelativePath(); const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js'); const scriptsToRun = []; + let hasSubsetScript = false; + // Scripts specified with the `// META: script=` header if (meta.script) { for (const script of meta.script) { + if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') { + hasSubsetScript = true; + } const obj = { filename: this.resource.toRealFilePath(relativePath, script), code: this.resource.read(relativePath, script, false), @@ -480,54 +608,65 @@ class WPTRunner { this.scriptsModifier?.(obj); scriptsToRun.push(obj); - const workerPath = path.join(__dirname, 'wpt/worker.js'); - const worker = new Worker(workerPath, { - execArgv: this.flags, - workerData: { - testRelativePath: relativePath, - wptRunner: __filename, - wptPath: this.path, - initScript: this.fullInitScript, - harness: { - code: fs.readFileSync(harnessPath, 'utf8'), - filename: harnessPath, + /** + * Example test with no META variant + * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4 + * + * Example test with multiple META variants + * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9 + */ + for (const variant of meta.variant || ['']) { + const workerPath = path.join(__dirname, 'wpt/worker.js'); + const worker = new Worker(workerPath, { + execArgv: this.flags, + workerData: { + testRelativePath: relativePath, + wptRunner: __filename, + wptPath: this.path, + initScript: this.fullInitScript(hasSubsetScript, variant), + harness: { + code: fs.readFileSync(harnessPath, 'utf8'), + filename: harnessPath, + }, + scriptsToRun, }, - scriptsToRun, - }, - }); - this.workers.set(testFileName, worker); - - worker.on('message', (message) => { - switch (message.type) { - case 'result': - return this.resultCallback(testFileName, message.result); - case 'completion': - return this.completionCallback(testFileName, message.status); - default: - throw new Error(`Unexpected message from worker: ${message.type}`); - } - }); + }); + this.workers.set(testFileName, worker); + + let reportResult; + worker.on('message', (message) => { + switch (message.type) { + case 'result': + reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK'); + return this.resultCallback(testFileName, message.result, reportResult); + case 'completion': + return this.completionCallback(testFileName, message.status); + default: + throw new Error(`Unexpected message from worker: ${message.type}`); + } + }); - worker.on('error', (err) => { - if (!this.inProgress.has(testFileName)) { - // The test is already finished. Ignore errors that occur after it. - // This can happen normally, for example in timers tests. - return; - } - this.fail( - testFileName, - { - status: NODE_UNCAUGHT, - name: 'evaluation in WPTRunner.runJsTests()', - message: err.message, - stack: inspect(err), - }, - kUncaught, - ); - this.inProgress.delete(testFileName); - }); + worker.on('error', (err) => { + if (!this.inProgress.has(testFileName)) { + // The test is already finished. Ignore errors that occur after it. + // This can happen normally, for example in timers tests. + return; + } + this.fail( + testFileName, + { + status: NODE_UNCAUGHT, + name: 'evaluation in WPTRunner.runJsTests()', + message: err.message, + stack: inspect(err), + }, + kUncaught, + ); + this.inProgress.delete(testFileName); + }); - await events.once(worker, 'exit').catch(() => {}); + await events.once(worker, 'exit').catch(() => {}); + } } process.on('exit', () => { @@ -587,6 +726,8 @@ class WPTRunner { } } + this.report?.write(); + const ran = queue.length; const total = ran + skipped; const passed = ran - expectedFailures - failures.length; @@ -611,8 +752,7 @@ class WPTRunner { getTestTitle(filename) { const spec = this.specMap.get(filename); - const title = spec.meta && spec.meta.title; - return title ? `${filename} : ${title}` : filename; + return spec.meta?.title || filename; } // Map WPT test status to strings @@ -638,14 +778,14 @@ class WPTRunner { * @param {string} filename * @param {Test} test The Test object returned by WPT harness */ - resultCallback(filename, test) { + resultCallback(filename, test, reportResult) { const status = this.getTestStatus(test.status); const title = this.getTestTitle(filename); console.log(`---- ${title} ----`); if (status !== kPass) { - this.fail(filename, test, status); + this.fail(filename, test, status, reportResult); } else { - this.succeed(filename, test, status); + this.succeed(filename, test, status, reportResult); } } @@ -693,11 +833,12 @@ class WPTRunner { } } - succeed(filename, test, status) { + succeed(filename, test, status, reportResult) { console.log(`[${status.toUpperCase()}] ${test.name}`); + reportResult?.addSubtest(test.name, 'PASS'); } - fail(filename, test, status) { + fail(filename, test, status, reportResult) { const spec = this.specMap.get(filename); const expected = spec.failedTests.includes(test.name); if (expected) { @@ -713,6 +854,9 @@ class WPTRunner { const command = `${process.execPath} ${process.execArgv}` + ` ${require.main.filename} ${filename}`; console.log(`Command: ${command}\n`); + + reportResult?.addSubtest(test.name, 'FAIL', test.message); + this.addTestResult(filename, { name: test.name, expected, @@ -742,7 +886,7 @@ class WPTRunner { const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/); const key = parts[1]; const value = parts[2]; - if (key === 'script') { + if (key === 'script' || key === 'variant') { if (result[key]) { result[key].push(value); } else { diff --git a/test/wpt/test-encoding.js b/test/wpt/test-encoding.js index 901f53e00e22f0..b297648d9cb7ee 100644 --- a/test/wpt/test-encoding.js +++ b/test/wpt/test-encoding.js @@ -4,7 +4,6 @@ const { WPTRunner } = require('../common/wpt'); const runner = new WPTRunner('encoding'); runner.setInitScript(` - globalThis.location ||= {}; const { MessageChannel } = require('worker_threads'); global.MessageChannel = MessageChannel; `); diff --git a/test/wpt/test-webcrypto.js b/test/wpt/test-webcrypto.js index fc253ca05d6783..0d53a51901bbb9 100644 --- a/test/wpt/test-webcrypto.js +++ b/test/wpt/test-webcrypto.js @@ -8,10 +8,6 @@ const { WPTRunner } = require('../common/wpt'); const runner = new WPTRunner('WebCryptoAPI'); -runner.setInitScript(` - global.location = {}; -`); - runner.pretendGlobalThisAs('Window'); runner.runJsTests(); From 3b07a30e4253f23f13b8e1a1184fdea0eb68ad61 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Sat, 4 Feb 2023 09:53:05 +0100 Subject: [PATCH 2/2] tools: add a daily wpt.fyi synchronized report upload --- .github/workflows/daily-wpt-fyi.yml | 110 ++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 .github/workflows/daily-wpt-fyi.yml diff --git a/.github/workflows/daily-wpt-fyi.yml b/.github/workflows/daily-wpt-fyi.yml new file mode 100644 index 00000000000000..0240dd6d404814 --- /dev/null +++ b/.github/workflows/daily-wpt-fyi.yml @@ -0,0 +1,110 @@ +# This workflow runs every night and tests various releases of Node.js +# (latest nightly, current, and two latest LTS release lines) against the +# `epochs/daily` branch of WPT. + +name: Daily WPT report + +on: + workflow_dispatch: + inputs: + node-versions: + description: Node.js versions (as supported by actions/setup-node) to test as JSON array + required: false + default: '["current", "lts/*", "lts/-1"]' + schedule: + # This is 20 minutes after `epochs/daily` branch is triggered to be created + # in WPT repo. + # https://github.com/web-platform-tests/wpt/blob/master/.github/workflows/epochs.yml + - cron: 30 0 * * * + +env: + PYTHON_VERSION: '3.11' + +permissions: + contents: read + +jobs: + report: + if: github.repository == 'nodejs/node' || github.event_name == 'workflow_dispatch' + strategy: + matrix: + node-version: ${{ fromJSON(github.event.inputs.node-versions || '["latest-nightly", "current", "lts/*", "lts/-1"]') }} + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Environment Information + run: npx envinfo + + # install a version and checkout + - name: Get latest nightly + if: matrix.node-version == 'latest-nightly' + run: echo "NIGHTLY=$(curl -s https://nodejs.org/download/nightly/index.json | jq -r '.[0].version')" >> $GITHUB_ENV + - name: Install Node.js + id: setup-node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NIGHTLY || matrix.node-version }} + - name: Get nightly ref + if: contains(matrix.node-version, 'nightly') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + SHORT_SHA=$(node -p 'process.version.split(/-nightly\d{8}/)[1]') + echo "NIGHTLY_REF=$(gh api /repos/nodejs/node/commits/$SHORT_SHA | jq -r '.sha')" >> $GITHUB_ENV + - name: Checkout ${{ steps.setup-node.outputs.node-version }} + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ env.NIGHTLY_REF || steps.setup-node.outputs.node-version }} + - name: Set env.NODE + run: echo "NODE=$(which node)" >> $GITHUB_ENV + + # replace checked out WPT with the synchronized branch + - name: Remove stale WPT + run: rm -rf wpt + working-directory: test/fixtures + - name: Checkout epochs/daily WPT + uses: actions/checkout@v3 + with: + repository: web-platform-tests/wpt + persist-credentials: false + path: test/fixtures/wpt + clean: false + ref: epochs/daily + - name: Set env.WPT_REVISION + run: echo "WPT_REVISION=$(git rev-parse HEAD)" >> $GITHUB_ENV + working-directory: test/fixtures/wpt + + - name: Run WPT and generate report + run: make test-wpt-report || true + - name: Clone report for upload + run: | + if [ -e out/wpt/wptreport.json ]; then + cd out/wpt + cp wptreport.json wptreport-${{ steps.setup-node.outputs.node-version }}.json + fi + - name: Upload GitHub Actions artifact + uses: actions/upload-artifact@v3 + with: + path: out/wpt/wptreport-*.json + name: WPT Reports + if-no-files-found: warn + - name: Upload WPT Report to wpt.fyi API + env: + WPT_FYI_ENDPOINT: ${{ vars.WPT_FYI_ENDPOINT }} + WPT_FYI_USERNAME: ${{ vars.WPT_FYI_USERNAME }} + WPT_FYI_PASSWORD: ${{ secrets.WPT_FYI_PASSWORD }} + run: | + if [ -e out/wpt/wptreport.json ]; then + cd out/wpt + gzip wptreport.json + curl \ + -u "$WPT_FYI_USERNAME:$WPT_FYI_PASSWORD" \ + -F "result_file=@wptreport.json.gz" \ + -F "labels=master" \ + $WPT_FYI_ENDPOINT + fi