-
Notifications
You must be signed in to change notification settings - Fork 30.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test_runner: add initial code coverage support
This commit adds code coverage functionality to the node:test module. When node:test is used in conjunction with the new --test-coverage CLI flag, a coverage report is created when the test runner finishes. The coverage summary is forwarded to any test runner reporters so that the display can be customized as desired. This new functionality is compatible with the existing NODE_V8_COVERAGE environment variable as well. There are still several limitations, which will be addressed in subsequent pull requests: - Coverage is only reported for a single process. It is possible to merge coverage reports together. Once this is done, the --test flag will be supported as well. - Source maps are not currently supported. - Excluding specific files or directories from the coverage report is not currently supported. Node core modules and node_modules/ are excluded though. PR-URL: #46017 Reviewed-By: Moshe Atlow <[email protected]> Reviewed-By: Geoffrey Booth <[email protected]>
1 parent
c9d5dfd
commit 6119289
Showing
16 changed files
with
796 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,371 @@ | ||
'use strict'; | ||
const { | ||
ArrayPrototypeMap, | ||
ArrayPrototypePush, | ||
JSONParse, | ||
MathFloor, | ||
NumberParseInt, | ||
RegExp, | ||
RegExpPrototypeExec, | ||
RegExpPrototypeSymbolSplit, | ||
StringPrototypeIncludes, | ||
StringPrototypeLocaleCompare, | ||
StringPrototypeStartsWith, | ||
} = primordials; | ||
const { | ||
copyFileSync, | ||
mkdirSync, | ||
mkdtempSync, | ||
opendirSync, | ||
readFileSync, | ||
} = require('fs'); | ||
const { setupCoverageHooks } = require('internal/util'); | ||
const { tmpdir } = require('os'); | ||
const { join, resolve } = require('path'); | ||
const { fileURLToPath } = require('url'); | ||
const kCoveragePattern = | ||
`^coverage\\-${process.pid}\\-(\\d{13})\\-(\\d+)\\.json$`; | ||
const kCoverageFileRegex = new RegExp(kCoveragePattern); | ||
const kIgnoreRegex = /\/\* node:coverage ignore next (?<count>\d+ )?\*\//; | ||
const kLineEndingRegex = /\r?\n$/u; | ||
const kLineSplitRegex = /(?<=\r?\n)/u; | ||
const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//; | ||
|
||
class CoverageLine { | ||
#covered; | ||
|
||
constructor(line, src, startOffset) { | ||
const newlineLength = | ||
RegExpPrototypeExec(kLineEndingRegex, src)?.[0].length ?? 0; | ||
|
||
this.line = line; | ||
this.src = src; | ||
this.startOffset = startOffset; | ||
this.endOffset = startOffset + src.length - newlineLength; | ||
this.ignore = false; | ||
this.#covered = true; | ||
} | ||
|
||
get covered() { | ||
return this.#covered; | ||
} | ||
|
||
set covered(isCovered) { | ||
// V8 can generate multiple ranges that span the same line. | ||
if (!this.#covered) { | ||
return; | ||
} | ||
|
||
this.#covered = isCovered; | ||
} | ||
} | ||
|
||
class TestCoverage { | ||
constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) { | ||
this.coverageDirectory = coverageDirectory; | ||
this.originalCoverageDirectory = originalCoverageDirectory; | ||
this.workingDirectory = workingDirectory; | ||
} | ||
|
||
summary() { | ||
internalBinding('profiler').takeCoverage(); | ||
const coverage = getCoverageFromDirectory(this.coverageDirectory); | ||
const coverageSummary = { | ||
__proto__: null, | ||
workingDirectory: this.workingDirectory, | ||
files: [], | ||
totals: { | ||
__proto__: null, | ||
totalLineCount: 0, | ||
totalBranchCount: 0, | ||
totalFunctionCount: 0, | ||
coveredLineCount: 0, | ||
coveredBranchCount: 0, | ||
coveredFunctionCount: 0, | ||
coveredLinePercent: 0, | ||
coveredBranchPercent: 0, | ||
coveredFunctionPercent: 0, | ||
}, | ||
}; | ||
|
||
if (!coverage) { | ||
return coverageSummary; | ||
} | ||
|
||
for (let i = 0; i < coverage.length; ++i) { | ||
const { functions, url } = coverage[i]; | ||
|
||
if (StringPrototypeStartsWith(url, 'node:') || | ||
StringPrototypeIncludes(url, '/node_modules/') || | ||
// On Windows some generated coverages are invalid. | ||
!StringPrototypeStartsWith(url, 'file:')) { | ||
continue; | ||
} | ||
|
||
// Split the file source into lines. Make sure the lines maintain their | ||
// original line endings because those characters are necessary for | ||
// determining offsets in the file. | ||
const filePath = fileURLToPath(url); | ||
const source = readFileSync(filePath, 'utf8'); | ||
const linesWithBreaks = | ||
RegExpPrototypeSymbolSplit(kLineSplitRegex, source); | ||
let ignoreCount = 0; | ||
let enabled = true; | ||
let offset = 0; | ||
let totalBranches = 0; | ||
let totalFunctions = 0; | ||
let branchesCovered = 0; | ||
let functionsCovered = 0; | ||
|
||
const lines = ArrayPrototypeMap(linesWithBreaks, (line, i) => { | ||
const startOffset = offset; | ||
const coverageLine = new CoverageLine(i + 1, line, startOffset); | ||
|
||
offset += line.length; | ||
|
||
// Determine if this line is being ignored. | ||
if (ignoreCount > 0) { | ||
ignoreCount--; | ||
coverageLine.ignore = true; | ||
} else if (!enabled) { | ||
coverageLine.ignore = true; | ||
} | ||
|
||
if (!coverageLine.ignore) { | ||
// If this line is not already being ignored, check for ignore | ||
// comments. | ||
const match = RegExpPrototypeExec(kIgnoreRegex, line); | ||
|
||
if (match !== null) { | ||
ignoreCount = NumberParseInt(match.groups?.count ?? 1, 10); | ||
} | ||
} | ||
|
||
// Check for comments to enable/disable coverage no matter what. These | ||
// take precedence over ignore comments. | ||
const match = RegExpPrototypeExec(kStatusRegex, line); | ||
const status = match?.groups?.status; | ||
|
||
if (status) { | ||
ignoreCount = 0; | ||
enabled = status === 'enable'; | ||
} | ||
|
||
return coverageLine; | ||
}); | ||
|
||
for (let j = 0; j < functions.length; ++j) { | ||
const { functionName, isBlockCoverage, ranges } = functions[j]; | ||
|
||
for (let k = 0; k < ranges.length; ++k) { | ||
const range = ranges[k]; | ||
|
||
mapRangeToLines(range, lines); | ||
|
||
if (isBlockCoverage) { | ||
if (range.count !== 0 || | ||
range.ignoredLines === range.lines.length) { | ||
branchesCovered++; | ||
} | ||
|
||
totalBranches++; | ||
} | ||
} | ||
|
||
if (functionName.length > 0 && ranges.length > 0) { | ||
const range = ranges[0]; | ||
|
||
if (range.count !== 0 || range.ignoredLines === range.lines.length) { | ||
functionsCovered++; | ||
} | ||
|
||
totalFunctions++; | ||
} | ||
} | ||
|
||
let coveredCnt = 0; | ||
const uncoveredLineNums = []; | ||
|
||
for (let j = 0; j < lines.length; ++j) { | ||
const line = lines[j]; | ||
|
||
if (line.covered || line.ignore) { | ||
coveredCnt++; | ||
} else { | ||
ArrayPrototypePush(uncoveredLineNums, line.line); | ||
} | ||
} | ||
|
||
ArrayPrototypePush(coverageSummary.files, { | ||
__proto__: null, | ||
path: filePath, | ||
totalLineCount: lines.length, | ||
totalBranchCount: totalBranches, | ||
totalFunctionCount: totalFunctions, | ||
coveredLineCount: coveredCnt, | ||
coveredBranchCount: branchesCovered, | ||
coveredFunctionCount: functionsCovered, | ||
coveredLinePercent: toPercentage(coveredCnt, lines.length), | ||
coveredBranchPercent: toPercentage(branchesCovered, totalBranches), | ||
coveredFunctionPercent: toPercentage(functionsCovered, totalFunctions), | ||
uncoveredLineNumbers: uncoveredLineNums, | ||
}); | ||
|
||
coverageSummary.totals.totalLineCount += lines.length; | ||
coverageSummary.totals.totalBranchCount += totalBranches; | ||
coverageSummary.totals.totalFunctionCount += totalFunctions; | ||
coverageSummary.totals.coveredLineCount += coveredCnt; | ||
coverageSummary.totals.coveredBranchCount += branchesCovered; | ||
coverageSummary.totals.coveredFunctionCount += functionsCovered; | ||
} | ||
|
||
coverageSummary.totals.coveredLinePercent = toPercentage( | ||
coverageSummary.totals.coveredLineCount, | ||
coverageSummary.totals.totalLineCount | ||
); | ||
coverageSummary.totals.coveredBranchPercent = toPercentage( | ||
coverageSummary.totals.coveredBranchCount, | ||
coverageSummary.totals.totalBranchCount | ||
); | ||
coverageSummary.totals.coveredFunctionPercent = toPercentage( | ||
coverageSummary.totals.coveredFunctionCount, | ||
coverageSummary.totals.totalFunctionCount | ||
); | ||
coverageSummary.files.sort(sortCoverageFiles); | ||
|
||
return coverageSummary; | ||
} | ||
|
||
cleanup() { | ||
// Restore the original value of process.env.NODE_V8_COVERAGE. Then, copy | ||
// all of the created coverage files to the original coverage directory. | ||
if (this.originalCoverageDirectory === undefined) { | ||
delete process.env.NODE_V8_COVERAGE; | ||
return; | ||
} | ||
|
||
process.env.NODE_V8_COVERAGE = this.originalCoverageDirectory; | ||
let dir; | ||
|
||
try { | ||
mkdirSync(this.originalCoverageDirectory, { recursive: true }); | ||
dir = opendirSync(this.coverageDirectory); | ||
|
||
for (let entry; (entry = dir.readSync()) !== null;) { | ||
const src = join(this.coverageDirectory, entry.name); | ||
const dst = join(this.originalCoverageDirectory, entry.name); | ||
copyFileSync(src, dst); | ||
} | ||
} finally { | ||
if (dir) { | ||
dir.closeSync(); | ||
} | ||
} | ||
} | ||
} | ||
|
||
function toPercentage(covered, total) { | ||
return total === 0 ? 100 : (covered / total) * 100; | ||
} | ||
|
||
function sortCoverageFiles(a, b) { | ||
return StringPrototypeLocaleCompare(a.path, b.path); | ||
} | ||
|
||
function setupCoverage() { | ||
let originalCoverageDirectory = process.env.NODE_V8_COVERAGE; | ||
const cwd = process.cwd(); | ||
|
||
if (originalCoverageDirectory) { | ||
// NODE_V8_COVERAGE was already specified. Convert it to an absolute path | ||
// and store it for later. The test runner will use a temporary directory | ||
// so that no preexisting coverage files interfere with the results of the | ||
// coverage report. Then, once the coverage is computed, move the coverage | ||
// files back to the original NODE_V8_COVERAGE directory. | ||
originalCoverageDirectory = resolve(cwd, originalCoverageDirectory); | ||
} | ||
|
||
const coverageDirectory = mkdtempSync(join(tmpdir(), 'node-coverage-')); | ||
const enabled = setupCoverageHooks(coverageDirectory); | ||
|
||
if (!enabled) { | ||
return null; | ||
} | ||
|
||
// Ensure that NODE_V8_COVERAGE is set so that coverage can propagate to | ||
// child processes. | ||
process.env.NODE_V8_COVERAGE = coverageDirectory; | ||
|
||
return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd); | ||
} | ||
|
||
function mapRangeToLines(range, lines) { | ||
const { startOffset, endOffset, count } = range; | ||
const mappedLines = []; | ||
let ignoredLines = 0; | ||
let start = 0; | ||
let end = lines.length; | ||
let mid; | ||
|
||
while (start <= end) { | ||
mid = MathFloor((start + end) / 2); | ||
let line = lines[mid]; | ||
|
||
if (startOffset >= line.startOffset && startOffset <= line.endOffset) { | ||
while (endOffset > line?.startOffset) { | ||
// If the range is not covered, and the range covers the entire line, | ||
// then mark that line as not covered. | ||
if (count === 0 && startOffset <= line.startOffset && | ||
endOffset >= line.endOffset) { | ||
line.covered = false; | ||
} | ||
|
||
ArrayPrototypePush(mappedLines, line); | ||
|
||
if (line.ignore) { | ||
ignoredLines++; | ||
} | ||
|
||
mid++; | ||
line = lines[mid]; | ||
} | ||
|
||
break; | ||
} else if (startOffset >= line.endOffset) { | ||
start = mid + 1; | ||
} else { | ||
end = mid - 1; | ||
} | ||
} | ||
|
||
// Add some useful data to the range. The test runner has read these ranges | ||
// from a file, so we own the data structures and can do what we want. | ||
range.lines = mappedLines; | ||
range.ignoredLines = ignoredLines; | ||
} | ||
|
||
function getCoverageFromDirectory(coverageDirectory) { | ||
// TODO(cjihrig): Instead of only reading the coverage file for this process, | ||
// combine all coverage files in the directory into a single data structure. | ||
let dir; | ||
|
||
try { | ||
dir = opendirSync(coverageDirectory); | ||
|
||
for (let entry; (entry = dir.readSync()) !== null;) { | ||
if (RegExpPrototypeExec(kCoverageFileRegex, entry.name) === null) { | ||
continue; | ||
} | ||
|
||
const coverageFile = join(coverageDirectory, entry.name); | ||
const coverage = JSONParse(readFileSync(coverageFile, 'utf8')); | ||
return coverage.result; | ||
} | ||
} finally { | ||
if (dir) { | ||
dir.closeSync(); | ||
} | ||
} | ||
} | ||
|
||
module.exports = { setupCoverage }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
'use strict'; | ||
const http = require('node:http'); // node:* should be filtered out. | ||
|
||
try { | ||
// This block does not throw. | ||
} catch { /* node:coverage ignore next 3 */ | ||
// So this block is uncovered. | ||
|
||
/* node:coverage disable */ | ||
|
||
/* node:coverage enable */ | ||
|
||
/* node:coverage ignore next */ | ||
} | ||
|
||
function uncalledTopLevelFunction() { | ||
if (true) { | ||
return 5; | ||
} | ||
|
||
return 9; | ||
} | ||
|
||
const test = require('node:test'); | ||
|
||
if (false) { | ||
console.log('this does not execute'); | ||
} else { | ||
require('./invalid-tap.js'); | ||
} | ||
|
||
test('a test', function testHandler() { | ||
const uncalled = () => {}; | ||
|
||
function fnWithControlFlow(val) { | ||
if (val < 0) { | ||
return -1; | ||
} else if (val === 0) { | ||
return 0; | ||
} else if (val < 100) { | ||
return 1; | ||
} else { | ||
return Infinity; | ||
} | ||
} | ||
|
||
fnWithControlFlow(-10); | ||
fnWithControlFlow(99); | ||
}); | ||
|
||
try { | ||
require('test-nm'); // node_modules should be filtered out. | ||
} catch { | ||
|
||
} | ||
|
||
async function main() { | ||
if (false) { console.log('boo'); } else { /* console.log('yay'); */ } | ||
|
||
if (false) { | ||
console.log('not happening'); | ||
} | ||
|
||
if (true) { | ||
if (false) { | ||
console.log('not printed'); | ||
} | ||
|
||
// Nothing to see here. | ||
} else { | ||
console.log('also not printed'); | ||
} | ||
|
||
try { | ||
const foo = {}; | ||
|
||
foo.x = 1; | ||
require('../v8-coverage/throw.js'); | ||
foo.y = 2; | ||
return foo; | ||
} catch (err) { | ||
let bar = []; | ||
bar.push(5); | ||
} finally { | ||
const baz = 1; | ||
} | ||
} | ||
|
||
main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
'use strict'; | ||
const common = require('../common'); | ||
const assert = require('node:assert'); | ||
const { spawnSync } = require('node:child_process'); | ||
const { readdirSync } = require('node:fs'); | ||
const { test } = require('node:test'); | ||
const fixtures = require('../common/fixtures'); | ||
const tmpdir = require('../common/tmpdir'); | ||
|
||
tmpdir.refresh(); | ||
|
||
function findCoverageFileForPid(pid) { | ||
const pattern = `^coverage\\-${pid}\\-(\\d{13})\\-(\\d+)\\.json$`; | ||
const regex = new RegExp(pattern); | ||
|
||
return readdirSync(tmpdir.path).find((file) => { | ||
return regex.test(file); | ||
}); | ||
} | ||
|
||
function getCoverageFixtureReport() { | ||
const report = [ | ||
'# start of coverage report', | ||
'# file | line % | branch % | funcs % | uncovered lines', | ||
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12, ' + | ||
'13, 16, 17, 18, 19, 20, 21, 22, 27, 39, 43, 44, 61, 62, 66, 67, 71, 72', | ||
'# test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', | ||
'# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5, 6', | ||
'# all files | 78.35 | 43.75 | 60.00 |', | ||
'# end of coverage report', | ||
].join('\n'); | ||
|
||
if (common.isWindows) { | ||
return report.replaceAll('/', '\\'); | ||
} | ||
|
||
return report; | ||
} | ||
|
||
test('--test-coverage and --test cannot be combined', () => { | ||
// TODO(cjihrig): This test can be removed once multi-process code coverage | ||
// is supported. | ||
const result = spawnSync(process.execPath, ['--test', '--test-coverage']); | ||
|
||
// 9 is the documented exit code for an invalid CLI argument. | ||
assert.strictEqual(result.status, 9); | ||
assert.match( | ||
result.stderr.toString(), /--test-coverage cannot be used with --test/ | ||
); | ||
}); | ||
|
||
test('handles the inspector not being available', (t) => { | ||
if (process.features.inspector) { | ||
return; | ||
} | ||
|
||
const fixture = fixtures.path('test-runner', 'coverage.js'); | ||
const args = ['--test-coverage', fixture]; | ||
const result = spawnSync(process.execPath, args); | ||
|
||
assert(!result.stdout.toString().includes('# start of coverage report')); | ||
assert(result.stderr.toString().includes('coverage could not be collected')); | ||
assert.strictEqual(result.status, 0); | ||
assert(!findCoverageFileForPid(result.pid)); | ||
}); | ||
|
||
test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => { | ||
if (!process.features.inspector) { | ||
return; | ||
} | ||
|
||
const fixture = fixtures.path('test-runner', 'coverage.js'); | ||
const args = ['--test-coverage', fixture]; | ||
const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; | ||
const result = spawnSync(process.execPath, args, options); | ||
const report = getCoverageFixtureReport(); | ||
|
||
assert(result.stdout.toString().includes(report)); | ||
assert.strictEqual(result.stderr.toString(), ''); | ||
assert.strictEqual(result.status, 0); | ||
assert(findCoverageFileForPid(result.pid)); | ||
}); | ||
|
||
test('coverage is reported without NODE_V8_COVERAGE present', (t) => { | ||
if (!process.features.inspector) { | ||
return; | ||
} | ||
|
||
const fixture = fixtures.path('test-runner', 'coverage.js'); | ||
const args = ['--test-coverage', fixture]; | ||
const result = spawnSync(process.execPath, args); | ||
const report = getCoverageFixtureReport(); | ||
|
||
assert(result.stdout.toString().includes(report)); | ||
assert.strictEqual(result.stderr.toString(), ''); | ||
assert.strictEqual(result.status, 0); | ||
assert(!findCoverageFileForPid(result.pid)); | ||
}); |