From 5adeea49a007f06d8ebeda1a51129505a55650a4 Mon Sep 17 00:00:00 2001 From: ConnectDotz Date: Mon, 29 Jun 2020 22:22:18 -0400 Subject: [PATCH] added a context-based matching algorithm (#593) --- CHANGELOG.md | 1 + jest.config.js | 3 +- package.json | 2 +- src/JestExt.ts | 2 +- src/TestResults/TestResult.ts | 17 +- src/TestResults/TestResultProvider.ts | 153 +------ src/TestResults/match-by-context.ts | 258 +++++++++++ src/TestResults/match-node.ts | 125 +++++ tests/TestResults/TestResultProvider.test.ts | 451 ++++++++----------- tests/TestResults/match-by-context.test.ts | 287 ++++++++++++ tests/diagnostics.test.ts | 36 +- tests/test-helper.ts | 68 +++ yarn.lock | 24 +- 13 files changed, 983 insertions(+), 444 deletions(-) create mode 100644 src/TestResults/match-by-context.ts create mode 100644 src/TestResults/match-node.ts create mode 100644 tests/TestResults/match-by-context.test.ts create mode 100644 tests/test-helper.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d1733764..4350fdcf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Bug-fixes within the same version aren't needed * upgrade all dependencies to the latest, except istanbul-lib-xxx, which requires more code change and will be handled in a separate coverage PR. - @connectdotz * code base clean up: migrate from tslint to eslint and adopted the latest recommended coding style, adopt semi-colon, added more ci check... - @connectdotz * resolve coverage map merge issue, upgrade istanbul dependencies to the latest and move to async coverageMap update. @connectdotz +* introducing a new matching algorithm to use context relationship in matching test blocks and assertions - @connectdotz --> ### 3.2.0 diff --git a/jest.config.js b/jest.config.js index c5050e7ab..9428ac266 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,7 +1,8 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - testRegex: 'tests/.*\\.ts$', + testRegex: 'tests/.*\\.test\\.ts$', + collectCoverageFrom: ['src/**/*.ts'], automock: true, moduleFileExtensions: ['ts', 'js', 'json'], unmockedModulePathPatterns: [ diff --git a/package.json b/package.json index 231bbee88..144eddfb7 100644 --- a/package.json +++ b/package.json @@ -279,7 +279,7 @@ "dependencies": { "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-source-maps": "^4.0.0", - "jest-editor-support": "^27.2.0", + "jest-editor-support": "^28.0.0-beta", "jest-snapshot": "^25.5.0" }, "devDependencies": { diff --git a/src/JestExt.ts b/src/JestExt.ts index b0af914ce..a1e7d6f8d 100644 --- a/src/JestExt.ts +++ b/src/JestExt.ts @@ -204,7 +204,7 @@ export class JestExt { this.pluginSettings = updatedSettings; this.jestWorkspace.rootPath = updatedSettings.rootPath; - this.jestWorkspace.pathToJest = pathToJest(updatedSettings); + this.jestWorkspace.jestCommandLine = pathToJest(updatedSettings); this.jestWorkspace.pathToConfig = pathToConfig(updatedSettings); // debug diff --git a/src/TestResults/TestResult.ts b/src/TestResults/TestResult.ts index 8d2eeab7e..75171b919 100644 --- a/src/TestResults/TestResult.ts +++ b/src/TestResults/TestResult.ts @@ -4,7 +4,7 @@ import { FileCoverage } from 'istanbul-lib-coverage'; import * as path from 'path'; import { cleanAnsi } from '../helpers'; -interface Position { +export interface Location { /** Zero-based column number */ column: number; @@ -12,10 +12,13 @@ interface Position { line: number; } -export interface TestResult { +export interface LocationRange { + start: Location; + end: Location; +} + +export interface TestResult extends LocationRange { name: string; - start: Position; - end: Position; status: TestReconciliationState; shortMessage?: string; @@ -88,7 +91,9 @@ export const coverageMapWithLowerCaseWindowsDriveLetters = (data: JestTotalResul * * @param data Parsed JSON results */ -export const resultsWithLowerCaseWindowsDriveLetters = (data: JestTotalResults) => { +export const resultsWithLowerCaseWindowsDriveLetters = ( + data: JestTotalResults +): JestTotalResults => { if (path.sep === '\\') { return { ...data, @@ -103,7 +108,7 @@ export const resultsWithLowerCaseWindowsDriveLetters = (data: JestTotalResults) /** * Removes ANSI escape sequence characters from test results in order to get clean messages */ -export const resultsWithoutAnsiEscapeSequence = (data: JestTotalResults) => { +export const resultsWithoutAnsiEscapeSequence = (data: JestTotalResults): JestTotalResults => { if (!data || !data.testResults) { return data; } diff --git a/src/TestResults/TestResultProvider.ts b/src/TestResults/TestResultProvider.ts index 235f865f7..fd47c42b3 100644 --- a/src/TestResults/TestResultProvider.ts +++ b/src/TestResults/TestResultProvider.ts @@ -1,13 +1,8 @@ -import { - TestReconciler, - JestTotalResults, - TestAssertionStatus, - TestFileAssertionStatus, - ItBlock, -} from 'jest-editor-support'; +import { TestReconciler, JestTotalResults, TestFileAssertionStatus } from 'jest-editor-support'; import { TestReconciliationState } from './TestReconciliationState'; import { TestResult } from './TestResult'; import { parseTest } from '../TestParser'; +import * as match from './match-by-context'; interface TestResultsMap { [filePath: string]: TestResult[]; @@ -24,9 +19,6 @@ interface SortedTestResultsMap { [filePath: string]: SortedTestResults; } -type IsMatched = (test: ItBlock, assertion: TestAssertionStatus) => boolean; -type OnMatchError = (test: ItBlock, matched: TestAssertionStatus[]) => string | undefined; - export class TestResultProvider { verbose: boolean; private reconciler: TestReconciler; @@ -39,141 +31,40 @@ export class TestResultProvider { this.verbose = verbose; } - resetCache() { + resetCache(): void { this.resultsByFilePath = {}; this.sortedResultsByFilePath = {}; } getResults(filePath: string): TestResult[] { - const toMatchResult = (test: ItBlock, assertion?: TestAssertionStatus, err?: string) => ({ - // Note the shift from one-based to zero-based line number and columns - name: test.name, - start: { - column: test.start.column - 1, - line: test.start.line - 1, - }, - end: { - column: test.end.column - 1, - line: test.end.line - 1, - }, - - status: assertion ? assertion.status : TestReconciliationState.Unknown, - shortMessage: assertion ? assertion.shortMessage : err, - terseMessage: assertion ? assertion.terseMessage : undefined, - lineNumberOfError: - assertion && - assertion.line && - assertion.line >= test.start.line && - assertion.line <= test.end.line - ? assertion.line - 1 - : test.end.line - 1, - }); - - const matchTests = ( - _itBlocks: ItBlock[], - _assertions: TestAssertionStatus[], - _isMatched: IsMatched[], - _onMatchError?: OnMatchError, - trackRemaining?: boolean - ): [TestResult[], ItBlock[], TestAssertionStatus[]] => { - const results: TestResult[] = []; - const remainingAssertions = Array.from(_assertions); - const remainingTests: ItBlock[] = []; - const _trackRemaining = trackRemaining === undefined ? true : trackRemaining; - - _itBlocks.forEach((test) => { - const matched = remainingAssertions.filter((a) => _isMatched.every((m) => m(test, a))); - if (matched.length === 1) { - const aIndex = remainingAssertions.indexOf(matched[0]); - if (aIndex < 0) { - throw new Error(`can't find assertion in the list`); - } - results.push(toMatchResult(test, matched[0])); - if (_trackRemaining) { - remainingAssertions.splice(aIndex, 1); - } - return; - } - - let err: string; - if (_onMatchError) { - err = _onMatchError(test, matched); - } - // if there is an error string, create a test result with it - if (err) { - results.push(toMatchResult(test, undefined, err)); - return; - } - - if (_trackRemaining) { - remainingTests.push(test); - } - }); - return [results, remainingTests, remainingAssertions]; - }; - if (this.resultsByFilePath[filePath]) { return this.resultsByFilePath[filePath]; } - const matchPos = (t: ItBlock, a: TestAssertionStatus): boolean => - (a.line !== undefined && a.line >= t.start.line && a.line <= t.end.line) || - (a.location && a.location.line >= t.start.line && a.location.line <= t.end.line); - const matchName = (t: ItBlock, a: TestAssertionStatus): boolean => t.name === a.title; - const templateLiteralPattern = /\${.*?}/; // template literal pattern - const matchTemplateLiteral = (t: ItBlock, a: TestAssertionStatus): boolean => { - if (!t.name.match(templateLiteralPattern)) { - return false; - } - const parts = t.name.split(templateLiteralPattern); - const r = parts.every((p) => a.title.includes(p)); - return r; - }; - const onMatchError: OnMatchError = (t: ItBlock, match: TestAssertionStatus[]) => { - let err: string; - if (match.length <= 0 && t.name.match(templateLiteralPattern)) { - err = 'no test result found, could be caused by template literals?'; - } - if (match.length > 1) { - err = - 'found multiple potential matches, could be caused by duplicate test names or template literals?'; - } - if (err && this.verbose) { - // tslint:disable-next-line: no-console - console.log(`'${t.name}' failed to find test result: ${err}`); - } - return err; - }; - - let { itBlocks } = parseTest(filePath); - let assertions = this.reconciler.assertionsForTestFile(filePath) || []; - const totalResult: TestResult[] = []; + let matchResult: TestResult[] = []; - if (assertions.length > 0 && itBlocks.length > 0) { - const algorithms: Array<[IsMatched[], OnMatchError]> = [ - [[matchName, matchPos], undefined], - [[matchTemplateLiteral, matchPos], undefined], - [[matchTemplateLiteral], undefined], - [[matchName], onMatchError], - ]; - for (const [matchers, onError] of algorithms) { - let result: TestResult[]; - [result, itBlocks, assertions] = matchTests(itBlocks, assertions, matchers, onError); - totalResult.push(...result); - if (itBlocks.length <= 0 || assertions.length <= 0) { - break; + try { + const assertions = this.reconciler.assertionsForTestFile(filePath); + if (!assertions) { + if (this.verbose) { + console.log(`no assertion found, perhaps not a test file? '${filePath}'`); } + } else if (assertions.length <= 0) { + // no assertion, all tests are unknown + const { itBlocks } = parseTest(filePath); + matchResult = itBlocks.map((t) => match.toMatchResult(t, 'no assertion found')); + } else { + const { root } = parseTest(filePath); + matchResult = match.matchTestAssertions(filePath, root, assertions, this.verbose); } + } catch (e) { + console.warn(`failed to get test result for ${filePath}:`, e); } - - // convert remaining itBlocks to unmatched result - itBlocks.forEach((t) => totalResult.push(toMatchResult(t))); - - this.resultsByFilePath[filePath] = totalResult; - return totalResult; + this.resultsByFilePath[filePath] = matchResult; + return matchResult; } - getSortedResults(filePath: string) { + getSortedResults(filePath: string): SortedTestResults { if (this.sortedResultsByFilePath[filePath]) { return this.sortedResultsByFilePath[filePath]; } @@ -207,7 +98,7 @@ export class TestResultProvider { return this.reconciler.updateFileWithJestStatus(data); } - removeCachedResults(filePath: string) { + removeCachedResults(filePath: string): void { this.resultsByFilePath[filePath] = null; this.sortedResultsByFilePath[filePath] = null; } diff --git a/src/TestResults/match-by-context.ts b/src/TestResults/match-by-context.ts new file mode 100644 index 000000000..0a8ef7b06 --- /dev/null +++ b/src/TestResults/match-by-context.ts @@ -0,0 +1,258 @@ +/** + * matching tests with assertion by its `context`, i.e. describe/test block structure + * as well as sequence (by relative line position) in its context. + * + * The assumption is both source parser and jest have generated correct output, + * while the names of the test might not always match (such as template-literal, jest.each use cases), + * nor the line numbers (due to source map difference and other transpile complication), + * the relative position should be the same, i.e. the first assertion in describe-block X + * should always match the first test block under the same describe block and so on. + */ + +import { + ItBlock, + TestAssertionStatus, + ParsedNode, + DescribeBlock, + Location, +} from 'jest-editor-support'; +import { TestReconciliationState } from './TestReconciliationState'; +import { TestResult } from './TestResult'; +import { DataNode, ContainerNode, ContextType, ChildNodeType } from './match-node'; + +const ROOT_NODE_NAME = '__root__'; +export const buildAssertionContainer = ( + assertions: TestAssertionStatus[] +): ContainerNode => { + const root = new ContainerNode(ROOT_NODE_NAME); + if (assertions.length > 0) { + assertions.forEach((a) => { + const container = root.findContainer(a.ancestorTitles, true); + container.addDataNode(new DataNode(a.title, a.location?.line ?? 0, a)); + }); + // group by line since there could be multiple assertions for the same test block, such + // as in the jest.each use case + root.sort(true); + } + return root; +}; + +export const buildSourceContainer = (sourceRoot: ParsedNode): ContainerNode => { + const isDescribeBlock = (node: ParsedNode): node is DescribeBlock => node.type === 'describe'; + const isItBlock = (node: ParsedNode): node is ItBlock => node.type === 'it'; + const buildNode = (node: ParsedNode, parent: ContainerNode): void => { + let container = parent; + if (isDescribeBlock(node)) { + container = new ContainerNode(node.name); + parent.addContainerNode(container); + } else if (isItBlock(node)) { + parent.addDataNode(new DataNode(node.name, node.start.line - 1, node)); + } + + node.children?.forEach((n) => buildNode(n, container)); + }; + + const root = new ContainerNode(ROOT_NODE_NAME); + buildNode(sourceRoot, root); + // do not need to do grouping since there can't be test blocks that share lines + root.sort(false); + return root; +}; + +const matchPos = (t: ItBlock, a?: TestAssertionStatus, forError = false): boolean => { + const line = forError ? a?.line : a?.line ?? a.location?.line; + return line >= t.start.line && line <= t.end.line; +}; +export const toMatchResult = ( + test: ItBlock, + assertionOrErr: TestAssertionStatus | string +): TestResult => { + const assertion = typeof assertionOrErr === 'string' ? undefined : assertionOrErr; + const err = typeof assertionOrErr === 'string' ? assertionOrErr : undefined; + const adjustLocation = (l: Location): Location => ({ column: l.column - 1, line: l.line - 1 }); + + // Note the shift from one-based to zero-based line number and columns + return { + name: test.name, + start: adjustLocation(test.start), + end: adjustLocation(test.end), + status: assertion?.status ?? TestReconciliationState.Unknown, + shortMessage: assertion?.shortMessage ?? err, + terseMessage: assertion?.terseMessage, + lineNumberOfError: matchPos(test, assertion, true) ? assertion.line - 1 : test.end.line - 1, + }; +}; + +/** mark all data and child containers unmatched */ +const toUnmatchedResults = (tContainer: ContainerNode, err: string): TestResult[] => { + const results = tContainer.childData.map((n) => toMatchResult(n.only(), err)); + tContainer.childContainers.forEach((c) => results.push(...toUnmatchedResults(c, err))); + return results; +}; + +type MessageType = 'context-mismatch' | 'match-failed' | 'unusual-match' | 'duplicate-name'; +const createMessaging = (fileName: string, verbose: boolean) => ( + messageType: MessageType, + contextType: ContextType, + source: ContainerNode | DataNode, + assertion?: ContainerNode | DataNode, + extraReason?: string +): void => { + if (!verbose) { + return; + } + + const output = (message: string): void => + console.warn(`[${fileName}] ${message} \n source=`, source, `\n assertion=`, assertion); + + const blockType = contextType === 'container' ? 'describe' : 'test'; + switch (messageType) { + case 'context-mismatch': + output( + `!! context mismatched !! ${contextType} nodes are different under "${source.name}": ` + ); + break; + case 'match-failed': { + output(`!! match failed !! ${blockType}: "${source.name}": ${extraReason} `); + break; + } + case 'duplicate-name': { + output( + `duplicate names in the same (describe) block is not recommanded and might not be matched reliably: ${source.name}` + ); + break; + } + case 'unusual-match': { + output(`unusual match: ${extraReason} : ${blockType} ${source.name}: `); + break; + } + } +}; +type Messaging = ReturnType; + +interface ContextMatchAlgorithm { + match: ( + tContainer: ContainerNode, + aContainer: ContainerNode + ) => TestResult[]; +} + +const ContextMatch = (messaging: Messaging): ContextMatchAlgorithm => { + const handleTestBlockMatch = ( + t: DataNode, + matched: DataNode[] + ): TestResult => { + if (matched.length !== 1) { + return toMatchResult(t.only(), `found ${matched.length} matched assertion(s)`); + } + const a = matched[0]; + const itBlock = t.only(); + switch (a.data.length) { + case 0: + throw new TypeError(`invalid state: assertion data should not be empty if it is a match!`); + case 1: { + const assertion = a.only(); + if (a.name !== t.name && !matchPos(itBlock, assertion)) { + messaging('unusual-match', 'data', t, a, 'neither name nor line matched'); + } + return toMatchResult(itBlock, assertion); + } + default: { + // 1-to-many + messaging('unusual-match', 'data', t, a, '1-to-many match, jest.each perhaps?'); + + // TODO: support multiple errorLine + // until we support multiple errors, choose the first error assertion, if any + const assertions = + a.data.find((assertion) => assertion.status === 'KnownFail') || a.first(); + return toMatchResult(itBlock, assertions); + } + } + }; + + const handleDescribeBlockMatch = ( + t: ContainerNode, + matched: ContainerNode[] + ): TestResult[] => { + if (matched.length !== 1) { + messaging('match-failed', 'container', t); + // if we can't find corresponding container to match, the whole container will be considered unmatched + return toUnmatchedResults(t, `can not find matching assertion for block ${t.name}`); + } + // eslint-disable-next-line @typescript-eslint/no-use-before-define + return matchContainers(t, matched[0]); + }; + + const matchChildren = ( + contextType: C, + tContainer: ContainerNode, + aContainer: ContainerNode, + onResult: (t: ChildNodeType, a: ChildNodeType[]) => void + ): void => { + const tList = tContainer.getChildren(contextType); + const aList = aContainer.getChildren(contextType); + + if (tList.length === aList.length) { + tList.forEach((t, idx) => onResult(t, [aList[idx]])); + } else { + messaging('context-mismatch', contextType, tContainer, aContainer); + tList.forEach((t) => { + // duplicate names under the same layer is really illegal jest practice, they can not + // be reliably resolved with name-based matching + if (tList.filter((t1) => t1.name === t.name).length > 1) { + messaging('duplicate-name', contextType, t); + onResult(t, []); + } else { + const found = aList.filter((a) => a.name === t.name); + onResult(t, found); + } + }); + } + }; + + /** + * this is where the actual test-block => assertion(s) match occurred. + * Each test and assertion container pair will try to match both its + * child-data and child-container list recursively. + * + * @param tContainer + * @param aContainer + * @returns a list of TestResult collected from all its children (data + containers) + */ + + const matchContainers = ( + tContainer: ContainerNode, + aContainer: ContainerNode + ): TestResult[] => { + const matchResults: TestResult[] = []; + matchChildren('data', tContainer, aContainer, (t, a) => + matchResults.push(handleTestBlockMatch(t, a)) + ); + matchChildren('container', tContainer, aContainer, (t, a) => + matchResults.push(...handleDescribeBlockMatch(t, a)) + ); + + return matchResults; + }; + return { match: matchContainers }; +}; + +/** + * match tests container with assertion container by their context structure. + * @param fileName + * @param tContainer + * @param aContainer + * @param verbose turns on/off the debugging warning messages. + */ + +export const matchTestAssertions = ( + fileName: string, + sourceRoot: ParsedNode, + assertions: TestAssertionStatus[], + verbose = true +): TestResult[] => { + const tContainer = buildSourceContainer(sourceRoot); + const aContainer = buildAssertionContainer(assertions); + const { match } = ContextMatch(createMessaging(fileName, verbose)); + return match(tContainer, aContainer); +}; diff --git a/src/TestResults/match-node.ts b/src/TestResults/match-node.ts new file mode 100644 index 000000000..ee7835191 --- /dev/null +++ b/src/TestResults/match-node.ts @@ -0,0 +1,125 @@ +/** + * internal classes used by `match-by-context` + */ + +export interface BaseNodeType { + zeroBasedLine: number; + name: string; +} + +/* interface implementation */ +export class DataNode implements BaseNodeType { + name: string; + zeroBasedLine: number; + data: T[]; + + constructor(name: string, zeroBasedLine: number, data: T) { + this.name = name; + this.zeroBasedLine = zeroBasedLine; + this.data = [data]; + } + + merge(another: DataNode): boolean { + if (another.zeroBasedLine !== this.zeroBasedLine) { + return false; + } + this.data.push(...another.data); + return true; + } + + /** return the only element in the list, exception otherwise */ + only(): T { + if (this.data.length !== 1) { + throw new TypeError(`expect 1 element but got ${this.data.length} elements`); + } + return this.data[0]; + } + /** return the first element, if no element, returns undefined */ + first(): T | undefined { + if (this.data.length > 0) { + return this.data[0]; + } + } +} + +export type ContextType = 'container' | 'data'; +export class ContainerNode implements BaseNodeType { + public childContainers: ContainerNode[] = []; + public childData: DataNode[] = []; + public zeroBasedLine: number; + public name: string; + + constructor(name: string) { + this.name = name; + this.zeroBasedLine = -1; + } + + public addContainerNode(container: ContainerNode): void { + this.childContainers.push(container); + } + + public addDataNode(dataNode: DataNode): void { + this.childData.push(dataNode); + } + + public findContainer(path: string[], createIfMissing = true): ContainerNode | undefined { + if (path.length <= 0) { + return this; + } + const [target, ...remaining] = path; + let container = this.childContainers.find((c) => c.name === target); + if (!container && createIfMissing) { + container = new ContainerNode(target); + this.addContainerNode(container); + } + return container?.findContainer(remaining, createIfMissing); + } + + /** + * deeply sort inline all child-data and child-containers by line position. + * it will update this.zeroBasedLine based on its top child, if it is undefined. + * @param grouping if true, will try to merge child-data with the same line + */ + public sort(grouping = false): void { + const sortByLine = (n1: BaseNodeType, n2: BaseNodeType): number => + n1.zeroBasedLine - n2.zeroBasedLine; + const groupData = (list: DataNode[], data: DataNode): DataNode[] => { + if (list.length <= 0) { + return [data]; + } + // if not able to merge with previous node, i.e . can not group, add it to the list + if (!list[list.length - 1].merge(data)) { + list.push(data); + } + return list; + }; + + this.childData.sort(sortByLine); + if (grouping) { + this.childData = this.childData.reduce(groupData, []); + } + + // recursive to sort childContainers, which will update its lineNumber and then sort the list itself + this.childContainers.forEach((c) => c.sort(grouping)); + this.childContainers.sort(sortByLine); + + // if container doesn't have valid line info, use the first child's + if (this.zeroBasedLine < 0) { + const topLines = [this.childData, this.childContainers] + .filter((l) => l.length > 0) + .map((l) => l[0].zeroBasedLine); + this.zeroBasedLine = Math.min(...topLines); + } + } + // use conditional type to narrow down exactly the type + public getChildren(type: C): ChildNodeType[] { + const children = type === 'container' ? this.childContainers : this.childData; + // has to cast explicitly due to the issue: + // https://github.com/microsoft/TypeScript/issues/24929 + return children as ChildNodeType[]; + } +} +export type NodeType = ContainerNode | DataNode; +export type ChildNodeType = C extends 'container' + ? ContainerNode + : DataNode; diff --git a/tests/TestResults/TestResultProvider.test.ts b/tests/TestResults/TestResultProvider.test.ts index 82a1ea1b9..0f8457716 100644 --- a/tests/TestResults/TestResultProvider.test.ts +++ b/tests/TestResults/TestResultProvider.test.ts @@ -1,4 +1,7 @@ jest.unmock('../../src/TestResults/TestResultProvider'); +jest.unmock('../../src/TestResults/match-node'); +jest.unmock('../../src/TestResults/match-by-context'); +jest.unmock('../test-helper'); const updateFileWithJestStatus = jest.fn(); const assertionsForTestFile = jest.fn(); @@ -33,28 +36,31 @@ jest.mock('path', () => { import { TestResultProvider } from '../../src/TestResults/TestResultProvider'; import { TestReconciliationState } from '../../src/TestResults'; import { parseTest } from '../../src/TestParser'; +import * as helper from '../test-helper'; +import { ItBlock } from 'jest-editor-support'; + +const mockParseTest = (itBlocks: ItBlock[]) => { + ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ + root: helper.makeRoot(itBlocks), + itBlocks, + }); +}; describe('TestResultProvider', () => { describe('getResults()', () => { const filePath = 'file.js'; - const testBlock = { - name: 'test name', - start: { - line: 2, - column: 3, - }, - end: { - line: 4, - column: 5, - }, - }; - const assertion = { - title: testBlock.name, - status: TestReconciliationState.KnownFail, - terseMessage: 'terseMessage', - shortMessage: 'shortMesage', - line: 3, - }; + const testBlock = helper.makeItBlock('test name', [2, 3, 4, 5]); + const assertion = helper.makeAssertion( + testBlock.name, + TestReconciliationState.KnownFail, + undefined, + undefined, + { + terseMessage: 'terseMessage', + shortMessage: 'shortMesage', + line: 3, + } + ); beforeEach(() => { jest.resetAllMocks(); @@ -62,9 +68,7 @@ describe('TestResultProvider', () => { it('should return the cached results if possible', () => { const sut = new TestResultProvider(); - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [], - }); + mockParseTest([]); assertionsForTestFile.mockReturnValueOnce([]); const expected = sut.getResults(filePath); @@ -73,9 +77,7 @@ describe('TestResultProvider', () => { it('should re-index the line and column number to zero-based', () => { const sut = new TestResultProvider(); - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock], - }); + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([assertion]); const actual = sut.getResults(filePath); @@ -91,24 +93,20 @@ describe('TestResultProvider', () => { }); }); - it('should look up the test result by line number only if the name matches', () => { + it('if context are the same, test will match even if name does not', () => { const sut = new TestResultProvider(); - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock], - }); + mockParseTest([testBlock]); const assertionC = { ...assertion }; assertionC.title = 'xxx'; assertionsForTestFile.mockReturnValueOnce([assertionC]); const actual = sut.getResults(filePath); expect(actual).toHaveLength(1); - expect(actual[0].status).toBe(TestReconciliationState.Unknown); + expect(actual[0].status).toBe(TestReconciliationState.KnownFail); }); it('should look up the test result by test name', () => { const sut = new TestResultProvider(); - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock], - }); + mockParseTest([testBlock]); const assertionC = { ...assertion }; assertionC.line = undefined; assertionsForTestFile.mockReturnValueOnce([assertionC]); @@ -130,55 +128,27 @@ describe('TestResultProvider', () => { }); }); - it('should use default values for unmatched assertions', () => { + it('unmatched test should report the reason', () => { const sut = new TestResultProvider(); - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock], - }); + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([]); const actual = sut.getResults(filePath); expect(actual).toHaveLength(1); expect(actual[0].status).toBe(TestReconciliationState.Unknown); - expect(actual[0].shortMessage).toBeUndefined(); + expect(actual[0].shortMessage).not.toBeUndefined(); expect(actual[0].terseMessage).toBeUndefined(); }); describe('duplicate test names', () => { - const testBlock2 = { - ...testBlock, - start: { - line: 5, - column: 3, - }, - end: { - line: 7, - column: 5, - }, - }; - const testBlock3 = { - ...testBlock, - start: { - line: 20, - column: 3, - }, - end: { - line: 25, - column: 5, - }, - }; + const testBlock2 = helper.makeItBlock(testBlock.name, [5, 3, 7, 5]); beforeEach(() => {}); - it('can resolve by matching error line', () => { - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock, testBlock2], - }); + it('can resolve as long as they have the same context structure', () => { + mockParseTest([testBlock, testBlock2]); const sut = new TestResultProvider(); assertionsForTestFile.mockReturnValueOnce([ - assertion, - { - title: testBlock.name, - status: TestReconciliationState.KnownSuccess, - }, + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownFail, [], [1, 0]), + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [10, 0]), ]); const actual = sut.getResults(filePath); @@ -186,255 +156,176 @@ describe('TestResultProvider', () => { expect(actual[0].status).toBe(TestReconciliationState.KnownFail); expect(actual[1].status).toBe(TestReconciliationState.KnownSuccess); }); - it('can resolve even if these tests pass, i.e. no line number', () => { - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock2, testBlock3], - }); - - const sut = new TestResultProvider(); - assertionsForTestFile.mockReturnValueOnce([ - { - title: testBlock.name, - status: TestReconciliationState.KnownSuccess, - }, - { - title: testBlock.name, - status: TestReconciliationState.KnownSuccess, - location: { colum: 3, line: 22 }, - }, - ]); - const actual = sut.getResults(filePath); - - expect(actual).toHaveLength(2); - expect(actual.every((r) => r.status === TestReconciliationState.KnownSuccess)).toEqual( - true - ); - }); - it('default to unknown if failed to match by line or location', () => { - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock2, testBlock3], - }); + it('however when context structures are different, duplicate names within the same layer can not be resolved.', () => { + mockParseTest([testBlock, testBlock2]); const sut = new TestResultProvider(); + // note: these 2 assertions have the same line number, therefore will be merge + // into a group-node, which made the context difference: source: 2 nodes, assertion: 1 node. + // but since the 2 assertions' name matched the testBlock, it will still be considered as 1-to-many match assertionsForTestFile.mockReturnValueOnce([ - { - title: testBlock.name, - status: TestReconciliationState.KnownSuccess, - }, - { - title: testBlock.name, - status: TestReconciliationState.KnownSuccess, - }, + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 0]), + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownFail, [], [1, 0]), ]); const actual = sut.getResults(filePath); expect(actual).toHaveLength(2); - expect(actual.every((r) => r.status === TestReconciliationState.Unknown)).toEqual(true); - expect( - actual.every((r) => r.shortMessage && r.shortMessage.includes('duplicate test names')) - ).toEqual(true); + expect(actual[0].status).toBe(TestReconciliationState.Unknown); + expect(actual[1].status).toBe(TestReconciliationState.Unknown); }); }); + it('should only mark error line number if it is within the right itBlock', () => { const sut = new TestResultProvider(); - const testBlock2 = { - name: 'test2', - start: { - line: 5, - column: 3, - }, - end: { - line: 7, - column: 5, - }, - }; - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks: [testBlock, testBlock2], - }); + const testBlock2 = helper.makeItBlock('test2', [5, 3, 7, 5]); + mockParseTest([testBlock, testBlock2]); assertionsForTestFile.mockReturnValueOnce([ - { - title: testBlock2.name, - status: TestReconciliationState.KnownSuccess, + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 1]), + helper.makeAssertion(testBlock2.name, TestReconciliationState.KnownFail, [], [2, 2], { line: 3, - }, + }), ]); const actual = sut.getResults(filePath); expect(actual).toHaveLength(2); + expect(actual.map((a) => a.name)).toEqual([testBlock.name, testBlock2.name]); + expect(actual.map((a) => a.status)).toEqual([ + TestReconciliationState.KnownSuccess, + TestReconciliationState.KnownFail, + ]); expect( - actual.some( - (a) => a.name === testBlock.name && a.status === TestReconciliationState.Unknown - ) - ).toEqual(true); - expect( - actual.some( - (a) => - a.name === testBlock2.name && - a.status === TestReconciliationState.KnownSuccess && - a.lineNumberOfError === testBlock2.end.line - 1 - ) - ).toEqual(true); + actual.find((a) => a.status === TestReconciliationState.KnownFail)?.lineNumberOfError + ).toEqual(testBlock2.end.line - 1); }); - describe('template literal handling', () => { - const testBlock2 = { - ...testBlock, - // tslint:disable-next-line no-invalid-template-strings - name: 'template literal ${num}', - start: { - line: 5, - column: 3, - }, - end: { - line: 7, - column: 5, - }, - }; - const useTests = (itBlocks = [testBlock, testBlock2]) => { - ((parseTest as unknown) as jest.Mock<{}>).mockReturnValueOnce({ - itBlocks, - }); - }; + it('can handle template literal in the context', () => { + const sut = new TestResultProvider(); + const testBlock2 = helper.makeItBlock('template literal I got ${str}', [6, 0, 7, 20]); + const testBlock3 = helper.makeItBlock('template literal ${i}, ${k}: {something}', [ + 10, + 5, + 20, + 5, + ]); + + mockParseTest([testBlock, testBlock3, testBlock2]); + assertionsForTestFile.mockReturnValueOnce([ + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 0]), + helper.makeAssertion( + 'template literal I got something like this', + TestReconciliationState.KnownFail, + [], + [2, 0] + ), + helper.makeAssertion( + 'template literal 1, 2: {something}', + TestReconciliationState.KnownSuccess, + [], + [3, 0] + ), + ]); + const actual = sut.getResults(filePath); + expect(actual).toHaveLength(3); + expect(actual.map((a) => a.name)).toEqual([testBlock.name, testBlock2.name, testBlock3.name]); + expect(actual.map((a) => a.status)).toEqual([ + TestReconciliationState.KnownSuccess, + TestReconciliationState.KnownFail, + TestReconciliationState.KnownSuccess, + ]); + }); + + describe('safe-guard warnings', () => { + const consoleWarning = jest.spyOn(console, 'warn').mockImplementation(() => {}); beforeEach(() => { jest.resetAllMocks(); }); - it(`find test by assertion error line`, () => { - const sut = new TestResultProvider(); + describe('when contexts does not align', () => { + beforeEach(() => { + mockParseTest([testBlock]); + assertionsForTestFile.mockReturnValueOnce([ + helper.makeAssertion('whatever', TestReconciliationState.KnownSuccess, [], [12, 19]), + helper.makeAssertion('whatever', TestReconciliationState.KnownSuccess, [], [20, 25]), + ]); + }); + it('reprots warning when verbose is true', () => { + const sut = new TestResultProvider(); + sut.verbose = true; + + const actual = sut.getResults(filePath); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.Unknown); + expect(actual[0].shortMessage).not.toBeUndefined(); + expect(consoleWarning).toHaveBeenCalled(); + }); + it('not warning if verbose is off', () => { + const sut = new TestResultProvider(); + sut.verbose = false; - useTests(); + const actual = sut.getResults(filePath); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.Unknown); + expect(actual[0].shortMessage).not.toBeUndefined(); + expect(consoleWarning).not.toHaveBeenCalled(); + }); + }); + it('report warning if context match but neither name nor location matched', () => { + const sut = new TestResultProvider(); + sut.verbose = true; + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([ - { - title: 'template literal 2', - status: TestReconciliationState.KnownFail, - line: 6, - }, + helper.makeAssertion('another name', TestReconciliationState.KnownSuccess, [], [20, 25]), ]); const actual = sut.getResults(filePath); - - expect(actual).toHaveLength(2); - expect( - actual.some( - (a) => a.status === TestReconciliationState.Unknown && a.name === testBlock.name - ) - ).toEqual(true); - expect( - actual.some( - (a) => a.status === TestReconciliationState.KnownFail && a.name === testBlock2.name - ) - ).toEqual(true); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.KnownSuccess); + expect(actual[0].shortMessage).toBeUndefined(); + expect(consoleWarning).toHaveBeenCalled(); }); - it(`find test by assertion location`, () => { + it('report warning if match failed', () => { const sut = new TestResultProvider(); - - useTests(); + sut.verbose = true; + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([ - { - title: 'template literal 2', - status: TestReconciliationState.KnownSuccess, - location: { colum: 3, line: 6 }, - }, + helper.makeAssertion( + 'another name', + TestReconciliationState.KnownSuccess, + ['d-1'], + [20, 25] + ), ]); const actual = sut.getResults(filePath); - - expect(actual).toHaveLength(2); - expect( - actual.some( - (a) => a.status === TestReconciliationState.Unknown && a.name === testBlock.name - ) - ).toEqual(true); - expect( - actual.some( - (a) => a.status === TestReconciliationState.KnownSuccess && a.name === testBlock2.name - ) - ).toEqual(true); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.Unknown); + expect(actual[0].shortMessage).not.toBeUndefined(); + expect(consoleWarning).toHaveBeenCalled(); }); - it(`find test by partial name match`, () => { + it('1-many match (jest.each) detected', () => { const sut = new TestResultProvider(); - useTests(); - + sut.verbose = true; + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([ - { - title: 'template literals ok', - status: TestReconciliationState.KnownFail, - }, - { - title: 'template literal 2', - status: TestReconciliationState.KnownSuccess, - }, + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 12]), + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 12]), + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownSuccess, [], [1, 12]), ]); const actual = sut.getResults(filePath); - expect(actual).toHaveLength(2); - expect( - actual.some( - (a) => a.status === TestReconciliationState.Unknown && a.name === testBlock.name - ) - ).toEqual(true); - expect( - actual.some( - (a) => a.status === TestReconciliationState.KnownSuccess && a.name === testBlock2.name - ) - ).toEqual(true); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.KnownSuccess); + expect(actual[0].shortMessage).toBeUndefined(); + expect(consoleWarning).toHaveBeenCalled(); }); - it(`multiple template literals`, () => { + it('when all goes according to plan, no warning', () => { const sut = new TestResultProvider(); - // tslint:disable-next-line: no-invalid-template-strings - const testBlock3 = { ...testBlock2, name: 'template literal ${i}, ${k}: {something}' }; - useTests([testBlock3, testBlock2]); + sut.verbose = true; + mockParseTest([testBlock]); assertionsForTestFile.mockReturnValueOnce([ - { - title: 'template literal I got something like this', - status: TestReconciliationState.KnownFail, - }, - { - title: 'template literal 1, 2: {something}', - status: TestReconciliationState.KnownSuccess, - }, + helper.makeAssertion(testBlock.name, TestReconciliationState.KnownFail, [], [1, 12]), ]); const actual = sut.getResults(filePath); - expect(actual).toHaveLength(2); - expect( - actual.some( - (a) => a.status === TestReconciliationState.KnownSuccess && a.name === testBlock3.name - ) - ).toEqual(true); - expect( - actual.some( - (a) => a.status === TestReconciliationState.KnownFail && a.name === testBlock2.name - ) - ).toEqual(true); - }); - - describe('when match failed', () => { - const consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); - beforeEach(() => { - jest.resetAllMocks(); - useTests([testBlock2]); - assertionsForTestFile.mockReturnValueOnce([ - { - title: 'template literals 2', - status: TestReconciliationState.KnownSuccess, - }, - ]); - }); - - it(`will report error`, () => { - const sut = new TestResultProvider(); - const actual = sut.getResults(filePath); - - expect(actual).toHaveLength(1); - expect(actual[0].status).toBe(TestReconciliationState.Unknown); - expect(actual[0].shortMessage).not.toBeUndefined(); - expect(consoleSpy).not.toHaveBeenCalled(); - }); - it('will also output to console in verbose mode', () => { - const sut = new TestResultProvider(true); - const actual = sut.getResults(filePath); - - expect(actual).toHaveLength(1); - expect(actual[0].status).toBe(TestReconciliationState.Unknown); - expect(actual[0].shortMessage).not.toBeUndefined(); - expect(consoleSpy).toHaveBeenCalled(); - }); + expect(actual).toHaveLength(1); + expect(actual[0].status).toBe(TestReconciliationState.KnownFail); + expect(actual[0].shortMessage).toBeUndefined(); + expect(consoleWarning).not.toHaveBeenCalled(); }); }); }); @@ -490,4 +381,14 @@ describe('TestResultProvider', () => { expect(updateFileWithJestStatus).toBeCalledWith(results); }); }); + it('match exception should just returns empty array and not cause the whole system to crash', () => { + const sut = new TestResultProvider(); + mockParseTest([]); + assertionsForTestFile.mockImplementation(() => { + throw new Error('whatever'); + }); + + const actual = sut.getResults('whatever'); + expect(actual).toEqual([]); + }); }); diff --git a/tests/TestResults/match-by-context.test.ts b/tests/TestResults/match-by-context.test.ts new file mode 100644 index 000000000..5ebd8f4f2 --- /dev/null +++ b/tests/TestResults/match-by-context.test.ts @@ -0,0 +1,287 @@ +jest.unmock('../../src/TestResults/match-node'); +jest.unmock('../../src/TestResults/match-by-context'); +jest.unmock('../test-helper'); + +import * as helper from '../test-helper'; +import * as match from '../../src/TestResults/match-by-context'; +import { TestReconciliationState } from '../../src/TestResults'; +import { TestAssertionStatus, ParsedNode } from 'jest-editor-support'; + +describe('buildAssertionContainer', () => { + it('can build and sort assertions without ancestors', () => { + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [1, 0]); + const a2 = helper.makeAssertion('test-2', 'KnownSuccess', [], [2, 0]); + const a3 = helper.makeAssertion('test-3', 'KnownSuccess', [], [3, 0]); + const root = match.buildAssertionContainer([a1, a3, a2]); + expect(root.childContainers).toHaveLength(0); + expect(root.childData).toHaveLength(3); + expect(root.childData.map((n) => n.zeroBasedLine)).toEqual([1, 2, 3]); + }); + it('can build and sort assertions with ancestors', () => { + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [1, 0]); + const a2 = helper.makeAssertion('test-2', 'KnownSuccess', ['d-1'], [2, 0]); + const a3 = helper.makeAssertion('test-3', 'KnownSuccess', ['d-1', 'd-1-1'], [3, 0]); + const a4 = helper.makeAssertion('test-4', 'KnownSuccess', ['d-1'], [4, 0]); + const a5 = helper.makeAssertion('test-4', 'KnownFail', ['d-2'], [5, 0]); + const a6 = helper.makeAssertion('test-4', 'KnownFail', ['d-2'], [8, 0]); + + // ensure the assertion hierarchical integrity before building the container + expect( + [a1, a5, a3, a2, a4, a6].every((a) => a.fullName === a.title || a.ancestorTitles.length > 0) + ).toBe(true); + + const root = match.buildAssertionContainer([a1, a5, a3, a2, a4, a6]); + expect(root.childContainers).toHaveLength(2); + expect(root.childData).toHaveLength(1); + expect(root.childContainers.map((n) => [n.name, n.zeroBasedLine])).toEqual([ + ['d-1', 2], + ['d-2', 5], + ]); + expect(root.childData.map((n) => [n.name, n.zeroBasedLine])).toEqual([['test-1', 1]]); + // the original assertion integrity should not be changed + expect( + [a1, a5, a3, a2, a4, a6].every((a) => a.fullName === a.title || a.ancestorTitles.length > 0) + ).toBe(true); + }); + it('can group assertions with the same line', () => { + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [2, 0]); + const a2 = helper.makeAssertion('test-2', 'KnownSuccess', [], [2, 0]); + const a3 = helper.makeAssertion('test-3', 'KnownSuccess', [], [2, 0]); + const a4 = helper.makeAssertion('test-4', 'KnownSuccess', [], [5, 0]); + const root = match.buildAssertionContainer([a1, a3, a4, a2]); + expect(root.childContainers).toHaveLength(0); + expect(root.childData).toHaveLength(2); + expect(root.childData.map((n) => n.zeroBasedLine)).toEqual([2, 5]); + const groupNode = root.childData[0]; + expect(groupNode.data).toHaveLength(3); + expect(groupNode.data.map((n) => n.title)).toEqual(['test-1', 'test-3', 'test-2']); + }); + + it('create a container based on assertion ancestorTitles structure', () => { + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [1, 0]); + const a2 = helper.makeAssertion('test-2', 'KnownSuccess', ['d-1'], [10, 0]); + const a3 = helper.makeAssertion('test-3', 'KnownSuccess', ['d-1', 'd-2'], [15, 0]); + const a4 = helper.makeAssertion('test-4', 'KnownFail', ['d-1'], [20, 0]); + const root = match.buildAssertionContainer([a4, a3, a1, a2]); + expect(root.childData.map((n) => (n as any).name)).toEqual(['test-1']); + expect(root.childContainers).toHaveLength(1); + const d1 = root.findContainer(['d-1']); + expect(d1.childContainers).toHaveLength(1); + expect(d1.childData.map((n) => (n as any).name)).toEqual(['test-2', 'test-4']); + const d2 = d1.findContainer(['d-2']); + expect(d2.childContainers).toHaveLength(0); + expect(d2.childData.map((n) => (n as any).name)).toEqual(['test-3']); + }); +}); +describe('buildSourceContainer', () => { + it('can build and sort source container without ancestors', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); + const t3 = helper.makeItBlock('test-3', [8, 0, 10, 0]); + const sourceRoot = helper.makeRoot([t2, t1, t3]); + const root = match.buildSourceContainer(sourceRoot); + expect(root.childContainers).toHaveLength(0); + expect(root.childData.map((n) => (n as any).name)).toEqual(['test-1', 'test-2', 'test-3']); + }); + it('can build and sort source container with ancestors', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); + const t3 = helper.makeItBlock('test-3', [8, 0, 10, 0]); + const d1 = helper.makeDescribeBlock('d-1', [t2]); + const d2 = helper.makeDescribeBlock('d-2', [t3]); + const sourceRoot = helper.makeRoot([t1, d1, d2]); + const root = match.buildSourceContainer(sourceRoot); + expect(root.childContainers).toHaveLength(2); + expect(root.childData).toHaveLength(1); + expect(root.childData.map((n) => (n as any).name)).toEqual([t1.name]); + + const d1Container = root.findContainer(['d-1']); + expect(d1Container?.childData).toHaveLength(1); + expect(d1Container?.childContainers).toHaveLength(0); + + const d2Container = root.findContainer(['d-2']); + expect(d2Container?.childData).toHaveLength(1); + expect(d2Container?.childContainers).toHaveLength(0); + }); + it('lines will be converted to zeroBased', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); + const sourceRoot = helper.makeRoot([t2, t1]); + const root = match.buildSourceContainer(sourceRoot); + expect(root.childContainers).toHaveLength(0); + expect(root.childData.map((n) => n.zeroBasedLine)).toEqual([0, 5]); + }); + it('can build and sort container from describe and it blocks', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); + const t3 = helper.makeItBlock('test-3', [8, 0, 10, 0]); + const d1 = helper.makeDescribeBlock('d-1', [t1, t2]); + const sourceRoot = helper.makeRoot([t3, d1]); + const root = match.buildSourceContainer(sourceRoot); + expect(root.childData.map((n) => (n as any).name)).toEqual(['test-3']); + expect(root.childContainers).toHaveLength(1); + const container = root.childContainers[0]; + expect(container.childContainers).toHaveLength(0); + expect(container.childData.map((n) => (n as any).name)).toEqual(['test-1', 'test-2']); + }); + it('does not group itBlocks even if they have the same start line (wrongly)', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [1, 0, 7, 0]); + const sourceRoot = helper.makeRoot([t1, t2]); + const root = match.buildSourceContainer(sourceRoot); + expect(root.childData.map((n) => (n as any).name)).toEqual(['test-1', 'test-2']); + expect(root.childContainers).toHaveLength(0); + }); +}); +describe('matchTestAssertions', () => { + it('tests are matched by context position regardless name and line', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2-${num}', [6, 0, 7, 0]); + const sourceRoot = helper.makeRoot([t2, t1]); + + const a1 = helper.makeAssertion('test-1', 'KnownFail', [], [0, 0]); + const a2 = helper.makeAssertion('test-2-100', 'KnownSuccess', [], [7, 0]); + const matched = match.matchTestAssertions('a file', sourceRoot, [a1, a2]); + + expect(matched).toHaveLength(2); + expect(matched.map((m) => m.name)).toEqual(['test-1', 'test-2-${num}']); + expect(matched.map((m) => m.status)).toEqual(['KnownFail', 'KnownSuccess']); + }); + it('can match tests with the same name but in different describe blocks', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-1', [6, 0, 7, 0]); + const d1 = helper.makeDescribeBlock('d-1', [t2]); + const sourceRoot = helper.makeRoot([t1, d1]); + + const a1 = helper.makeAssertion('test-1', 'KnownFail', [], [0, 0]); + const a2 = helper.makeAssertion('test-1', 'KnownSuccess', ['d-1'], [5, 0]); + const matched = match.matchTestAssertions('a file', sourceRoot, [a1, a2]); + expect(matched.map((m) => m.name)).toEqual(['test-1', 'test-1']); + expect(matched.map((m) => m.status)).toEqual(['KnownFail', 'KnownSuccess']); + expect(matched.map((m) => m.start.line)).toEqual([0, 5]); + expect(matched.map((m) => m.end.line)).toEqual([4, 6]); + }); + describe(`context do not align`, () => { + it('when test block is missing assertion in the same container', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const sourceRoot = helper.makeRoot([t1]); + const matched = match.matchTestAssertions('a file', sourceRoot, []); + expect(matched.map((m) => m.name)).toEqual(['test-1']); + expect(matched.map((m) => m.status)).toEqual(['Unknown']); + expect(matched.map((m) => m.start.line)).toEqual([0]); + expect(matched.map((m) => m.end.line)).toEqual([4]); + }); + it('can still resolve by fallback to simple name match', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); + const t2 = helper.makeItBlock('test-2', [1, 0, 5, 0]); + const sourceRoot = helper.makeRoot([t1, t2]); + + const a1 = helper.makeAssertion('test-1', 'KnownFail', [], [0, 0]); + + const matched = match.matchTestAssertions('a file', sourceRoot, [a1]); + expect(matched.map((m) => [m.name, m.status])).toEqual([ + ['test-1', 'KnownFail'], + ['test-2', 'Unknown'], + ]); + }); + it('will continue match the child containers', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); // under root + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); // under d-1 + const t3 = helper.makeItBlock('test-3', [8, 0, 9, 0]); // under d-1 + const t4 = helper.makeItBlock('test-4', [10, 0, 12, 0]); // under d-1-1 + const d11 = helper.makeDescribeBlock('d-1-1', [t4]); + const d1 = helper.makeDescribeBlock('d-1', [t2, t3, d11]); + const sourceRoot = helper.makeRoot([t1, d1]); + + // assertion missing for 'd-1': t3 + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [6, 0]); + const a2 = helper.makeAssertion('test-2', 'KnownFail', ['d-1'], [6, 0]); + const a4 = helper.makeAssertion('test-4', 'KnownSuccess', ['d-1', 'd-1-1'], [9, 0]); + + const matched = match.matchTestAssertions('a file', sourceRoot, [a1, a2, a4]); + expect(matched.map((m) => [m.name, m.status])).toEqual([ + ['test-1', 'KnownSuccess'], + ['test-2', 'KnownFail'], + ['test-3', 'Unknown'], + ['test-4', 'KnownSuccess'], + ]); + }); + it('describe block will fail if context mismatch and name lookup failed', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); // under root + const t2 = helper.makeItBlock('test-2', [6, 0, 7, 0]); // under d-1 + const d1 = helper.makeDescribeBlock('d-1', [t2]); + const sourceRoot = helper.makeRoot([t1, d1]); + + // assertion missing for t3 + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [6, 0]); + + const matched = match.matchTestAssertions('a file', sourceRoot, [a1]); + expect(matched.map((m) => [m.name, m.status])).toEqual([ + ['test-1', 'KnownSuccess'], + ['test-2', 'Unknown'], + ]); + }); + it('empty desecribe block', () => { + const t1 = helper.makeItBlock('test-1', [1, 0, 5, 0]); // under root + const d11 = helper.makeDescribeBlock('d-1-1', []); + const d1 = helper.makeDescribeBlock('d-1', [d11]); + const sourceRoot = helper.makeRoot([d1, t1]); + const a1 = helper.makeAssertion('test-1', 'KnownSuccess', [], [6, 0]); + const matched = match.matchTestAssertions('a file', sourceRoot, [a1]); + expect(matched).toHaveLength(1); + expect(matched.map((m) => [m.name, m.status])).toEqual([['test-1', 'KnownSuccess']]); + }); + }); + describe('1-many (jest.each) match', () => { + const createTestData = ( + statusList: (TestReconciliationState | [TestReconciliationState, number])[] + ): [ParsedNode, TestAssertionStatus[]] => { + const t1 = helper.makeItBlock('', [12, 1, 20, 1]); + const sourceRoot = helper.makeRoot([t1]); + + // this match jest.each with 2 assertions + const assertions = statusList.map((s, idx) => { + let state: TestReconciliationState; + let override: Partial; + if (typeof s === 'string') { + state = s; + override = {}; + } else { + state = s[0]; + override = { line: s[1] }; + } + return helper.makeAssertion(`test-${idx}`, state, [], [11, 0], override); + }); + return [sourceRoot, assertions]; + }; + it('any failed assertion will fail the test', () => { + const [root, assertions] = createTestData([ + 'KnownSuccess', + ['KnownFail', 13], + 'KnownSuccess', + ]); + const matched = match.matchTestAssertions('a file', root, assertions); + expect(matched).toHaveLength(1); + expect(matched[0].status).toEqual('KnownFail'); + expect(matched[0].start).toEqual({ line: 11, column: 0 }); + expect(matched[0].end).toEqual({ line: 19, column: 0 }); + expect(matched[0].lineNumberOfError).toEqual(12); + }); + it('test is succeeded if all assertions are successful', () => { + const [root, assertions] = createTestData(['KnownSuccess', 'KnownSuccess', 'KnownSuccess']); + const matched = match.matchTestAssertions('a file', root, assertions); + expect(matched).toHaveLength(1); + expect(matched[0].status).toEqual('KnownSuccess'); + }); + it('test is skip when all assertions are skipped', () => { + const [root, assertions] = createTestData([ + TestReconciliationState.KnownSkip, + TestReconciliationState.KnownSkip, + TestReconciliationState.KnownSkip, + ]); + const matched = match.matchTestAssertions('a file', root, assertions); + expect(matched).toHaveLength(1); + expect(matched[0].status).toEqual(TestReconciliationState.KnownSkip); + }); + }); +}); diff --git a/tests/diagnostics.test.ts b/tests/diagnostics.test.ts index 64375008b..5246d7f64 100644 --- a/tests/diagnostics.test.ts +++ b/tests/diagnostics.test.ts @@ -1,4 +1,5 @@ jest.unmock('../src/diagnostics'); +jest.unmock('./test-helper'); import { updateDiagnostics, updateCurrentDiagnostics, @@ -12,6 +13,7 @@ import { TestAssertionStatus, } from 'jest-editor-support'; import { TestResult, TestReconciliationState } from '../src/TestResults'; +import * as helper from './test-helper'; class MockDiagnosticCollection implements vscode.DiagnosticCollection { name = 'test'; @@ -47,17 +49,15 @@ describe('test diagnostics', () => { let lineNumber = 17; function createAssertion(title: string, status: TestReconcilationState): TestAssertionStatus { - return { - title, - status, + return helper.makeAssertion(title, status, undefined, undefined, { message: `${title} ${status}`, line: lineNumber++, - }; + }); } function createTestResult( file: string, assertions: TestAssertionStatus[], - status: TestReconcilationState = 'KnownFail' + status: TestReconcilationState = TestReconciliationState.KnownFail ): TestFileAssertionStatus { return { file, message: `${file}:${status}`, status, assertions }; } @@ -84,12 +84,10 @@ describe('test diagnostics', () => { console.warn = jest.fn(); const testResult = createTestResult('mocked-test-file.js', [ - { - title: 'should be valid', - status: 'KnownFail', + helper.makeAssertion('should be valid', TestReconciliationState.KnownFail, [], undefined, { message: 'failing reason', line: -100, - }, + }), ]); updateDiagnostics([testResult], mockDiagnostics); expect(vscode.Range).toHaveBeenCalledWith(0, 0, 0, Number.MAX_SAFE_INTEGER); @@ -99,22 +97,26 @@ describe('test diagnostics', () => { const mockDiagnostics = new MockDiagnosticCollection(); const testResult = createTestResult('mocked-test-file.js', [ - { - title: 'should be valid', - status: 'KnownFail', - message: `expect(received).toBe(expected) // Object.is equality + helper.makeAssertion( + 'should be valid', + TestReconciliationState.KnownFail, + undefined, + undefined, + { + message: `expect(received).toBe(expected) // Object.is equality Expected: 2 Received: 1 at Object.toBe (src/pages/Home.test.tsx:6:13)`, - shortMessage: `expect(received).toBe(expected) // Object.is equality + shortMessage: `expect(received).toBe(expected) // Object.is equality Expected: 2 Received: 1`, - terseMessage: `Expected: 2, Received: 1`, - line: 123, - }, + terseMessage: `Expected: 2, Received: 1`, + line: 123, + } + ), ]); updateDiagnostics([testResult], mockDiagnostics); expect(vscode.Diagnostic).toHaveBeenCalledTimes(1); diff --git a/tests/test-helper.ts b/tests/test-helper.ts new file mode 100644 index 000000000..e6730f385 --- /dev/null +++ b/tests/test-helper.ts @@ -0,0 +1,68 @@ +/* istanbul ignore file */ +import { Location, LocationRange, TestResult } from '../src/TestResults/TestResult'; +import { TestReconciliationState } from '../src/TestResults'; +import { ItBlock, TestAssertionStatus } from 'jest-editor-support'; + +export const EmptyLocation = { + line: 0, + column: 0, +}; +export const EmptyLocationRange = { + start: EmptyLocation, + end: EmptyLocation, +}; +export const makeLocation = (pos: [number, number]): Location => ({ + line: pos[0], + column: pos[1], +}); +export const makePositionRange = (pos: [number, number, number, number]) => ({ + start: makeLocation([pos[0], pos[1]]), + end: makeLocation([pos[2], pos[3]]), +}); +export const isSameLocation = (p1: Location, p2: Location): boolean => + p1.line === p2.line && p1.column === p2.column; +export const isSameLocationRange = (r1: LocationRange, r2: LocationRange): boolean => + isSameLocation(r1.start, r2.start) && isSameLocation(r1.end, r2.end); + +export const makeZeroBased = (r: LocationRange): LocationRange => ({ + start: { line: r.start.line - 1, column: r.start.column - 1 }, + end: { line: r.end.line - 1, column: r.end.column - 1 }, +}); +export const findResultForTest = (results: TestResult[], itBlock: ItBlock): TestResult[] => { + const zeroBasedRange = makeZeroBased(itBlock); + return results.filter((r) => r.name === itBlock.name && isSameLocationRange(r, zeroBasedRange)); +}; + +// factory method +export const makeItBlock = (name?: string, pos?: [number, number, number, number]): any => { + const loc = pos ? makePositionRange(pos) : EmptyLocationRange; + return { + type: 'it', + name, + ...loc, + }; +}; +export const makeDescribeBlock = (name: string, itBlocks: any[]): any => ({ + type: 'describe', + name, + children: itBlocks, +}); +export const makeRoot = (children: any[]): any => ({ + type: 'root', + children, +}); +export const makeAssertion = ( + title: string, + status: TestReconciliationState, + ancestorTitles: string[] = [], + location?: [number, number], + override?: Partial +): TestAssertionStatus => + ({ + title, + ancestorTitles, + fullName: [...ancestorTitles, title].join(' '), + status, + location: location ? makeLocation(location) : EmptyLocation, + ...(override || {}), + } as TestAssertionStatus); diff --git a/yarn.lock b/yarn.lock index dcefb784c..0d3bd17d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -149,6 +149,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.9.6.tgz#3b1bbb30dabe600cd72db58720998376ff653bc7" integrity sha512-AoeIEJn8vt+d/6+PXDRPaksYhnlbMIiejioBZvvMQsOjW/JYK6k/0dKnvvP3EhK5GfMBWDPtrxRtegWdAcdq9Q== +"@babel/parser@^7.8.3": + version "7.10.2" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.2.tgz#871807f10442b92ff97e4783b9b54f6a0ca812d0" + integrity sha512-PApSXlNMJyB4JiGVhCOlzKIif+TKFTvu0aQAhnTvfP/z3vVSN6ZypH5bfUNwFXXjRQtUEBNFd2PtmCmG2Py3qQ== + "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" @@ -1247,11 +1252,6 @@ babel-preset-jest@^25.5.0: babel-plugin-jest-hoist "^25.5.0" babel-preset-current-node-syntax "^0.1.2" -babylon@^6.14.1: - version "6.18.0" - resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" - integrity sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ== - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -3621,17 +3621,17 @@ jest-each@^25.5.0: jest-util "^25.5.0" pretty-format "^25.5.0" -jest-editor-support@^27.2.0: - version "27.2.0" - resolved "https://registry.yarnpkg.com/jest-editor-support/-/jest-editor-support-27.2.0.tgz#4f4308180ce9b77529a77503882c32cbe0381bca" - integrity sha512-sJ0aAcKq81HFACyCJUfyuJ+9SFMU71UH9o3XHbbX95jCdoZ21SWyTHkGdn8afi7xRbccKPguCLqGSaMph6SoYA== +jest-editor-support@^28.0.0-beta: + version "28.0.0-beta.0" + resolved "https://registry.yarnpkg.com/jest-editor-support/-/jest-editor-support-28.0.0-beta.0.tgz#2c7277be185f17393d08bebeb2e4f4c0ba7df62f" + integrity sha512-9n/nCnsMwnfujhfzaJ9RH19/E1NVjJZoU+sNic79mKPH74ihprN8gAzH25gt5HFpJnuaRlVmOMEMjynHwDSyww== dependencies: + "@babel/parser" "^7.8.3" "@babel/traverse" "^7.6.2" + "@babel/types" "^7.8.3" "@jest/types" "^24.8.0" - babylon "^6.14.1" core-js "^3.2.1" jest-snapshot "^24.7.0" - typescript "^3.4.3" jest-environment-jsdom@^25.5.0: version "25.5.0" @@ -6434,7 +6434,7 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^3.4.3, typescript@^3.8.3: +typescript@^3.8.3: version "3.8.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.8.3.tgz#409eb8544ea0335711205869ec458ab109ee1061" integrity sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==