diff --git a/news/2 Fixes/1070.md b/news/2 Fixes/1070.md new file mode 100644 index 000000000000..5f9100ebc192 --- /dev/null +++ b/news/2 Fixes/1070.md @@ -0,0 +1 @@ +Improvements to the logic used to parse the arguments passed into the test frameworks. diff --git a/package-lock.json b/package-lock.json index 060f25faeccc..12cd5948e3f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -256,6 +256,12 @@ "integrity": "sha512-Tt7w/ylBS/OEAlSCwzB0Db1KbxnkycP/1UkQpbvKFYoUuRn4uYsC3xh5TRPrOjTy0i8TIkSz1JdNL4GPVdf3KQ==", "dev": true }, + "@types/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha1-EHPEvIJHVK49EM+riKsCN7qWTk0=", + "dev": true + }, "@types/tough-cookie": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.3.tgz", diff --git a/package.json b/package.json index 854b86447122..4f6382ef049d 100644 --- a/package.json +++ b/package.json @@ -1955,6 +1955,7 @@ "@types/semver": "^5.5.0", "@types/shortid": "^0.0.29", "@types/sinon": "^4.3.0", + "@types/tmp": "0.0.33", "@types/untildify": "^3.0.0", "@types/uuid": "^3.4.3", "@types/winreg": "^1.2.30", diff --git a/src/client/activation/downloader.ts b/src/client/activation/downloader.ts index 62e063c83da8..adf5b81afbf5 100644 --- a/src/client/activation/downloader.ts +++ b/src/client/activation/downloader.ts @@ -7,7 +7,7 @@ import * as request from 'request'; import * as requestProgress from 'request-progress'; import { OutputChannel, ProgressLocation, window } from 'vscode'; import { STANDARD_OUTPUT_CHANNEL } from '../common/constants'; -import { createDeferred, createTemporaryFile } from '../common/helpers'; +import { createDeferred } from '../common/helpers'; import { IFileSystem, IPlatformService } from '../common/platform/types'; import { IExtensionContext, IOutputChannel } from '../common/types'; import { IServiceContainer } from '../ioc/types'; @@ -58,14 +58,14 @@ export class AnalysisEngineDownloader { private async downloadFile(location: string, fileName: string, title: string): Promise { const uri = `${location}/${fileName}`; this.output.append(`Downloading ${uri}... `); - const tempFile = await createTemporaryFile(downloadFileExtension); + const tempFile = await this.fs.createTemporaryFile(downloadFileExtension); const deferred = createDeferred(); const fileStream = fileSystem.createWriteStream(tempFile.filePath); fileStream.on('finish', () => { fileStream.close(); }).on('error', (err) => { - tempFile.cleanupCallback(); + tempFile.dispose(); deferred.reject(err); }); diff --git a/src/client/application/diagnostics/applicationDiagnostics.ts b/src/client/application/diagnostics/applicationDiagnostics.ts index 84422846a19b..c3eee0d55fe3 100644 --- a/src/client/application/diagnostics/applicationDiagnostics.ts +++ b/src/client/application/diagnostics/applicationDiagnostics.ts @@ -20,7 +20,7 @@ export class ApplicationDiagnostics implements IApplicationDiagnostics { const diagnostics = await envHealthCheck.diagnose(); this.log(diagnostics); if (diagnostics.length > 0) { - envHealthCheck.handle(diagnostics); + await envHealthCheck.handle(diagnostics); } } private log(diagnostics: IDiagnostic[]): void { diff --git a/src/client/application/diagnostics/checks/envPathVariable.ts b/src/client/application/diagnostics/checks/envPathVariable.ts index ee8712cfa4f4..c2b09b70c0dc 100644 --- a/src/client/application/diagnostics/checks/envPathVariable.ts +++ b/src/client/application/diagnostics/checks/envPathVariable.ts @@ -54,7 +54,7 @@ export class EnvironmentPathVariableDiagnosticsService extends BaseDiagnosticsSe return; } const diagnostic = diagnostics[0]; - if (this.filterService.shouldIgnoreDiagnostic(diagnostic.code)) { + if (await this.filterService.shouldIgnoreDiagnostic(diagnostic.code)) { return; } const commandFactory = this.serviceContainer.get(IDiagnosticsCommandFactory); diff --git a/src/client/common/application/types.ts b/src/client/common/application/types.ts index d3d67245feb2..c6546ce056fe 100644 --- a/src/client/common/application/types.ts +++ b/src/client/common/application/types.ts @@ -407,7 +407,7 @@ export interface IDocumentManager { showTextDocument(uri: Uri, options?: TextDocumentShowOptions): Thenable; } -export const IWorkspaceService = Symbol('IWorkspace'); +export const IWorkspaceService = Symbol('IWorkspaceService'); export interface IWorkspaceService { /** diff --git a/src/client/common/platform/fileSystem.ts b/src/client/common/platform/fileSystem.ts index 645aad5a2577..ede57f3290e0 100644 --- a/src/client/common/platform/fileSystem.ts +++ b/src/client/common/platform/fileSystem.ts @@ -7,8 +7,9 @@ import * as fs from 'fs-extra'; import * as glob from 'glob'; import { inject, injectable } from 'inversify'; import * as path from 'path'; +import * as tmp from 'tmp'; import { createDeferred } from '../helpers'; -import { IFileSystem, IPlatformService } from './types'; +import { IFileSystem, IPlatformService, TemporaryFile } from './types'; @injectable() export class FileSystem implements IFileSystem { @@ -141,4 +142,15 @@ export class FileSystem implements IFileSystem { }); }); } + public createTemporaryFile(extension: string): Promise { + return new Promise((resolve, reject) => { + tmp.file({ postfix: extension }, (err, tmpFile, _, cleanupCallback) => { + if (err) { + return reject(err); + } + resolve({ filePath: tmpFile, dispose: cleanupCallback }); + }); + }); + + } } diff --git a/src/client/common/platform/types.ts b/src/client/common/platform/types.ts index 28b2af9ce5b2..ec08fd7d284c 100644 --- a/src/client/common/platform/types.ts +++ b/src/client/common/platform/types.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import * as fs from 'fs'; +import { Disposable } from 'vscode'; export enum Architecture { Unknown = 1, @@ -28,6 +29,8 @@ export interface IPlatformService { virtualEnvBinName: 'bin' | 'scripts'; } +export type TemporaryFile = { filePath: string } & Disposable; + export const IFileSystem = Symbol('IFileSystem'); export interface IFileSystem { directorySeparatorChar: string; @@ -48,4 +51,5 @@ export interface IFileSystem { deleteFile(filename: string): Promise; getFileHash(filePath: string): Promise; search(globPattern: string): Promise; + createTemporaryFile(extension: string): Promise; } diff --git a/src/client/unittests/common/argumentsHelper.ts b/src/client/unittests/common/argumentsHelper.ts new file mode 100644 index 000000000000..3842a9a6874e --- /dev/null +++ b/src/client/unittests/common/argumentsHelper.ts @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { ILogger } from '../../common/types'; +import { IServiceContainer } from '../../ioc/types'; +import { IArgumentsHelper } from '../types'; + +@injectable() +export class ArgumentsHelper implements IArgumentsHelper { + private readonly logger: ILogger; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { + this.logger = serviceContainer.get(ILogger); + } + public getOptionValues(args: string[], option: string): string | string[] | undefined { + const values: string[] = []; + let returnNextValue = false; + for (const arg of args) { + if (returnNextValue) { + values.push(arg); + returnNextValue = false; + continue; + } + if (arg.startsWith(`${option}=`)) { + values.push(arg.substring(`${option}=`.length)); + continue; + } + if (arg === option) { + returnNextValue = true; + } + } + switch (values.length) { + case 0: { + return; + } + case 1: { + return values[0]; + } + default: { + return values; + } + } + } + public getPositionalArguments(args: string[], optionsWithArguments: string[] = [], optionsWithoutArguments: string[] = []): string[] { + let lastIndexOfOption = -1; + args.forEach((arg, index) => { + if (optionsWithoutArguments.indexOf(arg) !== -1) { + lastIndexOfOption = index; + return; + } else if (optionsWithArguments.indexOf(arg) !== -1) { + // Cuz the next item is the value. + lastIndexOfOption = index + 1; + } else if (optionsWithArguments.findIndex(item => arg.startsWith(`${item}=`)) !== -1) { + lastIndexOfOption = index; + return; + } else if (arg.startsWith('-')) { + // Ok this is an unknown option, lets treat this as one without values. + this.logger.logWarning(`Unknown command line option passed into args parser for tests '${arg}'. Please report on https://github.com/Microsoft/vscode-python/issues/new`); + lastIndexOfOption = index; + return; + } else if (args.indexOf('=') > 0) { + // Ok this is an unknown option with a value + this.logger.logWarning(`Unknown command line option passed into args parser for tests '${arg}'. Please report on https://github.com/Microsoft/vscode-python/issues/new`); + lastIndexOfOption = index; + } + }); + return args.slice(lastIndexOfOption + 1); + } + public filterArguments(args: string[], optionsWithArguments: string[] = [], optionsWithoutArguments: string[] = []): string[] { + let ignoreIndex = -1; + return args.filter((arg, index) => { + if (ignoreIndex === index) { + return false; + } + // Options can use willd cards (with trailing '*') + if (optionsWithoutArguments.indexOf(arg) >= 0 || + optionsWithoutArguments.filter(option => option.endsWith('*') && arg.startsWith(option.slice(0, -1))).length > 0) { + return false; + } + // Ignore args that match exactly. + if (optionsWithArguments.indexOf(arg) >= 0) { + ignoreIndex = index + 1; + return false; + } + // Ignore args that match exactly with wild cards & do not have inline values. + if (optionsWithArguments.filter(option => arg.startsWith(`${option}=`)).length > 0) { + return false; + } + // Ignore args that match a wild card (ending with *) and no ineline values. + // Eg. arg='--log-cli-level' and optionsArguments=['--log-*'] + if (arg.indexOf('=') === -1 && optionsWithoutArguments.filter(option => option.endsWith('*') && arg.startsWith(option.slice(0, -1))).length > 0) { + ignoreIndex = index + 1; + return false; + } + // Ignore args that match a wild card (ending with *) and have ineline values. + // Eg. arg='--log-cli-level=XYZ' and optionsArguments=['--log-*'] + if (arg.indexOf('=') >= 0 && optionsWithoutArguments.filter(option => option.endsWith('*') && arg.startsWith(option.slice(0, -1))).length > 0) { + return false; + } + return true; + }); + } +} diff --git a/src/client/unittests/common/managers/baseTestManager.ts b/src/client/unittests/common/managers/baseTestManager.ts index 437044aa94e3..89a54b12e40e 100644 --- a/src/client/unittests/common/managers/baseTestManager.ts +++ b/src/client/unittests/common/managers/baseTestManager.ts @@ -1,7 +1,7 @@ -import { CancellationToken, CancellationTokenSource, Disposable, OutputChannel, Uri, workspace } from 'vscode'; -import { PythonSettings } from '../../../common/configSettings'; +import { CancellationToken, CancellationTokenSource, Disposable, OutputChannel, Uri } from 'vscode'; +import { IWorkspaceService } from '../../../common/application/types'; import { isNotInstalledError } from '../../../common/helpers'; -import { IDisposableRegistry, IInstaller, IOutputChannel, IPythonSettings, Product } from '../../../common/types'; +import { IConfigurationService, IDisposableRegistry, IInstaller, IOutputChannel, IPythonSettings, Product } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; import { UNITTEST_DISCOVER, UNITTEST_RUN } from '../../../telemetry/constants'; import { sendTelemetryEvent } from '../../../telemetry/index'; @@ -25,9 +25,9 @@ export abstract class BaseTestManager implements ITestManager { } private testCollectionStorage: ITestCollectionStorageService; private _testResultsService: ITestResultsService; + private workspaceService: IWorkspaceService; private _outputChannel: OutputChannel; private tests?: Tests; - // tslint:disable-next-line:variable-name private _status: TestStatus = TestStatus.Unknown; private testDiscoveryCancellationTokenSource?: CancellationTokenSource; private testRunnerCancellationTokenSource?: CancellationTokenSource; @@ -42,12 +42,14 @@ export abstract class BaseTestManager implements ITestManager { constructor(public readonly testProvider: TestProvider, private product: Product, public readonly workspaceFolder: Uri, protected rootDirectory: string, protected serviceContainer: IServiceContainer) { this._status = TestStatus.Unknown; - this.settings = PythonSettings.getInstance(this.rootDirectory ? Uri.file(this.rootDirectory) : undefined); + const configService = serviceContainer.get(IConfigurationService); + this.settings = configService.getSettings(this.rootDirectory ? Uri.file(this.rootDirectory) : undefined); const disposables = serviceContainer.get(IDisposableRegistry); - disposables.push(this); this._outputChannel = this.serviceContainer.get(IOutputChannel, TEST_OUTPUT_CHANNEL); this.testCollectionStorage = this.serviceContainer.get(ITestCollectionStorageService); this._testResultsService = this.serviceContainer.get(ITestResultsService); + this.workspaceService = this.serviceContainer.get(IWorkspaceService); + disposables.push(this); } protected get testDiscoveryCancellationToken(): CancellationToken | undefined { return this.testDiscoveryCancellationTokenSource ? this.testDiscoveryCancellationTokenSource.token : undefined; @@ -62,8 +64,7 @@ export abstract class BaseTestManager implements ITestManager { return this._status; } public get workingDirectory(): string { - const settings = PythonSettings.getInstance(Uri.file(this.rootDirectory)); - return settings.unitTest.cwd && settings.unitTest.cwd.length > 0 ? settings.unitTest.cwd : this.rootDirectory; + return this.settings.unitTest.cwd && this.settings.unitTest.cwd.length > 0 ? this.settings.unitTest.cwd : this.rootDirectory; } public stop() { if (this.testDiscoveryCancellationTokenSource) { @@ -132,7 +133,7 @@ export abstract class BaseTestManager implements ITestManager { const testsHelper = this.serviceContainer.get(ITestsHelper); testsHelper.displayTestErrorMessage('There were some errors in discovering unit tests'); } - const wkspace = workspace.getWorkspaceFolder(Uri.file(this.rootDirectory))!.uri; + const wkspace = this.workspaceService.getWorkspaceFolder(Uri.file(this.rootDirectory))!.uri; this.testCollectionStorage.storeTests(wkspace, tests); this.disposeCancellationToken(CancellationTokenType.testDiscovery); sendTelemetryEvent(UNITTEST_DISCOVER, undefined, telementryProperties); @@ -156,7 +157,7 @@ export abstract class BaseTestManager implements ITestManager { // tslint:disable-next-line:prefer-template this.outputChannel.appendLine(reason.toString()); } - const wkspace = workspace.getWorkspaceFolder(Uri.file(this.rootDirectory))!.uri; + const wkspace = this.workspaceService.getWorkspaceFolder(Uri.file(this.rootDirectory))!.uri; this.testCollectionStorage.storeTests(wkspace, null); this.disposeCancellationToken(CancellationTokenType.testDiscovery); return Promise.reject(reason); diff --git a/src/client/unittests/common/runner.ts b/src/client/unittests/common/runner.ts index 9bf612a33b58..a8b1d216423e 100644 --- a/src/client/unittests/common/runner.ts +++ b/src/client/unittests/common/runner.ts @@ -1,3 +1,4 @@ +import { inject, injectable } from 'inversify'; import * as path from 'path'; import { CancellationToken, OutputChannel, Uri } from 'vscode'; import { PythonSettings } from '../../common/configSettings'; @@ -13,15 +14,16 @@ import { import { ExecutionInfo, IPythonSettings } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; import { NOSETEST_PROVIDER, PYTEST_PROVIDER, UNITTEST_PROVIDER } from './constants'; -import { ITestsHelper, TestProvider } from './types'; +import { ITestRunner, ITestsHelper, Options, TestProvider } from './types'; +export { Options } from './types'; -export type Options = { - workspaceFolder: Uri; - cwd: string; - args: string[]; - outChannel?: OutputChannel; - token: CancellationToken; -}; +@injectable() +export class TestRunner implements ITestRunner { + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { } + public run(testProvider: TestProvider, options: Options): Promise { + return run(this.serviceContainer, testProvider, options); + } +} export async function run(serviceContainer: IServiceContainer, testProvider: TestProvider, options: Options): Promise { const testExecutablePath = getExecutablePath(testProvider, PythonSettings.getInstance(options.workspaceFolder)); diff --git a/src/client/unittests/common/services/storageService.ts b/src/client/unittests/common/services/storageService.ts index 14945216bf78..859cf2939cbf 100644 --- a/src/client/unittests/common/services/storageService.ts +++ b/src/client/unittests/common/services/storageService.ts @@ -6,7 +6,7 @@ import { ITestCollectionStorageService, Tests } from './../types'; @injectable() export class TestCollectionStorageService implements ITestCollectionStorageService { private testsIndexedByWorkspaceUri = new Map(); - constructor( @inject(IDisposableRegistry) disposables: Disposable[]) { + constructor(@inject(IDisposableRegistry) disposables: Disposable[]) { disposables.push(this); } public getTests(wkspace: Uri): Tests | undefined { diff --git a/src/client/unittests/common/testUtils.ts b/src/client/unittests/common/testUtils.ts index 535f6c28c1a7..feb4d7c9be50 100644 --- a/src/client/unittests/common/testUtils.ts +++ b/src/client/unittests/common/testUtils.ts @@ -171,4 +171,39 @@ export class TestsHelper implements ITestsHelper { } }); } + public mergeTests(items: Tests[]): Tests { + return items.reduce((tests, otherTests, index) => { + if (index === 0) { + return tests; + } + + tests.summary.errors += otherTests.summary.errors; + tests.summary.failures += otherTests.summary.failures; + tests.summary.passed += otherTests.summary.passed; + tests.summary.skipped += otherTests.summary.skipped; + tests.rootTestFolders.push(...otherTests.rootTestFolders); + tests.testFiles.push(...otherTests.testFiles); + tests.testFolders.push(...otherTests.testFolders); + tests.testFunctions.push(...otherTests.testFunctions); + tests.testSuites.push(...otherTests.testSuites); + + return tests; + }, items[0]); + } + + public shouldRunAllTests(testsToRun?: TestsToRun) { + if (!testsToRun) { + return true; + } + if ( + (Array.isArray(testsToRun.testFile) && testsToRun.testFile.length > 0) || + (Array.isArray(testsToRun.testFolder) && testsToRun.testFolder.length > 0) || + (Array.isArray(testsToRun.testFunction) && testsToRun.testFunction.length > 0) || + (Array.isArray(testsToRun.testSuite) && testsToRun.testSuite.length > 0) + ) { + return false; + } + + return true; + } } diff --git a/src/client/unittests/common/types.ts b/src/client/unittests/common/types.ts index e5ad14fa6fd7..67e2663efab9 100644 --- a/src/client/unittests/common/types.ts +++ b/src/client/unittests/common/types.ts @@ -161,6 +161,8 @@ export interface ITestsHelper { flattenTestFiles(testFiles: TestFile[]): Tests; placeTestFilesIntoFolders(tests: Tests): void; displayTestErrorMessage(message: string): void; + shouldRunAllTests(testsToRun?: TestsToRun): boolean; + mergeTests(items: Tests[]): Tests; } export const ITestVisitor = Symbol('ITestVisitor'); @@ -244,3 +246,26 @@ export interface IUnitTestSocketServer extends Disposable { start(options?: { port?: number; host?: string }): Promise; stop(): void; } + +export type Options = { + workspaceFolder: Uri; + cwd: string; + args: string[]; + outChannel?: OutputChannel; + token: CancellationToken; +}; + +export const ITestRunner = Symbol('ITestRunner'); +export interface ITestRunner { + run(testProvider: TestProvider, options: Options): Promise; +} + +export enum PassCalculationFormulae { + pytest, + nosetests +} + +export const IXUnitParser = Symbol('IXUnitParser'); +export interface IXUnitParser { + updateResultsFromXmlLogFile(tests: Tests, outputXmlFile: string, passCalculationFormulae: PassCalculationFormulae): Promise; +} diff --git a/src/client/unittests/common/xUnitParser.ts b/src/client/unittests/common/xUnitParser.ts index 6318060a94f7..091ad9e87db2 100644 --- a/src/client/unittests/common/xUnitParser.ts +++ b/src/client/unittests/common/xUnitParser.ts @@ -1,11 +1,7 @@ import * as fs from 'fs'; +import { injectable } from 'inversify'; import * as xml2js from 'xml2js'; -import { Tests, TestStatus } from './types'; - -export enum PassCalculationFormulae { - pytest, - nosetests -} +import { IXUnitParser, PassCalculationFormulae, Tests, TestStatus } from './types'; type TestSuiteResult = { $: { errors: string; @@ -28,15 +24,15 @@ type TestCaseResult = { }; failure: { _: string; - $: { message: string, type: string } + $: { message: string; type: string }; }[]; error: { _: string; - $: { message: string, type: string } + $: { message: string; type: string }; }[]; skipped: { _: string; - $: { message: string, type: string } + $: { message: string; type: string }; }[]; }; @@ -46,7 +42,14 @@ function getSafeInt(value: string, defaultValue: any = 0): number { if (isNaN(num)) { return defaultValue; } return num; } -export function updateResultsFromXmlLogFile(tests: Tests, outputXmlFile: string, passCalculationFormulae: PassCalculationFormulae): Promise<{}> { + +@injectable() +export class XUnitParser implements IXUnitParser { + public updateResultsFromXmlLogFile(tests: Tests, outputXmlFile: string, passCalculationFormulae: PassCalculationFormulae): Promise { + return updateResultsFromXmlLogFile(tests, outputXmlFile, passCalculationFormulae); + } +} +export function updateResultsFromXmlLogFile(tests: Tests, outputXmlFile: string, passCalculationFormulae: PassCalculationFormulae): Promise { // tslint:disable-next-line:no-any return new Promise((resolve, reject) => { fs.readFile(outputXmlFile, 'utf8', (err, data) => { @@ -127,7 +130,7 @@ export function updateResultsFromXmlLogFile(tests: Tests, outputXmlFile: string, if (testcase.skipped) { result.testFunction.status = TestStatus.Skipped; - result.testFunction.passed = null; + result.testFunction.passed = undefined; result.testFunction.message = testcase.skipped[0].$.message; result.testFunction.traceback = ''; } diff --git a/src/client/unittests/nosetest/main.ts b/src/client/unittests/nosetest/main.ts index 5ffae0b80cd3..19aa8b125e9a 100644 --- a/src/client/unittests/nosetest/main.ts +++ b/src/client/unittests/nosetest/main.ts @@ -1,20 +1,26 @@ import { inject, injectable } from 'inversify'; import { Uri } from 'vscode'; -import { PythonSettings } from '../../common/configSettings'; import { Product } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; +import { NOSETEST_PROVIDER } from '../common/constants'; import { BaseTestManager } from '../common/managers/baseTestManager'; -import { TestDiscoveryOptions, TestRunOptions, Tests, TestsToRun } from '../common/types'; -import { runTest } from './runner'; +import { ITestsHelper, TestDiscoveryOptions, TestRunOptions, Tests, TestsToRun } from '../common/types'; +import { IArgumentsService, ITestManagerRunner, TestFilter } from '../types'; @injectable() export class TestManager extends BaseTestManager { + private readonly argsService: IArgumentsService; + private readonly helper: ITestsHelper; + private readonly runner: ITestManagerRunner; public get enabled() { - return PythonSettings.getInstance(this.workspaceFolder).unitTest.nosetestsEnabled; + return this.settings.unitTest.nosetestsEnabled; } constructor(workspaceFolder: Uri, rootDirectory: string, @inject(IServiceContainer) serviceContainer: IServiceContainer) { - super('nosetest', Product.nosetest, workspaceFolder, rootDirectory, serviceContainer); + super(NOSETEST_PROVIDER, Product.nosetest, workspaceFolder, rootDirectory, serviceContainer); + this.argsService = this.serviceContainer.get(IArgumentsService, this.testProvider); + this.helper = this.serviceContainer.get(ITestsHelper); + this.runner = this.serviceContainer.get(ITestManagerRunner, this.testProvider); } public getDiscoveryOptions(ignoreCache: boolean): TestDiscoveryOptions { const args = this.settings.unitTest.nosetestArgs.slice(0); @@ -25,14 +31,21 @@ export class TestManager extends BaseTestManager { outChannel: this.outputChannel }; } - // tslint:disable-next-line:no-any - public runTestImpl(tests: Tests, testsToRun?: TestsToRun, runFailedTests?: boolean, debug?: boolean): Promise { - const args = this.settings.unitTest.nosetestArgs.slice(0); + public runTestImpl(tests: Tests, testsToRun?: TestsToRun, runFailedTests?: boolean, debug?: boolean): Promise { + let args: string[]; + + const runAllTests = this.helper.shouldRunAllTests(testsToRun); + if (debug) { + args = this.argsService.filterArguments(this.settings.unitTest.nosetestArgs, runAllTests ? TestFilter.debugAll : TestFilter.debugSpecific); + } else { + args = this.argsService.filterArguments(this.settings.unitTest.nosetestArgs, runAllTests ? TestFilter.runAll : TestFilter.runSpecific); + } + if (runFailedTests === true && args.indexOf('--failed') === -1) { - args.push('--failed'); + args.splice(0, 0, '--failed'); } if (!runFailedTests && args.indexOf('--with-id') === -1) { - args.push('--with-id'); + args.splice(0, 0, '--with-id'); } const options: TestRunOptions = { workspaceFolder: Uri.file(this.rootDirectory), @@ -42,6 +55,6 @@ export class TestManager extends BaseTestManager { outChannel: this.outputChannel, debug }; - return runTest(this.serviceContainer, this.testResultsService, options); + return this.runner.runTest(this.testResultsService, options, this); } } diff --git a/src/client/unittests/nosetest/runner.ts b/src/client/unittests/nosetest/runner.ts index 64a170dd5150..a6ba3cc148e0 100644 --- a/src/client/unittests/nosetest/runner.ts +++ b/src/client/unittests/nosetest/runner.ts @@ -1,99 +1,99 @@ 'use strict'; -import { createTemporaryFile } from '../../common/helpers'; + +import { inject, injectable } from 'inversify'; +import { noop } from '../../common/core.utils'; +import { IFileSystem, TemporaryFile } from '../../common/platform/types'; import { IServiceContainer } from '../../ioc/types'; -import { Options, run } from '../common/runner'; -import { ITestDebugLauncher, ITestResultsService, LaunchOptions, TestRunOptions, Tests } from '../common/types'; -import { PassCalculationFormulae, updateResultsFromXmlLogFile } from '../common/xUnitParser'; +import { NOSETEST_PROVIDER } from '../common/constants'; +import { Options } from '../common/runner'; +import { ITestDebugLauncher, ITestManager, ITestResultsService, ITestRunner, IXUnitParser, LaunchOptions, PassCalculationFormulae, TestRunOptions, Tests } from '../common/types'; +import { IArgumentsHelper, IArgumentsService, ITestManagerRunner } from '../types'; const WITH_XUNIT = '--with-xunit'; const XUNIT_FILE = '--xunit-file'; -// tslint:disable-next-line:no-any -export function runTest(serviceContainer: IServiceContainer, testResultsService: ITestResultsService, options: TestRunOptions): Promise { - let testPaths: string[] = []; - if (options.testsToRun && options.testsToRun.testFolder) { - testPaths = testPaths.concat(options.testsToRun.testFolder.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testFile) { - testPaths = testPaths.concat(options.testsToRun.testFile.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testSuite) { - testPaths = testPaths.concat(options.testsToRun.testSuite.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testFunction) { - testPaths = testPaths.concat(options.testsToRun.testFunction.map(f => f.nameToRun)); - } - - let xmlLogFile = ''; - // tslint:disable-next-line:no-empty - let xmlLogFileCleanup: Function = () => { }; - - // Check if '--with-xunit' is in args list - const noseTestArgs = options.args.slice(); - if (noseTestArgs.indexOf(WITH_XUNIT) === -1) { - noseTestArgs.push(WITH_XUNIT); +@injectable() +export class TestManagerRunner implements ITestManagerRunner { + private readonly argsService: IArgumentsService; + private readonly argsHelper: IArgumentsHelper; + private readonly testRunner: ITestRunner; + private readonly xUnitParser: IXUnitParser; + private readonly fs: IFileSystem; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { + this.argsService = serviceContainer.get(IArgumentsService, NOSETEST_PROVIDER); + this.argsHelper = serviceContainer.get(IArgumentsHelper); + this.testRunner = serviceContainer.get(ITestRunner); + this.xUnitParser = this.serviceContainer.get(IXUnitParser); + this.fs = this.serviceContainer.get(IFileSystem); } + public async runTest(testResultsService: ITestResultsService, options: TestRunOptions, _: ITestManager): Promise { + let testPaths: string[] = []; + if (options.testsToRun && options.testsToRun.testFolder) { + testPaths = testPaths.concat(options.testsToRun.testFolder.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testFile) { + testPaths = testPaths.concat(options.testsToRun.testFile.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testSuite) { + testPaths = testPaths.concat(options.testsToRun.testSuite.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testFunction) { + testPaths = testPaths.concat(options.testsToRun.testFunction.map(f => f.nameToRun)); + } - // Check if '--xunit-file' exists, if not generate random xml file - const indexOfXUnitFile = noseTestArgs.findIndex(value => value.indexOf(XUNIT_FILE) === 0); - let promiseToGetXmlLogFile: Promise; - if (indexOfXUnitFile === -1) { - promiseToGetXmlLogFile = createTemporaryFile('.xml').then(xmlLogResult => { - xmlLogFileCleanup = xmlLogResult.cleanupCallback; - xmlLogFile = xmlLogResult.filePath; - - noseTestArgs.push(`${XUNIT_FILE}=${xmlLogFile}`); - return xmlLogResult.filePath; - }); - } else { - if (noseTestArgs[indexOfXUnitFile].indexOf('=') === -1) { - xmlLogFile = noseTestArgs[indexOfXUnitFile + 1]; - } else { - xmlLogFile = noseTestArgs[indexOfXUnitFile].substring(noseTestArgs[indexOfXUnitFile].indexOf('=') + 1).trim(); + let deleteJUnitXmlFile: Function = noop; + const args = options.args; + // Check if '--with-xunit' is in args list + if (args.indexOf(WITH_XUNIT) === -1) { + args.splice(0, 0, WITH_XUNIT); } - promiseToGetXmlLogFile = Promise.resolve(xmlLogFile); - } + try { + const xmlLogResult = await this.getUnitXmlFile(args); + const xmlLogFile = xmlLogResult.filePath; + deleteJUnitXmlFile = xmlLogResult.dispose; + // Remove the '--unixml' if it exists, and add it with our path. + const testArgs = this.argsService.filterArguments(args, [XUNIT_FILE]); + testArgs.splice(0, 0, `${XUNIT_FILE}=${xmlLogFile}`); - return promiseToGetXmlLogFile.then(() => { - if (options.debug === true) { - const debugLauncher = serviceContainer.get(ITestDebugLauncher); - const nosetestlauncherargs = [options.cwd, 'nose']; - const debuggerArgs = nosetestlauncherargs.concat(noseTestArgs.concat(testPaths)); - const launchOptions: LaunchOptions = { cwd: options.cwd, args: debuggerArgs, token: options.token, outChannel: options.outChannel, testProvider: 'nosetest' }; - // tslint:disable-next-line:prefer-type-cast no-any - return debugLauncher.launchDebugger(launchOptions) as Promise; - } else { - // tslint:disable-next-line:prefer-type-cast no-any - const runOptions: Options = { - args: noseTestArgs.concat(testPaths), - cwd: options.cwd, - outChannel: options.outChannel, - token: options.token, - workspaceFolder: options.workspaceFolder - }; + // Positional arguments control the tests to be run. + testArgs.push(...testPaths); - // Remove the directory argument, as we'll provide tests to be run. - if (testPaths.length > 0 && runOptions.args.length > 0 && !runOptions.args[0].trim().startsWith('-')) { - runOptions.args.shift(); + if (options.debug === true) { + const debugLauncher = this.serviceContainer.get(ITestDebugLauncher); + const debuggerArgs = [options.cwd, 'nose'].concat(testArgs); + const launchOptions: LaunchOptions = { cwd: options.cwd, args: debuggerArgs, token: options.token, outChannel: options.outChannel, testProvider: NOSETEST_PROVIDER }; + await debugLauncher.launchDebugger(launchOptions); + } else { + const runOptions: Options = { + args: testArgs.concat(testPaths), + cwd: options.cwd, + outChannel: options.outChannel, + token: options.token, + workspaceFolder: options.workspaceFolder + }; + await this.testRunner.run(NOSETEST_PROVIDER, runOptions); } - return run(serviceContainer, 'nosetest', runOptions); + + return options.debug ? options.tests : await this.updateResultsFromLogFiles(options.tests, xmlLogFile, testResultsService); + } catch (ex) { + return Promise.reject(ex); + } finally { + deleteJUnitXmlFile(); } - }).then(() => { - return options.debug ? options.tests : updateResultsFromLogFiles(options.tests, xmlLogFile, testResultsService); - }).then(result => { - xmlLogFileCleanup(); - return result; - }).catch(reason => { - xmlLogFileCleanup(); - return Promise.reject(reason); - }); -} + } -// tslint:disable-next-line:no-any -export function updateResultsFromLogFiles(tests: Tests, outputXmlFile: string, testResultsService: ITestResultsService): Promise { - return updateResultsFromXmlLogFile(tests, outputXmlFile, PassCalculationFormulae.nosetests).then(() => { + private async updateResultsFromLogFiles(tests: Tests, outputXmlFile: string, testResultsService: ITestResultsService): Promise { + await this.xUnitParser.updateResultsFromXmlLogFile(tests, outputXmlFile, PassCalculationFormulae.nosetests); testResultsService.updateResults(tests); return tests; - }); + } + private async getUnitXmlFile(args: string[]): Promise { + const xmlFile = this.argsHelper.getOptionValues(args, XUNIT_FILE); + if (typeof xmlFile === 'string') { + return { filePath: xmlFile, dispose: noop }; + } + + return this.fs.createTemporaryFile('.xml'); + } } diff --git a/src/client/unittests/nosetest/services/argsService.ts b/src/client/unittests/nosetest/services/argsService.ts new file mode 100644 index 000000000000..8fbe92f41006 --- /dev/null +++ b/src/client/unittests/nosetest/services/argsService.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { IServiceContainer } from '../../../ioc/types'; +import { IArgumentsHelper, IArgumentsService, TestFilter } from '../../types'; + +const OptionsWithArguments = ['--attr', '--config', '--cover-html-dir', '--cover-min-percentage', + '--cover-package', '--cover-xml-file', '--debug', '--debug-log', '--doctest-extension', + '--doctest-fixtures', '--doctest-options', '--doctest-result-variable', '--eval-attr', + '--exclude', '--id-file', '--ignore-files', '--include', '--log-config', '--logging-config', + '--logging-datefmt', '--logging-filter', '--logging-format', '--logging-level', '--match', + '--process-timeout', '--processes', '--py3where', '--testmatch', '--tests', '--verbosity', + '--where', '--xunit-file', '--xunit-testsuite-name', + '-A', '-a', '-c', '-e', '-i', '-I', '-l', '-m', '-w', + '--profile-restrict', '--profile-sort', '--profile-stats-file']; + +const OptionsWithoutArguments = ['-h', '--help', '-V', '--version', '-p', '--plugins', + '-v', '--verbose', '--quiet', '-x', '--stop', '-P', '--no-path-adjustment', + '--exe', '--noexe', '--traverse-namespace', '--first-package-wins', '--first-pkg-wins', + '--1st-pkg-wins', '--no-byte-compile', '-s', '--nocapture', '--nologcapture', + '--logging-clear-handlers', '--with-coverage', '--cover-erase', '--cover-tests', + '--cover-inclusive', '--cover-html', '--cover-branches', '--cover-xml', '--pdb', + '--pdb-failures', '--pdb-errors', '--no-deprecated', '--with-doctest', '--doctest-tests', + '--with-isolation', '-d', '--detailed-errors', '--failure-detail', '--no-skip', + '--with-id', '--failed', '--process-restartworker', '--with-xunit', + '--all-modules', '--collect-only', '--with-profile']; + +@injectable() +export class ArgumentsService implements IArgumentsService { + private readonly helper: IArgumentsHelper; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { + this.helper = serviceContainer.get(IArgumentsHelper); + } + public getKnownOptions(): { withArgs: string[]; withoutArgs: string[] } { + return { + withArgs: OptionsWithArguments, + withoutArgs: OptionsWithoutArguments + }; + } + public getOptionValue(args: string[], option: string): string | string[] | undefined { + return this.helper.getOptionValues(args, option); + } + // tslint:disable-next-line:max-func-body-length + public filterArguments(args: string[], argumentToRemoveOrFilter: string[] | TestFilter): string[] { + const optionsWithoutArgsToRemove: string[] = []; + const optionsWithArgsToRemove: string[] = []; + // Positional arguments in nosetest are test directories and files. + // So if we want to run a specific test, then remove positional args. + let removePositionalArgs = false; + if (Array.isArray(argumentToRemoveOrFilter)) { + argumentToRemoveOrFilter.forEach(item => { + if (OptionsWithArguments.indexOf(item) >= 0) { + optionsWithArgsToRemove.push(item); + } + if (OptionsWithoutArguments.indexOf(item) >= 0) { + optionsWithoutArgsToRemove.push(item); + } + }); + } else { + switch (argumentToRemoveOrFilter) { + case TestFilter.removeTests: { + removePositionalArgs = true; + break; + } + case TestFilter.discovery: { + optionsWithoutArgsToRemove.push(...[ + '-v', '--verbose', '-q', '--quiet', + '-x', '--stop', + '--with-coverage', + ...OptionsWithoutArguments.filter(item => item.startsWith('--cover')), + ...OptionsWithoutArguments.filter(item => item.startsWith('--logging')), + ...OptionsWithoutArguments.filter(item => item.startsWith('--pdb')), + ...OptionsWithoutArguments.filter(item => item.indexOf('xunit') >= 0) + ]); + optionsWithArgsToRemove.push(...[ + '--verbosity', '-l', '--debug', '--cover-package', + ...OptionsWithoutArguments.filter(item => item.startsWith('--cover')), + ...OptionsWithArguments.filter(item => item.startsWith('--logging')), + ...OptionsWithoutArguments.filter(item => item.indexOf('xunit') >= 0) + ]); + break; + } + case TestFilter.debugAll: + case TestFilter.runAll: { + break; + } + case TestFilter.debugSpecific: + case TestFilter.runSpecific: { + removePositionalArgs = true; + break; + } + default: { + throw new Error(`Unsupported Filter '${argumentToRemoveOrFilter}'`); + } + } + } + + let filteredArgs = args.slice(); + if (removePositionalArgs) { + const positionalArgs = this.helper.getPositionalArguments(filteredArgs, OptionsWithArguments, OptionsWithoutArguments); + filteredArgs = filteredArgs.filter(item => positionalArgs.indexOf(item) === -1); + } + return this.helper.filterArguments(filteredArgs, optionsWithArgsToRemove, optionsWithoutArgsToRemove); + } + public getTestFolders(args: string[]): string[] { + return this.helper.getPositionalArguments(args, OptionsWithArguments, OptionsWithoutArguments); + } +} diff --git a/src/client/unittests/nosetest/services/discoveryService.ts b/src/client/unittests/nosetest/services/discoveryService.ts index acd651189b6d..157b24d11257 100644 --- a/src/client/unittests/nosetest/services/discoveryService.ts +++ b/src/client/unittests/nosetest/services/discoveryService.ts @@ -5,43 +5,33 @@ import { inject, injectable, named } from 'inversify'; import { CancellationTokenSource } from 'vscode'; import { IServiceContainer } from '../../../ioc/types'; import { NOSETEST_PROVIDER } from '../../common/constants'; -import { Options, run } from '../../common/runner'; -import { ITestDiscoveryService, ITestsParser, TestDiscoveryOptions, Tests } from '../../common/types'; - -const argsToExcludeForDiscovery = ['-v', '--verbose', - '-q', '--quiet', '-x', '--stop', - '--with-coverage', '--cover-erase', '--cover-tests', - '--cover-inclusive', '--cover-html', '--cover-branches', '--cover-xml', - '--pdb', '--pdb-failures', '--pdb-errors', - '--failed', '--process-restartworker', '--with-xunit']; -const settingsInArgsToExcludeForDiscovery = ['--verbosity']; +import { Options } from '../../common/runner'; +import { ITestDiscoveryService, ITestRunner, ITestsParser, TestDiscoveryOptions, Tests } from '../../common/types'; +import { IArgumentsService, TestFilter } from '../../types'; @injectable() export class TestDiscoveryService implements ITestDiscoveryService { - constructor( @inject(IServiceContainer) private serviceContainer: IServiceContainer, - @inject(ITestsParser) @named(NOSETEST_PROVIDER) private testParser: ITestsParser) { } + private argsService: IArgumentsService; + private runner: ITestRunner; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer, + @inject(ITestsParser) @named(NOSETEST_PROVIDER) private testParser: ITestsParser) { + this.argsService = this.serviceContainer.get(IArgumentsService, NOSETEST_PROVIDER); + this.runner = this.serviceContainer.get(ITestRunner); + } public async discoverTests(options: TestDiscoveryOptions): Promise { - // Remove unwanted arguments - const args = options.args.filter(arg => { - if (argsToExcludeForDiscovery.indexOf(arg.trim()) !== -1) { - return false; - } - if (settingsInArgsToExcludeForDiscovery.some(setting => setting.indexOf(arg.trim()) === 0)) { - return false; - } - return true; - }); + // Remove unwanted arguments. + const args = this.argsService.filterArguments(options.args, TestFilter.discovery); const token = options.token ? options.token : new CancellationTokenSource().token; const runOptions: Options = { - args: args.concat(['--collect-only', '-vvv']), + args: ['--collect-only', '-vvv'].concat(args), cwd: options.cwd, workspaceFolder: options.workspaceFolder, token, outChannel: options.outChannel }; - const data = await run(this.serviceContainer, NOSETEST_PROVIDER, runOptions); + const data = await this.runner.run(NOSETEST_PROVIDER, runOptions); if (options.token && options.token.isCancellationRequested) { return Promise.reject('cancelled'); } diff --git a/src/client/unittests/nosetest/services/parserService.ts b/src/client/unittests/nosetest/services/parserService.ts index fc706ec67606..634369db83c7 100644 --- a/src/client/unittests/nosetest/services/parserService.ts +++ b/src/client/unittests/nosetest/services/parserService.ts @@ -13,7 +13,7 @@ const NOSE_WANT_FILE_SUFFIX_WITHOUT_EXT = '? True'; @injectable() export class TestsParser implements ITestsParser { - constructor( @inject(ITestsHelper) private testsHelper: ITestsHelper) { } + constructor(@inject(ITestsHelper) private testsHelper: ITestsHelper) { } public parse(content: string, options: ParserOptions): Tests { let testFiles = this.getTestFiles(content, options); // Exclude tests that don't have any functions or test suites. @@ -121,9 +121,10 @@ export class TestsParser implements ITestsParser { time: 0, functionsFailed: 0, functionsPassed: 0 }; - // tslint:disable-next-line:no-non-null-assertion - const cls = testFile.suites.find(suite => suite.name === clsName)!; - cls.functions.push(fn); + const cls = testFile.suites.find(suite => suite.name === clsName); + if (cls) { + cls.functions.push(fn); + } return; } if (line.startsWith('nose.selector: DEBUG: wantFunction (IArgumentsService, this.testProvider); + this.helper = this.serviceContainer.get(ITestsHelper); + this.runner = this.serviceContainer.get(ITestManagerRunner, this.testProvider); } public getDiscoveryOptions(ignoreCache: boolean): TestDiscoveryOptions { const args = this.settings.unitTest.pyTestArgs.slice(0); @@ -24,10 +31,18 @@ export class TestManager extends BaseTestManager { outChannel: this.outputChannel }; } - public async runTestImpl(tests: Tests, testsToRun?: TestsToRun, runFailedTests?: boolean, debug?: boolean): Promise<{}> { - const args = this.settings.unitTest.pyTestArgs.slice(0); + public async runTestImpl(tests: Tests, testsToRun?: TestsToRun, runFailedTests?: boolean, debug?: boolean): Promise { + let args: string[]; + + const runAllTests = this.helper.shouldRunAllTests(testsToRun); + if (debug) { + args = this.argsService.filterArguments(this.settings.unitTest.pyTestArgs, runAllTests ? TestFilter.debugAll : TestFilter.debugSpecific); + } else { + args = this.argsService.filterArguments(this.settings.unitTest.pyTestArgs, runAllTests ? TestFilter.runAll : TestFilter.runSpecific); + } + if (runFailedTests === true && args.indexOf('--lf') === -1 && args.indexOf('--last-failed') === -1) { - args.push('--last-failed'); + args.splice(0, 0, '--last-failed'); } const options: TestRunOptions = { workspaceFolder: this.workspaceFolder, @@ -36,6 +51,6 @@ export class TestManager extends BaseTestManager { token: this.testRunnerCancellationToken!, outChannel: this.outputChannel }; - return runTest(this.serviceContainer, this.testResultsService, options); + return this.runner.runTest(this.testResultsService, options, this); } } diff --git a/src/client/unittests/pytest/runner.ts b/src/client/unittests/pytest/runner.ts index fee6b1e86619..8bff79d5a9b3 100644 --- a/src/client/unittests/pytest/runner.ts +++ b/src/client/unittests/pytest/runner.ts @@ -1,68 +1,92 @@ 'use strict'; -import { createTemporaryFile } from '../../common/helpers'; +import { inject, injectable } from 'inversify'; +import { noop } from '../../common/core.utils'; +import { IFileSystem, TemporaryFile } from '../../common/platform/types'; import { IServiceContainer } from '../../ioc/types'; -import { Options, run } from '../common/runner'; -import { ITestDebugLauncher, ITestResultsService, LaunchOptions, TestRunOptions, Tests } from '../common/types'; -import { PassCalculationFormulae, updateResultsFromXmlLogFile } from '../common/xUnitParser'; +import { PYTEST_PROVIDER } from '../common/constants'; +import { Options } from '../common/runner'; +import { ITestDebugLauncher, ITestManager, ITestResultsService, ITestRunner, IXUnitParser, LaunchOptions, PassCalculationFormulae, TestRunOptions, Tests } from '../common/types'; +import { IArgumentsHelper, IArgumentsService, ITestManagerRunner } from '../types'; -export function runTest(serviceContainer: IServiceContainer, testResultsService: ITestResultsService, options: TestRunOptions): Promise { - let testPaths: string[] = []; - if (options.testsToRun && options.testsToRun.testFolder) { - testPaths = testPaths.concat(options.testsToRun.testFolder.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testFile) { - testPaths = testPaths.concat(options.testsToRun.testFile.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testSuite) { - testPaths = testPaths.concat(options.testsToRun.testSuite.map(f => f.nameToRun)); - } - if (options.testsToRun && options.testsToRun.testFunction) { - testPaths = testPaths.concat(options.testsToRun.testFunction.map(f => f.nameToRun)); +const JunitXmlArg = '--junitxml'; +@injectable() +export class TestManagerRunner implements ITestManagerRunner { + private readonly argsService: IArgumentsService; + private readonly argsHelper: IArgumentsHelper; + private readonly testRunner: ITestRunner; + private readonly xUnitParser: IXUnitParser; + private readonly fs: IFileSystem; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { + this.argsService = serviceContainer.get(IArgumentsService, PYTEST_PROVIDER); + this.argsHelper = serviceContainer.get(IArgumentsHelper); + this.testRunner = serviceContainer.get(ITestRunner); + this.xUnitParser = this.serviceContainer.get(IXUnitParser); + this.fs = this.serviceContainer.get(IFileSystem); } + public async runTest(testResultsService: ITestResultsService, options: TestRunOptions, _: ITestManager): Promise { + let testPaths: string[] = []; + if (options.testsToRun && options.testsToRun.testFolder) { + testPaths = testPaths.concat(options.testsToRun.testFolder.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testFile) { + testPaths = testPaths.concat(options.testsToRun.testFile.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testSuite) { + testPaths = testPaths.concat(options.testsToRun.testSuite.map(f => f.nameToRun)); + } + if (options.testsToRun && options.testsToRun.testFunction) { + testPaths = testPaths.concat(options.testsToRun.testFunction.map(f => f.nameToRun)); + } - let xmlLogFile = ''; - let xmlLogFileCleanup: Function; - let args = options.args; + let deleteJUnitXmlFile: Function = noop; + const args = options.args; + try { + const xmlLogResult = await this.getJUnitXmlFile(args); + const xmlLogFile = xmlLogResult.filePath; + deleteJUnitXmlFile = xmlLogResult.dispose; + // Remove the '--junixml' if it exists, and add it with our path. + const testArgs = this.argsService.filterArguments(args, [JunitXmlArg]); + testArgs.splice(0, 0, `${JunitXmlArg}=${xmlLogFile}`); - return createTemporaryFile('.xml').then(xmlLogResult => { - xmlLogFile = xmlLogResult.filePath; - xmlLogFileCleanup = xmlLogResult.cleanupCallback; - if (testPaths.length > 0) { - // Ignore the test directories, as we're running a specific test - args = args.filter(arg => arg.trim().startsWith('-')); - } - const testArgs = testPaths.concat(args, [`--junitxml=${xmlLogFile}`]); - if (options.debug) { - const debugLauncher = serviceContainer.get(ITestDebugLauncher); - const pytestlauncherargs = [options.cwd, 'pytest']; - const debuggerArgs = pytestlauncherargs.concat(testArgs); - const launchOptions: LaunchOptions = { cwd: options.cwd, args: debuggerArgs, token: options.token, outChannel: options.outChannel, testProvider: 'pytest' }; - // tslint:disable-next-line:prefer-type-cast no-any - return debugLauncher.launchDebugger(launchOptions) as Promise; - } else { - const runOptions: Options = { - args: testArgs, - cwd: options.cwd, - outChannel: options.outChannel, - token: options.token, - workspaceFolder: options.workspaceFolder - }; - return run(serviceContainer, 'pytest', runOptions); + // Positional arguments control the tests to be run. + testArgs.push(...testPaths); + + if (options.debug) { + const debugLauncher = this.serviceContainer.get(ITestDebugLauncher); + const debuggerArgs = [options.cwd, 'pytest'].concat(testArgs); + const launchOptions: LaunchOptions = { cwd: options.cwd, args: debuggerArgs, token: options.token, outChannel: options.outChannel, testProvider: PYTEST_PROVIDER }; + await debugLauncher.launchDebugger(launchOptions); + } else { + const runOptions: Options = { + args: testArgs, + cwd: options.cwd, + outChannel: options.outChannel, + token: options.token, + workspaceFolder: options.workspaceFolder + }; + await this.testRunner.run(PYTEST_PROVIDER, runOptions); + } + + return options.debug ? options.tests : await this.updateResultsFromLogFiles(options.tests, xmlLogFile, testResultsService); + } catch (ex) { + return Promise.reject(ex); + } finally { + deleteJUnitXmlFile(); } - }).then(() => { - return options.debug ? options.tests : updateResultsFromLogFiles(options.tests, xmlLogFile, testResultsService); - }).then(result => { - xmlLogFileCleanup(); - return result; - }).catch(reason => { - xmlLogFileCleanup(); - return Promise.reject(reason); - }); -} + } -export function updateResultsFromLogFiles(tests: Tests, outputXmlFile: string, testResultsService: ITestResultsService): Promise { - return updateResultsFromXmlLogFile(tests, outputXmlFile, PassCalculationFormulae.pytest).then(() => { + private async updateResultsFromLogFiles(tests: Tests, outputXmlFile: string, testResultsService: ITestResultsService): Promise { + await this.xUnitParser.updateResultsFromXmlLogFile(tests, outputXmlFile, PassCalculationFormulae.pytest); testResultsService.updateResults(tests); return tests; - }); + } + + private async getJUnitXmlFile(args: string[]): Promise { + const xmlFile = this.argsHelper.getOptionValues(args, JunitXmlArg); + if (typeof xmlFile === 'string') { + return { filePath: xmlFile, dispose: noop }; + } + return this.fs.createTemporaryFile('.xml'); + } + } diff --git a/src/client/unittests/pytest/services/argsService.ts b/src/client/unittests/pytest/services/argsService.ts new file mode 100644 index 000000000000..c539fdbae265 --- /dev/null +++ b/src/client/unittests/pytest/services/argsService.ts @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { IServiceContainer } from '../../../ioc/types'; +import { IArgumentsHelper, IArgumentsService, TestFilter } from '../../types'; + +const OptionsWithArguments = ['-c', '-k', '-m', '-o', '-p', '-r', '-W', + '--assert', '--basetemp', '--capture', '--color', '--confcutdir', + '--deselect', '--dist', '--doctest-glob', + '--doctest-report', '--durations', '--ignore', '--import-mode', + '--junit-prefix', '--junit-xml', '--last-failed-no-failures', + '--lfnf', '--log-cli-date-format', '--log-cli-format', + '--log-cli-level', '--log-date-format', '--log-file', + '--log-file-date-format', '--log-file-format', '--log-file-level', + '--log-format', '--log-level', '--maxfail', '--override-ini', + '--pastebin', '--pdbcls', '--pythonwarnings', '--result-log', + '--rootdir', '--show-capture', '--tb', '--verbosity', '--max-slave-restart', + '--numprocesses', '--rsyncdir', '--rsyncignore', '--tx']; + +const OptionsWithoutArguments = ['--cache-clear', '--cache-show', '--collect-in-virtualenv', + '--collect-only', '--continue-on-collection-errors', '--debug', '--disable-pytest-warnings', + '--disable-warnings', '--doctest-continue-on-failure', '--doctest-ignore-import-errors', + '--doctest-modules', '--exitfirst', '--failed-first', '--ff', '--fixtures', + '--fixtures-per-test', '--force-sugar', '--full-trace', '--funcargs', '--help', + '--keep-duplicates', '--last-failed', '--lf', '--markers', '--new-first', '--nf', + '--no-print-logs', '--noconftest', '--old-summary', '--pdb', '--pyargs', + '--quiet', '--runxfail', '--setup-only', '--setup-plan', '--setup-show', '--showlocals', + '--strict', '--trace-config', '--verbose', '--version', '-h', '-l', '-q', '-s', '-v', '-x', + '--boxed', '--forked', '--looponfail', '--tx', '-d']; + +@injectable() +export class ArgumentsService implements IArgumentsService { + private readonly helper: IArgumentsHelper; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { + this.helper = serviceContainer.get(IArgumentsHelper); + } + public getKnownOptions(): { withArgs: string[]; withoutArgs: string[] } { + return { + withArgs: OptionsWithArguments, + withoutArgs: OptionsWithoutArguments + }; + } + public getOptionValue(args: string[], option: string): string | string[] | undefined { + return this.helper.getOptionValues(args, option); + } + public filterArguments(args: string[], argumentToRemoveOrFilter: string[] | TestFilter): string[] { + const optionsWithoutArgsToRemove: string[] = []; + const optionsWithArgsToRemove: string[] = []; + // Positional arguments in pytest are test directories and files. + // So if we want to run a specific test, then remove positional args. + let removePositionalArgs = false; + if (Array.isArray(argumentToRemoveOrFilter)) { + argumentToRemoveOrFilter.forEach(item => { + if (OptionsWithArguments.indexOf(item) >= 0) { + optionsWithArgsToRemove.push(item); + } + if (OptionsWithoutArguments.indexOf(item) >= 0) { + optionsWithoutArgsToRemove.push(item); + } + }); + } else { + switch (argumentToRemoveOrFilter) { + case TestFilter.removeTests: { + optionsWithoutArgsToRemove.push(...[ + '--lf', '--last-failed', + '--ff', '--failed-first', + '--nf', '--new-first' + ]); + optionsWithArgsToRemove.push(...[ + '-k', '-m', + '--lfnf', '--last-failed-no-failures' + ]); + removePositionalArgs = true; + break; + } + case TestFilter.discovery: { + optionsWithoutArgsToRemove.push(...[ + '-x', '--exitfirst', + '--fixtures', '--funcargs', + '--fixtures-per-test', '--pdb', + '--lf', '--last-failed', + '--ff', '--failed-first', + '--nf', '--new-first', + '--cache-show', + '-v', '--verbose', '-q', '-quiet', + '-l', '--showlocals', + '--no-print-logs', + '--debug', + '--setup-only', '--setup-show', '--setup-plan' + ]); + optionsWithArgsToRemove.push(...[ + '-m', '--maxfail', + '--pdbcls', '--capture', + '--lfnf', '--last-failed-no-failures', + '--verbosity', '-r', + '--tb', + '--rootdir', '--show-capture', + '--durations', + '--junit-xml', '--junit-prefix', '--result-log', + '-W', '--pythonwarnings', + '--log-*' + ]); + removePositionalArgs = true; + break; + } + case TestFilter.debugAll: + case TestFilter.runAll: { + optionsWithoutArgsToRemove.push('--collect-only'); + break; + } + case TestFilter.debugSpecific: + case TestFilter.runSpecific: { + optionsWithoutArgsToRemove.push(...[ + '--collect-only', + '--lf', '--last-failed', + '--ff', '--failed-first', + '--nf', '--new-first' + ]); + optionsWithArgsToRemove.push(...[ + '-k', '-m', + '--lfnf', '--last-failed-no-failures' + ]); + removePositionalArgs = true; + break; + } + default: { + throw new Error(`Unsupported Filter '${argumentToRemoveOrFilter}'`); + } + } + } + + let filteredArgs = args.slice(); + if (removePositionalArgs) { + const positionalArgs = this.helper.getPositionalArguments(filteredArgs, OptionsWithArguments, OptionsWithoutArguments); + filteredArgs = filteredArgs.filter(item => positionalArgs.indexOf(item) === -1); + } + return this.helper.filterArguments(filteredArgs, optionsWithArgsToRemove, optionsWithoutArgsToRemove); + } + public getTestFolders(args: string[]): string[] { + const testDirs = this.helper.getOptionValues(args, '--rootdir'); + if (typeof testDirs === 'string') { + return [testDirs]; + } + if (Array.isArray(testDirs) && testDirs.length > 0) { + return testDirs; + } + const positionalArgs = this.helper.getPositionalArguments(args, OptionsWithArguments, OptionsWithoutArguments); + // Positional args in pytest are files or directories. + // Remove files from the args, and what's left are test directories. + // If users enter test modules/methods, then its not supported. + return positionalArgs.filter(arg => !arg.toUpperCase().endsWith('.PY')); + } +} diff --git a/src/client/unittests/pytest/services/discoveryService.ts b/src/client/unittests/pytest/services/discoveryService.ts index c3fc02e131be..57d9a3902e5d 100644 --- a/src/client/unittests/pytest/services/discoveryService.ts +++ b/src/client/unittests/pytest/services/discoveryService.ts @@ -4,48 +4,67 @@ import { inject, injectable, named } from 'inversify'; import { CancellationTokenSource } from 'vscode'; import { IServiceContainer } from '../../../ioc/types'; -import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../../common/constants'; -import { Options, run } from '../../common/runner'; -import { ITestDiscoveryService, ITestsParser, TestDiscoveryOptions, Tests } from '../../common/types'; - -const argsToExcludeForDiscovery = ['-x', '--exitfirst', - '--fixtures-per-test', '--pdb', '--runxfail', - '--lf', '--last-failed', '--ff', '--failed-first', - '--cache-show', '--cache-clear', - '-v', '--verbose', '-q', '-quiet', - '--disable-pytest-warnings', '-l', '--showlocals']; - -type PytestDiscoveryOptions = TestDiscoveryOptions & { - startDirectory: string; - pattern: string; -}; +import { PYTEST_PROVIDER } from '../../common/constants'; +import { ITestDiscoveryService, ITestRunner, ITestsHelper, ITestsParser, Options, TestDiscoveryOptions, Tests } from '../../common/types'; +import { IArgumentsService, TestFilter } from '../../types'; @injectable() export class TestDiscoveryService implements ITestDiscoveryService { - constructor( @inject(IServiceContainer) private serviceContainer: IServiceContainer, - @inject(ITestsParser) @named(PYTEST_PROVIDER) private testParser: ITestsParser) { } + private argsService: IArgumentsService; + private helper: ITestsHelper; + private runner: ITestRunner; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer, + @inject(ITestsParser) @named(PYTEST_PROVIDER) private testParser: ITestsParser) { + this.argsService = this.serviceContainer.get(IArgumentsService, PYTEST_PROVIDER); + this.helper = this.serviceContainer.get(ITestsHelper); + this.runner = this.serviceContainer.get(ITestRunner); + } public async discoverTests(options: TestDiscoveryOptions): Promise { - // Remove unwanted arguments - const args = options.args.filter(arg => { - if (argsToExcludeForDiscovery.indexOf(arg.trim()) !== -1) { - return false; - } - return true; - }); - if (options.ignoreCache && args.indexOf('--cache-clear') === -1) { - args.push('--cache-clear'); + const args = this.buildTestCollectionArgs(options); + + // Collect tests for each test directory separately and merge. + const testDirectories = this.argsService.getTestFolders(options.args); + if (testDirectories.length === 0) { + const opts = { + ...options, + args + }; + return this.discoverTestsInTestDirectory(opts); } + const results = await Promise.all(testDirectories.map(testDir => { + // Add test directory as a positional argument. + const opts = { + ...options, + args: [...args, testDir] + }; + return this.discoverTestsInTestDirectory(opts); + })); + return this.helper.mergeTests(results); + } + private buildTestCollectionArgs(options: TestDiscoveryOptions) { + // Remove unwnted arguments (which happen to be test directories & test specific args). + const args = this.argsService.filterArguments(options.args, TestFilter.discovery); + if (options.ignoreCache && args.indexOf('--cache-clear') === -1) { + args.splice(0, 0, '--cache-clear'); + } + if (args.indexOf('-s') === -1) { + args.splice(0, 0, '-s'); + } + args.splice(0, 0, '--collect-only'); + return args; + } + private async discoverTestsInTestDirectory(options: TestDiscoveryOptions): Promise { const token = options.token ? options.token : new CancellationTokenSource().token; const runOptions: Options = { - args: args.concat(['--collect-only']), + args: options.args, cwd: options.cwd, workspaceFolder: options.workspaceFolder, token, outChannel: options.outChannel }; - const data = await run(this.serviceContainer, PYTEST_PROVIDER, runOptions); + const data = await this.runner.run(PYTEST_PROVIDER, runOptions); if (options.token && options.token.isCancellationRequested) { return Promise.reject('cancelled'); } diff --git a/src/client/unittests/pytest/services/parserService.ts b/src/client/unittests/pytest/services/parserService.ts index f0fad3b88b12..822ab930ae04 100644 --- a/src/client/unittests/pytest/services/parserService.ts +++ b/src/client/unittests/pytest/services/parserService.ts @@ -5,13 +5,13 @@ import { inject, injectable } from 'inversify'; import * as os from 'os'; import * as path from 'path'; import { convertFileToPackage, extractBetweenDelimiters } from '../../common/testUtils'; -import { ITestsHelper, ITestsParser, ParserOptions, TestDiscoveryOptions, TestFile, TestFunction, Tests, TestStatus, TestSuite } from '../../common/types'; +import { ITestsHelper, ITestsParser, ParserOptions, TestFile, TestFunction, Tests, TestSuite } from '../../common/types'; const DELIMITER = '\''; @injectable() export class TestsParser implements ITestsParser { - constructor( @inject(ITestsHelper) private testsHelper: ITestsHelper) { } + constructor(@inject(ITestsHelper) private testsHelper: ITestsHelper) { } public parse(content: string, options: ParserOptions): Tests { const testFiles = this.getTestFiles(content, options); return this.testsHelper.flattenTestFiles(testFiles); @@ -20,7 +20,7 @@ export class TestsParser implements ITestsParser { private getTestFiles(content: string, options: ParserOptions) { let logOutputLines: string[] = ['']; const testFiles: TestFile[] = []; - const parentNodes: { indent: number, item: TestFile | TestSuite }[] = []; + const parentNodes: { indent: number; item: TestFile | TestSuite }[] = []; const errorLine = /==*( *)ERRORS( *)=*/; const errorFileLine = /__*( *)ERROR collecting (.*)/; @@ -75,7 +75,7 @@ export class TestsParser implements ITestsParser { } private parsePyTestModuleCollectionError(rootDirectory: string, lines: string[], testFiles: TestFile[], - parentNodes: { indent: number, item: TestFile | TestSuite }[]) { + parentNodes: { indent: number; item: TestFile | TestSuite }[]) { lines = lines.filter(line => line.trim().length > 0); if (lines.length <= 1) { @@ -98,7 +98,7 @@ export class TestsParser implements ITestsParser { return; } - private parsePyTestModuleCollectionResult(rootDirectory: string, lines: string[], testFiles: TestFile[], parentNodes: { indent: number, item: TestFile | TestSuite }[]) { + private parsePyTestModuleCollectionResult(rootDirectory: string, lines: string[], testFiles: TestFile[], parentNodes: { indent: number; item: TestFile | TestSuite }[]) { let currentPackage: string = ''; lines.forEach(line => { @@ -146,7 +146,7 @@ export class TestsParser implements ITestsParser { }); } - private findParentOfCurrentItem(indentOfCurrentItem: number, parentNodes: { indent: number, item: TestFile | TestSuite }[]): { indent: number, item: TestFile | TestSuite } | undefined { + private findParentOfCurrentItem(indentOfCurrentItem: number, parentNodes: { indent: number; item: TestFile | TestSuite }[]): { indent: number; item: TestFile | TestSuite } | undefined { while (parentNodes.length > 0) { const parentNode = parentNodes[parentNodes.length - 1]; if (parentNode.indent < indentOfCurrentItem) { diff --git a/src/client/unittests/serviceRegistry.ts b/src/client/unittests/serviceRegistry.ts index 92a4f1b5f6e7..10b0a22cf140 100644 --- a/src/client/unittests/serviceRegistry.ts +++ b/src/client/unittests/serviceRegistry.ts @@ -3,8 +3,10 @@ import { Uri } from 'vscode'; import { IServiceContainer, IServiceManager } from '../ioc/types'; +import { ArgumentsHelper } from './common/argumentsHelper'; import { NOSETEST_PROVIDER, PYTEST_PROVIDER, UNITTEST_PROVIDER } from './common/constants'; import { DebugLauncher } from './common/debugLauncher'; +import { TestRunner } from './common/runner'; import { TestConfigSettingsService } from './common/services/configSettingService'; import { TestCollectionStorageService } from './common/services/storageService'; import { TestManagerService } from './common/services/testManagerService'; @@ -16,21 +18,29 @@ import { TestFolderGenerationVisitor } from './common/testVisitors/folderGenerat import { TestResultResetVisitor } from './common/testVisitors/resultResetVisitor'; import { ITestCollectionStorageService, ITestConfigSettingsService, ITestDebugLauncher, ITestDiscoveryService, ITestManager, ITestManagerFactory, ITestManagerService, ITestManagerServiceFactory, - ITestResultsService, ITestsHelper, ITestsParser, ITestVisitor, IUnitTestSocketServer, IWorkspaceTestManagerService, TestProvider + ITestResultsService, ITestRunner, ITestsHelper, ITestsParser, ITestVisitor, IUnitTestSocketServer, IWorkspaceTestManagerService, IXUnitParser, TestProvider } from './common/types'; +import { XUnitParser } from './common/xUnitParser'; import { UnitTestConfigurationService } from './configuration'; import { TestConfigurationManagerFactory } from './configurationFactory'; import { TestResultDisplay } from './display/main'; import { TestDisplay } from './display/picker'; import { UnitTestManagementService } from './main'; import { TestManager as NoseTestManager } from './nosetest/main'; +import { TestManagerRunner as NoseTestManagerRunner } from './nosetest/runner'; +import { ArgumentsService as NoseTestArgumentsService } from './nosetest/services/argsService'; import { TestDiscoveryService as NoseTestDiscoveryService } from './nosetest/services/discoveryService'; import { TestsParser as NoseTestTestsParser } from './nosetest/services/parserService'; import { TestManager as PyTestTestManager } from './pytest/main'; +import { TestManagerRunner as PytestManagerRunner } from './pytest/runner'; +import { ArgumentsService as PyTestArgumentsService } from './pytest/services/argsService'; import { TestDiscoveryService as PytestTestDiscoveryService } from './pytest/services/discoveryService'; import { TestsParser as PytestTestsParser } from './pytest/services/parserService'; -import { ITestConfigurationManagerFactory, ITestDisplay, ITestResultDisplay, IUnitTestConfigurationService, IUnitTestManagementService } from './types'; +import { IArgumentsHelper, IArgumentsService, ITestConfigurationManagerFactory, ITestDisplay, ITestManagerRunner, ITestResultDisplay, IUnitTestConfigurationService, IUnitTestHelper, IUnitTestManagementService } from './types'; +import { UnitTestHelper } from './unittest/helper'; import { TestManager as UnitTestTestManager } from './unittest/main'; +import { TestManagerRunner as UnitTestTestManagerRunner } from './unittest/runner'; +import { ArgumentsService as UnitTestArgumentsService } from './unittest/services/argsService'; import { TestDiscoveryService as UnitTestTestDiscoveryService } from './unittest/services/discoveryService'; import { TestsParser as UnitTestTestsParser } from './unittest/services/parserService'; import { UnitTestSocketServer } from './unittest/socketServer'; @@ -57,6 +67,18 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.add(ITestDiscoveryService, PytestTestDiscoveryService, PYTEST_PROVIDER); serviceManager.add(ITestDiscoveryService, NoseTestDiscoveryService, NOSETEST_PROVIDER); + serviceManager.add(IArgumentsHelper, ArgumentsHelper); + serviceManager.add(ITestRunner, TestRunner); + serviceManager.add(IXUnitParser, XUnitParser); + serviceManager.add(IUnitTestHelper, UnitTestHelper); + + serviceManager.add(IArgumentsService, PyTestArgumentsService, PYTEST_PROVIDER); + serviceManager.add(IArgumentsService, NoseTestArgumentsService, NOSETEST_PROVIDER); + serviceManager.add(IArgumentsService, UnitTestArgumentsService, UNITTEST_PROVIDER); + serviceManager.add(ITestManagerRunner, PytestManagerRunner, PYTEST_PROVIDER); + serviceManager.add(ITestManagerRunner, NoseTestManagerRunner, NOSETEST_PROVIDER); + serviceManager.add(ITestManagerRunner, UnitTestTestManagerRunner, UNITTEST_PROVIDER); + serviceManager.addSingleton(IUnitTestConfigurationService, UnitTestConfigurationService); serviceManager.addSingleton(IUnitTestManagementService, UnitTestManagementService); serviceManager.addSingleton(ITestResultDisplay, TestResultDisplay); diff --git a/src/client/unittests/types.ts b/src/client/unittests/types.ts index ebf063581354..b03bda2856eb 100644 --- a/src/client/unittests/types.ts +++ b/src/client/unittests/types.ts @@ -7,7 +7,7 @@ import { Disposable, Event, TextDocument, Uri } from 'vscode'; import { Product } from '../common/types'; import { PythonSymbolProvider } from '../providers/symbolProvider'; import { CommandSource } from './common/constants'; -import { FlattenedTestFunction, ITestManager, TestFile, TestFunction, Tests, TestsToRun, UnitTestProduct } from './common/types'; +import { FlattenedTestFunction, ITestManager, ITestResultsService, TestFile, TestFunction, TestRunOptions, Tests, TestsToRun, UnitTestProduct } from './common/types'; export const IUnitTestConfigurationService = Symbol('IUnitTestConfigurationService'); export interface IUnitTestConfigurationService { @@ -67,3 +67,38 @@ export const ITestConfigurationManagerFactory = Symbol('ITestConfigurationManage export interface ITestConfigurationManagerFactory { create(wkspace: Uri, product: Product): ITestConfigurationManager; } + +export enum TestFilter { + removeTests = 'removeTests', + discovery = 'discovery', + runAll = 'runAll', + runSpecific = 'runSpecific', + debugAll = 'debugAll', + debugSpecific = 'debugSpecific' +} +export const IArgumentsService = Symbol('IArgumentsService'); +export interface IArgumentsService { + getKnownOptions(): { withArgs: string[]; withoutArgs: string[] }; + getOptionValue(args: string[], option: string): string | string[] | undefined; + filterArguments(args: string[], argumentToRemove: string[]): string[]; + // tslint:disable-next-line:unified-signatures + filterArguments(args: string[], filter: TestFilter): string[]; + getTestFolders(args: string[]): string[]; +} +export const IArgumentsHelper = Symbol('IArgumentsHelper'); +export interface IArgumentsHelper { + getOptionValues(args: string[], option: string): string | string[] | undefined; + filterArguments(args: string[], optionsWithArguments?: string[], optionsWithoutArguments?: string[]): string[]; + getPositionalArguments(args: string[], optionsWithArguments?: string[], optionsWithoutArguments?: string[]): string[]; +} + +export const ITestManagerRunner = Symbol('ITestManagerRunner'); +export interface ITestManagerRunner { + runTest(testResultsService: ITestResultsService, options: TestRunOptions, testManager: ITestManager): Promise; +} + +export const IUnitTestHelper = Symbol('IUnitTestHelper'); +export interface IUnitTestHelper { + getStartDirectory(args: string[]): string; + getIdsOfTestsToRun(tests: Tests, testsToRun: TestsToRun): string[]; +} diff --git a/src/client/unittests/unittest/helper.ts b/src/client/unittests/unittest/helper.ts new file mode 100644 index 000000000000..a97aed3102cf --- /dev/null +++ b/src/client/unittests/unittest/helper.ts @@ -0,0 +1,52 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { IServiceContainer } from '../../ioc/types'; +import { Tests, TestsToRun } from '../common/types'; +import { IArgumentsHelper, IUnitTestHelper } from '../types'; + +@injectable() +export class UnitTestHelper implements IUnitTestHelper { + private readonly argsHelper: IArgumentsHelper; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { + this.argsHelper = serviceContainer.get(IArgumentsHelper); + } + public getStartDirectory(args: string[]): string { + const shortValue = this.argsHelper.getOptionValues(args, '-s'); + if (typeof shortValue === 'string') { + return shortValue; + } + const longValue = this.argsHelper.getOptionValues(args, '--start-directory'); + if (typeof longValue === 'string') { + return longValue; + } + return '.'; + } + public getIdsOfTestsToRun(tests: Tests, testsToRun: TestsToRun): string[] { + const testIds: string[] = []; + if (testsToRun && testsToRun.testFolder) { + // Get test ids of files in these folders. + testsToRun.testFolder.map(folder => { + tests.testFiles.forEach(f => { + if (f.fullPath.startsWith(folder.name)) { + testIds.push(f.nameToRun); + } + }); + }); + } + if (testsToRun && testsToRun.testFile) { + testIds.push(...testsToRun.testFile.map(f => f.nameToRun)); + } + if (testsToRun && testsToRun.testSuite) { + testIds.push(...testsToRun.testSuite.map(f => f.nameToRun)); + } + if (testsToRun && testsToRun.testFunction) { + testIds.push(...testsToRun.testFunction.map(f => f.nameToRun)); + } + return testIds; + } +} diff --git a/src/client/unittests/unittest/main.ts b/src/client/unittests/unittest/main.ts index ef9a8f134879..b9056579ea0b 100644 --- a/src/client/unittests/unittest/main.ts +++ b/src/client/unittests/unittest/main.ts @@ -1,20 +1,27 @@ import { Uri } from 'vscode'; -import { PythonSettings } from '../../common/configSettings'; +import { noop } from '../../common/core.utils'; import { Product } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; +import { UNITTEST_PROVIDER } from '../common/constants'; import { BaseTestManager } from '../common/managers/baseTestManager'; -import { TestDiscoveryOptions, TestRunOptions, Tests, TestStatus, TestsToRun } from '../common/types'; -import { runTest } from './runner'; +import { ITestsHelper, TestDiscoveryOptions, TestRunOptions, Tests, TestStatus, TestsToRun } from '../common/types'; +import { IArgumentsService, ITestManagerRunner, TestFilter } from '../types'; export class TestManager extends BaseTestManager { + private readonly argsService: IArgumentsService; + private readonly helper: ITestsHelper; + private readonly runner: ITestManagerRunner; public get enabled() { - return PythonSettings.getInstance(this.workspaceFolder).unitTest.unittestEnabled; + return this.settings.unitTest.unittestEnabled; } constructor(workspaceFolder: Uri, rootDirectory: string, serviceContainer: IServiceContainer) { - super('unittest', Product.unittest, workspaceFolder, rootDirectory, serviceContainer); + super(UNITTEST_PROVIDER, Product.unittest, workspaceFolder, rootDirectory, serviceContainer); + this.argsService = this.serviceContainer.get(IArgumentsService, this.testProvider); + this.helper = this.serviceContainer.get(ITestsHelper); + this.runner = this.serviceContainer.get(ITestManagerRunner, this.testProvider); } - // tslint:disable-next-line:no-empty public configure() { + noop(); } public getDiscoveryOptions(ignoreCache: boolean): TestDiscoveryOptions { const args = this.settings.unitTest.unittestArgs.slice(0); @@ -26,7 +33,15 @@ export class TestManager extends BaseTestManager { }; } public async runTestImpl(tests: Tests, testsToRun?: TestsToRun, runFailedTests?: boolean, debug?: boolean): Promise<{}> { - const args = this.settings.unitTest.unittestArgs.slice(0); + let args: string[]; + + const runAllTests = this.helper.shouldRunAllTests(testsToRun); + if (debug) { + args = this.argsService.filterArguments(this.settings.unitTest.unittestArgs, runAllTests ? TestFilter.debugAll : TestFilter.debugSpecific); + } else { + args = this.argsService.filterArguments(this.settings.unitTest.unittestArgs, runAllTests ? TestFilter.runAll : TestFilter.runSpecific); + } + if (runFailedTests === true) { testsToRun = { testFile: [], testFolder: [], testSuite: [], testFunction: [] }; testsToRun.testFunction = tests.testFunctions.filter(fn => { @@ -40,6 +55,6 @@ export class TestManager extends BaseTestManager { token: this.testRunnerCancellationToken!, outChannel: this.outputChannel }; - return runTest(this.serviceContainer, this, this.testResultsService, options); + return this.runner.runTest(this.testResultsService, options, this); } } diff --git a/src/client/unittests/unittest/runner.ts b/src/client/unittests/unittest/runner.ts index 472ba315137f..214eab94a93a 100644 --- a/src/client/unittests/unittest/runner.ts +++ b/src/client/unittests/unittest/runner.ts @@ -1,9 +1,14 @@ 'use strict'; +import { inject, injectable } from 'inversify'; import * as path from 'path'; +import { EXTENSION_ROOT_DIR } from '../../common/constants'; +import { noop } from '../../common/core.utils'; +import { ILogger } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; -import { BaseTestManager } from '../common/managers/baseTestManager'; -import { Options, run } from '../common/runner'; -import { ITestDebugLauncher, ITestResultsService, IUnitTestSocketServer, LaunchOptions, TestRunOptions, Tests, TestStatus, TestsToRun } from '../common/types'; +import { UNITTEST_PROVIDER } from '../common/constants'; +import { Options } from '../common/runner'; +import { ITestDebugLauncher, ITestManager, ITestResultsService, ITestRunner, IUnitTestSocketServer, LaunchOptions, TestRunOptions, Tests, TestStatus, TestsToRun } from '../common/types'; +import { IArgumentsHelper, ITestManagerRunner, IUnitTestHelper } from '../types'; type TestStatusMap = { status: TestStatus; @@ -11,13 +16,9 @@ type TestStatusMap = { }; const outcomeMapping = new Map(); -// tslint:disable-next-line:no-backbone-get-set-outside-model outcomeMapping.set('passed', { status: TestStatus.Pass, summaryProperty: 'passed' }); -// tslint:disable-next-line:no-backbone-get-set-outside-model outcomeMapping.set('failed', { status: TestStatus.Fail, summaryProperty: 'failures' }); -// tslint:disable-next-line:no-backbone-get-set-outside-model outcomeMapping.set('error', { status: TestStatus.Error, summaryProperty: 'errors' }); -// tslint:disable-next-line:no-backbone-get-set-outside-model outcomeMapping.set('skipped', { status: TestStatus.Skipped, summaryProperty: 'skipped' }); interface ITestData { @@ -27,64 +28,63 @@ interface ITestData { traceback: string; } -// tslint:disable-next-line:max-func-body-length -export async function runTest(serviceContainer: IServiceContainer, testManager: BaseTestManager, testResultsService: ITestResultsService, options: TestRunOptions): Promise { - options.tests.summary.errors = 0; - options.tests.summary.failures = 0; - options.tests.summary.passed = 0; - options.tests.summary.skipped = 0; - let failFast = false; - const testLauncherFile = path.join(__dirname, '..', '..', '..', '..', 'pythonFiles', 'PythonTools', 'visualstudio_py_testlauncher.py'); - const server = serviceContainer.get(IUnitTestSocketServer); - server.on('error', (message: string, ...data: string[]) => { - // tslint:disable-next-line:no-console - console.log(`${message} ${data.join(' ')}`); - }); - // tslint:disable-next-line:no-empty - server.on('log', (message: string, ...data: string[]) => { - }); - // tslint:disable-next-line:no-empty no-any - server.on('connect', (data: any) => { - }); - // tslint:disable-next-line:no-empty - server.on('start', (data: { test: string }) => { - }); - server.on('result', (data: ITestData) => { - const test = options.tests.testFunctions.find(t => t.testFunction.nameToRun === data.test); - const statusDetails = outcomeMapping.get(data.outcome)!; - if (test) { - test.testFunction.status = statusDetails.status; - test.testFunction.message = data.message; - test.testFunction.traceback = data.traceback; - options.tests.summary[statusDetails.summaryProperty] += 1; - - if (failFast && (statusDetails.summaryProperty === 'failures' || statusDetails.summaryProperty === 'errors')) { - testManager.stop(); - } - } else { - if (statusDetails) { +@injectable() +export class TestManagerRunner implements ITestManagerRunner { + private readonly argsHelper: IArgumentsHelper; + private readonly helper: IUnitTestHelper; + private readonly testRunner: ITestRunner; + private readonly server: IUnitTestSocketServer; + private readonly logger: ILogger; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { + this.argsHelper = serviceContainer.get(IArgumentsHelper); + this.testRunner = serviceContainer.get(ITestRunner); + this.server = this.serviceContainer.get(IUnitTestSocketServer); + this.logger = this.serviceContainer.get(ILogger); + this.helper = this.serviceContainer.get(IUnitTestHelper); + } + public async runTest(testResultsService: ITestResultsService, options: TestRunOptions, testManager: ITestManager): Promise { + options.tests.summary.errors = 0; + options.tests.summary.failures = 0; + options.tests.summary.passed = 0; + options.tests.summary.skipped = 0; + let failFast = false; + const testLauncherFile = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'PythonTools', 'visualstudio_py_testlauncher.py'); + this.server.on('error', (message: string, ...data: string[]) => this.logger.logError(`${message} ${data.join(' ')}`)); + this.server.on('log', noop); + this.server.on('connect', noop); + this.server.on('start', noop); + this.server.on('socket.disconnected', noop); + this.server.on('result', (data: ITestData) => { + const test = options.tests.testFunctions.find(t => t.testFunction.nameToRun === data.test); + const statusDetails = outcomeMapping.get(data.outcome)!; + if (test) { + test.testFunction.status = statusDetails.status; + test.testFunction.message = data.message; + test.testFunction.traceback = data.traceback; options.tests.summary[statusDetails.summaryProperty] += 1; + + if (failFast && (statusDetails.summaryProperty === 'failures' || statusDetails.summaryProperty === 'errors')) { + testManager.stop(); + } + } else { + if (statusDetails) { + options.tests.summary[statusDetails.summaryProperty] += 1; + } } - } - }); - // tslint:disable-next-line:no-empty no-any - server.on('socket.disconnected', (data: any) => { - }); + }); - return server.start().then(port => { - const testPaths: string[] = getIdsOfTestsToRun(options.tests, options.testsToRun!); + const port = await this.server.start(); + const testPaths: string[] = this.helper.getIdsOfTestsToRun(options.tests, options.testsToRun!); for (let counter = 0; counter < testPaths.length; counter += 1) { testPaths[counter] = `-t${testPaths[counter].trim()}`; } - const startTestDiscoveryDirectory = getStartDirectory(options.args); - function runTestInternal(testFile: string = '', testId: string = '') { - let testArgs = buildTestArgs(options.args); + const runTestInternal = async (testFile: string = '', testId: string = '') => { + let testArgs = this.buildTestArgs(options.args); failFast = testArgs.indexOf('--uf') >= 0; testArgs = testArgs.filter(arg => arg !== '--uf'); testArgs.push(`--result-port=${port}`); - testArgs.push(`--us=${startTestDiscoveryDirectory}`); if (testId.length > 0) { testArgs.push(`-t${testId}`); } @@ -92,13 +92,11 @@ export async function runTest(serviceContainer: IServiceContainer, testManager: testArgs.push(`--testFile=${testFile}`); } if (options.debug === true) { - const debugLauncher = serviceContainer.get(ITestDebugLauncher); - testArgs.push(...['--debug']); - const launchOptions: LaunchOptions = { cwd: options.cwd, args: testArgs, token: options.token, outChannel: options.outChannel, testProvider: 'unittest' }; - // tslint:disable-next-line:prefer-type-cast no-any + const debugLauncher = this.serviceContainer.get(ITestDebugLauncher); + testArgs.push('--debug'); + const launchOptions: LaunchOptions = { cwd: options.cwd, args: testArgs, token: options.token, outChannel: options.outChannel, testProvider: UNITTEST_PROVIDER }; return debugLauncher.launchDebugger(launchOptions); } else { - // tslint:disable-next-line:prefer-type-cast no-any const runOptions: Options = { args: [testLauncherFile].concat(testArgs), cwd: options.cwd, @@ -106,109 +104,57 @@ export async function runTest(serviceContainer: IServiceContainer, testManager: token: options.token, workspaceFolder: options.workspaceFolder }; - return run(serviceContainer, 'unittest', runOptions); + await this.testRunner.run(UNITTEST_PROVIDER, runOptions); } - } + }; - // Test everything + // Test everything. if (testPaths.length === 0) { - return runTestInternal(); + await runTestInternal(); } - // Ok, the ptvs test runner can only work with one test at a time - let promise = Promise.resolve(''); - if (Array.isArray(options.testsToRun!.testFile)) { - options.testsToRun!.testFile!.forEach(testFile => { - // tslint:disable-next-line:prefer-type-cast no-any - promise = promise.then(() => runTestInternal(testFile.fullPath, testFile.nameToRun) as Promise); - }); - } - if (Array.isArray(options.testsToRun!.testSuite)) { - options.testsToRun!.testSuite!.forEach(testSuite => { - const testFileName = options.tests.testSuites.find(t => t.testSuite === testSuite)!.parentTestFile.fullPath; - // tslint:disable-next-line:prefer-type-cast no-any - promise = promise.then(() => runTestInternal(testFileName, testSuite.nameToRun) as Promise); - }); - } - if (Array.isArray(options.testsToRun!.testFunction)) { - options.testsToRun!.testFunction!.forEach(testFn => { - const testFileName = options.tests.testFunctions.find(t => t.testFunction === testFn)!.parentTestFile.fullPath; - // tslint:disable-next-line:prefer-type-cast no-any - promise = promise.then(() => runTestInternal(testFileName, testFn.nameToRun) as Promise); - }); + // Ok, the test runner can only work with one test at a time. + if (options.testsToRun) { + let promise = Promise.resolve(undefined); + if (Array.isArray(options.testsToRun.testFile)) { + options.testsToRun.testFile.forEach(testFile => { + promise = promise.then(() => runTestInternal(testFile.fullPath, testFile.nameToRun)); + }); + } + if (Array.isArray(options.testsToRun.testSuite)) { + options.testsToRun.testSuite.forEach(testSuite => { + const testFileName = options.tests.testSuites.find(t => t.testSuite === testSuite)!.parentTestFile.fullPath; + promise = promise.then(() => runTestInternal(testFileName, testSuite.nameToRun)); + }); + } + if (Array.isArray(options.testsToRun.testFunction)) { + options.testsToRun.testFunction.forEach(testFn => { + const testFileName = options.tests.testFunctions.find(t => t.testFunction === testFn)!.parentTestFile.fullPath; + promise = promise.then(() => runTestInternal(testFileName, testFn.nameToRun)); + }); + } + await promise; } - // tslint:disable-next-line:prefer-type-cast no-any - return promise as Promise; - }).then(() => { + testResultsService.updateResults(options.tests); return options.tests; - }).catch(reason => { - return Promise.reject(reason); - }); -} -function getStartDirectory(args: string[]): string { - let startDirectory = '.'; - const indexOfStartDir = args.findIndex(arg => arg.indexOf('-s') === 0 || arg.indexOf('--start-directory') === 0); - if (indexOfStartDir >= 0) { - const startDir = args[indexOfStartDir].trim(); - if ((startDir.trim() === '-s' || startDir.trim() === '--start-directory') && args.length >= indexOfStartDir) { - // Assume the next items is the directory - startDirectory = args[indexOfStartDir + 1]; - } else { - const lenToStartFrom = startDir.startsWith('-s') ? '-s'.length : '--start-directory'.length; - startDirectory = startDir.substring(lenToStartFrom).trim(); - if (startDirectory.startsWith('=')) { - startDirectory = startDirectory.substring(1); - } - } } - return startDirectory; -} -function buildTestArgs(args: string[]): string[] { - const startTestDiscoveryDirectory = getStartDirectory(args); - let pattern = 'test*.py'; - const indexOfPattern = args.findIndex(arg => arg.indexOf('-p') === 0 || arg.indexOf('--pattern') === 0); - if (indexOfPattern >= 0) { - const patternValue = args[indexOfPattern].trim(); - if ((patternValue.trim() === '-p' || patternValue.trim() === '--pattern') && args.length >= indexOfPattern) { - // Assume the next items is the directory - pattern = args[indexOfPattern + 1]; - } else { - const lenToStartFrom = patternValue.startsWith('-p') ? '-p'.length : '--pattern'.length; - pattern = patternValue.substring(lenToStartFrom).trim(); - if (pattern.startsWith('=')) { - pattern = pattern.substring(1); - } + private buildTestArgs(args: string[]): string[] { + const startTestDiscoveryDirectory = this.helper.getStartDirectory(args); + let pattern = 'test*.py'; + const shortValue = this.argsHelper.getOptionValues(args, '-p'); + const longValueValue = this.argsHelper.getOptionValues(args, '-pattern'); + if (typeof shortValue === 'string') { + pattern = shortValue; + } else if (typeof longValueValue === 'string') { + pattern = longValueValue; } + const failFast = args.some(arg => arg.trim() === '-f' || arg.trim() === '--failfast'); + const verbosity = args.some(arg => arg.trim().indexOf('-v') === 0) ? 2 : 1; + const testArgs = [`--us=${startTestDiscoveryDirectory}`, `--up=${pattern}`, `--uvInt=${verbosity}`]; + if (failFast) { + testArgs.push('--uf'); + } + return testArgs; } - const failFast = args.some(arg => arg.trim() === '-f' || arg.trim() === '--failfast'); - const verbosity = args.some(arg => arg.trim().indexOf('-v') === 0) ? 2 : 1; - const testArgs = [`--us=${startTestDiscoveryDirectory}`, `--up=${pattern}`, `--uvInt=${verbosity}`]; - if (failFast) { - testArgs.push('--uf'); - } - return testArgs; -} -function getIdsOfTestsToRun(tests: Tests, testsToRun: TestsToRun): string[] { - const testIds: string[] = []; - if (testsToRun && testsToRun.testFolder) { - // Get test ids of files in these folders - testsToRun.testFolder.map(folder => { - tests.testFiles.forEach(f => { - if (f.fullPath.startsWith(folder.name)) { - testIds.push(f.nameToRun); - } - }); - }); - } - if (testsToRun && testsToRun.testFile) { - testIds.push(...testsToRun.testFile.map(f => f.nameToRun)); - } - if (testsToRun && testsToRun.testSuite) { - testIds.push(...testsToRun.testSuite.map(f => f.nameToRun)); - } - if (testsToRun && testsToRun.testFunction) { - testIds.push(...testsToRun.testFunction.map(f => f.nameToRun)); - } - return testIds; } diff --git a/src/client/unittests/unittest/services/argsService.ts b/src/client/unittests/unittest/services/argsService.ts new file mode 100644 index 000000000000..6a8cf7b1d625 --- /dev/null +++ b/src/client/unittests/unittest/services/argsService.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { IServiceContainer } from '../../../ioc/types'; +import { IArgumentsHelper, IArgumentsService, TestFilter } from '../../types'; + +const OptionsWithArguments = ['-p', '-s', '-t', '--pattern', + '--start-directory', '--top-level-directory']; + +const OptionsWithoutArguments = ['-b', '-c', '-f', '-h', '-q', '-v', + '--buffer', '--catch', '--failfast', '--help', '--locals', + '--quiet', '--verbose']; + +@injectable() +export class ArgumentsService implements IArgumentsService { + private readonly helper: IArgumentsHelper; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { + this.helper = serviceContainer.get(IArgumentsHelper); + } + public getKnownOptions(): { withArgs: string[]; withoutArgs: string[] } { + return { + withArgs: OptionsWithArguments, + withoutArgs: OptionsWithoutArguments + }; + } + public getOptionValue(args: string[], option: string): string | string[] | undefined { + return this.helper.getOptionValues(args, option); + } + public filterArguments(args: string[], argumentToRemoveOrFilter: string[] | TestFilter): string[] { + const optionsWithoutArgsToRemove: string[] = []; + const optionsWithArgsToRemove: string[] = []; + // Positional arguments in pytest positional args are test directories and files. + // So if we want to run a specific test, then remove positional args. + let removePositionalArgs = false; + if (Array.isArray(argumentToRemoveOrFilter)) { + argumentToRemoveOrFilter.forEach(item => { + if (OptionsWithArguments.indexOf(item) >= 0) { + optionsWithArgsToRemove.push(item); + } + if (OptionsWithoutArguments.indexOf(item) >= 0) { + optionsWithoutArgsToRemove.push(item); + } + }); + } else { + removePositionalArgs = true; + } + + let filteredArgs = args.slice(); + if (removePositionalArgs) { + const positionalArgs = this.helper.getPositionalArguments(filteredArgs, OptionsWithArguments, OptionsWithoutArguments); + filteredArgs = filteredArgs.filter(item => positionalArgs.indexOf(item) === -1); + } + return this.helper.filterArguments(filteredArgs, optionsWithArgsToRemove, optionsWithoutArgsToRemove); + } + public getTestFolders(args: string[]): string[] { + const shortValue = this.helper.getOptionValues(args, '-s'); + if (typeof shortValue === 'string') { + return [shortValue]; + } + const longValue = this.helper.getOptionValues(args, '--start-directory'); + if (typeof longValue === 'string') { + return [longValue]; + } + return ['.']; + } +} diff --git a/src/client/unittests/unittest/services/discoveryService.ts b/src/client/unittests/unittest/services/discoveryService.ts index cff7561c4420..fdf28cff3c93 100644 --- a/src/client/unittests/unittest/services/discoveryService.ts +++ b/src/client/unittests/unittest/services/discoveryService.ts @@ -4,8 +4,9 @@ import { inject, injectable, named } from 'inversify'; import { IServiceContainer } from '../../../ioc/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; -import { Options, run } from '../../common/runner'; -import { ITestDiscoveryService, ITestsParser, TestDiscoveryOptions, Tests } from '../../common/types'; +import { Options } from '../../common/runner'; +import { ITestDiscoveryService, ITestRunner, ITestsParser, TestDiscoveryOptions, Tests } from '../../common/types'; +import { IArgumentsHelper } from '../../types'; type UnitTestDiscoveryOptions = TestDiscoveryOptions & { startDirectory: string; @@ -14,8 +15,13 @@ type UnitTestDiscoveryOptions = TestDiscoveryOptions & { @injectable() export class TestDiscoveryService implements ITestDiscoveryService { - constructor( @inject(IServiceContainer) private serviceContainer: IServiceContainer, - @inject(ITestsParser) @named(UNITTEST_PROVIDER) private testParser: ITestsParser) { } + private readonly argsHelper: IArgumentsHelper; + private readonly runner: ITestRunner; + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer, + @inject(ITestsParser) @named(UNITTEST_PROVIDER) private testParser: ITestsParser) { + this.argsHelper = serviceContainer.get(IArgumentsHelper); + this.runner = serviceContainer.get(ITestRunner); + } public async discoverTests(options: TestDiscoveryOptions): Promise { const pythonScript = this.getDiscoveryScript(options); const unitTestOptions = this.translateOptions(options); @@ -27,7 +33,7 @@ export class TestDiscoveryService implements ITestDiscoveryService { outChannel: options.outChannel }; - const data = await run(this.serviceContainer, UNITTEST_PROVIDER, runOptions); + const data = await this.runner.run(UNITTEST_PROVIDER, runOptions); if (options.token && options.token.isCancellationRequested) { return Promise.reject('cancelled'); @@ -51,43 +57,32 @@ for suite in suites._tests: pass`; } public translateOptions(options: TestDiscoveryOptions): UnitTestDiscoveryOptions { - const unitTestOptions = { ...options } as UnitTestDiscoveryOptions; - unitTestOptions.startDirectory = this.getStartDirectory(options); - unitTestOptions.pattern = this.getTestPattern(options); - return unitTestOptions; + return { + ...options, + startDirectory: this.getStartDirectory(options), + pattern: this.getTestPattern(options) + }; } private getStartDirectory(options: TestDiscoveryOptions) { - let startDirectory = '.'; - const indexOfStartDir = options.args.findIndex(arg => arg.indexOf('-s') === 0); - if (indexOfStartDir >= 0) { - const startDir = options.args[indexOfStartDir].trim(); - if (startDir.trim() === '-s' && options.args.length >= indexOfStartDir) { - // Assume the next items is the directory - startDirectory = options.args[indexOfStartDir + 1]; - } else { - startDirectory = startDir.substring(2).trim(); - if (startDirectory.startsWith('=') || startDirectory.startsWith(' ')) { - startDirectory = startDirectory.substring(1); - } - } + const shortValue = this.argsHelper.getOptionValues(options.args, '-s'); + if (typeof shortValue === 'string') { + return shortValue; } - return startDirectory; + const longValue = this.argsHelper.getOptionValues(options.args, '--start-directory'); + if (typeof longValue === 'string') { + return longValue; + } + return '.'; } private getTestPattern(options: TestDiscoveryOptions) { - let pattern = 'test*.py'; - const indexOfPattern = options.args.findIndex(arg => arg.indexOf('-p') === 0); - if (indexOfPattern >= 0) { - const patternValue = options.args[indexOfPattern].trim(); - if (patternValue.trim() === '-p' && options.args.length >= indexOfPattern) { - // Assume the next items is the directory - pattern = options.args[indexOfPattern + 1]; - } else { - pattern = patternValue.substring(2).trim(); - if (pattern.startsWith('=')) { - pattern = pattern.substring(1); - } - } + const shortValue = this.argsHelper.getOptionValues(options.args, '-p'); + if (typeof shortValue === 'string') { + return shortValue; + } + const longValue = this.argsHelper.getOptionValues(options.args, '--pattern'); + if (typeof longValue === 'string') { + return longValue; } - return pattern; + return 'test*.py'; } } diff --git a/src/client/unittests/unittest/services/parserService.ts b/src/client/unittests/unittest/services/parserService.ts index dd746be00694..1b4acdd194c9 100644 --- a/src/client/unittests/unittest/services/parserService.ts +++ b/src/client/unittests/unittest/services/parserService.ts @@ -3,7 +3,7 @@ import { inject, injectable } from 'inversify'; import * as path from 'path'; -import { ITestsHelper, ITestsParser, TestDiscoveryOptions, TestFile, TestFunction, Tests, TestStatus, TestSuite } from '../../common/types'; +import { ITestsHelper, ITestsParser, TestDiscoveryOptions, TestFile, TestFunction, Tests, TestStatus } from '../../common/types'; type UnitTestParserOptions = TestDiscoveryOptions & { startDirectory: string }; @@ -21,7 +21,7 @@ export class TestsParser implements ITestsParser { private getTestIds(content: string): string[] { let startedCollecting = false; return content.split(/\r?\n/g) - .map((line, index) => { + .map(line => { if (!startedCollecting) { if (line === 'start') { startedCollecting = true; @@ -34,13 +34,22 @@ export class TestsParser implements ITestsParser { } private parseTestIds(rootDirectory: string, testIds: string[]): Tests { const testFiles: TestFile[] = []; - testIds.forEach(testId => { - this.addTestId(rootDirectory, testId, testFiles); - }); + testIds.forEach(testId => this.addTestId(rootDirectory, testId, testFiles)); return this.testsHelper.flattenTestFiles(testFiles); } + /** + * Add the test Ids into the array provided. + * TestIds are fully qualified including the method names. + * E.g. tone_test.Failing2Tests.test_failure + * Where tone_test = folder, Failing2Tests = class/suite, test_failure = method. + * @private + * @param {string} rootDirectory + * @param {string[]} testIds + * @returns {Tests} + * @memberof TestsParser + */ private addTestId(rootDirectory: string, testId: string, testFiles: TestFile[]) { const testIdParts = testId.split('.'); // We must have a file, class and function name @@ -59,10 +68,8 @@ export class TestsParser implements ITestsParser { testFile = { name: path.basename(filePath), fullPath: filePath, - // tslint:disable-next-line:prefer-type-cast - functions: [] as TestFunction[], - // tslint:disable-next-line:prefer-type-cast - suites: [] as TestSuite[], + functions: [], + suites: [], nameToRun: `${className}.${functionName}`, xmlName: '', status: TestStatus.Idle, @@ -77,10 +84,8 @@ export class TestsParser implements ITestsParser { if (!testSuite) { testSuite = { name: className, - // tslint:disable-next-line:prefer-type-cast - functions: [] as TestFunction[], - // tslint:disable-next-line:prefer-type-cast - suites: [] as TestSuite[], + functions: [], + suites: [], isUnitTest: true, isInstance: false, nameToRun: `${path.parse(filePath).name}.${classNameToRun}`, diff --git a/src/test/common.ts b/src/test/common.ts index 755d1a41a6f2..d2645fd5e31c 100644 --- a/src/test/common.ts +++ b/src/test/common.ts @@ -2,6 +2,7 @@ import * as fs from 'fs-extra'; import * as path from 'path'; import { ConfigurationTarget, Uri, workspace } from 'vscode'; import { PythonSettings } from '../client/common/configSettings'; +import { EXTENSION_ROOT_DIR } from '../client/common/constants'; import { sleep } from './core'; import { IS_MULTI_ROOT_TEST } from './initialize'; @@ -9,7 +10,7 @@ export * from './core'; // tslint:disable:no-non-null-assertion no-unsafe-any await-promise no-any no-use-before-declare no-string-based-set-timeout no-unsafe-any no-any no-invalid-this -const fileInNonRootWorkspace = path.join(__dirname, '..', '..', 'src', 'test', 'pythonFiles', 'dummy.py'); +const fileInNonRootWorkspace = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'dummy.py'); export const rootWorkspaceUri = getWorkspaceRoot(); export const PYTHON_PATH = getPythonPath(); @@ -40,7 +41,7 @@ export async function updateSetting(setting: PythonSettingKeys, value: {} | unde function getWorkspaceRoot() { if (!Array.isArray(workspace.workspaceFolders) || workspace.workspaceFolders.length === 0) { - return Uri.file(path.join(__dirname, '..', '..', 'src', 'test')); + return Uri.file(path.join(EXTENSION_ROOT_DIR, 'src', 'test')); } if (workspace.workspaceFolders.length === 1) { return workspace.workspaceFolders[0].uri; diff --git a/src/test/common/platform/filesystem.test.ts b/src/test/common/platform/filesystem.unit.test.ts similarity index 96% rename from src/test/common/platform/filesystem.test.ts rename to src/test/common/platform/filesystem.unit.test.ts index b434bd5d9c96..eb29e10b22b9 100644 --- a/src/test/common/platform/filesystem.test.ts +++ b/src/test/common/platform/filesystem.unit.test.ts @@ -1,12 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { expect } from 'chai'; +import { expect, use } from 'chai'; import * as fs from 'fs-extra'; import * as path from 'path'; import * as TypeMoq from 'typemoq'; import { FileSystem } from '../../../client/common/platform/fileSystem'; import { IFileSystem, IPlatformService } from '../../../client/common/platform/types'; +// tslint:disable-next-line:no-require-imports no-var-requires +const assertArrays = require('chai-arrays'); +use(assertArrays); // tslint:disable-next-line:max-func-body-length suite('FileSystem', () => { diff --git a/src/test/index.ts b/src/test/index.ts index 5a2ebe364582..58eb65ffd46c 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -4,9 +4,11 @@ if ((Reflect as any).metadata === undefined) { require('reflect-metadata'); } -import { IS_CI_SERVER, IS_CI_SERVER_TEST_DEBUGGER, - IS_MULTI_ROOT_TEST, IS_VSTS, MOCHA_CI_PROPERTIES, - MOCHA_CI_REPORTFILE, MOCHA_REPORTER_JUNIT } from './constants'; +import { + IS_CI_SERVER, IS_CI_SERVER_TEST_DEBUGGER, + IS_MULTI_ROOT_TEST, IS_VSTS, MOCHA_CI_PROPERTIES, + MOCHA_CI_REPORTFILE, MOCHA_REPORTER_JUNIT +} from './constants'; import * as testRunner from './testRunner'; process.env.VSC_PYTHON_CI_TEST = '1'; diff --git a/src/test/unittests/argsService.unit.test.ts b/src/test/unittests/argsService.unit.test.ts new file mode 100644 index 000000000000..e0ca1043c84e --- /dev/null +++ b/src/test/unittests/argsService.unit.test.ts @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length no-any no-conditional-assignment no-increment-decrement no-invalid-this insecure-random +import { fail } from 'assert'; +import { expect } from 'chai'; +import { spawnSync } from 'child_process'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { EnumEx } from '../../client/common/enumUtils'; +import { ILogger, Product } from '../../client/common/types'; +import { IServiceContainer } from '../../client/ioc/types'; +import { ArgumentsHelper } from '../../client/unittests/common/argumentsHelper'; +import { ArgumentsService as NoseTestArgumentsService } from '../../client/unittests/nosetest/services/argsService'; +import { ArgumentsService as PyTestArgumentsService } from '../../client/unittests/pytest/services/argsService'; +import { IArgumentsHelper, IArgumentsService } from '../../client/unittests/types'; +import { ArgumentsService as UnitTestArgumentsService } from '../../client/unittests/unittest/services/argsService'; +import { PYTHON_PATH } from '../common'; + +suite('Unit Tests - argsService', () => { + [Product.unittest, Product.nosetest, Product.pytest] + .forEach(product => { + const productNames = EnumEx.getNamesAndValues(Product); + const productName = productNames.find(item => item.value === product)!.name; + suite(productName, () => { + let argumentsService: IArgumentsService; + let moduleName = ''; + let expectedWithArgs: string[] = []; + let expectedWithoutArgs: string[] = []; + + suiteSetup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + const logger = typeMoq.Mock.ofType(); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(ILogger), typeMoq.It.isAny())) + .returns(() => logger.object); + + const argsHelper = new ArgumentsHelper(serviceContainer.object); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(IArgumentsHelper), typeMoq.It.isAny())) + .returns(() => argsHelper); + + switch (product) { + case Product.unittest: { + argumentsService = new UnitTestArgumentsService(serviceContainer.object); + moduleName = 'unittest'; + break; + } + case Product.nosetest: { + argumentsService = new NoseTestArgumentsService(serviceContainer.object); + moduleName = 'nose'; + break; + } + case Product.pytest: { + moduleName = 'pytest'; + argumentsService = new PyTestArgumentsService(serviceContainer.object); + break; + } + default: { + throw new Error('Unrecognized Test Framework'); + } + } + + expectedWithArgs = getOptions(product, moduleName, true); + expectedWithoutArgs = getOptions(product, moduleName, false); + }); + + test('Check for new/unrecognized options with values', () => { + const options = argumentsService.getKnownOptions(); + const optionsNotFound = expectedWithArgs.filter(item => options.withArgs.indexOf(item) === -1); + + if (optionsNotFound.length > 0) { + fail('', optionsNotFound.join(', '), 'Options not found'); + } + }); + test('Check for new/unrecognized options without values', () => { + const options = argumentsService.getKnownOptions(); + const optionsNotFound = expectedWithoutArgs.filter(item => options.withoutArgs.indexOf(item) === -1); + + if (optionsNotFound.length > 0) { + fail('', optionsNotFound.join(', '), 'Options not found'); + } + }); + test('Test getting value for an option with a single value', () => { + for (const option of expectedWithArgs) { + const args = ['--some-option-with-a-value', '1234', '--another-value-with-inline=1234', option, 'abcd']; + const value = argumentsService.getOptionValue(args, option); + expect(value).to.equal('abcd'); + } + }); + test('Test getting value for an option with a multiple value', () => { + for (const option of expectedWithArgs) { + const args = ['--some-option-with-a-value', '1234', '--another-value-with-inline=1234', option, 'abcd', option, 'xyz']; + const value = argumentsService.getOptionValue(args, option); + expect(value).to.deep.equal(['abcd', 'xyz']); + } + }); + test('Test getting the test folder in unittest with -s', function () { + if (product !== Product.unittest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '-s', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with -s in the middle', function () { + if (product !== Product.unittest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '-s', dir, 'some other', '--value', '1234']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with --start-directory', function () { + if (product !== Product.unittest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '--start-directory', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in unittest with --start-directory in the middle', function () { + if (product !== Product.unittest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', '--start-directory', dir, 'some other', '--value', '1234']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in nosetest', function () { + if (product !== Product.nosetest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--three', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in nosetest (with multiple dirs)', function () { + if (product !== Product.nosetest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--three', dir, dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', '--rootdir', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in pytest (with multiple dirs)', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with multiple dirs in the middle)', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', '--rootdir', dir, '--rootdir', dir2, '-xyz']; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with single positional dir)', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const args = ['anzy', '--one', dir]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(1); + expect(testDirs[0]).to.equal(dir); + }); + test('Test getting the test folder in pytest (with multiple positional dirs)', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', dir, dir2]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test getting the test folder in pytest (with multiple dirs excluding python files)', function () { + if (product !== Product.pytest) { + return this.skip(); + } + const dir = path.join('a', 'b', 'c'); + const dir2 = path.join('a', 'b', '2'); + const args = ['anzy', '--one', dir, dir2, path.join(dir, 'one.py')]; + const testDirs = argumentsService.getTestFolders(args); + expect(testDirs).to.be.lengthOf(2); + expect(testDirs[0]).to.equal(dir); + expect(testDirs[1]).to.equal(dir2); + }); + test('Test filtering of arguments', () => { + const args: string[] = []; + const knownOptions = argumentsService.getKnownOptions(); + const argumentsToRemove: string[] = []; + const expectedFilteredArgs: string[] = []; + // Generate some random arguments. + for (let i = 0; i < 5; i += 1) { + args.push(knownOptions.withArgs[i], `Random Value ${i}`); + args.push(knownOptions.withoutArgs[i]); + + if (i % 2 === 0) { + argumentsToRemove.push(knownOptions.withArgs[i], knownOptions.withoutArgs[i]); + } else { + expectedFilteredArgs.push(knownOptions.withArgs[i], `Random Value ${i}`); + expectedFilteredArgs.push(knownOptions.withoutArgs[i]); + } + } + + const filteredArgs = argumentsService.filterArguments(args, argumentsToRemove); + expect(filteredArgs).to.be.deep.equal(expectedFilteredArgs); + }); + }); + }); +}); + +function getOptions(product: Product, moduleName: string, withValues: boolean) { + // const result = spawnSync('/Users/donjayamanne/Desktop/Development/PythonStuff/vscodePythonTesting/testingFolder/venv/bin/python', ['-m', moduleName, '-h']); + const result = spawnSync(PYTHON_PATH, ['-m', moduleName, '-h']); + const output = result.stdout.toString(); + + // Our regex isn't the best, so lets exclude stuff that shouldn't be captured. + const knownOptionsWithoutArgs: string[] = []; + const knownOptionsWithArgs: string[] = []; + if (product === Product.pytest) { + knownOptionsWithArgs.push(...['-c', '-p', '-r']); + } + + if (withValues) { + return getOptionsWithArguments(output) + .concat(...knownOptionsWithArgs) + .filter(item => knownOptionsWithoutArgs.indexOf(item) === -1) + .sort(); + } else { + return getOptionsWithoutArguments(output) + .concat(...knownOptionsWithoutArgs) + .filter(item => knownOptionsWithArgs.indexOf(item) === -1) + // In pytest, any option begining with --log- is known to have args. + .filter(item => product === Product.pytest ? !item.startsWith('--log-') : true) + .sort(); + } +} + +function getOptionsWithoutArguments(output: string) { + return getMatches('\\s{1,}(-{1,2}[A-Za-z0-9-]+)(?:,|\\s{2,})', output); +} +function getOptionsWithArguments(output: string) { + return getMatches('\\s{1,}(-{1,2}[A-Za-z0-9-]+)(?:=|\\s{0,1}[A-Z])', output); +} + +function getMatches(pattern, str) { + const matches: string[] = []; + const regex = new RegExp(pattern, 'gm'); + let result; + while ((result = regex.exec(str)) !== null) { + if (result.index === regex.lastIndex) { + regex.lastIndex++; + } + matches.push(result[1].trim()); + } + return matches + .sort() + .reduce((items, item) => items.indexOf(item) === -1 ? items.concat([item]) : items, []); +} diff --git a/src/test/unittests/common/argsHelper.unit.test.ts b/src/test/unittests/common/argsHelper.unit.test.ts new file mode 100644 index 000000000000..29a9ac3261d1 --- /dev/null +++ b/src/test/unittests/common/argsHelper.unit.test.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length no-any no-conditional-assignment no-increment-decrement no-invalid-this no-require-imports no-var-requires +import { expect, use } from 'chai'; +import * as typeMoq from 'typemoq'; +import { ILogger } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { ArgumentsHelper } from '../../../client/unittests/common/argumentsHelper'; +import { IArgumentsHelper } from '../../../client/unittests/types'; +const assertArrays = require('chai-arrays'); +use(assertArrays); + +suite('Unit Tests - Arguments Helper', () => { + let argsHelper: IArgumentsHelper; + setup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + const logger = typeMoq.Mock.ofType(); + + serviceContainer + .setup(s => s.get(typeMoq.It.isValue(ILogger), typeMoq.It.isAny())) + .returns(() => logger.object); + + argsHelper = new ArgumentsHelper(serviceContainer.object); + }); + + test('Get Option Value', () => { + const args = ['-abc', '1234', 'zys', '--root', 'value']; + const value = argsHelper.getOptionValues(args, '--root'); + expect(value).to.not.be.array(); + expect(value).to.be.deep.equal('value'); + }); + test('Get Option Value when using =', () => { + const args = ['-abc', '1234', 'zys', '--root=value']; + const value = argsHelper.getOptionValues(args, '--root'); + expect(value).to.not.be.array(); + expect(value).to.be.deep.equal('value'); + }); + test('Get Option Values', () => { + const args = ['-abc', '1234', 'zys', '--root', 'value1', '--root', 'value2']; + const values = argsHelper.getOptionValues(args, '--root'); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(2); + expect(values).to.be.deep.equal(['value1', 'value2']); + }); + test('Get Option Values when using =', () => { + const args = ['-abc', '1234', 'zys', '--root=value1', '--root=value2']; + const values = argsHelper.getOptionValues(args, '--root'); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(2); + expect(values).to.be.deep.equal(['value1', 'value2']); + }); + test('Get Positional options', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--no-value-option', 'value2']; + const values = argsHelper.getPositionalArguments(args, ['--value-option', '-abc'], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(1); + expect(values).to.be.deep.equal(['value2']); + }); + test('Get multiple Positional options', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--no-value-option', 'value2', 'value3']; + const values = argsHelper.getPositionalArguments(args, ['--value-option', '-abc'], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(2); + expect(values).to.be.deep.equal(['value2', 'value3']); + }); + test('Get multiple Positional options and ineline values', () => { + const args = ['-abc=1234', '--value-option=value1', '--no-value-option', 'value2', 'value3']; + const values = argsHelper.getPositionalArguments(args, ['--value-option', '-abc'], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(2); + expect(values).to.be.deep.equal(['value2', 'value3']); + }); + test('Get Positional options with trailing value option', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--value-option', 'value2', 'value3']; + const values = argsHelper.getPositionalArguments(args, ['--value-option', '-abc'], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(1); + expect(values).to.be.deep.equal(['value3']); + }); + test('Get multiplle Positional options with trailing value option', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--value-option', 'value2', 'value3', '4']; + const values = argsHelper.getPositionalArguments(args, ['--value-option', '-abc'], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(2); + expect(values).to.be.deep.equal(['value3', '4']); + }); + test('Filter to remove those with values', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--value-option', 'value2', 'value3', '4']; + const values = argsHelper.filterArguments(args, ['--value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(4); + expect(values).to.be.deep.equal(['-abc', '1234', 'value3', '4']); + }); + test('Filter to remove those without values', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--no-value-option', 'value2', 'value3', '4']; + const values = argsHelper.filterArguments(args, [], ['--no-value-option']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(7); + expect(values).to.be.deep.equal(['-abc', '1234', '--value-option', 'value1', 'value2', 'value3', '4']); + }); + test('Filter to remove those with and without values', () => { + const args = ['-abc', '1234', '--value-option', 'value1', '--value-option', 'value2', 'value3', '4']; + const values = argsHelper.filterArguments(args, ['--value-option'], ['-abc']); + expect(values).to.be.array(); + expect(values).to.be.lengthOf(3); + expect(values).to.be.deep.equal(['1234', 'value3', '4']); + }); +}); diff --git a/src/test/unittests/debugger.test.ts b/src/test/unittests/debugger.test.ts index c526961527a9..d687de8b6755 100644 --- a/src/test/unittests/debugger.test.ts +++ b/src/test/unittests/debugger.test.ts @@ -3,8 +3,19 @@ import * as chaiAsPromised from 'chai-as-promised'; import * as path from 'path'; import { ConfigurationTarget } from 'vscode'; import { createDeferred } from '../../client/common/helpers'; -import { CANCELLATION_REASON, CommandSource } from '../../client/unittests/common/constants'; -import { ITestDebugLauncher, ITestManagerFactory, TestProvider } from '../../client/unittests/common/types'; +import { TestManagerRunner as NoseTestManagerRunner } from '../../client/unittests//nosetest/runner'; +import { TestManagerRunner as PytestManagerRunner } from '../../client/unittests//pytest/runner'; +import { TestManagerRunner as UnitTestTestManagerRunner } from '../../client/unittests//unittest/runner'; +import { ArgumentsHelper } from '../../client/unittests/common/argumentsHelper'; +import { CANCELLATION_REASON, CommandSource, NOSETEST_PROVIDER, PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../../client/unittests/common/constants'; +import { TestRunner } from '../../client/unittests/common/runner'; +import { ITestDebugLauncher, ITestManagerFactory, ITestRunner, IXUnitParser, TestProvider } from '../../client/unittests/common/types'; +import { XUnitParser } from '../../client/unittests/common/xUnitParser'; +import { ArgumentsService as NoseTestArgumentsService } from '../../client/unittests/nosetest/services/argsService'; +import { ArgumentsService as PyTestArgumentsService } from '../../client/unittests/pytest/services/argsService'; +import { IArgumentsHelper, IArgumentsService, ITestManagerRunner, IUnitTestHelper } from '../../client/unittests/types'; +import { UnitTestHelper } from '../../client/unittests/unittest/helper'; +import { ArgumentsService as UnitTestArgumentsService } from '../../client/unittests/unittest/services/argsService'; import { deleteDirectory, rootWorkspaceUri, updateSetting } from '../common'; import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; import { MockDebugLauncher } from './mocks'; @@ -60,6 +71,16 @@ suite('Unit Tests - debugging', () => { ioc.registerTestsHelper(); ioc.registerTestManagers(); ioc.registerMockUnitTestSocketServer(); + ioc.serviceManager.add(IArgumentsHelper, ArgumentsHelper); + ioc.serviceManager.add(ITestRunner, TestRunner); + ioc.serviceManager.add(IXUnitParser, XUnitParser); + ioc.serviceManager.add(IUnitTestHelper, UnitTestHelper); + ioc.serviceManager.add(IArgumentsService, NoseTestArgumentsService, NOSETEST_PROVIDER); + ioc.serviceManager.add(IArgumentsService, PyTestArgumentsService, PYTEST_PROVIDER); + ioc.serviceManager.add(IArgumentsService, UnitTestArgumentsService, UNITTEST_PROVIDER); + ioc.serviceManager.add(ITestManagerRunner, PytestManagerRunner, PYTEST_PROVIDER); + ioc.serviceManager.add(ITestManagerRunner, NoseTestManagerRunner, NOSETEST_PROVIDER); + ioc.serviceManager.add(ITestManagerRunner, UnitTestTestManagerRunner, UNITTEST_PROVIDER); ioc.serviceManager.addSingleton(ITestDebugLauncher, MockDebugLauncher); } @@ -77,8 +98,8 @@ suite('Unit Tests - debugging', () => { // This promise should never resolve nor reject. runningPromise - .then(() => deferred.reject('Debugger stopped when it shouldn\'t have')) - .catch(error => deferred.reject(error)); + .then(() => deferred.reject('Debugger stopped when it shouldn\'t have')) + .catch(error => deferred.reject(error)); mockDebugLauncher.launched .then((launched) => { @@ -87,14 +108,14 @@ suite('Unit Tests - debugging', () => { } else { deferred.reject('Debugger not launched'); } - }) .catch(error => deferred.reject(error)); + }).catch(error => deferred.reject(error)); await deferred.promise; } test('Debugger should start (unittest)', async () => { await updateSetting('unitTest.unittestArgs', ['-s=./tests', '-p=test_*.py'], rootWorkspaceUri, configTarget); - await testStartingDebugger('unittest'); + await testStartingDebugger('unittest'); }); test('Debugger should start (pytest)', async () => { diff --git a/src/test/unittests/nosetest/nosetest.discovery.unit.test.ts b/src/test/unittests/nosetest/nosetest.discovery.unit.test.ts new file mode 100644 index 000000000000..4ea9b9d54556 --- /dev/null +++ b/src/test/unittests/nosetest/nosetest.discovery.unit.test.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable-next-line:max-func-body-length + +import { expect, use } from 'chai'; +import * as chaipromise from 'chai-as-promised'; +import * as typeMoq from 'typemoq'; +import { CancellationToken } from 'vscode'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { NOSETEST_PROVIDER } from '../../../client/unittests/common/constants'; +import { ITestDiscoveryService, ITestRunner, ITestsParser, Options, TestDiscoveryOptions, Tests } from '../../../client/unittests/common/types'; +import { TestDiscoveryService } from '../../../client/unittests/nosetest/services/discoveryService'; +import { IArgumentsService, TestFilter } from '../../../client/unittests/types'; + +use(chaipromise); + +suite('Unit Tests - nose - Discovery', () => { + let discoveryService: ITestDiscoveryService; + let argsService: typeMoq.IMock; + let testParser: typeMoq.IMock; + let runner: typeMoq.IMock; + setup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + argsService = typeMoq.Mock.ofType(); + testParser = typeMoq.Mock.ofType(); + runner = typeMoq.Mock.ofType(); + + serviceContainer.setup(s => s.get(typeMoq.It.isValue(IArgumentsService), typeMoq.It.isAny())) + .returns(() => argsService.object); + serviceContainer.setup(s => s.get(typeMoq.It.isValue(ITestRunner), typeMoq.It.isAny())) + .returns(() => runner.object); + + discoveryService = new TestDiscoveryService(serviceContainer.object, testParser.object); + }); + test('Ensure discovery is invoked with the right args', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsService.setup(a => a.filterArguments(typeMoq.It.isValue(args), typeMoq.It.isValue(TestFilter.discovery))) + .returns(() => []) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(NOSETEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('--collect-only'); + expect(opts.args).to.include('-vvv'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + argsService.verifyAll(); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is cancelled', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsService.setup(a => a.filterArguments(typeMoq.It.isValue(args), typeMoq.It.isValue(TestFilter.discovery))) + .returns(() => []) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(NOSETEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('--collect-only'); + expect(opts.args).to.include('-vvv'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.never()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + token.setup(t => t.isCancellationRequested) + .returns(() => true) + .verifiable(typeMoq.Times.once()); + + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + const promise = discoveryService.discoverTests(options.object); + + await expect(promise).to.eventually.be.rejectedWith('cancelled'); + argsService.verifyAll(); + runner.verifyAll(); + testParser.verifyAll(); + }); +}); diff --git a/src/test/unittests/nosetest.disovery.test.ts b/src/test/unittests/nosetest/nosetest.disovery.test.ts similarity index 88% rename from src/test/unittests/nosetest.disovery.test.ts rename to src/test/unittests/nosetest/nosetest.disovery.test.ts index 51001b1ab568..5bda99d7c657 100644 --- a/src/test/unittests/nosetest.disovery.test.ts +++ b/src/test/unittests/nosetest/nosetest.disovery.test.ts @@ -5,18 +5,19 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as path from 'path'; import * as vscode from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { lookForTestFile } from './helper'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { lookForTestFile } from '../helper'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const PYTHON_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles'); -const UNITTEST_TEST_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); -const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'single'); +const PYTHON_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles'); +const UNITTEST_TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); +const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'single'); const filesToDelete = [ path.join(UNITTEST_TEST_FILES_PATH, '.noseids'), path.join(UNITTEST_SINGLE_TEST_FILE_PATH, '.noseids') diff --git a/src/test/unittests/nosetest.run.test.ts b/src/test/unittests/nosetest/nosetest.run.test.ts similarity index 90% rename from src/test/unittests/nosetest.run.test.ts rename to src/test/unittests/nosetest/nosetest.run.test.ts index 37d26716a31e..88e2adc63b8e 100644 --- a/src/test/unittests/nosetest.run.test.ts +++ b/src/test/unittests/nosetest/nosetest.run.test.ts @@ -5,16 +5,17 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as path from 'path'; import * as vscode from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory, TestsToRun } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory, TestsToRun } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const UNITTEST_TEST_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); -const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'single'); +const UNITTEST_TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); +const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'single'); const filesToDelete = [ path.join(UNITTEST_TEST_FILES_PATH, '.noseids'), path.join(UNITTEST_SINGLE_TEST_FILE_PATH, '.noseids') @@ -65,7 +66,7 @@ suite('Unit Tests - nose - run against actual python process', () => { procService.onExecObservable((file, args, options, callback) => { if (args.indexOf('--collect-only') >= 0) { callback({ - out: fs.readFileSync(path.join(UNITTEST_TEST_FILES_PATH, outputFileName), 'utf8'), + out: fs.readFileSync(path.join(UNITTEST_TEST_FILES_PATH, outputFileName), 'utf8').replace(/\/Users\/donjayamanne\/.vscode\/extensions\/pythonVSCode\/src\/test\/pythonFiles\/testFiles\/noseFiles/g, UNITTEST_TEST_FILES_PATH), source: 'stdout' }); } diff --git a/src/test/unittests/nosetest.test.ts b/src/test/unittests/nosetest/nosetest.test.ts similarity index 77% rename from src/test/unittests/nosetest.test.ts rename to src/test/unittests/nosetest/nosetest.test.ts index 4ef31a0b1266..2f3581a0b682 100644 --- a/src/test/unittests/nosetest.test.ts +++ b/src/test/unittests/nosetest/nosetest.test.ts @@ -2,15 +2,16 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as path from 'path'; import * as vscode from 'vscode'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { lookForTestFile } from './helper'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { lookForTestFile } from '../helper'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const UNITTEST_TEST_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); -const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'single'); +const UNITTEST_TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'noseFiles'); +const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'single'); const filesToDelete = [ path.join(UNITTEST_TEST_FILES_PATH, '.noseids'), path.join(UNITTEST_SINGLE_TEST_FILE_PATH, '.noseids') diff --git a/src/test/unittests/pytest.discovery.test.ts b/src/test/unittests/pytest/pytest.discovery.test.ts similarity index 92% rename from src/test/unittests/pytest.discovery.test.ts rename to src/test/unittests/pytest/pytest.discovery.test.ts index bdd6002ab81c..d2e0ef7ca93b 100644 --- a/src/test/unittests/pytest.discovery.test.ts +++ b/src/test/unittests/pytest/pytest.discovery.test.ts @@ -4,18 +4,19 @@ import * as assert from 'assert'; import * as path from 'path'; import * as vscode from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const UNITTEST_TEST_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'standard'); -const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'single'); -const UNITTEST_TEST_FILES_PATH_WITH_CONFIGS = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'unitestsWithConfigs'); -const unitTestTestFilesCwdPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'cwd', 'src'); +const UNITTEST_TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'standard'); +const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'single'); +const UNITTEST_TEST_FILES_PATH_WITH_CONFIGS = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'unitestsWithConfigs'); +const unitTestTestFilesCwdPath = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'cwd', 'src'); // tslint:disable-next-line:max-func-body-length suite('Unit Tests - pytest - discovery with mocked process output', () => { diff --git a/src/test/unittests/pytest/pytest.discovery.unit.test.ts b/src/test/unittests/pytest/pytest.discovery.unit.test.ts new file mode 100644 index 000000000000..a7255308739f --- /dev/null +++ b/src/test/unittests/pytest/pytest.discovery.unit.test.ts @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length + +import { expect, use } from 'chai'; +import * as chaipromise from 'chai-as-promised'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { CancellationToken } from 'vscode'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { PYTEST_PROVIDER } from '../../../client/unittests/common/constants'; +import { ITestDiscoveryService, ITestRunner, ITestsHelper, ITestsParser, Options, TestDiscoveryOptions, Tests } from '../../../client/unittests/common/types'; +import { TestDiscoveryService } from '../../../client/unittests/pytest/services/discoveryService'; +import { IArgumentsService, TestFilter } from '../../../client/unittests/types'; + +use(chaipromise); + +suite('Unit Tests - PyTest - Discovery', () => { + let discoveryService: ITestDiscoveryService; + let argsService: typeMoq.IMock; + let testParser: typeMoq.IMock; + let runner: typeMoq.IMock; + let helper: typeMoq.IMock; + setup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + argsService = typeMoq.Mock.ofType(); + testParser = typeMoq.Mock.ofType(); + runner = typeMoq.Mock.ofType(); + helper = typeMoq.Mock.ofType(); + + serviceContainer.setup(s => s.get(typeMoq.It.isValue(IArgumentsService), typeMoq.It.isAny())) + .returns(() => argsService.object); + serviceContainer.setup(s => s.get(typeMoq.It.isValue(ITestRunner), typeMoq.It.isAny())) + .returns(() => runner.object); + serviceContainer.setup(s => s.get(typeMoq.It.isValue(ITestsHelper), typeMoq.It.isAny())) + .returns(() => helper.object); + + discoveryService = new TestDiscoveryService(serviceContainer.object, testParser.object); + }); + test('Ensure discovery is invoked with the right args and single dir', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const dir = path.join('a', 'b', 'c'); + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsService.setup(a => a.filterArguments(typeMoq.It.isValue(args), typeMoq.It.isValue(TestFilter.discovery))) + .returns(() => []) + .verifiable(typeMoq.Times.once()); + argsService.setup(a => a.getTestFolders(typeMoq.It.isValue(args))) + .returns(() => [dir]) + .verifiable(typeMoq.Times.once()); + helper.setup(a => a.mergeTests(typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(PYTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('--cache-clear'); + expect(opts.args).to.include('-s'); + expect(opts.args).to.include('--collect-only'); + expect(opts.args[opts.args.length - 1]).to.equal(dir); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + argsService.verifyAll(); + runner.verifyAll(); + testParser.verifyAll(); + helper.verifyAll(); + }); + test('Ensure discovery is invoked with the right args and multiple dirs', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const dirs = [path.join('a', 'b', '1'), path.join('a', 'b', '2')]; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsService.setup(a => a.filterArguments(typeMoq.It.isValue(args), typeMoq.It.isValue(TestFilter.discovery))) + .returns(() => []) + .verifiable(typeMoq.Times.once()); + argsService.setup(a => a.getTestFolders(typeMoq.It.isValue(args))) + .returns(() => dirs) + .verifiable(typeMoq.Times.once()); + helper.setup(a => a.mergeTests(typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(PYTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('--cache-clear'); + expect(opts.args).to.include('-s'); + expect(opts.args).to.include('--collect-only'); + const dir = opts.args[opts.args.length - 1]; + expect(dirs).to.include(dir); + dirs.splice(dirs.indexOf(dir) - 1, 1); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + argsService.verifyAll(); + runner.verifyAll(); + testParser.verifyAll(); + helper.verifyAll(); + }); + test('Ensure discovery is cancelled', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsService.setup(a => a.filterArguments(typeMoq.It.isValue(args), typeMoq.It.isValue(TestFilter.discovery))) + .returns(() => []) + .verifiable(typeMoq.Times.once()); + argsService.setup(a => a.getTestFolders(typeMoq.It.isValue(args))) + .returns(() => ['']) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(PYTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('--cache-clear'); + expect(opts.args).to.include('-s'); + expect(opts.args).to.include('--collect-only'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.never()); + helper.setup(a => a.mergeTests(typeMoq.It.isAny())) + .returns(() => tests); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + token.setup(t => t.isCancellationRequested) + .returns(() => true) + .verifiable(typeMoq.Times.once()); + + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + const promise = discoveryService.discoverTests(options.object); + + await expect(promise).to.eventually.be.rejectedWith('cancelled'); + argsService.verifyAll(); + runner.verifyAll(); + testParser.verifyAll(); + }); +}); diff --git a/src/test/unittests/pytest.run.test.ts b/src/test/unittests/pytest/pytest.run.test.ts similarity index 89% rename from src/test/unittests/pytest.run.test.ts rename to src/test/unittests/pytest/pytest.run.test.ts index 57cc69f7bf97..09941039ed74 100644 --- a/src/test/unittests/pytest.run.test.ts +++ b/src/test/unittests/pytest/pytest.run.test.ts @@ -5,16 +5,17 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as path from 'path'; import * as vscode from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory, TestFile, TestsToRun } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory, TestFile, TestsToRun } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const UNITTEST_TEST_FILES_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'standard'); -const PYTEST_RESULTS_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'pytestFiles', 'results'); +const UNITTEST_TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'standard'); +const PYTEST_RESULTS_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'pytestFiles', 'results'); // tslint:disable-next-line:max-func-body-length suite('Unit Tests - pytest - run with mocked process output', () => { @@ -48,7 +49,7 @@ suite('Unit Tests - pytest - run with mocked process output', () => { procService.onExecObservable((file, args, options, callback) => { if (args.indexOf('--collect-only') >= 0) { callback({ - out: fs.readFileSync(path.join(PYTEST_RESULTS_PATH, outputFileName), 'utf8'), + out: fs.readFileSync(path.join(PYTEST_RESULTS_PATH, outputFileName), 'utf8').replace(/\/Users\/donjayamanne\/.vscode\/extensions\/pythonVSCode\/src\/test\/pythonFiles\/testFiles\/noseFiles/g, PYTEST_RESULTS_PATH), source: 'stdout' }); } diff --git a/src/test/unittests/pytest.test.ts b/src/test/unittests/pytest/pytest.test.ts similarity index 79% rename from src/test/unittests/pytest.test.ts rename to src/test/unittests/pytest/pytest.test.ts index d8cea72441d5..210c3c5d8f84 100644 --- a/src/test/unittests/pytest.test.ts +++ b/src/test/unittests/pytest/pytest.test.ts @@ -1,13 +1,14 @@ import * as assert from 'assert'; import * as path from 'path'; import * as vscode from 'vscode'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles', 'single'); +const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles', 'single'); // tslint:disable-next-line:max-func-body-length suite('Unit Tests - pytest - discovery against actual python process', () => { diff --git a/src/test/unittests/unittest.discovery.test.ts b/src/test/unittests/unittest/unittest.discovery.test.ts similarity index 92% rename from src/test/unittests/unittest.discovery.test.ts rename to src/test/unittests/unittest/unittest.discovery.test.ts index e573726c2b4e..d3e8c5b477b5 100644 --- a/src/test/unittests/unittest.discovery.test.ts +++ b/src/test/unittests/unittest/unittest.discovery.test.ts @@ -6,15 +6,16 @@ import * as fs from 'fs-extra'; import { EOL } from 'os'; import * as path from 'path'; import { ConfigurationTarget } from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const testFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles'); +const testFilesPath = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles'); const UNITTEST_TEST_FILES_PATH = path.join(testFilesPath, 'standard'); const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(testFilesPath, 'single'); const unitTestTestFilesCwdPath = path.join(testFilesPath, 'cwd', 'src'); diff --git a/src/test/unittests/unittest/unittest.discovery.unit.test.ts b/src/test/unittests/unittest/unittest.discovery.unit.test.ts new file mode 100644 index 000000000000..b70ad76845d5 --- /dev/null +++ b/src/test/unittests/unittest/unittest.discovery.unit.test.ts @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length + +import { expect, use } from 'chai'; +import * as chaipromise from 'chai-as-promised'; +import * as path from 'path'; +import * as typeMoq from 'typemoq'; +import { CancellationToken } from 'vscode'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { UNITTEST_PROVIDER } from '../../../client/unittests/common/constants'; +import { ITestDiscoveryService, ITestRunner, ITestsParser, Options, TestDiscoveryOptions, Tests } from '../../../client/unittests/common/types'; +import { IArgumentsHelper } from '../../../client/unittests/types'; +import { TestDiscoveryService } from '../../../client/unittests/unittest/services/discoveryService'; + +use(chaipromise); + +suite('Unit Tests - Unittest - Discovery', () => { + let discoveryService: ITestDiscoveryService; + let argsHelper: typeMoq.IMock; + let testParser: typeMoq.IMock; + let runner: typeMoq.IMock; + const dir = path.join('a', 'b', 'c'); + const pattern = 'Pattern_To_Search_For'; + setup(() => { + const serviceContainer = typeMoq.Mock.ofType(); + argsHelper = typeMoq.Mock.ofType(); + testParser = typeMoq.Mock.ofType(); + runner = typeMoq.Mock.ofType(); + + serviceContainer.setup(s => s.get(typeMoq.It.isValue(IArgumentsHelper), typeMoq.It.isAny())) + .returns(() => argsHelper.object); + serviceContainer.setup(s => s.get(typeMoq.It.isValue(ITestRunner), typeMoq.It.isAny())) + .returns(() => runner.object); + + discoveryService = new TestDiscoveryService(serviceContainer.object, testParser.object); + }); + test('Ensure discovery is invoked with the right args with start directory defined with -s', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-s'))) + .returns(() => dir) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.contain(dir); + expect(opts.args[1]).to.not.contain('loader.discover("."'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is invoked with the right args with start directory defined with --start-directory', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-s'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('--start-directory'))) + .returns(() => dir) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.contain(dir); + expect(opts.args[1]).to.not.contain('loader.discover("."'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is invoked with the right args without a start directory', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-s'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('--start-directory'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.not.contain(dir); + expect(opts.args[1]).to.contain('loader.discover("."'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is invoked with the right args without a pattern defined with -p', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-p'))) + .returns(() => pattern) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.contain(pattern); + expect(opts.args[1]).to.not.contain('test*.py'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is invoked with the right args without a pattern defined with ---pattern', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-p'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('--pattern'))) + .returns(() => pattern) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.contain(pattern); + expect(opts.args[1]).to.not.contain('test*.py'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is invoked with the right args without a pattern not defined', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-p'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('--pattern'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .callback((_, opts: Options) => { + expect(opts.args).to.include('-c'); + expect(opts.args[1]).to.not.contain(pattern); + expect(opts.args[1]).to.contain('test*.py'); + }) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.once()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => false); + + const result = await discoveryService.discoverTests(options.object); + + expect(result).to.be.equal(tests); + runner.verifyAll(); + testParser.verifyAll(); + }); + test('Ensure discovery is cancelled', async () => { + const args: string[] = []; + const runOutput = 'xyz'; + const tests: Tests = { + summary: { errors: 1, failures: 0, passed: 0, skipped: 0 }, + testFiles: [], testFunctions: [], testSuites: [], + rootTestFolders: [], testFolders: [] + }; + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('-p'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + argsHelper.setup(a => a.getOptionValues(typeMoq.It.isValue(args), typeMoq.It.isValue('--pattern'))) + .returns(() => undefined) + .verifiable(typeMoq.Times.once()); + runner.setup(r => r.run(typeMoq.It.isValue(UNITTEST_PROVIDER), typeMoq.It.isAny())) + .returns(() => Promise.resolve(runOutput)) + .verifiable(typeMoq.Times.once()); + testParser.setup(t => t.parse(typeMoq.It.isValue(runOutput), typeMoq.It.isAny())) + .returns(() => tests) + .verifiable(typeMoq.Times.never()); + + const options = typeMoq.Mock.ofType(); + const token = typeMoq.Mock.ofType(); + options.setup(o => o.args).returns(() => args); + options.setup(o => o.token).returns(() => token.object); + token.setup(t => t.isCancellationRequested) + .returns(() => true); + + const promise = discoveryService.discoverTests(options.object); + + await expect(promise).to.eventually.be.rejectedWith('cancelled'); + runner.verifyAll(); + testParser.verifyAll(); + }); +}); diff --git a/src/test/unittests/unittest.run.test.ts b/src/test/unittests/unittest/unittest.run.test.ts similarity index 90% rename from src/test/unittests/unittest.run.test.ts rename to src/test/unittests/unittest/unittest.run.test.ts index 757ecf836199..45ecc8a9e939 100644 --- a/src/test/unittests/unittest.run.test.ts +++ b/src/test/unittests/unittest/unittest.run.test.ts @@ -6,16 +6,23 @@ import * as fs from 'fs-extra'; import { EOL } from 'os'; import * as path from 'path'; import { ConfigurationTarget } from 'vscode'; -import { IProcessServiceFactory } from '../../client/common/process/types'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory, IUnitTestSocketServer, TestsToRun } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { MockProcessService } from '../mocks/proc'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { MockUnitTestSocketServer } from './mocks'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { IProcessServiceFactory } from '../../../client/common/process/types'; +import { ArgumentsHelper } from '../../../client/unittests/common/argumentsHelper'; +import { CommandSource, UNITTEST_PROVIDER } from '../../../client/unittests/common/constants'; +import { TestRunner } from '../../../client/unittests/common/runner'; +import { ITestManagerFactory, ITestRunner, IUnitTestSocketServer, TestsToRun } from '../../../client/unittests/common/types'; +import { IArgumentsHelper, IArgumentsService, ITestManagerRunner, IUnitTestHelper } from '../../../client/unittests/types'; +import { UnitTestHelper } from '../../../client/unittests/unittest/helper'; +import { TestManagerRunner } from '../../../client/unittests/unittest/runner'; +import { ArgumentsService } from '../../../client/unittests/unittest/services/argsService'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { MockProcessService } from '../../mocks/proc'; +import { MockUnitTestSocketServer } from '../mocks'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const testFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles'); +const testFilesPath = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles'); const UNITTEST_TEST_FILES_PATH = path.join(testFilesPath, 'standard'); const unitTestSpecificTestFilesPath = path.join(testFilesPath, 'specificTest'); const defaultUnitTestArgs = [ @@ -68,6 +75,11 @@ suite('Unit Tests - unittest - run with mocked process output', () => { ioc.registerTestsHelper(); ioc.registerTestStorage(); ioc.registerTestVisitors(); + ioc.serviceManager.add(IArgumentsService, ArgumentsService, UNITTEST_PROVIDER); + ioc.serviceManager.add(IArgumentsHelper, ArgumentsHelper); + ioc.serviceManager.add(ITestManagerRunner, TestManagerRunner, UNITTEST_PROVIDER); + ioc.serviceManager.add(ITestRunner, TestRunner); + ioc.serviceManager.add(IUnitTestHelper, UnitTestHelper); } async function ignoreTestLauncher() { diff --git a/src/test/unittests/unittest.test.ts b/src/test/unittests/unittest/unittest.test.ts similarity index 83% rename from src/test/unittests/unittest.test.ts rename to src/test/unittests/unittest/unittest.test.ts index 8465b7bdbb92..ebad6a6b5d18 100644 --- a/src/test/unittests/unittest.test.ts +++ b/src/test/unittests/unittest/unittest.test.ts @@ -2,13 +2,14 @@ import * as assert from 'assert'; import * as fs from 'fs-extra'; import * as path from 'path'; import { ConfigurationTarget } from 'vscode'; -import { CommandSource } from '../../client/unittests/common/constants'; -import { ITestManagerFactory } from '../../client/unittests/common/types'; -import { rootWorkspaceUri, updateSetting } from '../common'; -import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../initialize'; -import { UnitTestIocContainer } from './serviceRegistry'; +import { EXTENSION_ROOT_DIR } from '../../../client/common/constants'; +import { CommandSource } from '../../../client/unittests/common/constants'; +import { ITestManagerFactory } from '../../../client/unittests/common/types'; +import { rootWorkspaceUri, updateSetting } from '../../common'; +import { UnitTestIocContainer } from '../serviceRegistry'; +import { initialize, initializeTest, IS_MULTI_ROOT_TEST } from './../../initialize'; -const testFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'testFiles'); +const testFilesPath = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'testFiles'); const UNITTEST_TEST_FILES_PATH = path.join(testFilesPath, 'standard'); const UNITTEST_SINGLE_TEST_FILE_PATH = path.join(testFilesPath, 'single'); const defaultUnitTestArgs = [