diff --git a/packages/cspell-gitignore/src/GitIgnore.ts b/packages/cspell-gitignore/src/GitIgnore.ts index 769436f27371..9ed800cb3241 100644 --- a/packages/cspell-gitignore/src/GitIgnore.ts +++ b/packages/cspell-gitignore/src/GitIgnore.ts @@ -51,18 +51,23 @@ export class GitIgnore { this.resolvedGitIgnoreHierarchies.set(directory, found); return find; } + filterOutIgnored(files: string[]): Promise; + filterOutIgnored(files: Iterable): Promise; + filterOutIgnored(files: AsyncIterable): AsyncIterable; + filterOutIgnored(files: Iterable | AsyncIterable): Promise | AsyncIterable; + filterOutIgnored(files: Iterable & AsyncIterable): AsyncIterable; + filterOutIgnored(files: Iterable | AsyncIterable): Promise | AsyncIterable { + const iter = this.filterOutIgnoredAsync(files); + return isAsyncIterable(files) ? iter : asyncIterableToArray(iter); + } - async filterOutIgnored(files: string[]): Promise { - const result: string[] = []; - - for (const file of files) { + async *filterOutIgnoredAsync(files: Iterable | AsyncIterable): AsyncIterable { + for await (const file of files) { const isIgnored = this.isIgnoredQuick(file) ?? (await this.isIgnored(file)); if (!isIgnored) { - result.push(file); + yield file; } } - - return result; } get roots(): string[] { @@ -125,3 +130,17 @@ function sortRoots(roots: string[]): string[] { roots.sort((a, b) => a.length - b.length); return roots; } + +function isAsyncIterable(i: Iterable | AsyncIterable): i is AsyncIterable { + const as = >i; + return typeof as[Symbol.asyncIterator] === 'function'; +} + +async function asyncIterableToArray(iter: Iterable | AsyncIterable): Promise[]> { + const r: Awaited[] = []; + + for await (const t of iter) { + r.push(t); + } + return r; +} diff --git a/packages/cspell/src/app.test.ts b/packages/cspell/src/app.test.ts index 2ad61647568e..723761ac3469 100644 --- a/packages/cspell/src/app.test.ts +++ b/packages/cspell/src/app.test.ts @@ -7,7 +7,7 @@ import * as Util from 'util'; import { URI } from 'vscode-uri'; import * as app from './app'; import * as Link from './link'; -import { mergeAsyncIterables } from './util/util'; +import { mergeAsyncIterables } from './util/async'; jest.mock('readline'); const mockCreateInterface = jest.mocked(readline.createInterface); diff --git a/packages/cspell/src/application.test.ts b/packages/cspell/src/application.test.ts index 451e462a3a10..aeb2e1226696 100644 --- a/packages/cspell/src/application.test.ts +++ b/packages/cspell/src/application.test.ts @@ -6,7 +6,7 @@ import { TraceOptions } from '.'; import * as App from './application'; import { LinterOptions } from './options'; import { InMemoryReporter } from './util/InMemoryReporter'; -import { asyncIterableToArray } from './util/util'; +import { asyncIterableToArray } from './util/async'; const getStdinResult = { value: '', diff --git a/packages/cspell/src/application.ts b/packages/cspell/src/application.ts index 37f43dfedbc0..d6c095f56845 100644 --- a/packages/cspell/src/application.ts +++ b/packages/cspell/src/application.ts @@ -2,11 +2,12 @@ import type { CSpellReporter, RunResult } from '@cspell/cspell-types'; import * as cspell from 'cspell-lib'; import { CheckTextInfo, TraceResult, traceWordsAsync, suggestionsForWords, SuggestionError } from 'cspell-lib'; import * as path from 'path'; -import { calcFinalConfigInfo, readConfig, readFile } from './fileHelper'; +import { calcFinalConfigInfo, readConfig, readFile } from './util/fileHelper'; import { LintRequest, runLint } from './lint'; import { BaseOptions, fixLegacy, LegacyOptions, LinterOptions, SuggestionOptions, TraceOptions } from './options'; import { readStdin } from './util/stdin'; import * as util from './util/util'; +import * as async from './util/async'; export { IncludeExcludeFlag } from 'cspell-lib'; export type { TraceResult } from 'cspell-lib'; @@ -20,7 +21,7 @@ export function lint(fileGlobs: string[], options: LinterOptions, emitters: CSpe export async function* trace(words: string[], options: TraceOptions): AsyncIterableIterator { options = fixLegacy(options); - const iWords = options.stdin ? util.mergeAsyncIterables(words, readStdin()) : words; + const iWords = options.stdin ? async.mergeAsyncIterables(words, readStdin()) : words; const { languageId, locale, allowCompoundWords, ignoreCase } = options; const configFile = await readConfig(options.config, undefined); const config = cspell.mergeSettings(cspell.getDefaultSettings(), cspell.getGlobalSettings(), configFile.config); @@ -47,7 +48,7 @@ export async function* suggestions( ): AsyncIterable { options = fixLegacy(options); const configFile = await readConfig(options.config, undefined); - const iWords = options.useStdin ? util.mergeAsyncIterables(words, readStdin()) : words; + const iWords = options.useStdin ? async.mergeAsyncIterables(words, readStdin()) : words; try { const results = suggestionsForWords(iWords, options, configFile.config); yield* results; diff --git a/packages/cspell/src/lint/lint.ts b/packages/cspell/src/lint/lint.ts index d88ce5c79323..866c89c04c77 100644 --- a/packages/cspell/src/lint/lint.ts +++ b/packages/cspell/src/lint/lint.ts @@ -16,7 +16,7 @@ import { readConfig, readFileInfo, readFileListFiles, -} from '../fileHelper'; +} from '../util/fileHelper'; import type { CSpellLintResultCache } from '../util/cache'; import { calcCacheSettings, createCache, CreateCacheSettings } from '../util/cache'; import { toApplicationError, toError } from '../util/errors'; @@ -25,11 +25,11 @@ import { buildGlobMatcher, extractGlobsFromMatcher, extractPatterns, normalizeGl import { loadReporters, mergeReporters } from '../util/reporters'; import { getTimeMeasurer } from '../util/timer'; import * as util from '../util/util'; +import { pipeAsync, isAsyncIterable, filter, pipeSync } from '../util/async'; import { LintRequest } from './LintRequest'; export async function runLint(cfg: LintRequest): Promise { let { reporter } = cfg; - const { fileLists } = cfg; cspell.setLogger(getLoggerFromReporter(reporter)); const configErrors = new Set(); @@ -121,11 +121,11 @@ export async function runLint(cfg: LintRequest): Promise { } async function processFiles( - files: string[], + files: string[] | AsyncIterable, configInfo: ConfigInfo, cacheSettings: CreateCacheSettings ): Promise { - const fileCount = files.length; + const fileCount = files instanceof Array ? files.length : undefined; const status: RunResult = runResult(); const cache = createCache(cacheSettings); const failFast = cfg.options.failFast ?? configInfo.config.failFast ?? false; @@ -134,7 +134,7 @@ export async function runLint(cfg: LintRequest): Promise { reporter.progress({ type: 'ProgressFileComplete', fileNum, - fileCount, + fileCount: fileCount ?? fileNum, filename, elapsedTimeMs: result?.elapsedTimeMs, processed: result?.processed, @@ -143,10 +143,11 @@ export async function runLint(cfg: LintRequest): Promise { }); async function* loadAndProcessFiles() { - for (let i = 0; i < files.length; i++) { - const filename = files[i]; + let i = 0; + for await (const filename of files) { + ++i; const result = await processFile(filename, configInfo, cache); - yield { filename, fileNum: i + 1, result }; + yield { filename, fileNum: i, result }; } } @@ -222,19 +223,9 @@ export async function runLint(cfg: LintRequest): Promise { reporter = mergeReporters(cfg.reporter, ...loadReporters(configInfo.config)); cspell.setLogger(getLoggerFromReporter(reporter)); - const useGitignore = cfg.options.gitignore ?? configInfo.config.useGitignore ?? false; - const gitignoreRoots = cfg.options.gitignoreRoot ?? configInfo.config.gitignoreRoot; - const gitIgnore = useGitignore ? await generateGitIgnore(gitignoreRoots) : undefined; - - const cliGlobs: Glob[] = cfg.fileGlobs; - const allGlobs: Glob[] = cliGlobs.length ? cliGlobs : configInfo.config.files || []; - const combinedGlobs = normalizeGlobsToRoot(allGlobs, cfg.root, false); - const cliExcludeGlobs = extractPatterns(cfg.excludes).map((p) => p.glob); - const normalizedExcludes = normalizeGlobsToRoot(cliExcludeGlobs, cfg.root, true); - const includeGlobs = combinedGlobs.filter((g) => !g.startsWith('!')); - const excludeGlobs = combinedGlobs.filter((g) => g.startsWith('!')).concat(normalizedExcludes); - const fileGlobs: string[] = includeGlobs; - const hasFileLists = !!fileLists.length; + const globInfo = await determineGlobs(configInfo, cfg); + const { fileGlobs, excludeGlobs } = globInfo; + const hasFileLists = !!cfg.fileLists.length; if (!fileGlobs.length && !hasFileLists) { // Nothing to do. return runResult(); @@ -248,28 +239,10 @@ export async function runLint(cfg: LintRequest): Promise { // Get Exclusions from the config files. const { root } = cfg; - const globsToExclude = (configInfo.config.ignorePaths || []).concat(excludeGlobs); - const globMatcher = buildGlobMatcher(globsToExclude, root, true); - const ignoreGlobs = extractGlobsFromMatcher(globMatcher); - // cspell:word nodir - const globOptions: GlobOptions = { - root, - cwd: root, - ignore: ignoreGlobs.concat(normalizedExcludes), - nodir: true, - }; - const enableGlobDot = cfg.enableGlobDot ?? configInfo.config.enableGlobDot; - if (enableGlobDot !== undefined) { - globOptions.dot = enableGlobDot; - } try { const cacheSettings = await calcCacheSettings(configInfo.config, cfg.options, root); - const foundFiles = await (hasFileLists - ? useFileLists(fileLists, allGlobs, root, enableGlobDot) - : findFiles(fileGlobs, globOptions)); - const filtered = gitIgnore ? await gitIgnore.filterOutIgnored(foundFiles) : foundFiles; - const files = filterFiles(filtered, globMatcher); + const files = await determineFilesToCheck(configInfo, cfg, reporter, globInfo); return await processFiles(files, configInfo, cacheSettings); } catch (e) { @@ -297,6 +270,76 @@ Options: MessageTypes.Info ); } +} + +interface AppGlobInfo { + /** globs from cli or config.files */ + allGlobs: Glob[]; + /** GitIgnore config to use. */ + gitIgnore: GitIgnore | undefined; + /** file globs used to search for matching files. */ + fileGlobs: string[]; + /** globs to exclude files found. */ + excludeGlobs: string[]; + /** normalized cli exclude globs */ + normalizedExcludes: string[]; +} + +async function determineGlobs(configInfo: ConfigInfo, cfg: LintRequest): Promise { + const useGitignore = cfg.options.gitignore ?? configInfo.config.useGitignore ?? false; + const gitignoreRoots = cfg.options.gitignoreRoot ?? configInfo.config.gitignoreRoot; + const gitIgnore = useGitignore ? await generateGitIgnore(gitignoreRoots) : undefined; + + const cliGlobs: Glob[] = cfg.fileGlobs; + const allGlobs: Glob[] = cliGlobs.length ? cliGlobs : configInfo.config.files || []; + const combinedGlobs = normalizeGlobsToRoot(allGlobs, cfg.root, false); + const cliExcludeGlobs = extractPatterns(cfg.excludes).map((p) => p.glob); + const normalizedExcludes = normalizeGlobsToRoot(cliExcludeGlobs, cfg.root, true); + const includeGlobs = combinedGlobs.filter((g) => !g.startsWith('!')); + const excludeGlobs = combinedGlobs.filter((g) => g.startsWith('!')).concat(normalizedExcludes); + const fileGlobs: string[] = includeGlobs; + + return { allGlobs, gitIgnore, fileGlobs, excludeGlobs, normalizedExcludes }; +} + +async function determineFilesToCheck( + configInfo: ConfigInfo, + cfg: LintRequest, + reporter: CSpellReporter, + globInfo: AppGlobInfo +): Promise> { + async function _determineFilesToCheck(): Promise> { + const { fileLists } = cfg; + const hasFileLists = !!fileLists.length; + const { allGlobs, gitIgnore, fileGlobs, excludeGlobs, normalizedExcludes } = globInfo; + + // Get Exclusions from the config files. + const { root } = cfg; + const globsToExclude = (configInfo.config.ignorePaths || []).concat(excludeGlobs); + const globMatcher = buildGlobMatcher(globsToExclude, root, true); + const ignoreGlobs = extractGlobsFromMatcher(globMatcher); + // cspell:word nodir + const globOptions: GlobOptions = { + root, + cwd: root, + ignore: ignoreGlobs.concat(normalizedExcludes), + nodir: true, + }; + const enableGlobDot = cfg.enableGlobDot ?? configInfo.config.enableGlobDot; + if (enableGlobDot !== undefined) { + globOptions.dot = enableGlobDot; + } + + const filterFiles = filter(filterFilesFn(globMatcher)); + const foundFiles = await (hasFileLists + ? useFileLists(fileLists, allGlobs, root, enableGlobDot) + : findFiles(fileGlobs, globOptions)); + const filtered = gitIgnore ? await gitIgnore.filterOutIgnored(foundFiles) : foundFiles; + const files = isAsyncIterable(filtered) + ? pipeAsync(filtered, filterFiles) + : [...pipeSync(filtered, filterFiles)]; + return files; + } function isExcluded(filename: string, globMatcherExclude: GlobMatcher) { if (cspell.isBinaryFile(URI.file(filename))) { @@ -317,24 +360,17 @@ Options: return r.matched; } - function extractGlobSource(g: GlobPatternWithRoot | GlobPatternNormalized) { - const { glob, rawGlob, source } = g; - return { - glob: rawGlob || glob, - source, - }; - } - - function filterFiles(files: string[], globMatcherExclude: GlobMatcher): string[] { + function filterFilesFn(globMatcherExclude: GlobMatcher): (file: string) => boolean { const patterns = globMatcherExclude.patterns; const excludeInfo = patterns .map(extractGlobSource) .map(({ glob, source }) => `Glob: ${glob} from ${source}`) .filter(util.uniqueFn()); reporter.info(`Exclusion Globs: \n ${excludeInfo.join('\n ')}\n`, MessageTypes.Info); - const result = files.filter(util.uniqueFn()).filter((filename) => !isExcluded(filename, globMatcherExclude)); - return result; + return (filename: string): boolean => !isExcluded(filename, globMatcherExclude); } + + return _determineFilesToCheck(); } function extractContext( @@ -374,6 +410,14 @@ function extractContext( return context; } +function extractGlobSource(g: GlobPatternWithRoot | GlobPatternNormalized) { + const { glob, rawGlob, source } = g; + return { + glob: rawGlob || glob, + source, + }; +} + function runResult(init: Partial = {}): RunResult { const { files = 0, filesWithIssues = new Set(), issues = 0, errors = 0, cachedFiles = 0 } = init; return { files, filesWithIssues, issues, errors, cachedFiles }; @@ -421,7 +465,7 @@ async function useFileLists( includeGlobPatterns: Glob[], root: string, dot: boolean | undefined -): Promise { +): Promise> { includeGlobPatterns = includeGlobPatterns.length ? includeGlobPatterns : ['**']; const options: GlobMatchOptions = { root, mode: 'include' }; if (dot !== undefined) { @@ -430,6 +474,6 @@ async function useFileLists( const globMatcher = new GlobMatcher(includeGlobPatterns, options); const files = await readFileListFiles(fileListFiles); - - return files.filter((file) => globMatcher.match(file)); + const filterFiles = (file: string) => globMatcher.match(file); + return files instanceof Array ? files.filter(filterFiles) : pipeAsync(files, filter(filterFiles)); } diff --git a/packages/cspell/src/util/async.test.ts b/packages/cspell/src/util/async.test.ts new file mode 100644 index 000000000000..57005fb43a24 --- /dev/null +++ b/packages/cspell/src/util/async.test.ts @@ -0,0 +1,81 @@ +import { asyncAwait, asyncIterableToArray, filter, map, pipeAsync, pipeSync, toAsyncIterable } from './async'; + +describe('Validate async', () => { + test('mergeAsyncIterables', async () => { + const a = 'hello'.split(''); + const b = 'there'.split(''); + expect(await asyncIterableToArray(toAsyncIterable(a, b))).toEqual([...a, ...b]); + }); + + test('toAsyncIterable', async () => { + const values = ['one', 'two', 'three']; + expect(await asyncIterableToArray(toAsyncIterable(values, wrapInPromise(values), toAsync(values)))).toEqual( + values.concat(values).concat(values) + ); + }); + + test('map', async () => { + const values = ['one', 'two', 'three']; + + const mapFn = (v: string) => v.length; + const mapFn2 = (v: number) => v * 2; + + const expected = values.map(mapFn).map(mapFn2); + + const mapToLen = map(mapFn); + + const s = pipeSync(values, mapToLen, map(mapFn2)); + const a = pipeAsync(values, mapToLen, map(mapFn2)); + + const sync = [...s]; + const async = await asyncIterableToArray(a); + + expect(sync).toEqual(expected); + expect(async).toEqual(expected); + }); + + test('asyncAwait', async () => { + const values = ['one', 'two', 'three']; + + const mapFn = (v: string) => v.length; + const mapFn2 = (v: number) => v * 2; + + const expected = values.map(mapFn).map(mapFn2); + const mapToLen = map(mapFn); + const a = pipeAsync(values, map(toPromise), asyncAwait(), mapToLen, map(mapFn2)); + const async = await asyncIterableToArray(a); + + expect(async).toEqual(expected); + }); + + test('filter', async () => { + const values = ['one', 'two', 'three']; + + const filterFn = (v: string) => v.length === 3; + + const expected = values.filter(filterFn); + + const filterToLen = filter(filterFn); + + const s = pipeSync(values, filterToLen); + const a = pipeAsync(toAsyncIterable(values), filterToLen); + + const sync = [...s]; + const async = await asyncIterableToArray(a); + + expect(sync).toEqual(expected); + expect(async).toEqual(expected); + }); +}); + +function toPromise(t: T): Promise { + return Promise.resolve(t); +} + +async function wrapInPromise(t: T): Promise { + return t; +} + +async function* toAsync(t: T[]) { + yield* t; +} diff --git a/packages/cspell/src/util/async.ts b/packages/cspell/src/util/async.ts new file mode 100644 index 000000000000..513faa613259 --- /dev/null +++ b/packages/cspell/src/util/async.ts @@ -0,0 +1,171 @@ +function asyncMap(mapFn: (v: T) => U): (iter: AsyncIterable) => AsyncIterable { + async function* fn(iter: Iterable | AsyncIterable) { + for await (const v of iter) { + yield mapFn(v); + } + } + + return fn; +} + +function syncMap(mapFn: (v: T) => U): (iter: Iterable) => Iterable { + function* fn(iter: Iterable) { + for (const v of iter) { + yield mapFn(v); + } + } + return fn; +} + +export const map = (fn: (v: T) => U) => toPipeFn(syncMap(fn), asyncMap(fn)); + +function asyncFilter(filterFn: (v: T) => boolean): (iter: AsyncIterable) => AsyncIterable { + async function* fn(iter: Iterable | AsyncIterable) { + for await (const v of iter) { + if (filterFn(v)) yield v; + } + } + + return fn; +} + +function syncFilter(filterFn: (v: T) => boolean): (iter: Iterable) => Iterable { + function* fn(iter: Iterable) { + for (const v of iter) { + if (filterFn(v)) yield v; + } + } + + return fn; +} + +async function* _asyncAwait(iter: AsyncIterable): AsyncIterable> { + for await (const v of iter) { + yield v; + } +} + +export function asyncAwait(): (iter: AsyncIterable) => AsyncIterable> { + return _asyncAwait; +} + +export const filter = (fn: (i: T) => boolean) => toPipeFn(syncFilter(fn), asyncFilter(fn)); + +interface PipeFnSync { + (iter: Iterable): Iterable; + /** This is just to help TypeScript figure out the type. */ + __PipeFnSync__?: [T, U]; +} + +interface PipeFnAsync { + (iter: AsyncIterable): AsyncIterable; + /** This is just to help TypeScript figure out the type. */ + __PipeFnAsync__?: [T, U]; +} + +type PipeFn = PipeFnSync & PipeFnAsync; + +function toPipeFn(syncFn: PipeFnSync, asyncFn: PipeFnAsync): PipeFn { + function _(i: Iterable): Iterable; + function _(i: AsyncIterable): AsyncIterable; + function _(i: Iterable | AsyncIterable): Iterable | AsyncIterable { + return isAsyncIterable(i) ? asyncFn(i) : syncFn(i); + } + return _; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +// type Rest = [...any]; + +export function pipeAsync(i: AnyIterable): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0]>): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0, T1]>): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0, T1, T2]>): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0, T1, T2, T3]>): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0, T1, T2, T3, T4]>): AsyncIterable; // prettier-ignore +export function pipeAsync(i: AnyIterable, ...f: PipeAsyncTx<[T, T0, T1, T2, T3, T4, T5]>): AsyncIterable; // prettier-ignore + +export function pipeAsync(i: AnyIterable, ...fns: PaFn[]): AsyncIterable { + let iter = toAsyncIterable(i); + for (const fn of fns) { + iter = fn(iter); + } + return iter; +} + +type PsFn = PipeFnSync | ((i: Iterable) => Iterable); + +export function pipeSync(i: Iterable): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0, T1]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0, T1, T2]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0, T1, T2, T3]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0, T1, T2, T3, T4]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...f: PipeSyncTx<[T, T0, T1, T2, T3, T4, T5]>): Iterable; // prettier-ignore +export function pipeSync(i: Iterable, ...fns: PsFn[]): Iterable { + let iter: Iterable = i; + for (const fn of fns) { + iter = fn(iter); + } + return iter; +} + +type AnyIterable = Iterable | AsyncIterable | Promise> | Iterable>; + +export function mergeAsyncIterables(iter: Iterable): AsyncIterable; +export function mergeAsyncIterables(iter: AsyncIterable): AsyncIterable; +export function mergeAsyncIterables(iter: Promise>): AsyncIterable; +export function mergeAsyncIterables(iter: AnyIterable): AsyncIterable; +export function mergeAsyncIterables(iter: AnyIterable, ...rest: AnyIterable[]): AsyncIterable; // prettier-ignore + +/** + * Merge multiple iterables into an AsyncIterable + * @param iter - initial iterable. + * @param rest - iterables to merge. + */ +export async function* mergeAsyncIterables( + iter: Iterable | AsyncIterable | Promise>, + ...rest: (Iterable | AsyncIterable | Promise>)[] +): AsyncIterableIterator { + for await (const i of [iter, ...rest]) { + yield* i; + } +} + +/** + * Convert one or more iterables to an AsyncIterable + */ +export const toAsyncIterable = mergeAsyncIterables; + +export async function asyncIterableToArray(iter: Iterable | AsyncIterable): Promise[]> { + const r: Awaited[] = []; + + for await (const t of iter) { + r.push(t); + } + return r; +} + +export function isAsyncIterable(i: AnyIterable): i is AsyncIterable { + return typeof (>i)[Symbol.asyncIterator] === 'function'; +} + +type PaFn = PipeFnAsync | ((i: AsyncIterable) => AsyncIterable); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type PipeAsyncTx = T extends [infer Left, infer Right, ...infer Rest] + ? // eslint-disable-next-line @typescript-eslint/no-explicit-any + Rest extends [any, ...any] + ? [PaFn, ...PipeAsyncTx<[Right, ...Rest]>] + : [PaFn] + : never; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type PipeSyncTx = T extends [infer Left, infer Right, ...infer Rest] + ? // eslint-disable-next-line @typescript-eslint/no-explicit-any + Rest extends [any, ...any] + ? [PsFn, ...PipeSyncTx<[Right, ...Rest]>] + : [PsFn] + : never; + +// type Last = T extends [infer U, ...infer R] ? (R extends [any, ...any] ? Last : U) : never; diff --git a/packages/cspell/src/util/cache/CSpellLintResultCache.ts b/packages/cspell/src/util/cache/CSpellLintResultCache.ts index a805f359eea4..34b2e89c86bb 100644 --- a/packages/cspell/src/util/cache/CSpellLintResultCache.ts +++ b/packages/cspell/src/util/cache/CSpellLintResultCache.ts @@ -1,4 +1,4 @@ -import { FileResult } from '../../fileHelper'; +import { FileResult } from '../../util/fileHelper'; export interface CSpellLintResultCache { /** * Retrieve cached lint results for a given file name, if present in the cache. diff --git a/packages/cspell/src/util/cache/DiskCache.test.ts b/packages/cspell/src/util/cache/DiskCache.test.ts index a98d3a966cd3..20b7871ad4f2 100644 --- a/packages/cspell/src/util/cache/DiskCache.test.ts +++ b/packages/cspell/src/util/cache/DiskCache.test.ts @@ -1,6 +1,6 @@ import * as FileEntryCacheModule from 'file-entry-cache'; import * as path from 'path'; -import * as fileHelper from '../../fileHelper'; +import * as fileHelper from '../../util/fileHelper'; import { CachedFileResult, DiskCache, CSpellCacheMeta } from './DiskCache'; jest.mock('./getConfigHash', () => ({ @@ -21,7 +21,7 @@ jest.mock('file-entry-cache', () => ({ })); const mockReadFileInfo = jest.spyOn(fileHelper, 'readFileInfo'); -jest.mock('../../fileHelper', () => ({ readFileInfo: jest.fn() })); +jest.mock('../../util/fileHelper', () => ({ readFileInfo: jest.fn() })); const RESULT_NO_ISSUES: CachedFileResult = { processed: true, diff --git a/packages/cspell/src/util/cache/DiskCache.ts b/packages/cspell/src/util/cache/DiskCache.ts index 35cee922e0da..e19965712c5b 100644 --- a/packages/cspell/src/util/cache/DiskCache.ts +++ b/packages/cspell/src/util/cache/DiskCache.ts @@ -1,8 +1,8 @@ import type { FileDescriptor, FileEntryCache } from 'file-entry-cache'; import * as fileEntryCache from 'file-entry-cache'; import { resolve as resolvePath } from 'path'; -import type { FileResult } from '../../fileHelper'; -import { readFileInfo } from '../../fileHelper'; +import type { FileResult } from '../../util/fileHelper'; +import { readFileInfo } from '../../util/fileHelper'; import type { CSpellLintResultCache } from './CSpellLintResultCache'; export type CachedFileResult = Omit; diff --git a/packages/cspell/src/util/cache/getConfigHash.ts b/packages/cspell/src/util/cache/getConfigHash.ts index 086d9fea7a6a..22eec87564d0 100644 --- a/packages/cspell/src/util/cache/getConfigHash.ts +++ b/packages/cspell/src/util/cache/getConfigHash.ts @@ -1,6 +1,6 @@ import stringify from 'fast-json-stable-stringify'; import path from 'path'; -import { ConfigInfo } from '../../fileHelper'; +import { ConfigInfo } from '../../util/fileHelper'; import { hash } from './hash'; // eslint-disable-next-line @typescript-eslint/no-var-requires diff --git a/packages/cspell/src/fileHelper.test.ts b/packages/cspell/src/util/fileHelper.test.ts similarity index 94% rename from packages/cspell/src/fileHelper.test.ts rename to packages/cspell/src/util/fileHelper.test.ts index fadc79a35abf..b3169668cb92 100644 --- a/packages/cspell/src/fileHelper.test.ts +++ b/packages/cspell/src/util/fileHelper.test.ts @@ -1,8 +1,9 @@ import { readFileInfo, readFileListFile, readFileListFiles } from './fileHelper'; import * as path from 'path'; -import { IOError } from './util/errors'; +import { IOError } from './errors'; -const fixtures = path.join(__dirname, '../fixtures/fileHelper'); +const packageRoot = path.join(__dirname, '../..'); +const fixtures = path.join(packageRoot, 'fixtures/fileHelper'); const fileListFile = path.join(fixtures, 'file-list.txt'); const fileListFile2 = path.join(fixtures, 'nested/file-list-2.txt'); diff --git a/packages/cspell/src/fileHelper.ts b/packages/cspell/src/util/fileHelper.ts similarity index 89% rename from packages/cspell/src/fileHelper.ts rename to packages/cspell/src/util/fileHelper.ts index 8e6b9fb699d5..d919c571811e 100644 --- a/packages/cspell/src/fileHelper.ts +++ b/packages/cspell/src/util/fileHelper.ts @@ -1,10 +1,12 @@ import * as cspell from 'cspell-lib'; import * as fsp from 'fs-extra'; import getStdin from 'get-stdin'; -import { GlobOptions, globP } from './util/glob'; +import { GlobOptions, globP } from './glob'; import * as path from 'path'; import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib'; -import { IOError, toApplicationError, toError } from './util/errors'; +import { IOError, toApplicationError, toError } from './errors'; +import { mergeAsyncIterables } from './async'; +import { readStdin } from './stdin'; const UTF8: BufferEncoding = 'utf8'; const STDIN = 'stdin'; @@ -134,8 +136,16 @@ export function calcFinalConfigInfo( * file will be resolved relative to the containing file. * @returns - a list of files to be processed. */ -export async function readFileListFiles(listFiles: string[]): Promise { - return flatten(await Promise.all(listFiles.map(readFileListFile))); +export async function readFileListFiles(listFiles: string[]): Promise> { + let useStdin = false; + const files = listFiles.filter((file) => { + const isStdin = file === 'stdin'; + useStdin = useStdin || isStdin; + return !isStdin; + }); + const found = flatten(await Promise.all(files.map(readFileListFile))); + // Move `stdin` to the end. + return useStdin ? mergeAsyncIterables(found, readStdin()) : found; } /** diff --git a/packages/cspell/src/util/stdin.test.ts b/packages/cspell/src/util/stdin.test.ts index 7832148a666d..ef791a1cca7a 100644 --- a/packages/cspell/src/util/stdin.test.ts +++ b/packages/cspell/src/util/stdin.test.ts @@ -1,6 +1,6 @@ import * as readline from 'readline'; import { readStdin } from './stdin'; -import { asyncIterableToArray, mergeAsyncIterables } from './util'; +import { asyncIterableToArray, mergeAsyncIterables } from './async'; jest.mock('readline'); const mockCreateInterface = jest.mocked(readline.createInterface); diff --git a/packages/cspell/src/util/util.test.ts b/packages/cspell/src/util/util.test.ts index d49fce706a2c..a110928c5c6a 100644 --- a/packages/cspell/src/util/util.test.ts +++ b/packages/cspell/src/util/util.test.ts @@ -53,10 +53,4 @@ describe('Validate util', () => { `('pad', ({ text, n, expected }) => { expect(util.pad(text, n)).toBe(expected); }); - - test('async', async () => { - const a = 'hello'.split(''); - const b = 'there'.split(''); - expect(await util.asyncIterableToArray(util.mergeAsyncIterables(a, b))).toEqual([...a, ...b]); - }); }); diff --git a/packages/cspell/src/util/util.ts b/packages/cspell/src/util/util.ts index 8f49b578aa40..de94eaeac476 100644 --- a/packages/cspell/src/util/util.ts +++ b/packages/cspell/src/util/util.ts @@ -32,25 +32,6 @@ export function clean(src: T): T { return r; } -export async function* mergeAsyncIterables( - iter: Iterable | AsyncIterable, - ...rest: (Iterable | AsyncIterable)[] -): AsyncIterableIterator { - yield* iter; - for await (const i of rest) { - yield* i; - } -} - -export async function asyncIterableToArray(iter: Iterable | AsyncIterable): Promise[]> { - const r: Awaited[] = []; - - for await (const t of iter) { - r.push(t); - } - return r; -} - export function pad(s: string, w: number): string { if (s.length >= w) return s; return (s + ' '.repeat(w)).slice(0, w);