diff --git a/docs/generated/devkit/README.md b/docs/generated/devkit/README.md index e3049681e7339..57f6790427404 100644 --- a/docs/generated/devkit/README.md +++ b/docs/generated/devkit/README.md @@ -129,6 +129,7 @@ It only uses language primitives and immutable objects - [getProjects](../../devkit/documents/getProjects) - [getWorkspaceLayout](../../devkit/documents/getWorkspaceLayout) - [glob](../../devkit/documents/glob) +- [globAsync](../../devkit/documents/globAsync) - [hashArray](../../devkit/documents/hashArray) - [installPackagesTask](../../devkit/documents/installPackagesTask) - [isWorkspacesEnabled](../../devkit/documents/isWorkspacesEnabled) diff --git a/docs/generated/devkit/glob.md b/docs/generated/devkit/glob.md index 08a29fcafbec7..fa9c895a72296 100644 --- a/docs/generated/devkit/glob.md +++ b/docs/generated/devkit/glob.md @@ -18,3 +18,7 @@ Paths should be unix-style with forward slashes. `string`[] Normalized paths in the workspace that match the provided glob patterns. + +**`Deprecated`** + +Use [globAsync](../../devkit/documents/globAsync) instead. diff --git a/docs/generated/devkit/globAsync.md b/docs/generated/devkit/globAsync.md new file mode 100644 index 0000000000000..6f1f766258ab0 --- /dev/null +++ b/docs/generated/devkit/globAsync.md @@ -0,0 +1,20 @@ +# Function: globAsync + +▸ **globAsync**(`tree`, `patterns`): `Promise`\<`string`[]\> + +Performs a tree-aware glob search on the files in a workspace. Able to find newly +created files and hides deleted files before the updates are committed to disk. +Paths should be unix-style with forward slashes. + +#### Parameters + +| Name | Type | Description | +| :--------- | :------------------------------------ | :---------------------- | +| `tree` | [`Tree`](../../devkit/documents/Tree) | The file system tree | +| `patterns` | `string`[] | A list of glob patterns | + +#### Returns + +`Promise`\<`string`[]\> + +Normalized paths in the workspace that match the provided glob patterns. diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md index e3049681e7339..57f6790427404 100644 --- a/docs/generated/packages/devkit/documents/nx_devkit.md +++ b/docs/generated/packages/devkit/documents/nx_devkit.md @@ -129,6 +129,7 @@ It only uses language primitives and immutable objects - [getProjects](../../devkit/documents/getProjects) - [getWorkspaceLayout](../../devkit/documents/getWorkspaceLayout) - [glob](../../devkit/documents/glob) +- [globAsync](../../devkit/documents/globAsync) - [hashArray](../../devkit/documents/hashArray) - [installPackagesTask](../../devkit/documents/installPackagesTask) - [isWorkspacesEnabled](../../devkit/documents/isWorkspacesEnabled) diff --git a/jest.preset.js b/jest.preset.js index 28490c0c213ff..28be337c9faf3 100644 --- a/jest.preset.js +++ b/jest.preset.js @@ -12,4 +12,5 @@ module.exports = { coverageReporters: ['html'], maxWorkers: 1, testEnvironment: 'node', + setupFiles: ['../../scripts/unit-test-setup.js'], }; diff --git a/packages/cypress/src/executors/cypress/cypress.impl.spec.ts b/packages/cypress/src/executors/cypress/cypress.impl.spec.ts index f60956aa88ca0..7d75a6a7bafa8 100644 --- a/packages/cypress/src/executors/cypress/cypress.impl.spec.ts +++ b/packages/cypress/src/executors/cypress/cypress.impl.spec.ts @@ -75,6 +75,11 @@ describe('Cypress builder', () => { configuration, }; }; + (devkit as any).logger = { + warn: jest.fn(), + log: jest.fn(), + info: jest.fn(), + }; cypressRun = jest .spyOn(Cypress, 'run') .mockReturnValue(Promise.resolve({})); diff --git a/packages/cypress/src/plugins/plugin.ts b/packages/cypress/src/plugins/plugin.ts index 630231b791dac..a7a466eaccc97 100644 --- a/packages/cypress/src/plugins/plugin.ts +++ b/packages/cypress/src/plugins/plugin.ts @@ -19,12 +19,13 @@ import { getLockFileName } from '@nx/js'; import { getNamedInputs } from '@nx/devkit/src/utils/get-named-inputs'; import { existsSync, readdirSync } from 'fs'; -import { globWithWorkspaceContext } from 'nx/src/utils/workspace-context'; + import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; import { projectGraphCacheDirectory } from 'nx/src/utils/cache-directory'; import { NX_PLUGIN_OPTIONS } from '../utils/constants'; import { loadConfigFile } from '@nx/devkit/src/utils/config-utils'; import { hashObject } from 'nx/src/devkit-internals'; +import { globWithWorkspaceContext } from 'nx/src/utils/workspace-context'; export interface CypressPluginOptions { ciTargetName?: string; @@ -98,9 +99,12 @@ async function createNodesInternal( return {}; } - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildCypressTargets( configFilePath, @@ -237,7 +241,7 @@ async function buildCypressTargets( : Array.isArray(cypressConfig.e2e.excludeSpecPattern) ? cypressConfig.e2e.excludeSpecPattern.map((p) => join(projectRoot, p)) : [join(projectRoot, cypressConfig.e2e.excludeSpecPattern)]; - const specFiles = globWithWorkspaceContext( + const specFiles = await globWithWorkspaceContext( context.workspaceRoot, specPatterns, excludeSpecPatterns diff --git a/packages/detox/src/plugins/plugin.ts b/packages/detox/src/plugins/plugin.ts index 3873b3e0b5ca9..cfe03e75a826a 100644 --- a/packages/detox/src/plugins/plugin.ts +++ b/packages/detox/src/plugins/plugin.ts @@ -42,7 +42,7 @@ export const createDependencies: CreateDependencies = () => { export const createNodes: CreateNodes = [ '**/{detox.config,.detoxrc}.{json,js}', - (configFilePath, options, context) => { + async (configFilePath, options, context) => { options = normalizeOptions(options); const projectRoot = dirname(configFilePath); @@ -52,9 +52,12 @@ export const createNodes: CreateNodes = [ return {}; } - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= buildDetoxTargets(projectRoot, options, context); diff --git a/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts b/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts index 641bc52bbd4c2..45b8779f9b286 100644 --- a/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts +++ b/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts @@ -3,14 +3,14 @@ import { CreateNodesContext, hashArray } from 'nx/src/devkit-exports'; import { hashObject, hashWithWorkspaceContext } from 'nx/src/devkit-internals'; -export function calculateHashForCreateNodes( +export async function calculateHashForCreateNodes( projectRoot: string, options: object, context: CreateNodesContext, additionalGlobs: string[] = [] -): string { +): Promise { return hashArray([ - hashWithWorkspaceContext(context.workspaceRoot, [ + await hashWithWorkspaceContext(context.workspaceRoot, [ join(projectRoot, '**/*'), ...additionalGlobs, ]), diff --git a/packages/eslint-plugin/src/migrations/update-17-2-6-rename-workspace-rules/rename-workspace-rules.spec.ts b/packages/eslint-plugin/src/migrations/update-17-2-6-rename-workspace-rules/rename-workspace-rules.spec.ts index c00e45fe28f70..57d2ba436d87b 100644 --- a/packages/eslint-plugin/src/migrations/update-17-2-6-rename-workspace-rules/rename-workspace-rules.spec.ts +++ b/packages/eslint-plugin/src/migrations/update-17-2-6-rename-workspace-rules/rename-workspace-rules.spec.ts @@ -3,6 +3,8 @@ import { createTreeWithEmptyWorkspace } from 'nx/src/devkit-testing-exports'; import { WORKSPACE_PLUGIN_DIR } from '../../constants'; import update from './rename-workspace-rules'; +import 'nx/src/internal-testing-utils/mock-project-graph'; + const rule1Name = 'test-rule'; const rule2Name = 'my-rule'; diff --git a/packages/eslint/src/generators/init/init.spec.ts b/packages/eslint/src/generators/init/init.spec.ts index bcf161b287aef..140bb2aeb7b30 100644 --- a/packages/eslint/src/generators/init/init.spec.ts +++ b/packages/eslint/src/generators/init/init.spec.ts @@ -3,6 +3,7 @@ import 'nx/src/internal-testing-utils/mock-project-graph'; import { NxJsonConfiguration, readJson, Tree, updateJson } from '@nx/devkit'; import { createTreeWithEmptyWorkspace } from '@nx/devkit/testing'; import { LinterInitOptions, lintInitGenerator } from './init'; +import { setWorkspaceRoot } from 'nx/src/utils/workspace-root'; describe('@nx/eslint:init', () => { let tree: Tree; @@ -10,6 +11,7 @@ describe('@nx/eslint:init', () => { beforeEach(() => { tree = createTreeWithEmptyWorkspace({ layout: 'apps-libs' }); + setWorkspaceRoot(tree.root); options = { addPlugin: true, }; diff --git a/packages/eslint/src/plugins/plugin.ts b/packages/eslint/src/plugins/plugin.ts index e89ca703d84f2..60943ba4474ca 100644 --- a/packages/eslint/src/plugins/plugin.ts +++ b/packages/eslint/src/plugins/plugin.ts @@ -51,7 +51,7 @@ export const createNodes: CreateNodes = [ } } - const projectFiles = globWithWorkspaceContext( + const projectFiles = await globWithWorkspaceContext( context.workspaceRoot, [ 'project.json', @@ -77,7 +77,7 @@ export const createNodes: CreateNodes = [ const nestedProjectRootPatterns = excludePatterns.slice(index + 1); // Ignore project roots where the project does not contain any lintable files - const lintableFiles = globWithWorkspaceContext( + const lintableFiles = await globWithWorkspaceContext( context.workspaceRoot, [join(childProjectRoot, `**/*.{${options.extensions.join(',')}}`)], // exclude nested eslint roots and nested project roots diff --git a/packages/expo/plugins/plugin.ts b/packages/expo/plugins/plugin.ts index d9f3b4bd812d2..7f7c7f70c7be3 100644 --- a/packages/expo/plugins/plugin.ts +++ b/packages/expo/plugins/plugin.ts @@ -72,9 +72,12 @@ export const createNodes: CreateNodes = [ return {}; } - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= buildExpoTargets(projectRoot, options, context); diff --git a/packages/gradle/src/plugin/nodes.ts b/packages/gradle/src/plugin/nodes.ts index 0d1021eae98d6..3f312e71e4d49 100644 --- a/packages/gradle/src/plugin/nodes.ts +++ b/packages/gradle/src/plugin/nodes.ts @@ -72,7 +72,7 @@ export const createNodesV2: CreateNodesV2 = [ ); const targetsCache = readTargetsCache(cachePath); - populateGradleReport(context.workspaceRoot); + await populateGradleReport(context.workspaceRoot); const gradleReport = getCurrentGradleReport(); try { @@ -93,14 +93,14 @@ export const makeCreateNodes = gradleReport: GradleReport, targetsCache: GradleTargets ): CreateNodesFunction => - ( + async ( gradleFilePath, options: GradlePluginOptions | undefined, context: CreateNodesContext ) => { const projectRoot = dirname(gradleFilePath); - const hash = calculateHashForCreateNodes( + const hash = await calculateHashForCreateNodes( projectRoot, options ?? {}, context @@ -128,14 +128,14 @@ export const makeCreateNodes = */ export const createNodes: CreateNodes = [ gradleConfigGlob, - (configFile, options, context) => { + async (configFile, options, context) => { logger.warn( '`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.' ); - populateGradleReport(context.workspaceRoot); + await populateGradleReport(context.workspaceRoot); const gradleReport = getCurrentGradleReport(); const internalCreateNodes = makeCreateNodes(gradleReport, {}); - return internalCreateNodes(configFile, options, context); + return await internalCreateNodes(configFile, options, context); }, ]; diff --git a/packages/gradle/src/utils/get-gradle-report.ts b/packages/gradle/src/utils/get-gradle-report.ts index 38c00fb9d8487..76919c29e264c 100644 --- a/packages/gradle/src/utils/get-gradle-report.ts +++ b/packages/gradle/src/utils/get-gradle-report.ts @@ -36,8 +36,10 @@ export function getCurrentGradleReport() { return gradleReportCache; } -export function populateGradleReport(workspaceRoot: string): void { - const gradleConfigHash = hashWithWorkspaceContext(workspaceRoot, [ +export async function populateGradleReport( + workspaceRoot: string +): Promise { + const gradleConfigHash = await hashWithWorkspaceContext(workspaceRoot, [ gradleConfigGlob, ]); if (gradleReportCache && gradleConfigHash === gradleCurrentConfigHash) { diff --git a/packages/jest/src/plugins/plugin.ts b/packages/jest/src/plugins/plugin.ts index 3a1b634c47670..fb23032dfe94f 100644 --- a/packages/jest/src/plugins/plugin.ts +++ b/packages/jest/src/plugins/plugin.ts @@ -127,7 +127,7 @@ async function createNodesInternal( options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context); + const hash = await calculateHashForCreateNodes(projectRoot, options, context); targetsCache[hash] ??= await buildJestTargets( configFilePath, projectRoot, diff --git a/packages/js/src/plugins/typescript/plugin.ts b/packages/js/src/plugins/typescript/plugin.ts index 3c16473a0da11..1473c672351eb 100644 --- a/packages/js/src/plugins/typescript/plugin.ts +++ b/packages/js/src/plugins/typescript/plugin.ts @@ -76,7 +76,7 @@ export const PLUGIN_NAME = '@nx/js/typescript'; export const createNodes: CreateNodes = [ '**/tsconfig*.json', - (configFilePath, options, context) => { + async (configFilePath, options, context) => { const pluginOptions = normalizePluginOptions(options); const projectRoot = dirname(configFilePath); const fullConfigPath = joinPathFragments( @@ -101,7 +101,7 @@ export const createNodes: CreateNodes = [ return {}; } - const nodeHash = calculateHashForCreateNodes( + const nodeHash = await calculateHashForCreateNodes( projectRoot, pluginOptions, context, diff --git a/packages/nest/jest.config.ts b/packages/nest/jest.config.ts index 8075e6159956b..9b464c25a7445 100644 --- a/packages/nest/jest.config.ts +++ b/packages/nest/jest.config.ts @@ -7,4 +7,5 @@ export default { globals: {}, displayName: 'nest', preset: '../../jest.preset.js', + setupFilesAfterEnv: ['/test-setup.ts'], }; diff --git a/packages/nest/test-setup.ts b/packages/nest/test-setup.ts new file mode 100644 index 0000000000000..5126f328f3f8f --- /dev/null +++ b/packages/nest/test-setup.ts @@ -0,0 +1,12 @@ +// If a test uses a util from devkit, but that util +// lives in the Nx package and creates the project graph, +// we need to mock the resolved value inside the Nx package +jest + .spyOn( + require('nx/src/project-graph/project-graph'), + 'createProjectGraphAsync' + ) + .mockResolvedValue({ + nodes: {}, + dependencies: {}, + }); diff --git a/packages/nest/tsconfig.lib.json b/packages/nest/tsconfig.lib.json index 1d2ddef3c1be3..b54ee18bfc3ee 100644 --- a/packages/nest/tsconfig.lib.json +++ b/packages/nest/tsconfig.lib.json @@ -11,7 +11,8 @@ "**/*.test.ts", "**/*_spec.ts", "**/*_test.ts", - "jest.config.ts" + "jest.config.ts", + "test-setup.ts" ], "include": ["**/*.ts"] } diff --git a/packages/nest/tsconfig.spec.json b/packages/nest/tsconfig.spec.json index 869c90c526a0f..aa3ea0fe52140 100644 --- a/packages/nest/tsconfig.spec.json +++ b/packages/nest/tsconfig.spec.json @@ -17,6 +17,7 @@ "**/*.spec.jsx", "**/*.test.jsx", "**/*.d.ts", - "jest.config.ts" + "jest.config.ts", + "test-setup.ts" ] } diff --git a/packages/next/src/plugins/plugin.ts b/packages/next/src/plugins/plugin.ts index 3bfa2b07caaf5..87de5aa084783 100644 --- a/packages/next/src/plugins/plugin.ts +++ b/packages/next/src/plugins/plugin.ts @@ -63,9 +63,12 @@ export const createNodes: CreateNodes = [ } options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildNextTargets( configFilePath, diff --git a/packages/nuxt/src/plugins/plugin.ts b/packages/nuxt/src/plugins/plugin.ts index 922f2d81e60bd..c1614e50456d4 100644 --- a/packages/nuxt/src/plugins/plugin.ts +++ b/packages/nuxt/src/plugins/plugin.ts @@ -62,9 +62,12 @@ export const createNodes: CreateNodes = [ options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildNuxtTargets( configFilePath, projectRoot, diff --git a/packages/nx/src/adapter/ngcli-adapter.spec.ts b/packages/nx/src/adapter/ngcli-adapter.spec.ts index cb055967a3ead..2e1946031d09f 100644 --- a/packages/nx/src/adapter/ngcli-adapter.spec.ts +++ b/packages/nx/src/adapter/ngcli-adapter.spec.ts @@ -5,6 +5,14 @@ import { import { createTreeWithEmptyWorkspace } from '../generators/testing-utils/create-tree-with-empty-workspace'; import { addProjectConfiguration } from '../generators/utils/project-configuration'; +jest.mock('../project-graph/project-graph', () => ({ + ...jest.requireActual('../project-graph/project-graph'), + createProjectGraphAsync: () => ({ + nodes: {}, + externalNodes: {}, + }), +})); + describe('ngcli-adapter', () => { it('arrayBufferToString should support large buffers', () => { const largeString = 'a'.repeat(1000000); diff --git a/packages/nx/src/command-line/init/implementation/react/index.ts b/packages/nx/src/command-line/init/implementation/react/index.ts index 73b7c2360747b..a62a3e146528e 100644 --- a/packages/nx/src/command-line/init/implementation/react/index.ts +++ b/packages/nx/src/command-line/init/implementation/react/index.ts @@ -290,7 +290,7 @@ async function addBundler(options: NormalizedOptions) { options.isStandalone, options.appIsJs ); - renameJsToJsx(options.reactAppName, options.isStandalone); + await renameJsToJsx(options.reactAppName, options.isStandalone); } else { output.log({ title: '🧑‍🔧 Setting up craco + Webpack' }); const { addCracoCommandsToPackageScripts } = await import( diff --git a/packages/nx/src/command-line/init/implementation/react/rename-js-to-jsx.ts b/packages/nx/src/command-line/init/implementation/react/rename-js-to-jsx.ts index 7cef84c375ea8..7b86c034481c9 100644 --- a/packages/nx/src/command-line/init/implementation/react/rename-js-to-jsx.ts +++ b/packages/nx/src/command-line/init/implementation/react/rename-js-to-jsx.ts @@ -3,8 +3,8 @@ import { globWithWorkspaceContext } from '../../../../utils/workspace-context'; import { fileExists } from '../../../../utils/fileutils'; // Vite cannot process JSX like
or
unless the file is named .jsx or .tsx -export function renameJsToJsx(appName: string, isStandalone: boolean) { - const files = globWithWorkspaceContext(process.cwd(), [ +export async function renameJsToJsx(appName: string, isStandalone: boolean) { + const files = await globWithWorkspaceContext(process.cwd(), [ isStandalone ? 'src/**/*.js' : `apps/${appName}/src/**/*.js`, ]); diff --git a/packages/nx/src/command-line/init/init-v2.ts b/packages/nx/src/command-line/init/init-v2.ts index f7cf3ec2a36d4..b3834d56489a0 100644 --- a/packages/nx/src/command-line/init/init-v2.ts +++ b/packages/nx/src/command-line/init/init-v2.ts @@ -168,7 +168,7 @@ async function detectPlugins(): Promise<{ updatePackageScripts: boolean; }> { let files = ['package.json'].concat( - globWithWorkspaceContext(process.cwd(), ['**/*/package.json']) + await globWithWorkspaceContext(process.cwd(), ['**/*/package.json']) ); const detectedPlugins = new Set(); diff --git a/packages/nx/src/config/to-project-name.spec.ts b/packages/nx/src/config/to-project-name.spec.ts index bcc2dedffadf5..d8995c22c076f 100644 --- a/packages/nx/src/config/to-project-name.spec.ts +++ b/packages/nx/src/config/to-project-name.spec.ts @@ -42,7 +42,7 @@ describe('Workspaces', () => { readNxJson(fs.tempDir).plugins, fs.tempDir ); - const res = retrieveProjectConfigurations( + const res = await retrieveProjectConfigurations( plugins, fs.tempDir, readNxJson(fs.tempDir) diff --git a/packages/nx/src/daemon/client/client.ts b/packages/nx/src/daemon/client/client.ts index 3bd933e685b0a..29f6524592002 100644 --- a/packages/nx/src/daemon/client/client.ts +++ b/packages/nx/src/daemon/client/client.ts @@ -4,7 +4,7 @@ import { readFileSync, statSync } from 'fs'; import { FileHandle, open } from 'fs/promises'; import { ensureDirSync, ensureFileSync } from 'fs-extra'; import { connect } from 'net'; -import { join } from 'path'; +import { extname, join } from 'path'; import { performance } from 'perf_hooks'; import { output } from '../../utils/output'; import { getFullOsSocketPath, killSocketOrPath } from '../socket-utils'; @@ -30,6 +30,21 @@ import { ProjectGraphError, } from '../../project-graph/error-types'; import { loadRootEnvFiles } from '../../utils/dotenv'; +import { HandleGlobMessage } from '../message-types/glob'; +import { + GET_NX_WORKSPACE_FILES, + HandleNxWorkspaceFilesMessage, +} from '../message-types/get-nx-workspace-files'; +import { + GET_CONTEXT_FILE_DATA, + HandleContextFileDataMessage, +} from '../message-types/get-context-file-data'; +import { + GET_FILES_IN_DIRECTORY, + HandleGetFilesInDirectoryMessage, +} from '../message-types/get-files-in-directory'; +import { HASH_GLOB, HandleHashGlobMessage } from '../message-types/hash-glob'; +import { NxWorkspaceFiles } from '../../native'; const DAEMON_ENV_SETTINGS = { NX_PROJECT_GLOB_CACHE: 'false', @@ -256,6 +271,49 @@ export class DaemonClient { }); } + glob(globs: string[], exclude?: string[]): Promise { + const message: HandleGlobMessage = { + type: 'GLOB', + globs, + exclude, + }; + return this.sendToDaemonViaQueue(message); + } + + getWorkspaceContextFileData(): Promise { + const message: HandleContextFileDataMessage = { + type: GET_CONTEXT_FILE_DATA, + }; + return this.sendToDaemonViaQueue(message); + } + + getWorkspaceFiles( + projectRootMap: Record + ): Promise { + const message: HandleNxWorkspaceFilesMessage = { + type: GET_NX_WORKSPACE_FILES, + projectRootMap, + }; + return this.sendToDaemonViaQueue(message); + } + + getFilesInDirectory(dir: string): Promise { + const message: HandleGetFilesInDirectoryMessage = { + type: GET_FILES_IN_DIRECTORY, + dir, + }; + return this.sendToDaemonViaQueue(message); + } + + hashGlob(globs: string[], exclude?: string[]): Promise { + const message: HandleHashGlobMessage = { + type: HASH_GLOB, + globs, + exclude, + }; + return this.sendToDaemonViaQueue(message); + } + async isServerAvailable(): Promise { return new Promise((resolve) => { try { @@ -414,14 +472,17 @@ export class DaemonClient { const backgroundProcess = spawn( process.execPath, - [join(__dirname, '../server/start.js')], + [join(__dirname, `../server/start.js`)], { cwd: workspaceRoot, stdio: ['ignore', this._out.fd, this._err.fd], detached: true, windowsHide: true, shell: false, - env: { ...process.env, ...DAEMON_ENV_SETTINGS }, + env: { + ...process.env, + ...DAEMON_ENV_SETTINGS, + }, } ); backgroundProcess.unref(); diff --git a/packages/nx/src/daemon/is-on-daemon.ts b/packages/nx/src/daemon/is-on-daemon.ts new file mode 100644 index 0000000000000..e105d3d292128 --- /dev/null +++ b/packages/nx/src/daemon/is-on-daemon.ts @@ -0,0 +1,3 @@ +export function isOnDaemon() { + return !!global.NX_DAEMON; +} diff --git a/packages/nx/src/daemon/message-types/get-context-file-data.ts b/packages/nx/src/daemon/message-types/get-context-file-data.ts new file mode 100644 index 0000000000000..935f1cbb87609 --- /dev/null +++ b/packages/nx/src/daemon/message-types/get-context-file-data.ts @@ -0,0 +1,16 @@ +export const GET_CONTEXT_FILE_DATA = 'GET_CONTEXT_FILE_DATA' as const; + +export type HandleContextFileDataMessage = { + type: typeof GET_CONTEXT_FILE_DATA; +}; + +export function isHandleContextFileDataMessage( + message: unknown +): message is HandleContextFileDataMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === GET_CONTEXT_FILE_DATA + ); +} diff --git a/packages/nx/src/daemon/message-types/get-files-in-directory.ts b/packages/nx/src/daemon/message-types/get-files-in-directory.ts new file mode 100644 index 0000000000000..022a2f8756622 --- /dev/null +++ b/packages/nx/src/daemon/message-types/get-files-in-directory.ts @@ -0,0 +1,17 @@ +export const GET_FILES_IN_DIRECTORY = 'GET_FILES_IN_DIRECTORY' as const; + +export type HandleGetFilesInDirectoryMessage = { + type: typeof GET_FILES_IN_DIRECTORY; + dir: string; +}; + +export function isHandleGetFilesInDirectoryMessage( + message: unknown +): message is HandleGetFilesInDirectoryMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === GET_FILES_IN_DIRECTORY + ); +} diff --git a/packages/nx/src/daemon/message-types/get-nx-workspace-files.ts b/packages/nx/src/daemon/message-types/get-nx-workspace-files.ts new file mode 100644 index 0000000000000..872157b091f9e --- /dev/null +++ b/packages/nx/src/daemon/message-types/get-nx-workspace-files.ts @@ -0,0 +1,17 @@ +export const GET_NX_WORKSPACE_FILES = 'GET_NX_WORKSPACE_FILES' as const; + +export type HandleNxWorkspaceFilesMessage = { + type: typeof GET_NX_WORKSPACE_FILES; + projectRootMap: Record; +}; + +export function isHandleNxWorkspaceFilesMessage( + message: unknown +): message is HandleNxWorkspaceFilesMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === GET_NX_WORKSPACE_FILES + ); +} diff --git a/packages/nx/src/daemon/message-types/glob.ts b/packages/nx/src/daemon/message-types/glob.ts new file mode 100644 index 0000000000000..50fd3071cdded --- /dev/null +++ b/packages/nx/src/daemon/message-types/glob.ts @@ -0,0 +1,18 @@ +export const GLOB = 'GLOB' as const; + +export type HandleGlobMessage = { + type: typeof GLOB; + globs: string[]; + exclude?: string[]; +}; + +export function isHandleGlobMessage( + message: unknown +): message is HandleGlobMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === GLOB + ); +} diff --git a/packages/nx/src/daemon/message-types/hash-glob.ts b/packages/nx/src/daemon/message-types/hash-glob.ts new file mode 100644 index 0000000000000..9bec1f566c14c --- /dev/null +++ b/packages/nx/src/daemon/message-types/hash-glob.ts @@ -0,0 +1,18 @@ +export const HASH_GLOB = 'HASH_GLOB' as const; + +export type HandleHashGlobMessage = { + type: typeof HASH_GLOB; + globs: string[]; + exclude?: string[]; +}; + +export function isHandleHashGlobMessage( + message: unknown +): message is HandleHashGlobMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === HASH_GLOB + ); +} diff --git a/packages/nx/src/daemon/message-types/update-context-files.ts b/packages/nx/src/daemon/message-types/update-context-files.ts new file mode 100644 index 0000000000000..42eecea94040e --- /dev/null +++ b/packages/nx/src/daemon/message-types/update-context-files.ts @@ -0,0 +1,18 @@ +export const GLOB = 'GLOB' as const; + +export type HandleUpdateContextMessage = { + type: typeof GLOB; + updatedFiles: string[]; + deletedFiles: string[]; +}; + +export function isHandleUpdateContextMessage( + message: unknown +): message is HandleUpdateContextMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === GLOB + ); +} diff --git a/packages/nx/src/daemon/server/handle-context-file-data.ts b/packages/nx/src/daemon/server/handle-context-file-data.ts new file mode 100644 index 0000000000000..52468d72033d8 --- /dev/null +++ b/packages/nx/src/daemon/server/handle-context-file-data.ts @@ -0,0 +1,11 @@ +import { getAllFileDataInContext } from '../../utils/workspace-context'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { HandlerResult } from './server'; + +export async function handleContextFileData(): Promise { + const files = await getAllFileDataInContext(workspaceRoot); + return { + response: JSON.stringify(files), + description: 'handleContextFileData', + }; +} diff --git a/packages/nx/src/daemon/server/handle-get-files-in-directory.ts b/packages/nx/src/daemon/server/handle-get-files-in-directory.ts new file mode 100644 index 0000000000000..adcfe93f82c4e --- /dev/null +++ b/packages/nx/src/daemon/server/handle-get-files-in-directory.ts @@ -0,0 +1,13 @@ +import { getFilesInDirectoryUsingContext } from '../../utils/workspace-context'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { HandlerResult } from './server'; + +export async function handleGetFilesInDirectory( + dir: string +): Promise { + const files = await getFilesInDirectoryUsingContext(workspaceRoot, dir); + return { + response: JSON.stringify(files), + description: 'handleNxWorkspaceFiles', + }; +} diff --git a/packages/nx/src/daemon/server/handle-glob.ts b/packages/nx/src/daemon/server/handle-glob.ts new file mode 100644 index 0000000000000..c163a33194b52 --- /dev/null +++ b/packages/nx/src/daemon/server/handle-glob.ts @@ -0,0 +1,14 @@ +import { workspaceRoot } from '../../utils/workspace-root'; +import { globWithWorkspaceContext } from '../../utils/workspace-context'; +import { HandlerResult } from './server'; + +export async function handleGlob( + globs: string[], + exclude?: string[] +): Promise { + const files = await globWithWorkspaceContext(workspaceRoot, globs, exclude); + return { + response: JSON.stringify(files), + description: 'handleGlob', + }; +} diff --git a/packages/nx/src/daemon/server/handle-hash-glob.ts b/packages/nx/src/daemon/server/handle-hash-glob.ts new file mode 100644 index 0000000000000..1c283005f31ce --- /dev/null +++ b/packages/nx/src/daemon/server/handle-hash-glob.ts @@ -0,0 +1,14 @@ +import { workspaceRoot } from '../../utils/workspace-root'; +import { hashWithWorkspaceContext } from '../../utils/workspace-context'; +import { HandlerResult } from './server'; + +export async function handleHashGlob( + globs: string[], + exclude?: string[] +): Promise { + const files = await hashWithWorkspaceContext(workspaceRoot, globs, exclude); + return { + response: JSON.stringify(files), + description: 'handleHashGlob', + }; +} diff --git a/packages/nx/src/daemon/server/handle-nx-workspace-files.ts b/packages/nx/src/daemon/server/handle-nx-workspace-files.ts new file mode 100644 index 0000000000000..c4a8c03b04025 --- /dev/null +++ b/packages/nx/src/daemon/server/handle-nx-workspace-files.ts @@ -0,0 +1,16 @@ +import { getNxWorkspaceFilesFromContext } from '../../utils/workspace-context'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { HandlerResult } from './server'; + +export async function handleNxWorkspaceFiles( + projectRootMap: Record +): Promise { + const files = await getNxWorkspaceFilesFromContext( + workspaceRoot, + projectRootMap + ); + return { + response: JSON.stringify(files), + description: 'handleNxWorkspaceFiles', + }; +} diff --git a/packages/nx/src/daemon/server/handle-request-file-data.ts b/packages/nx/src/daemon/server/handle-request-file-data.ts deleted file mode 100644 index 2bfd24e7e1696..0000000000000 --- a/packages/nx/src/daemon/server/handle-request-file-data.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { getAllFileDataInContext } from '../../utils/workspace-context'; -import { workspaceRoot } from '../../utils/workspace-root'; - -export async function handleRequestFileData() { - const response = JSON.stringify(getAllFileDataInContext(workspaceRoot)); - return { - response, - description: 'handleRequestFileData', - }; -} diff --git a/packages/nx/src/daemon/server/server.ts b/packages/nx/src/daemon/server/server.ts index c648f0d1cf838..dcd51bc464aa2 100644 --- a/packages/nx/src/daemon/server/server.ts +++ b/packages/nx/src/daemon/server/server.ts @@ -26,7 +26,6 @@ import { handleRecordOutputsHash, } from './handle-outputs-tracking'; import { handleProcessInBackground } from './handle-process-in-background'; -import { handleRequestFileData } from './handle-request-file-data'; import { handleRequestProjectGraph } from './handle-request-project-graph'; import { handleRequestShutdown } from './handle-request-shutdown'; import { serverLogger } from './logger'; @@ -52,11 +51,32 @@ import { watchOutputFiles, watchWorkspace, } from './watcher'; +import { handleGlob } from './handle-glob'; +import { GLOB, isHandleGlobMessage } from '../message-types/glob'; +import { + GET_NX_WORKSPACE_FILES, + isHandleNxWorkspaceFilesMessage, +} from '../message-types/get-nx-workspace-files'; +import { handleNxWorkspaceFiles } from './handle-nx-workspace-files'; +import { + GET_CONTEXT_FILE_DATA, + isHandleContextFileDataMessage, +} from '../message-types/get-context-file-data'; +import { handleContextFileData } from './handle-context-file-data'; +import { + GET_FILES_IN_DIRECTORY, + isHandleGetFilesInDirectoryMessage, +} from '../message-types/get-files-in-directory'; +import { handleGetFilesInDirectory } from './handle-get-files-in-directory'; +import { HASH_GLOB, isHandleHashGlobMessage } from '../message-types/hash-glob'; +import { handleHashGlob } from './handle-hash-glob'; let performanceObserver: PerformanceObserver | undefined; let workspaceWatcherError: Error | undefined; let outputsWatcherError: Error | undefined; +global.NX_DAEMON = true; + export type HandlerResult = { description: string; error?: any; @@ -111,11 +131,12 @@ async function handleMessage(socket, data: string) { ); } - if (daemonIsOutdated()) { + const outdated = daemonIsOutdated(); + if (outdated) { await respondWithErrorAndExit( socket, - `Lock files changed`, - new Error('LOCK-FILES-CHANGED') + `Daemon outdated`, + new Error(outdated) ); } @@ -143,10 +164,6 @@ async function handleMessage(socket, data: string) { ); } else if (payload.type === 'HASH_TASKS') { await handleResult(socket, 'HASH_TASKS', () => handleHashTasks(payload)); - } else if (payload.type === 'REQUEST_FILE_DATA') { - await handleResult(socket, 'REQUEST_FILE_DATA', () => - handleRequestFileData() - ); } else if (payload.type === 'PROCESS_IN_BACKGROUND') { await handleResult(socket, 'PROCESS_IN_BACKGROUND', () => handleProcessInBackground(payload) @@ -165,6 +182,26 @@ async function handleMessage(socket, data: string) { ); } else if (payload.type === 'REGISTER_FILE_WATCHER') { registeredFileWatcherSockets.push({ socket, config: payload.config }); + } else if (isHandleGlobMessage(payload)) { + await handleResult(socket, GLOB, () => + handleGlob(payload.globs, payload.exclude) + ); + } else if (isHandleNxWorkspaceFilesMessage(payload)) { + await handleResult(socket, GET_NX_WORKSPACE_FILES, () => + handleNxWorkspaceFiles(payload.projectRootMap) + ); + } else if (isHandleGetFilesInDirectoryMessage(payload)) { + await handleResult(socket, GET_FILES_IN_DIRECTORY, () => + handleGetFilesInDirectory(payload.dir) + ); + } else if (isHandleContextFileDataMessage(payload)) { + await handleResult(socket, GET_CONTEXT_FILE_DATA, () => + handleContextFileData() + ); + } else if (isHandleHashGlobMessage(payload)) { + await handleResult(socket, HASH_GLOB, () => + handleHashGlob(payload.globs, payload.exclude) + ); } else { await respondWithErrorAndExit( socket, @@ -233,8 +270,13 @@ function registerProcessTerminationListeners() { let existingLockHash: string | undefined; -function daemonIsOutdated(): boolean { - return nxVersionChanged() || lockFileHashChanged(); +function daemonIsOutdated(): string | null { + if (nxVersionChanged()) { + return 'NX_VERSION_CHANGED'; + } else if (lockFileHashChanged()) { + return 'LOCK_FILES_CHANGED'; + } + return null; } function nxVersionChanged(): boolean { @@ -291,10 +333,11 @@ const handleWorkspaceChanges: FileWatcherCallback = async ( try { resetInactivityTimeout(handleInactivityTimeout); - if (daemonIsOutdated()) { + const outdatedReason = daemonIsOutdated(); + if (outdatedReason) { await handleServerProcessTermination({ server, - reason: 'Lock file changed', + reason: outdatedReason, }); return; } diff --git a/packages/nx/src/devkit-exports.ts b/packages/nx/src/devkit-exports.ts index 046030bdb030c..2f205dce74a79 100644 --- a/packages/nx/src/devkit-exports.ts +++ b/packages/nx/src/devkit-exports.ts @@ -132,7 +132,7 @@ export { /** * @category Generators */ -export { glob } from './generators/utils/glob'; +export { glob, globAsync } from './generators/utils/glob'; /** * @category Generators diff --git a/packages/nx/src/generators/utils/glob.ts b/packages/nx/src/generators/utils/glob.ts index a3920fef4bf36..c97daccded69e 100644 --- a/packages/nx/src/generators/utils/glob.ts +++ b/packages/nx/src/generators/utils/glob.ts @@ -1,7 +1,10 @@ import { minimatch } from 'minimatch'; import { Tree } from '../tree'; import { combineGlobPatterns } from '../../utils/globs'; -import { globWithWorkspaceContext } from '../../utils/workspace-context'; +import { + globWithWorkspaceContext, + globWithWorkspaceContextSync, +} from '../../utils/workspace-context'; /** * Performs a tree-aware glob search on the files in a workspace. Able to find newly @@ -11,9 +14,42 @@ import { globWithWorkspaceContext } from '../../utils/workspace-context'; * @param tree The file system tree * @param patterns A list of glob patterns * @returns Normalized paths in the workspace that match the provided glob patterns. + * @deprecated Use {@link globAsync} instead. */ export function glob(tree: Tree, patterns: string[]): string[] { - const matches = new Set(globWithWorkspaceContext(tree.root, patterns)); + return combineGlobResultsWithTree( + tree, + patterns, + globWithWorkspaceContextSync(tree.root, patterns) + ); +} + +/** + * Performs a tree-aware glob search on the files in a workspace. Able to find newly + * created files and hides deleted files before the updates are committed to disk. + * Paths should be unix-style with forward slashes. + * + * @param tree The file system tree + * @param patterns A list of glob patterns + * @returns Normalized paths in the workspace that match the provided glob patterns. + */ +export async function globAsync( + tree: Tree, + patterns: string[] +): Promise { + return combineGlobResultsWithTree( + tree, + patterns, + await globWithWorkspaceContext(tree.root, patterns) + ); +} + +function combineGlobResultsWithTree( + tree: Tree, + patterns: string[], + results: string[] +) { + const matches = new Set(results); const combinedGlob = combineGlobPatterns(patterns); const matcher = minimatch.makeRe(combinedGlob); diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 1992d6d2e886a..52fde37740f54 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -4,12 +4,8 @@ import { basename, join, relative } from 'path'; import { buildProjectConfigurationFromPackageJson, getGlobPatternsFromPackageManagerWorkspaces, - createNodes as packageJsonWorkspacesCreateNodes, } from '../../plugins/package-json-workspaces'; -import { - buildProjectFromProjectJson, - ProjectJsonProjectsPlugin, -} from '../../plugins/project-json/build-nodes/project-json'; +import { buildProjectFromProjectJson } from '../../plugins/project-json/build-nodes/project-json'; import { renamePropertyWithStableKeys } from '../../adapter/angular-json'; import { ProjectConfiguration, @@ -19,8 +15,7 @@ import { mergeProjectConfigurationIntoRootMap, readProjectConfigurationsFromRootMap, } from '../../project-graph/utils/project-configuration-utils'; -import { configurationGlobs } from '../../project-graph/utils/retrieve-workspace-files'; -import { globWithWorkspaceContext } from '../../utils/workspace-context'; +import { globWithWorkspaceContextSync } from '../../utils/workspace-context'; import { output } from '../../utils/output'; import { PackageJson } from '../../utils/package-json'; import { joinPathFragments, normalizePath } from '../../utils/path'; @@ -28,7 +23,6 @@ import { readJson, writeJson } from './json'; import { readNxJson } from './nx-json'; import type { Tree } from '../tree'; -import { NxPlugin } from '../../project-graph/plugins'; export { readNxJson, updateNxJson } from './nx-json'; @@ -200,7 +194,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): { readJson(tree, p, { expectComments: true }) ), ]; - const globbedFiles = globWithWorkspaceContext(tree.root, patterns); + const globbedFiles = globWithWorkspaceContextSync(tree.root, patterns); const createdFiles = findCreatedProjectFiles(tree, patterns); const deletedFiles = findDeletedProjectFiles(tree, patterns); const projectFiles = [...globbedFiles, ...createdFiles].filter( diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts index 3849124542e8e..529caa2b39ce5 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-package-json-dependencies.spec.ts @@ -89,7 +89,7 @@ describe('explicit package json dependencies', () => { const fileMap = createFileMap( projectsConfigurations as any, - getAllFileDataInContext(tempFs.tempDir) + await getAllFileDataInContext(tempFs.tempDir) ).fileMap; const builder = new ProjectGraphBuilder(undefined, fileMap.projectFileMap); diff --git a/packages/nx/src/project-graph/file-map-utils.ts b/packages/nx/src/project-graph/file-map-utils.ts index 8ebed90d7144e..73d6eb9d795da 100644 --- a/packages/nx/src/project-graph/file-map-utils.ts +++ b/packages/nx/src/project-graph/file-map-utils.ts @@ -39,12 +39,7 @@ export async function createFileMapUsingProjectGraph( ): Promise { const configs = readProjectsConfigurationFromProjectGraph(graph); - let files: FileData[]; - if (daemonClient.enabled()) { - files = await daemonClient.getAllFileData(); - } else { - files = getAllFileDataInContext(workspaceRoot); - } + let files: FileData[] = await getAllFileDataInContext(workspaceRoot); return createFileMap(configs, files); } diff --git a/packages/nx/src/project-graph/file-utils.ts b/packages/nx/src/project-graph/file-utils.ts index 7107547ed9b4a..efba33e943feb 100644 --- a/packages/nx/src/project-graph/file-utils.ts +++ b/packages/nx/src/project-graph/file-utils.ts @@ -1,6 +1,6 @@ import { execSync } from 'child_process'; import { existsSync, readFileSync } from 'fs'; -import { extname, join, relative, sep } from 'path'; +import { basename, extname, join, relative, sep } from 'path'; import { readNxJson } from '../config/configuration'; import { FileData } from '../config/project-graph'; import { @@ -17,16 +17,18 @@ import { } from './project-graph'; import { toOldFormat } from '../adapter/angular-json'; import { getIgnoreObject } from '../utils/ignore'; -import { retrieveProjectConfigurationPaths } from './utils/retrieve-workspace-files'; import { mergeProjectConfigurationIntoRootMap, readProjectConfigurationsFromRootMap, } from './utils/project-configuration-utils'; -import { NxJsonConfiguration } from '../config/nx-json'; -import { getDefaultPluginsSync } from '../utils/nx-plugin.deprecated'; -import { minimatch } from 'minimatch'; -import { CreateNodesResult } from '../devkit-exports'; -import { PackageJsonProjectsNextToProjectJsonPlugin } from '../plugins/project-json/build-nodes/package-json-next-to-project-json'; +import { + buildProjectConfigurationFromPackageJson, + getGlobPatternsFromPackageManagerWorkspaces, +} from '../plugins/package-json-workspaces'; +import { globWithWorkspaceContextSync } from '../utils/workspace-context'; +import { buildProjectFromProjectJson } from '../plugins/project-json/build-nodes/project-json'; +import { PackageJson } from '../utils/package-json'; +import { NxJsonConfiguration } from '../devkit-exports'; export interface Change { type: string; @@ -151,7 +153,7 @@ export function readWorkspaceConfig(opts: { } catch { configuration = { version: 2, - projects: getProjectsSyncNoInference(root, nxJson).projects, + projects: getProjectsSync(root, nxJson), }; } if (opts.format === 'angularCli') { @@ -179,50 +181,59 @@ export { FileData }; /** * TODO(v20): Remove this function. */ -function getProjectsSyncNoInference(root: string, nxJson: NxJsonConfiguration) { - const allConfigFiles = retrieveProjectConfigurationPaths( - root, - getDefaultPluginsSync(root) - ); - const plugins = [ - PackageJsonProjectsNextToProjectJsonPlugin, - ...getDefaultPluginsSync(root), +function getProjectsSync( + root: string, + nxJson: NxJsonConfiguration +): { + [name: string]: ProjectConfiguration; +} { + /** + * We can't update projects that come from plugins anyways, so we are going + * to ignore them for now. Plugins should add their own add/create/update methods + * if they would like to use devkit to update inferred projects. + */ + const patterns = [ + '**/project.json', + 'project.json', + ...getGlobPatternsFromPackageManagerWorkspaces(root, readJsonFile), ]; - - const projectRootMap: Record = {}; - - // We iterate over plugins first - this ensures that plugins specified first take precedence. - for (const plugin of plugins) { - const [pattern, createNodes] = plugin.createNodes ?? []; - if (!pattern) { - continue; - } - const matchingConfigFiles = allConfigFiles.filter((file) => - minimatch(file, pattern, { dot: true }) - ); - for (const file of matchingConfigFiles) { - if (minimatch(file, pattern, { dot: true })) { - let r = createNodes( - file, - {}, + const projectFiles = globWithWorkspaceContextSync(root, patterns); + + const rootMap: Record = {}; + for (const projectFile of projectFiles) { + if (basename(projectFile) === 'project.json') { + const json = readJsonFile(projectFile); + const config = buildProjectFromProjectJson(json, projectFile); + mergeProjectConfigurationIntoRootMap( + rootMap, + config, + undefined, + undefined, + true + ); + } else if (basename(projectFile) === 'package.json') { + const packageJson = readJsonFile(projectFile); + const config = buildProjectConfigurationFromPackageJson( + packageJson, + projectFile, + nxJson + ); + if (!rootMap[config.root]) { + mergeProjectConfigurationIntoRootMap( + rootMap, + // Inferred targets, tags, etc don't show up when running generators + // This is to help avoid running into issues when trying to update the workspace { - nxJsonConfiguration: nxJson, - workspaceRoot: root, - configFiles: matchingConfigFiles, - } - ) as CreateNodesResult; - for (const node in r.projects) { - const project = { - root: node, - ...r.projects[node], - }; - mergeProjectConfigurationIntoRootMap(projectRootMap, project); - } + name: config.name, + root: config.root, + }, + undefined, + undefined, + true + ); } } } - return { - projects: readProjectConfigurationsFromRootMap(projectRootMap), - }; + return readProjectConfigurationsFromRootMap(rootMap); } diff --git a/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts b/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts index 221aefd323cfc..5f076c8ab920c 100644 --- a/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts +++ b/packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts @@ -28,17 +28,23 @@ export function loadRemoteNxPlugin( // but its typescript. const isWorkerTypescript = path.extname(__filename) === '.ts'; const workerPath = path.join(__dirname, 'plugin-worker'); + + const env: Record = { + ...process.env, + ...(isWorkerTypescript + ? { + // Ensures that the worker uses the same tsconfig as the main process + TS_NODE_PROJECT: path.join( + __dirname, + '../../../../tsconfig.lib.json' + ), + } + : {}), + }; + const worker = fork(workerPath, [], { stdio: ['ignore', 'inherit', 'inherit', 'ipc'], - env: { - ...process.env, - ...(isWorkerTypescript - ? { - // Ensures that the worker uses the same tsconfig as the main process - TS_NODE_PROJECT: path.join(__dirname, '../../../tsconfig.lib.json'), - } - : {}), - }, + env, execArgv: [ ...process.execArgv, // If the worker is typescript, we need to register ts-node diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts index 3d23230b11e69..4499637562e47 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.spec.ts @@ -25,7 +25,7 @@ describe('retrieveProjectConfigurationPaths', () => { }) ); - const configPaths = retrieveProjectConfigurationPaths(fs.tempDir, [ + const configPaths = await retrieveProjectConfigurationPaths(fs.tempDir, [ { createNodes: [ '{project.json,**/project.json}', diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index d61a11cef9a4b..d87792f1093fc 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -38,7 +38,7 @@ export async function retrieveWorkspaceFiles( performance.mark('get-workspace-files:start'); const { projectFileMap, globalFiles, externalReferences } = - getNxWorkspaceFilesFromContext(workspaceRoot, projectRootMap); + await getNxWorkspaceFilesFromContext(workspaceRoot, projectRootMap); performance.mark('get-workspace-files:end'); performance.measure( 'get-workspace-files', @@ -60,13 +60,16 @@ export async function retrieveWorkspaceFiles( * Walk through the workspace and return `ProjectConfigurations`. Only use this if the projectFileMap is not needed. */ -export function retrieveProjectConfigurations( +export async function retrieveProjectConfigurations( plugins: LoadedNxPlugin[], workspaceRoot: string, nxJson: NxJsonConfiguration ): Promise { const globPatterns = configurationGlobs(plugins); - const workspaceFiles = globWithWorkspaceContext(workspaceRoot, globPatterns); + const workspaceFiles = await globWithWorkspaceContext( + workspaceRoot, + globPatterns + ); return createProjectConfigurations( workspaceRoot, @@ -98,7 +101,11 @@ export async function retrieveProjectConfigurationsWithAngularProjects( workspaceRoot ); - const res = retrieveProjectConfigurations(plugins, workspaceRoot, nxJson); + const res = await retrieveProjectConfigurations( + plugins, + workspaceRoot, + nxJson + ); cleanup(); return res; } @@ -106,7 +113,7 @@ export async function retrieveProjectConfigurationsWithAngularProjects( export function retrieveProjectConfigurationPaths( root: string, plugins: Array<{ createNodes?: readonly [string, ...unknown[]] } & unknown> -): string[] { +): Promise { const projectGlobPatterns = configurationGlobs(plugins); return globWithWorkspaceContext(root, projectGlobPatterns); } @@ -122,7 +129,10 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( ): Promise> { const nxJson = readNxJson(root); const [plugins, cleanup] = await loadNxPlugins([]); // only load default plugins - const projectGlobPatterns = retrieveProjectConfigurationPaths(root, plugins); + const projectGlobPatterns = await retrieveProjectConfigurationPaths( + root, + plugins + ); const cacheKey = root + ',' + projectGlobPatterns.join(','); if (projectsWithoutPluginCache.has(cacheKey)) { @@ -130,7 +140,7 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( } const projectFiles = - globWithWorkspaceContext(root, projectGlobPatterns) ?? []; + (await globWithWorkspaceContext(root, projectGlobPatterns)) ?? []; const { projects } = await createProjectConfigurations( root, nxJson, diff --git a/packages/nx/src/utils/all-file-data.ts b/packages/nx/src/utils/all-file-data.ts index 4a6a36271b7c7..7ce3da07886b3 100644 --- a/packages/nx/src/utils/all-file-data.ts +++ b/packages/nx/src/utils/all-file-data.ts @@ -1,12 +1,7 @@ import { FileData } from '../config/project-graph'; -import { daemonClient } from '../daemon/client/client'; import { getAllFileDataInContext } from './workspace-context'; import { workspaceRoot } from './workspace-root'; export function allFileData(): Promise { - if (daemonClient.enabled()) { - return daemonClient.getAllFileData(); - } else { - return Promise.resolve(getAllFileDataInContext(workspaceRoot)); - } + return getAllFileDataInContext(workspaceRoot); } diff --git a/packages/nx/src/utils/workspace-context.ts b/packages/nx/src/utils/workspace-context.ts index f45b3026234cb..ca5f0914e1e76 100644 --- a/packages/nx/src/utils/workspace-context.ts +++ b/packages/nx/src/utils/workspace-context.ts @@ -1,6 +1,8 @@ import type { NxWorkspaceFilesExternals, WorkspaceContext } from '../native'; import { performance } from 'perf_hooks'; import { cacheDirectoryForWorkspace } from './cache-directory'; +import { isOnDaemon } from '../daemon/is-on-daemon'; +import { daemonClient } from '../daemon/client/client'; let workspaceContext: WorkspaceContext | undefined; @@ -20,15 +22,25 @@ export function setupWorkspaceContext(workspaceRoot: string) { ); } -export function getNxWorkspaceFilesFromContext( +export async function getNxWorkspaceFilesFromContext( workspaceRoot: string, projectRootMap: Record ) { - ensureContextAvailable(workspaceRoot); - return workspaceContext.getWorkspaceFiles(projectRootMap); + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.getWorkspaceFiles(projectRootMap); + } + return daemonClient.getWorkspaceFiles(projectRootMap); } -export function globWithWorkspaceContext( +/** + * Sync method to get files matching globs from workspace context. + * NOTE: This method will create the workspace context if it doesn't exist. + * It should only be used within Nx internal in code paths that **must** be sync. + * If used in an isolated plugin thread this will cause the workspace context + * to be recreated which is slow. + */ +export function globWithWorkspaceContextSync( workspaceRoot: string, globs: string[], exclude?: string[] @@ -37,13 +49,29 @@ export function globWithWorkspaceContext( return workspaceContext.glob(globs, exclude); } -export function hashWithWorkspaceContext( +export async function globWithWorkspaceContext( workspaceRoot: string, globs: string[], exclude?: string[] ) { - ensureContextAvailable(workspaceRoot); - return workspaceContext.hashFilesMatchingGlob(globs, exclude); + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.glob(globs, exclude); + } else { + return daemonClient.glob(globs, exclude); + } +} + +export async function hashWithWorkspaceContext( + workspaceRoot: string, + globs: string[], + exclude?: string[] +) { + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.hashFilesMatchingGlob(globs, exclude); + } + return daemonClient.hashGlob(globs, exclude); } export function updateFilesInContext( @@ -53,17 +81,23 @@ export function updateFilesInContext( return workspaceContext?.incrementalUpdate(updatedFiles, deletedFiles); } -export function getAllFileDataInContext(workspaceRoot: string) { - ensureContextAvailable(workspaceRoot); - return workspaceContext.allFileData(); +export async function getAllFileDataInContext(workspaceRoot: string) { + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.allFileData(); + } + return daemonClient.getWorkspaceContextFileData(); } -export function getFilesInDirectoryUsingContext( +export async function getFilesInDirectoryUsingContext( workspaceRoot: string, dir: string ) { - ensureContextAvailable(workspaceRoot); - return workspaceContext.getFilesInDirectory(dir); + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.getFilesInDirectory(dir); + } + return daemonClient.getFilesInDirectory(dir); } export function updateProjectFiles( diff --git a/packages/playwright/src/plugins/plugin.ts b/packages/playwright/src/plugins/plugin.ts index a1596b63cb612..2384d669e53e2 100644 --- a/packages/playwright/src/plugins/plugin.ts +++ b/packages/playwright/src/plugins/plugin.ts @@ -108,9 +108,12 @@ async function createNodesInternal( const normalizedOptions = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildPlaywrightTargets( configFilePath, @@ -199,7 +202,7 @@ async function buildPlaywrightTargets( playwrightConfig.testMatch ??= '**/*.@(spec|test).?(c|m)[jt]s?(x)'; const dependsOn: TargetConfiguration['dependsOn'] = []; - forEachTestFile( + await forEachTestFile( (testFile) => { const relativeSpecFilePath = normalizePath( relative(projectRoot, testFile) @@ -246,7 +249,7 @@ async function buildPlaywrightTargets( return { targets, metadata }; } -function forEachTestFile( +async function forEachTestFile( cb: (path: string) => void, opts: { context: CreateNodesContext; @@ -254,7 +257,7 @@ function forEachTestFile( config: PlaywrightTestConfig; } ) { - const files = getFilesInDirectoryUsingContext( + const files = await getFilesInDirectoryUsingContext( opts.context.workspaceRoot, opts.path ); diff --git a/packages/react-native/plugins/plugin.ts b/packages/react-native/plugins/plugin.ts index 6333d7dd82d7a..d56a08be9dc8d 100644 --- a/packages/react-native/plugins/plugin.ts +++ b/packages/react-native/plugins/plugin.ts @@ -70,9 +70,12 @@ export const createNodes: CreateNodes = [ return {}; } - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= buildReactNativeTargets( projectRoot, diff --git a/packages/remix/src/plugins/plugin.ts b/packages/remix/src/plugins/plugin.ts index e244d5c74e55e..faf019af2bb2d 100644 --- a/packages/remix/src/plugins/plugin.ts +++ b/packages/remix/src/plugins/plugin.ts @@ -66,9 +66,12 @@ export const createNodes: CreateNodes = [ options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildRemixTargets( configFilePath, projectRoot, diff --git a/packages/remix/src/utils/create-watch-paths.spec.ts b/packages/remix/src/utils/create-watch-paths.spec.ts index fe657316ce098..6139227dfc27f 100644 --- a/packages/remix/src/utils/create-watch-paths.spec.ts +++ b/packages/remix/src/utils/create-watch-paths.spec.ts @@ -6,10 +6,39 @@ import { describe('createWatchPaths', () => { it('should list root paths of dependencies relative to project root', async () => { - const testDir = joinPathFragments(workspaceRoot, 'e2e/remix'); + // This test is written based on the Nx repo's project graph. + jest + .spyOn(require('@nx/devkit'), 'createProjectGraphAsync') + .mockResolvedValue({ + nodes: { + parent: { + type: 'app', + name: 'parent', + data: { root: 'apps/parent' }, + }, + lib: { + type: 'lib', + name: 'lib', + data: { root: 'packages/lib' }, + }, + example: { + type: 'app', + name: 'example', + data: { root: 'examples/example' }, + }, + }, + dependencies: { + parent: [ + { type: 'static', source: 'parent', target: 'lib' }, + { type: 'static', source: 'parent', target: 'example' }, + ], + example: [{ type: 'static', source: 'example', target: 'lib' }], + }, + }); + const testDir = joinPathFragments(workspaceRoot, 'apps/parent'); const paths = await createWatchPaths(testDir); - expect(paths).toEqual(['../../packages', '../../graph', '../../e2e/utils']); + expect(paths).toEqual(['../../packages', '../../examples']); }); }); diff --git a/packages/rollup/src/plugins/plugin.ts b/packages/rollup/src/plugins/plugin.ts index a65b15b6968ba..c48a2a51d4d00 100644 --- a/packages/rollup/src/plugins/plugin.ts +++ b/packages/rollup/src/plugins/plugin.ts @@ -63,9 +63,12 @@ export const createNodes: CreateNodes = [ options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); targetsCache[hash] ??= await buildRollupTarget( configFilePath, diff --git a/packages/storybook/src/plugins/plugin.ts b/packages/storybook/src/plugins/plugin.ts index c93749fa1e8f7..597ac6a00f0eb 100644 --- a/packages/storybook/src/plugins/plugin.ts +++ b/packages/storybook/src/plugins/plugin.ts @@ -72,9 +72,12 @@ export const createNodes: CreateNodes = [ } options = normalizeOptions(options); - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); const projectName = buildProjectName(projectRoot, context.workspaceRoot); diff --git a/packages/vite/src/plugins/plugin.spec.ts b/packages/vite/src/plugins/plugin.spec.ts index 2c5561817322f..e7c99cf8b1fb0 100644 --- a/packages/vite/src/plugins/plugin.spec.ts +++ b/packages/vite/src/plugins/plugin.spec.ts @@ -44,9 +44,10 @@ describe('@nx/vite/plugin', () => { production: ['!{projectRoot}/**/*.spec.ts'], }, }, - workspaceRoot: '', + workspaceRoot: tempFs.tempDir, }; tempFs.createFileSync('index.html', ''); + tempFs.createFileSync('package.json', ''); }); afterEach(() => { diff --git a/packages/vite/src/plugins/plugin.ts b/packages/vite/src/plugins/plugin.ts index b823ce405fd13..ca05415845cf9 100644 --- a/packages/vite/src/plugins/plugin.ts +++ b/packages/vite/src/plugins/plugin.ts @@ -6,7 +6,6 @@ import { joinPathFragments, readJsonFile, TargetConfiguration, - workspaceRoot, writeJsonFile, } from '@nx/devkit'; import { dirname, isAbsolute, join, relative } from 'path'; @@ -118,7 +117,8 @@ async function buildViteTargets( const { buildOutputs, testOutputs, hasTest, isBuildable } = getOutputs( viteConfig, - projectRoot + projectRoot, + context.workspaceRoot ); const namedInputs = getNamedInputs(projectRoot, context); @@ -244,7 +244,8 @@ function serveStaticTarget(options: VitePluginOptions) { function getOutputs( viteConfig: Record | undefined, - projectRoot: string + projectRoot: string, + workspaceRoot: string ): { buildOutputs: string[]; testOutputs: string[]; @@ -256,6 +257,7 @@ function getOutputs( const buildOutputPath = normalizeOutputPath( build?.outDir, projectRoot, + workspaceRoot, 'dist' ); @@ -267,6 +269,7 @@ function getOutputs( const reportsDirectoryPath = normalizeOutputPath( test?.coverage?.reportsDirectory, projectRoot, + workspaceRoot, 'coverage' ); @@ -281,6 +284,7 @@ function getOutputs( function normalizeOutputPath( outputPath: string | undefined, projectRoot: string, + workspaceRoot: string, path: 'coverage' | 'dist' ): string | undefined { if (!outputPath) { diff --git a/packages/webpack/src/plugins/plugin.ts b/packages/webpack/src/plugins/plugin.ts index e60b83310fa8a..e9a2f59ca5ab5 100644 --- a/packages/webpack/src/plugins/plugin.ts +++ b/packages/webpack/src/plugins/plugin.ts @@ -69,9 +69,12 @@ export const createNodes: CreateNodes = [ return {}; } - const hash = calculateHashForCreateNodes(projectRoot, options, context, [ - getLockFileName(detectPackageManager(context.workspaceRoot)), - ]); + const hash = await calculateHashForCreateNodes( + projectRoot, + options, + context, + [getLockFileName(detectPackageManager(context.workspaceRoot))] + ); const targets = targetsCache[hash] ? targetsCache[hash] : await createWebpackTargets( diff --git a/scripts/unit-test-setup.js b/scripts/unit-test-setup.js new file mode 100644 index 0000000000000..8436818b05311 --- /dev/null +++ b/scripts/unit-test-setup.js @@ -0,0 +1,34 @@ +module.exports = () => { + /** + * When the daemon is enabled during unit tests, + * and the daemon is already running, the daemon-client.ts + * code will be used, but it will hit the already running + * daemon which is from the installed version of Nx. + * + * In the vast majority of cases, this is fine. However, + * if a new message type has been added to the daemon in + * the source code, and isn't yet in the installed version, + * any test that hits that codepath will fail. This is because + * the installed version of the daemon doesn't know how to + * handle the new message type. + * + * To prevent this, we disable the daemon during unit tests. + */ + process.env.NX_DAEMON = 'false'; + + /** + * When `createProjectGraphAsync` is called during tests, + * if its not mocked, it will return the Nx repo's project + * graph. We don't want any unit tests to depend on the structure + * of the Nx repo, so we mock it to return an empty project graph. + */ + jest.doMock('@nx/devkit', () => ({ + ...jest.requireActual('@nx/devkit'), + createProjectGraphAsync: jest.fn().mockImplementation(async () => { + return { + nodes: {}, + dependencies: {}, + }; + }), + })); +}; diff --git a/typedoc-theme/jest.config.ts b/typedoc-theme/jest.config.ts index 69cf4a39427b9..79217ac523e8e 100644 --- a/typedoc-theme/jest.config.ts +++ b/typedoc-theme/jest.config.ts @@ -1,5 +1,13 @@ +const nxPreset = require('@nx/jest/preset').default; + /* eslint-disable */ export default { + ...nxPreset, + testTimeout: 35000, + testMatch: ['**/+(*.)+(spec|test).+(ts|js)?(x)'], + coverageReporters: ['html'], + maxWorkers: 1, + testEnvironment: 'node', displayName: 'typedoc-theme', globals: {}, @@ -14,5 +22,5 @@ export default { resolver: '../scripts/patched-jest-resolver.js', moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'], coverageDirectory: '../coverage/typedoc-theme', - preset: '../jest.preset.js', + setupFiles: [], };