Skip to content

Commit

Permalink
feat(core): add NX_PLUGIN_ISOLATION flag to enable workers
Browse files Browse the repository at this point in the history
  • Loading branch information
AgentEnder committed Mar 26, 2024
1 parent 2c9796e commit 0e7232d
Show file tree
Hide file tree
Showing 29 changed files with 464 additions and 359 deletions.
5 changes: 3 additions & 2 deletions packages/nx/src/config/workspaces.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { TempFs } from '../internal-testing-utils/temp-fs';
import { withEnvironmentVariables } from '../internal-testing-utils/with-environment';
import { retrieveProjectConfigurations } from '../project-graph/utils/retrieve-workspace-files';
import { readNxJson } from './configuration';
import { loadNxPluginsInIsolation } from '../project-graph/plugins/internal-api';
import { loadNxPlugins } from '../project-graph/plugins/internal-api';

describe('Workspaces', () => {
let fs: TempFs;
Expand Down Expand Up @@ -38,7 +38,7 @@ describe('Workspaces', () => {
NX_WORKSPACE_ROOT_PATH: fs.tempDir,
},
async () => {
const [plugins, cleanup] = await loadNxPluginsInIsolation(
const [plugins, cleanup] = await loadNxPlugins(
readNxJson(fs.tempDir).plugins,
fs.tempDir
);
Expand All @@ -51,6 +51,7 @@ describe('Workspaces', () => {
return res;
}
);
console.log(projects);
expect(projects['my-package']).toEqual({
name: 'my-package',
root: 'packages/my-package',
Expand Down
8 changes: 4 additions & 4 deletions packages/nx/src/daemon/server/plugins.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import { readNxJson } from '../../config/nx-json';
import {
RemotePlugin,
loadNxPluginsInIsolation,
LoadedNxPlugin,
loadNxPlugins,
} from '../../project-graph/plugins/internal-api';
import { workspaceRoot } from '../../utils/workspace-root';

let loadedPlugins: Promise<RemotePlugin[]>;
let loadedPlugins: Promise<LoadedNxPlugin[]>;
let cleanup: () => void;

export async function getPlugins() {
if (loadedPlugins) {
return loadedPlugins;
}
const pluginsConfiguration = readNxJson().plugins ?? [];
const [result, cleanupFn] = await loadNxPluginsInIsolation(
const [result, cleanupFn] = await loadNxPlugins(
pluginsConfiguration,
workspaceRoot
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ import { serverLogger } from './logger';
import { NxWorkspaceFilesExternals } from '../../native';
import {
ConfigurationResult,
ProjectConfigurationsError,
} from '../../project-graph/utils/project-configuration-utils';
import { DaemonProjectGraphError } from '../daemon-project-graph-error';
import { LoadedNxPlugin } from '../../project-graph/plugins/internal-api';
import { getPlugins } from './plugins';
import { ProjectConfigurationsError } from 'nx/src/project-graph/error-types';

interface SerializedProjectGraph {
error: Error | null;
Expand Down
4 changes: 2 additions & 2 deletions packages/nx/src/executors/utils/convert-nx-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { readNxJson } from '../../config/nx-json';
import { Executor, ExecutorContext } from '../../config/misc-interfaces';
import { retrieveProjectConfigurations } from '../../project-graph/utils/retrieve-workspace-files';
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
import { loadNxPluginsInIsolation } from '../../project-graph/plugins/internal-api';
import { loadNxPlugins } from '../../project-graph/plugins/internal-api';

/**
* Convert an Nx Executor into an Angular Devkit Builder
Expand All @@ -19,7 +19,7 @@ export function convertNxExecutor(executor: Executor) {
const promise = async () => {
const nxJsonConfiguration = readNxJson(builderContext.workspaceRoot);

const [plugins, cleanup] = await loadNxPluginsInIsolation(
const [plugins, cleanup] = await loadNxPlugins(
nxJsonConfiguration.plugins,
builderContext.workspaceRoot
);
Expand Down
2 changes: 1 addition & 1 deletion packages/nx/src/generators/utils/project-configuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
];
const projectGlobPatterns = configurationGlobs([
ProjectJsonProjectsPlugin,
{ createNodes: packageJsonWorkspacesCreateNodes } as NxPlugin,
{ createNodes: packageJsonWorkspacesCreateNodes },
]);
const globbedFiles = globWithWorkspaceContext(tree.root, projectGlobPatterns);
const createdFiles = findCreatedProjectFiles(tree, patterns);
Expand Down
6 changes: 4 additions & 2 deletions packages/nx/src/migrations/update-15-1-0/set-project-names.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import { dirname } from 'path';
import { readJson, writeJson } from '../../generators/utils/json';
import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available';
import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files';
import { loadPlugins } from '../../project-graph/plugins/internal-api';
import { loadNxPlugins } from '../../project-graph/plugins/internal-api';

export default async function (tree: Tree) {
const nxJson = readNxJson(tree);
const [plugins, cleanup] = (await loadNxPlugins(nxJson?.plugins ?? [], tree.root))
const projectFiles = retrieveProjectConfigurationPaths(
tree.root,
(await loadPlugins(nxJson?.plugins ?? [], tree.root)).map((p) => p.plugin)
plugins
);
const projectJsons = projectFiles.filter((f) => f.endsWith('project.json'));

Expand All @@ -22,6 +23,7 @@ export default async function (tree: Tree) {
}
}
await formatChangedFilesWithPrettierIfAvailable(tree);
cleanup();
}

function toProjectName(directory: string, nxJson: any): string {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ import {
} from '../../../../project-graph/utils/retrieve-workspace-files';
import { CreateDependenciesContext } from '../../../../project-graph/plugins';
import { setupWorkspaceContext } from '../../../../utils/workspace-context';
import ProjectJsonProjectsPlugin from '../../../project-json/build-nodes/project-json';
import { loadNxPluginsInIsolation } from '../../../../project-graph/plugins/internal-api';
import { loadNxPlugins } from '../../../../project-graph/plugins/internal-api';

// projectName => tsconfig import path
const dependencyProjectNamesToImportPaths = {
Expand Down Expand Up @@ -566,7 +565,7 @@ async function createContext(

setupWorkspaceContext(tempFs.tempDir);

const [plugins, cleanup] = await loadNxPluginsInIsolation([], tempFs.tempDir);
const [plugins, cleanup] = await loadNxPlugins([], tempFs.tempDir);
const { projects, projectRootMap } = await retrieveProjectConfigurations(
plugins,
tempFs.tempDir,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const patterns = getGlobPatternsFromPackageManagerWorkspaces(
workspaceRoot,
readJson
);

const negativePatterns = patterns.filter((p) => p.startsWith('!'));
const positivePatterns = patterns.filter((p) => !p.startsWith('!'));
if (
Expand Down
18 changes: 18 additions & 0 deletions packages/nx/src/plugins/target-defaults/symbols.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/**
* This marks that a target provides information which should modify a target already registered
* on the project via other plugins. If the target has not already been registered, and this symbol is true,
* the information provided by it will be discarded.
*
* NOTE: This cannot be a symbol, as they are not serialized in JSON the communication
* between the plugin-worker and the main process.
*/
export const ONLY_MODIFIES_EXISTING_TARGET = 'NX_ONLY_MODIFIES_EXISTING_TARGET';

/**
* This is used to override the source file for the target defaults plugin.
* This allows the plugin to use the project files as the context, but point to nx.json as the source file.
*
* NOTE: This cannot be a symbol, as they are not serialized in JSON the communication
* between the plugin-worker and the main process.
*/
export const OVERRIDE_SOURCE_FILE = 'NX_OVERRIDE_SOURCE_FILE';
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,7 @@ import {
readTargetsFromPackageJson,
} from '../../utils/package-json';
import { getGlobPatternsFromPackageManagerWorkspaces } from '../package-json-workspaces';

/**
* This marks that a target provides information which should modify a target already registered
* on the project via other plugins. If the target has not already been registered, and this symbol is true,
* the information provided by it will be discarded.
*
* NOTE: This cannot be a symbol, as they are not serialized in JSON the communication
* between the plugin-worker and the main process.
*/
export const ONLY_MODIFIES_EXISTING_TARGET = 'NX_ONLY_MODIFIES_EXISTING_TARGET';

/**
* This is used to override the source file for the target defaults plugin.
* This allows the plugin to use the project files as the context, but point to nx.json as the source file.
*
* NOTE: This cannot be a symbol, as they are not serialized in JSON the communication
* between the plugin-worker and the main process.
*/
export const OVERRIDE_SOURCE_FILE = 'NX_OVERRIDE_SOURCE_FILE';
import { ONLY_MODIFIES_EXISTING_TARGET, OVERRIDE_SOURCE_FILE } from './symbols';

export const TargetDefaultsPlugin: NxPluginV2 = {
name: 'nx/core/target-defaults',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@ import { workspaceRoot } from '../../../utils/workspace-root';
import { join } from 'path';
import { existsSync } from 'fs';
import { configurationGlobs } from '../../utils/retrieve-workspace-files';
import { loadPlugins } from '../../plugins/internal-api';
import { loadNxPlugins } from '../../plugins/internal-api';
import { combineGlobPatterns } from '../../../utils/globs';

export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator =
async (touchedFiles, projectGraphNodes, nxJson): Promise<string[]> => {
const plugins = await loadPlugins(nxJson?.plugins ?? [], workspaceRoot);
const globPattern = combineGlobPatterns(
configurationGlobs(plugins.map((p) => p.plugin))
);
const [plugins] = await loadNxPlugins(nxJson?.plugins ?? [], workspaceRoot);
const globPattern = combineGlobPatterns(configurationGlobs(plugins));

const touchedProjects = new Set<string>();
for (const touchedFile of touchedFiles) {
Expand Down
9 changes: 4 additions & 5 deletions packages/nx/src/project-graph/build-project-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ async function updateProjectGraphWithPlugins(
) {
let graph = initProjectGraph;
const errors: Array<ProcessDependenciesError | ProcessProjectGraphError> = [];
for (const { plugin } of plugins) {
for (const plugin of plugins) {
try {
if (
isNxPluginV1(plugin) &&
Expand Down Expand Up @@ -307,15 +307,14 @@ async function updateProjectGraphWithPlugins(
);

const createDependencyPlugins = plugins.filter(
({plugin}) => isNxPluginV2(plugin) && plugin.createDependencies
(plugin) => isNxPluginV2(plugin) && plugin.createDependencies
);
await Promise.all(
createDependencyPlugins.map(async ({plugin, options}) => {
createDependencyPlugins.map(async (plugin) => {
performance.mark(`${plugin.name}:createDependencies - start`);

try {
// TODO: we shouldn't have to pass null here
const dependencies = await plugin.createDependencies(options, {
const dependencies = await plugin.createDependencies({
...context,
});

Expand Down
98 changes: 98 additions & 0 deletions packages/nx/src/project-graph/error-types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { CreateNodesResultWithContext } from './plugins/internal-api';
import { ConfigurationResult } from './utils/project-configuration-utils';

export class ProjectConfigurationsError extends Error {
constructor(
public readonly errors: Array<MergeNodesError | CreateNodesError>,
public readonly partialProjectConfigurationsResult: ConfigurationResult
) {
super('Failed to create project configurations');
this.name = this.constructor.name;
}
}

export class CreateNodesError extends Error {
file: string;
pluginName: string;

constructor({
file,
pluginName,
error,
}: {
file: string;
pluginName: string;
error: Error;
}) {
const msg = `The "${pluginName}" plugin threw an error while creating nodes from ${file}:`;

super(msg, { cause: error });
this.name = this.constructor.name;
this.file = file;
this.pluginName = pluginName;
this.stack = `${this.message}\n ${error.stack.split('\n').join('\n ')}`;
}
}

export class AggregateCreateNodesError extends Error {
constructor(
public readonly pluginName: string,
public readonly errors: Array<CreateNodesError>,
public readonly partialResults: Array<CreateNodesResultWithContext>
) {
super('Failed to create nodes');
this.name = this.constructor.name;
}
}

export class MergeNodesError extends Error {
file: string;
pluginName: string;

constructor({
file,
pluginName,
error,
}: {
file: string;
pluginName: string;
error: Error;
}) {
const msg = `The nodes created from ${file} by the "${pluginName}" could not be merged into the project graph:`;

super(msg, { cause: error });
this.name = this.constructor.name;
this.file = file;
this.pluginName = pluginName;
this.stack = `${this.message}\n ${error.stack.split('\n').join('\n ')}`;
}
}

export function isCreateNodesError(e: unknown): e is CreateNodesError {
return (
e instanceof CreateNodesError ||
(typeof e === 'object' &&
'name' in e &&
e?.name === CreateNodesError.prototype.name)
);
}

export function isAggregateCreateNodesError(
e: unknown
): e is AggregateCreateNodesError {
return (
e instanceof AggregateCreateNodesError ||
(typeof e === 'object' &&
'name' in e &&
e?.name === AggregateCreateNodesError.prototype.name)
);
}

export function isMergeNodesError(e: unknown): e is MergeNodesError {
return (
e instanceof MergeNodesError ||
(typeof e === 'object' &&
'name' in e &&
e?.name === MergeNodesError.prototype.name)
);
}
7 changes: 4 additions & 3 deletions packages/nx/src/project-graph/file-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { getDefaultPluginsSync } from '../utils/nx-plugin.deprecated';
import { minimatch } from 'minimatch';
import { CreateNodesResult } from '../devkit-exports';
import { PackageJsonProjectsNextToProjectJsonPlugin } from '../plugins/project-json/build-nodes/package-json-next-to-project-json';
import { LoadedNxPlugin } from './plugins/internal-api';

export interface Change {
type: string;
Expand Down Expand Up @@ -183,17 +184,17 @@ export { readNxJson, workspaceLayout } from '../config/configuration';
function getProjectsSyncNoInference(root: string, nxJson: NxJsonConfiguration) {
const allConfigFiles = retrieveProjectConfigurationPaths(
root,
getDefaultPluginsSync(root).map((p) => p.plugin)
getDefaultPluginsSync(root)
);
const plugins = [
{ plugin: PackageJsonProjectsNextToProjectJsonPlugin },
PackageJsonProjectsNextToProjectJsonPlugin,
...getDefaultPluginsSync(root),
];

const projectRootMap: Map<string, ProjectConfiguration> = new Map();

// We iterate over plugins first - this ensures that plugins specified first take precedence.
for (const { plugin } of plugins) {
for (const plugin of plugins) {
const [pattern, createNodes] = plugin.createNodes ?? [];
if (!pattern) {
continue;
Expand Down
2 changes: 1 addition & 1 deletion packages/nx/src/project-graph/plugins/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ export * from './public-api';
export {
readPluginPackageJson,
registerPluginTSTranspiler,
} from './worker-api';
} from './loader';
Loading

0 comments on commit 0e7232d

Please sign in to comment.