From d62acecec60955fb960a98ce178a08f1186e08c1 Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Fri, 13 Oct 2023 10:47:43 -0400 Subject: [PATCH] feat(core): allow using Nx Cloud without nx-cloud installed (#19553) --- docs/generated/cli/connect.md | 8 + docs/generated/manifests/menus.json | 17 + docs/generated/manifests/nx-api.json | 12 +- docs/generated/packages-metadata.json | 12 +- .../nx/documents/connect-to-nx-cloud.md | 8 + .../nx/generators/connect-to-nx-cloud.json | 34 ++ .../packages/workspace/generators/new.json | 5 - docs/shared/reference/sitemap.md | 2 + e2e/nx-run/src/nx-cloud.test.ts | 37 ++ .../src/utils/nx/nx-cloud.ts | 2 +- packages/nx/bin/nx-cloud.ts | 65 ++++ packages/nx/generators.json | 10 + packages/nx/migrations.json | 2 +- packages/nx/package.json | 4 +- .../command-line/connect/command-object.ts | 19 +- .../connect/connect-to-nx-cloud.ts | 30 +- .../nx/src/command-line/connect/view-logs.ts | 21 +- .../init/implementation/add-nx-to-monorepo.ts | 2 +- .../init/implementation/add-nx-to-nest.ts | 2 +- .../init/implementation/add-nx-to-npm-repo.ts | 2 +- .../init/implementation/angular/index.ts | 6 +- .../angular/legacy-angular-versions.ts | 8 - .../command-line/init/implementation/utils.ts | 16 +- .../nx/src/command-line/migrate/migrate.ts | 7 +- .../command-line/yargs-utils/documentation.ts | 5 +- ...al-config-for-tasks-runner-options.spec.ts | 10 + ...minimal-config-for-tasks-runner-options.ts | 29 ++ packages/nx/src/nx-cloud/debug-logger.ts | 5 + .../connect-to-nx-cloud.ts | 139 +++++++ .../connect-to-nx-cloud/schema.json | 22 ++ .../nx-cloud/nx-cloud-tasks-runner-shell.ts | 68 ++++ .../nx/src/nx-cloud/resolution-helpers.ts | 21 ++ packages/nx/src/nx-cloud/update-manager.ts | 351 ++++++++++++++++++ packages/nx/src/nx-cloud/utilities/axios.ts | 39 ++ .../nx/src/nx-cloud/utilities/environment.ts | 48 +++ .../nx-cloud/utilities/get-cloud-options.ts | 10 + packages/nx/src/tasks-runner/run-command.ts | 51 ++- packages/workspace/src/generators/new/new.ts | 18 - .../workspace/src/generators/new/schema.json | 5 - 39 files changed, 1040 insertions(+), 112 deletions(-) create mode 100644 docs/generated/packages/nx/generators/connect-to-nx-cloud.json create mode 100644 e2e/nx-run/src/nx-cloud.test.ts create mode 100644 packages/nx/bin/nx-cloud.ts create mode 100644 packages/nx/generators.json create mode 100644 packages/nx/src/nx-cloud/debug-logger.ts create mode 100644 packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts create mode 100644 packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json create mode 100644 packages/nx/src/nx-cloud/nx-cloud-tasks-runner-shell.ts create mode 100644 packages/nx/src/nx-cloud/resolution-helpers.ts create mode 100644 packages/nx/src/nx-cloud/update-manager.ts create mode 100644 packages/nx/src/nx-cloud/utilities/axios.ts create mode 100644 packages/nx/src/nx-cloud/utilities/environment.ts create mode 100644 packages/nx/src/nx-cloud/utilities/get-cloud-options.ts diff --git a/docs/generated/cli/connect.md b/docs/generated/cli/connect.md index 247c3df195843..12458a4f36125 100644 --- a/docs/generated/cli/connect.md +++ b/docs/generated/cli/connect.md @@ -23,6 +23,14 @@ Type: `boolean` Show help +### interactive + +Type: `boolean` + +Default: `true` + +Prompt for confirmation + ### version Type: `boolean` diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index 87c9a30d0a72a..df876433bb909 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -7666,6 +7666,23 @@ ], "isExternal": false, "disableCollapsible": false + }, + { + "id": "generators", + "path": "/nx-api/nx/generators", + "name": "generators", + "children": [ + { + "id": "connect-to-nx-cloud", + "path": "/nx-api/nx/generators/connect-to-nx-cloud", + "name": "connect-to-nx-cloud", + "children": [], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false } ], "isExternal": false, diff --git a/docs/generated/manifests/nx-api.json b/docs/generated/manifests/nx-api.json index 9e0f2de6b1057..3abd78d663b39 100644 --- a/docs/generated/manifests/nx-api.json +++ b/docs/generated/manifests/nx-api.json @@ -1815,7 +1815,17 @@ "type": "executor" } }, - "generators": {}, + "generators": { + "/nx-api/nx/generators/connect-to-nx-cloud": { + "description": "Connect a workspace to Nx Cloud", + "file": "generated/packages/nx/generators/connect-to-nx-cloud.json", + "hidden": false, + "name": "connect-to-nx-cloud", + "originalFilePath": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json", + "path": "/nx-api/nx/generators/connect-to-nx-cloud", + "type": "generator" + } + }, "path": "/nx-api/nx" }, "playwright": { diff --git a/docs/generated/packages-metadata.json b/docs/generated/packages-metadata.json index 0d75fd5d3df99..9d7e870e33909 100644 --- a/docs/generated/packages-metadata.json +++ b/docs/generated/packages-metadata.json @@ -1794,7 +1794,17 @@ "type": "executor" } ], - "generators": [], + "generators": [ + { + "description": "Connect a workspace to Nx Cloud", + "file": "generated/packages/nx/generators/connect-to-nx-cloud.json", + "hidden": false, + "name": "connect-to-nx-cloud", + "originalFilePath": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json", + "path": "nx/generators/connect-to-nx-cloud", + "type": "generator" + } + ], "githubRoot": "https://github.com/nrwl/nx/blob/master", "name": "nx", "packageName": "nx", diff --git a/docs/generated/packages/nx/documents/connect-to-nx-cloud.md b/docs/generated/packages/nx/documents/connect-to-nx-cloud.md index 247c3df195843..12458a4f36125 100644 --- a/docs/generated/packages/nx/documents/connect-to-nx-cloud.md +++ b/docs/generated/packages/nx/documents/connect-to-nx-cloud.md @@ -23,6 +23,14 @@ Type: `boolean` Show help +### interactive + +Type: `boolean` + +Default: `true` + +Prompt for confirmation + ### version Type: `boolean` diff --git a/docs/generated/packages/nx/generators/connect-to-nx-cloud.json b/docs/generated/packages/nx/generators/connect-to-nx-cloud.json new file mode 100644 index 0000000000000..df27a5d6760ae --- /dev/null +++ b/docs/generated/packages/nx/generators/connect-to-nx-cloud.json @@ -0,0 +1,34 @@ +{ + "name": "connect-to-nx-cloud", + "factory": "./src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud", + "schema": { + "$schema": "http://json-schema.org/schema", + "id": "NxCloudInit", + "title": "Add Nx Cloud Configuration to the workspace", + "description": "Connect a workspace to Nx Cloud.", + "type": "object", + "cli": "nx", + "properties": { + "analytics": { + "type": "boolean", + "description": "Anonymously store hashed machine ID for task runs", + "default": false + }, + "installationSource": { + "type": "string", + "description": "Name of Nx Cloud installation invoker (ex. user, add-nx-to-monorepo, create-nx-workspace, nx-upgrade", + "default": "user" + } + }, + "additionalProperties": false, + "required": [], + "presets": [] + }, + "description": "Connect a workspace to Nx Cloud", + "x-hidden": true, + "implementation": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts", + "aliases": [], + "hidden": false, + "path": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json", + "type": "generator" +} diff --git a/docs/generated/packages/workspace/generators/new.json b/docs/generated/packages/workspace/generators/new.json index 83420a5a10715..c4bb61ebfae93 100644 --- a/docs/generated/packages/workspace/generators/new.json +++ b/docs/generated/packages/workspace/generators/new.json @@ -44,11 +44,6 @@ "type": "string" }, "appName": { "type": "string", "description": "Application name." }, - "nxCloud": { - "description": "Connect the workspace to the free tier of the distributed cache provided by Nx Cloud.", - "type": "boolean", - "default": false - }, "linter": { "description": "The tool to use for running lint checks.", "type": "string", diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index a3eee7facd865..ea55f6f3cb716 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -511,6 +511,8 @@ - [noop](/nx-api/nx/executors/noop) - [run-commands](/nx-api/nx/executors/run-commands) - [run-script](/nx-api/nx/executors/run-script) + - [generators](/nx-api/nx/generators) + - [connect-to-nx-cloud](/nx-api/nx/generators/connect-to-nx-cloud) - [playwright](/nx-api/playwright) - [documents](/nx-api/playwright/documents) - [Overview](/nx-api/playwright/documents/overview) diff --git a/e2e/nx-run/src/nx-cloud.test.ts b/e2e/nx-run/src/nx-cloud.test.ts new file mode 100644 index 0000000000000..5ab3a649f1f3c --- /dev/null +++ b/e2e/nx-run/src/nx-cloud.test.ts @@ -0,0 +1,37 @@ +import { cleanupProject, newProject, runCLI } from '@nx/e2e/utils'; + +describe('Nx Cloud', () => { + beforeAll(() => + newProject({ + unsetProjectNameAndRootFormat: false, + }) + ); + + const libName = 'test-lib'; + beforeAll(() => { + runCLI('connect --no-interactive', { + env: { + ...process.env, + NX_CLOUD_API: 'https://staging.nx.app', + }, + }); + runCLI(`generate @nx/js:lib ${libName} --no-interactive`); + }); + + afterAll(() => cleanupProject()); + + it('should cache tests', async () => { + // Should be able to view logs with Nx Cloud + expect(runCLI(`test ${libName}`)).toContain( + `View logs and investigate cache misses at https://staging.nx.app` + ); + + // Reset Local cache + runCLI(`reset`); + + // Should be pull cache from Nx Cloud + expect(runCLI(`test ${libName}`)).toContain( + `Nx Cloud made it possible to reuse test-lib: https://staging.nx.app` + ); + }); +}); diff --git a/packages/create-nx-workspace/src/utils/nx/nx-cloud.ts b/packages/create-nx-workspace/src/utils/nx/nx-cloud.ts index 0d9174fbf13ba..984ece17ac7e7 100644 --- a/packages/create-nx-workspace/src/utils/nx/nx-cloud.ts +++ b/packages/create-nx-workspace/src/utils/nx/nx-cloud.ts @@ -13,7 +13,7 @@ export async function setupNxCloud( try { const pmc = getPackageManagerCommand(packageManager); const res = await execAndWait( - `${pmc.exec} nx g nx-cloud:init --no-analytics --installationSource=create-nx-workspace`, + `${pmc.exec} nx g nx:connect-to-nx-cloud --no-interactive --quiet`, directory ); nxCloudSpinner.succeed('NxCloud has been set up successfully'); diff --git a/packages/nx/bin/nx-cloud.ts b/packages/nx/bin/nx-cloud.ts new file mode 100644 index 0000000000000..054776d752277 --- /dev/null +++ b/packages/nx/bin/nx-cloud.ts @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +import { findAncestorNodeModules } from '../src/nx-cloud/resolution-helpers'; +import { getCloudOptions } from '../src/nx-cloud/utilities/get-cloud-options'; +import { + NxCloudClientUnavailableError, + NxCloudEnterpriseOutdatedError, + verifyOrUpdateNxCloudClient, +} from '../src/nx-cloud/update-manager'; +import type { CloudTaskRunnerOptions } from '../src/nx-cloud/nx-cloud-tasks-runner-shell'; +import { output } from '../src/utils/output'; + +const command = process.argv[2]; + +const options = getCloudOptions(); + +Promise.resolve().then(async () => invokeCommandWithNxCloudClient(options)); + +async function invokeCommandWithNxCloudClient(options: CloudTaskRunnerOptions) { + try { + const { nxCloudClient } = await verifyOrUpdateNxCloudClient(options); + + const paths = findAncestorNodeModules(__dirname, []); + nxCloudClient.configureLightClientRequire()(paths); + + if (command in nxCloudClient.commands) { + nxCloudClient.commands[command]() + .then(() => process.exit(0)) + .catch((e) => { + console.error(e); + process.exit(1); + }); + } else { + output.error({ + title: `Unknown Command "${command}"`, + }); + output.log({ + title: 'Available Commands:', + bodyLines: Object.keys(nxCloudClient.commands).map((c) => `- ${c}`), + }); + process.exit(1); + } + } catch (e: any) { + const body = ['Cannot run commands from the `nx-cloud` CLI.']; + + if (e instanceof NxCloudEnterpriseOutdatedError) { + body.push( + 'If you are an Nx Enterprise customer, please reach out to your assigned Developer Productivity Engineer.', + 'If you are NOT an Nx Enterprise customer but are seeing this message, please reach out to cloud-support@nrwl.io.' + ); + } + + if (e instanceof NxCloudClientUnavailableError) { + body.unshift( + 'You may be offline. Please try again when you are back online.' + ); + } + + output.error({ + title: e.message, + bodyLines: body, + }); + process.exit(1); + } +} diff --git a/packages/nx/generators.json b/packages/nx/generators.json new file mode 100644 index 0000000000000..d786a5b9d6de2 --- /dev/null +++ b/packages/nx/generators.json @@ -0,0 +1,10 @@ +{ + "generators": { + "connect-to-nx-cloud": { + "factory": "./src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud", + "schema": "./src/nx-cloud/generators/connect-to-nx-cloud/schema.json", + "description": "Connect a workspace to Nx Cloud", + "x-hidden": true + } + } +} diff --git a/packages/nx/migrations.json b/packages/nx/migrations.json index 1ea3b538560e5..c65caa6ee065d 100644 --- a/packages/nx/migrations.json +++ b/packages/nx/migrations.json @@ -68,7 +68,7 @@ }, "17.0.0-use-minimal-config-for-tasks-runner-options": { "cli": "nx", - "version": "17.0.0-beta.2", + "version": "17.0.0-beta.3", "description": "Use minimal config for tasksRunnerOptions", "implementation": "./src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options" } diff --git a/packages/nx/package.json b/packages/nx/package.json index 1beae3011ba91..70ecf6d6aa37b 100644 --- a/packages/nx/package.json +++ b/packages/nx/package.json @@ -23,7 +23,8 @@ "CLI" ], "bin": { - "nx": "./bin/nx.js" + "nx": "./bin/nx.js", + "nx-cloud": "./bin/nx-cloud.js" }, "author": "Victor Savkin", "license": "MIT", @@ -153,6 +154,7 @@ } ] }, + "generators": "./generators.json", "executors": "./executors.json", "builders": "./executors.json", "publishConfig": { diff --git a/packages/nx/src/command-line/connect/command-object.ts b/packages/nx/src/command-line/connect/command-object.ts index 2d220be37c9f6..1127d423964fe 100644 --- a/packages/nx/src/command-line/connect/command-object.ts +++ b/packages/nx/src/command-line/connect/command-object.ts @@ -1,13 +1,24 @@ import { CommandModule } from 'yargs'; import { linkToNxDevAndExamples } from '../yargs-utils/documentation'; +import type { ConnectToNxCloudOptions } from './connect-to-nx-cloud'; -export const yargsConnectCommand: CommandModule = { +export const yargsConnectCommand: CommandModule<{}, ConnectToNxCloudOptions> = { command: 'connect', aliases: ['connect-to-nx-cloud'], describe: `Connect workspace to Nx Cloud`, - builder: (yargs) => linkToNxDevAndExamples(yargs, 'connect-to-nx-cloud'), - handler: async () => { - await (await import('./connect-to-nx-cloud')).connectToNxCloudCommand(); + builder: (yargs) => + linkToNxDevAndExamples( + yargs.option('interactive', { + type: 'boolean', + description: 'Prompt for confirmation', + default: true, + }), + 'connect-to-nx-cloud' + ), + handler: async (options) => { + await ( + await import('./connect-to-nx-cloud') + ).connectToNxCloudCommand(options); process.exit(0); }, }; diff --git a/packages/nx/src/command-line/connect/connect-to-nx-cloud.ts b/packages/nx/src/command-line/connect/connect-to-nx-cloud.ts index 17351ae99ec88..ba0c1c26adb08 100644 --- a/packages/nx/src/command-line/connect/connect-to-nx-cloud.ts +++ b/packages/nx/src/command-line/connect/connect-to-nx-cloud.ts @@ -1,6 +1,4 @@ import { output } from '../../utils/output'; -import { getPackageManagerCommand } from '../../utils/package-manager'; -import { execSync } from 'child_process'; import { readNxJson } from '../../config/configuration'; import { getNxCloudToken, @@ -48,9 +46,15 @@ export async function connectToNxCloudIfExplicitlyAsked( } } -export async function connectToNxCloudCommand( - promptOverride?: string -): Promise { +export interface ConnectToNxCloudOptions { + interactive: boolean; + promptOverride?: string; +} + +export async function connectToNxCloudCommand({ + promptOverride, + interactive, +}: ConnectToNxCloudOptions): Promise { const nxJson = readNxJson(); if (isNxCloudUsed(nxJson)) { output.log({ @@ -68,21 +72,9 @@ export async function connectToNxCloudCommand( return false; } - const res = await connectToNxCloudPrompt(promptOverride); + const res = interactive ? await connectToNxCloudPrompt(promptOverride) : true; if (!res) return false; - const pmc = getPackageManagerCommand(); - if (pmc) { - execSync(`${pmc.addDev} nx-cloud@latest`); - } else { - const nxJson = readNxJson(); - if (nxJson.installation) { - nxJson.installation.plugins ??= {}; - nxJson.installation.plugins['nx-cloud'] = execSync( - `npm view nx-cloud@latest version` - ).toString(); - } - } - runNxSync(`g nx-cloud:init`, { + runNxSync(`g nx:connect-to-nx-cloud --quiet --no-interactive`, { stdio: [0, 1, 2], }); return true; diff --git a/packages/nx/src/command-line/connect/view-logs.ts b/packages/nx/src/command-line/connect/view-logs.ts index 3fc3df8d08da9..e8cb61c8048c4 100644 --- a/packages/nx/src/command-line/connect/view-logs.ts +++ b/packages/nx/src/command-line/connect/view-logs.ts @@ -42,26 +42,17 @@ export async function viewLogs(): Promise { if (!installCloud) return; const pmc = getPackageManagerCommand(); - try { - output.log({ - title: 'Installing nx-cloud', - }); - execSync(`${pmc.addDev} nx-cloud@latest`, { stdio: 'ignore' }); - } catch (e) { - output.log({ - title: 'Installation failed', - }); - console.log(e); - return 1; - } try { output.log({ title: 'Connecting to Nx Cloud', }); - runNxSync(`g nx-cloud:init --installation-source=view-logs`, { - stdio: 'ignore', - }); + runNxSync( + `g nx:connect-to-nx-cloud --installation-source=view-logs --quiet --no-interactive`, + { + stdio: 'ignore', + } + ); } catch (e) { output.log({ title: 'Failed to connect to Nx Cloud', diff --git a/packages/nx/src/command-line/init/implementation/add-nx-to-monorepo.ts b/packages/nx/src/command-line/init/implementation/add-nx-to-monorepo.ts index f2a3cabc329b6..b8653323780e3 100644 --- a/packages/nx/src/command-line/init/implementation/add-nx-to-monorepo.ts +++ b/packages/nx/src/command-line/init/implementation/add-nx-to-monorepo.ts @@ -88,7 +88,7 @@ export async function addNxToMonorepo(options: Options) { scriptOutputs ); - addDepsToPackageJson(repoRoot, useNxCloud); + addDepsToPackageJson(repoRoot); output.log({ title: '📦 Installing dependencies' }); runInstall(repoRoot); diff --git a/packages/nx/src/command-line/init/implementation/add-nx-to-nest.ts b/packages/nx/src/command-line/init/implementation/add-nx-to-nest.ts index 201756c9f1a78..8d9dab0119d33 100644 --- a/packages/nx/src/command-line/init/implementation/add-nx-to-nest.ts +++ b/packages/nx/src/command-line/init/implementation/add-nx-to-nest.ts @@ -117,7 +117,7 @@ export async function addNxToNest(options: Options, packageJson: PackageJson) { const pmc = getPackageManagerCommand(); - addDepsToPackageJson(repoRoot, useNxCloud); + addDepsToPackageJson(repoRoot); addNestPluginToPackageJson(repoRoot); markRootPackageJsonAsNxProject( repoRoot, diff --git a/packages/nx/src/command-line/init/implementation/add-nx-to-npm-repo.ts b/packages/nx/src/command-line/init/implementation/add-nx-to-npm-repo.ts index eb4cd1f2ce56a..a129d3070551c 100644 --- a/packages/nx/src/command-line/init/implementation/add-nx-to-npm-repo.ts +++ b/packages/nx/src/command-line/init/implementation/add-nx-to-npm-repo.ts @@ -72,7 +72,7 @@ export async function addNxToNpmRepo(options: Options) { const pmc = getPackageManagerCommand(); - addDepsToPackageJson(repoRoot, useNxCloud); + addDepsToPackageJson(repoRoot); markRootPackageJsonAsNxProject( repoRoot, cacheableOperations, diff --git a/packages/nx/src/command-line/init/implementation/angular/index.ts b/packages/nx/src/command-line/init/implementation/angular/index.ts index 4bcf05d490252..193a906791555 100644 --- a/packages/nx/src/command-line/init/implementation/angular/index.ts +++ b/packages/nx/src/command-line/init/implementation/angular/index.ts @@ -54,7 +54,7 @@ export async function addNxToAngularCliRepo(options: Options) { options.nxCloud ?? (options.interactive ? await askAboutNxCloud() : false); output.log({ title: '📦 Installing dependencies' }); - installDependencies(useNxCloud); + installDependencies(); output.log({ title: '📝 Setting up workspace' }); await setupWorkspace(cacheableOperations, options.integrated); @@ -107,8 +107,8 @@ async function collectCacheableOperations(options: Options): Promise { return cacheableOperations; } -function installDependencies(useNxCloud: boolean): void { - addDepsToPackageJson(repoRoot, useNxCloud); +function installDependencies(): void { + addDepsToPackageJson(repoRoot); addPluginDependencies(); runInstall(repoRoot); } diff --git a/packages/nx/src/command-line/init/implementation/angular/legacy-angular-versions.ts b/packages/nx/src/command-line/init/implementation/angular/legacy-angular-versions.ts index 2e36bcf08bcce..e5888f27d99fa 100644 --- a/packages/nx/src/command-line/init/implementation/angular/legacy-angular-versions.ts +++ b/packages/nx/src/command-line/init/implementation/angular/legacy-angular-versions.ts @@ -140,14 +140,6 @@ async function installDependencies( json.devDependencies[`${pkgInfo.pkgScope}/tao`] = pkgInfo.pkgVersion; } - if (useNxCloud) { - // get the latest nx-cloud version compatible with the Nx major - // version being installed - json.devDependencies['nx-cloud'] = await resolvePackageVersion( - 'nx-cloud', - `^${major(pkgInfo.pkgVersion)}.0.0` - ); - } json.devDependencies = sortObjectByKeys(json.devDependencies); if (pkgInfo.unscopedPkgName === 'angular') { diff --git a/packages/nx/src/command-line/init/implementation/utils.ts b/packages/nx/src/command-line/init/implementation/utils.ts index 8df9618c68fc1..4502dfe30323e 100644 --- a/packages/nx/src/command-line/init/implementation/utils.ts +++ b/packages/nx/src/command-line/init/implementation/utils.ts @@ -113,14 +113,11 @@ function deduceDefaultBase() { } } -export function addDepsToPackageJson(repoRoot: string, useCloud: boolean) { +export function addDepsToPackageJson(repoRoot: string) { const path = joinPathFragments(repoRoot, `package.json`); const json = readJsonFile(path); if (!json.devDependencies) json.devDependencies = {}; json.devDependencies['nx'] = nxVersion; - if (useCloud) { - json.devDependencies['nx-cloud'] = 'latest'; - } writeJsonFile(path, json); } @@ -140,10 +137,13 @@ export function initCloud( | 'nx-init-nest' | 'nx-init-npm-repo' ) { - runNxSync(`g nx-cloud:init --installationSource=${installationSource}`, { - stdio: [0, 1, 2], - cwd: repoRoot, - }); + runNxSync( + `g nx:connect-to-nx-cloud --installationSource=${installationSource} --quiet --no-interactive`, + { + stdio: [0, 1, 2], + cwd: repoRoot, + } + ); } export function addVsCodeRecommendedExtensions( diff --git a/packages/nx/src/command-line/migrate/migrate.ts b/packages/nx/src/command-line/migrate/migrate.ts index 5fcfbde7596cb..0045b2518638c 100644 --- a/packages/nx/src/command-line/migrate/migrate.ts +++ b/packages/nx/src/command-line/migrate/migrate.ts @@ -1218,9 +1218,10 @@ async function generateMigrationsJsonAndUpdatePackageJson( !isCI() && !isNxCloudUsed(originalNxJson) ) { - const useCloud = await connectToNxCloudCommand( - messages.getPromptMessage('nxCloudMigration') - ); + const useCloud = await connectToNxCloudCommand({ + promptOverride: messages.getPromptMessage('nxCloudMigration'), + interactive: true, + }); await recordStat({ command: 'migrate', nxVersion, diff --git a/packages/nx/src/command-line/yargs-utils/documentation.ts b/packages/nx/src/command-line/yargs-utils/documentation.ts index 90f1531e32d57..3466da3827c05 100644 --- a/packages/nx/src/command-line/yargs-utils/documentation.ts +++ b/packages/nx/src/command-line/yargs-utils/documentation.ts @@ -2,7 +2,10 @@ import chalk = require('chalk'); import yargs = require('yargs'); import { examples } from '../examples'; -export function linkToNxDevAndExamples(yargs: yargs.Argv, command: string) { +export function linkToNxDevAndExamples( + yargs: yargs.Argv, + command: string +) { (examples[command] || []).forEach((t) => { yargs = yargs.example(t.command, t.description); }); diff --git a/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.spec.ts b/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.spec.ts index c6e8555f1a187..fdd4ab9a36e6a 100644 --- a/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.spec.ts +++ b/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.spec.ts @@ -81,6 +81,12 @@ describe('use-minimal-config-for-tasks-runner-options migration', () => { }, }, }); + writeJson(tree, 'package.json', { + devDependencies: { + 'nx-cloud': 'latest', + nx: 'latest', + }, + }); await migrate(tree); @@ -89,6 +95,10 @@ describe('use-minimal-config-for-tasks-runner-options migration', () => { expect(nxJson.nxCloudUrl).toEqual('https://nx.app'); expect(nxJson.nxCloudEncryptionKey).toEqual('secret'); expect(nxJson.tasksRunnerOptions).not.toBeDefined(); + + expect(readJson(tree, 'package.json').devDependencies).toEqual({ + nx: 'latest', + }); }); it('should move nxCloudAccessToken and nxCloudUrl for @nrwl/nx-cloud', async () => { diff --git a/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.ts b/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.ts index 421c7c796faa8..d2d9ffa222b5a 100644 --- a/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.ts +++ b/packages/nx/src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options.ts @@ -1,6 +1,8 @@ import { updateJson } from '../../generators/utils/json'; import { Tree } from '../../generators/tree'; import { NxJsonConfiguration } from '../../config/nx-json'; +import { PackageJson } from '../../utils/package-json'; +import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available'; export default async function migrate(tree: Tree) { if (!tree.exists('nx.json')) { @@ -33,6 +35,19 @@ export default async function migrate(tree: Tree) { if (options.url) { nxJson.nxCloudUrl = options.url; delete options.url; + + if ( + [ + 'https://nx.app', + 'https://cloud.nx.app', + 'https://staging.nx.app', + 'https://snapshot.nx.app', + ].includes(nxJson.nxCloudUrl) + ) { + removeNxCloudDependency(tree); + } + } else { + removeNxCloudDependency(tree); } if (options.encryptionKey) { nxJson.nxCloudEncryptionKey = options.encryptionKey; @@ -69,4 +84,18 @@ export default async function migrate(tree: Tree) { } return nxJson; }); + + await formatChangedFilesWithPrettierIfAvailable(tree); +} + +function removeNxCloudDependency(tree: Tree) { + if (tree.exists('package.json')) { + updateJson(tree, 'package.json', (packageJson) => { + delete packageJson.dependencies?.['nx-cloud']; + delete packageJson.devDependencies?.['nx-cloud']; + delete packageJson.dependencies?.['@nrwl/nx-cloud']; + delete packageJson.devDependencies?.['@nrwl/nx-cloud']; + return packageJson; + }); + } } diff --git a/packages/nx/src/nx-cloud/debug-logger.ts b/packages/nx/src/nx-cloud/debug-logger.ts new file mode 100644 index 0000000000000..ad4eb6f0366ef --- /dev/null +++ b/packages/nx/src/nx-cloud/debug-logger.ts @@ -0,0 +1,5 @@ +export function debugLog(...args: any[]) { + if (process.env['NX_VERBOSE_LOGGING'] === 'true') { + console.log('[NX CLOUD]', ...args); + } +} diff --git a/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts b/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts new file mode 100644 index 0000000000000..0a4534e616265 --- /dev/null +++ b/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts @@ -0,0 +1,139 @@ +import { execSync } from 'child_process'; +import { URL } from 'node:url'; +import { output } from '../../../utils/output'; +import { Tree } from '../../../generators/tree'; +import { readJson } from '../../../generators/utils/json'; +import { NxJsonConfiguration } from '../../../config/nx-json'; +import { readNxJson, updateNxJson } from '../../../generators/utils/nx-json'; +import { formatChangedFilesWithPrettierIfAvailable } from '../../../generators/internal-utils/format-changed-files-with-prettier-if-available'; + +function printCloudConnectionDisabledMessage() { + output.error({ + title: `Connections to Nx Cloud are disabled for this workspace`, + bodyLines: [ + `This was an intentional decision by someone on your team.`, + `Nx Cloud cannot and will not be enabled.`, + ``, + `To allow connections to Nx Cloud again, remove the 'neverConnectToCloud'`, + `property in nx.json.`, + ], + }); +} + +function getRootPackageName(tree: Tree): string { + let packageJson; + try { + packageJson = readJson(tree, 'package.json'); + } catch (e) {} + return packageJson?.name ?? 'my-workspace'; +} +function removeTrailingSlash(apiUrl: string) { + return apiUrl[apiUrl.length - 1] === '/' + ? apiUrl.substr(0, apiUrl.length - 1) + : apiUrl; +} + +function getNxInitDate(): string | null { + try { + const nxInitIso = execSync( + 'git log --diff-filter=A --follow --format=%aI -- nx.json | tail -1', + { stdio: 'pipe' } + ) + .toString() + .trim(); + const nxInitDate = new Date(nxInitIso); + return nxInitDate.toISOString(); + } catch (e) { + return null; + } +} + +async function createNxCloudWorkspace( + workspaceName: string, + installationSource: string, + nxInitDate: string | null +): Promise<{ token: string; url: string }> { + const apiUrl = removeTrailingSlash( + process.env.NX_CLOUD_API || process.env.NRWL_API || `https://cloud.nx.app` + ); + const response = await require('axios').post( + `${apiUrl}/nx-cloud/create-org-and-workspace`, + { + workspaceName, + installationSource, + nxInitDate, + } + ); + + if (response.data.message) { + throw new Error(response.data.message); + } + + return response.data; +} + +function printSuccessMessage(url: string) { + let host = 'nx.app'; + try { + host = new URL(url).host; + } catch (e) {} + + output.note({ + title: `Distributed caching via Nx Cloud has been enabled`, + bodyLines: [ + `In addition to the caching, Nx Cloud provides config-free distributed execution,`, + `UI for viewing complex runs and GitHub integration. Learn more at https://nx.app`, + ``, + `Your workspace is currently unclaimed. Run details from unclaimed workspaces can be viewed on ${host} by anyone`, + `with the link. Claim your workspace at the following link to restrict access.`, + ``, + `${url}`, + ], + }); +} + +interface ConnectToNxCloudOptions { + analytics: boolean; + installationSource: string; +} + +function addNxCloudOptionsToNxJson( + tree: Tree, + nxJson: NxJsonConfiguration, + token: string +) { + nxJson.nxCloudAccessToken = token; + const overrideUrl = process.env.NX_CLOUD_API || process.env.NRWL_API; + if (overrideUrl) { + (nxJson as any).nxCloudUrl = overrideUrl; + } + updateNxJson(tree, nxJson); +} + +export async function connectToNxCloud( + tree: Tree, + schema: ConnectToNxCloudOptions +) { + const nxJson = readNxJson(tree); + + if ((nxJson as any).neverConnectToCloud) { + return () => { + printCloudConnectionDisabledMessage(); + }; + } else { + // TODO: Change to using loading light client when that is enabled by default + const r = await createNxCloudWorkspace( + getRootPackageName(tree), + schema.installationSource, + getNxInitDate() + ); + + addNxCloudOptionsToNxJson(tree, nxJson, r.token); + + await formatChangedFilesWithPrettierIfAvailable(tree); + + return () => printSuccessMessage(r.url); + } +} + +export default connectToNxCloud; diff --git a/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json b/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json new file mode 100644 index 0000000000000..21afa4edd4c22 --- /dev/null +++ b/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/schema", + "id": "NxCloudInit", + "title": "Add Nx Cloud Configuration to the workspace", + "description": "Connect a workspace to Nx Cloud.", + "type": "object", + "cli": "nx", + "properties": { + "analytics": { + "type": "boolean", + "description": "Anonymously store hashed machine ID for task runs", + "default": false + }, + "installationSource": { + "type": "string", + "description": "Name of Nx Cloud installation invoker (ex. user, add-nx-to-monorepo, create-nx-workspace, nx-upgrade", + "default": "user" + } + }, + "additionalProperties": false, + "required": [] +} diff --git a/packages/nx/src/nx-cloud/nx-cloud-tasks-runner-shell.ts b/packages/nx/src/nx-cloud/nx-cloud-tasks-runner-shell.ts new file mode 100644 index 0000000000000..8195cda3e6581 --- /dev/null +++ b/packages/nx/src/nx-cloud/nx-cloud-tasks-runner-shell.ts @@ -0,0 +1,68 @@ +import { findAncestorNodeModules } from './resolution-helpers'; +import { + NxCloudClientUnavailableError, + NxCloudEnterpriseOutdatedError, + verifyOrUpdateNxCloudClient, +} from './update-manager'; +import { + defaultTasksRunner, + DefaultTasksRunnerOptions, +} from '../tasks-runner/default-tasks-runner'; +import { TasksRunner } from '../tasks-runner/tasks-runner'; +import { output } from '../utils/output'; +import { Task } from '../config/task-graph'; + +export interface CloudTaskRunnerOptions extends DefaultTasksRunnerOptions { + accessToken?: string; + canTrackAnalytics?: boolean; + encryptionKey?: string; + maskedProperties?: string[]; + showUsageWarnings?: boolean; + customProxyConfigPath?: string; + useLatestApi?: boolean; + url?: string; + useLightClient?: boolean; + clientVersion?: string; +} + +export const nxCloudTasksRunnerShell: TasksRunner< + CloudTaskRunnerOptions +> = async (tasks: Task[], options: CloudTaskRunnerOptions, context) => { + try { + const { nxCloudClient, version } = await verifyOrUpdateNxCloudClient( + options + ); + + options.clientVersion = version; + + const paths = findAncestorNodeModules(__dirname, []); + nxCloudClient.configureLightClientRequire()(paths); + + return nxCloudClient.nxCloudTasksRunner(tasks, options, context); + } catch (e: any) { + const body = + e instanceof NxCloudEnterpriseOutdatedError + ? [ + 'If you are an Nx Enterprise customer, please reach out to your assigned Developer Productivity Engineer.', + 'If you are NOT an Nx Enterprise customer but are seeing this message, please reach out to cloud-support@nrwl.io.', + ] + : e instanceof NxCloudClientUnavailableError + ? [ + 'You might be offline. Nx Cloud will be re-enabled when you are back online.', + ] + : []; + + if (e instanceof NxCloudEnterpriseOutdatedError) { + output.warn({ + title: e.message, + bodyLines: ['Nx Cloud will not used for this command.', ...body], + }); + } + const results = await defaultTasksRunner(tasks, options, context); + output.warn({ + title: e.message, + bodyLines: ['Nx Cloud was not used for this command.', ...body], + }); + return results; + } +}; diff --git a/packages/nx/src/nx-cloud/resolution-helpers.ts b/packages/nx/src/nx-cloud/resolution-helpers.ts new file mode 100644 index 0000000000000..8e20e2b40a9c2 --- /dev/null +++ b/packages/nx/src/nx-cloud/resolution-helpers.ts @@ -0,0 +1,21 @@ +import { existsSync } from 'fs'; +import { dirname, isAbsolute, join, resolve } from 'path'; + +export function findAncestorNodeModules(startPath, collector) { + let currentPath = isAbsolute(startPath) ? startPath : resolve(startPath); + + while (currentPath !== dirname(currentPath)) { + const potentialNodeModules = join(currentPath, 'node_modules'); + if (existsSync(potentialNodeModules)) { + collector.push(potentialNodeModules); + } + + if (existsSync(join(currentPath, 'nx.json'))) { + break; + } + + currentPath = dirname(currentPath); + } + + return collector; +} diff --git a/packages/nx/src/nx-cloud/update-manager.ts b/packages/nx/src/nx-cloud/update-manager.ts new file mode 100644 index 0000000000000..6d8fadae9070f --- /dev/null +++ b/packages/nx/src/nx-cloud/update-manager.ts @@ -0,0 +1,351 @@ +import { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'; +import { + createWriteStream, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'fs'; +import { createGunzip } from 'zlib'; +import { join } from 'path'; +import { createApiAxiosInstance } from './utilities/axios'; +import { debugLog } from './debug-logger'; +import type { CloudTaskRunnerOptions } from './nx-cloud-tasks-runner-shell'; +import * as tar from 'tar-stream'; +import { cacheDir } from '../utils/cache-directory'; +import { createHash } from 'crypto'; +import { TasksRunner } from '../tasks-runner/tasks-runner'; + +interface CloudBundleInstall { + version: string; + fullPath: string; +} + +type ValidVerifyClientBundleResponse = { + valid: true; + url: null; + version: null; +}; + +type InvalidVerifyClientBundleResponse = { + valid: false; + url: string; + version: string; +}; + +type VerifyClientBundleResponse = + | ValidVerifyClientBundleResponse + | InvalidVerifyClientBundleResponse; + +export class NxCloudEnterpriseOutdatedError extends Error { + constructor(url: string) { + super(`Nx Cloud instance hosted at ${url} is outdated`); + } +} +export class NxCloudClientUnavailableError extends Error { + constructor() { + super('No existing Nx Cloud client and failed to download new version'); + } +} + +export interface NxCloudClient { + configureLightClientRequire: () => (paths: string[]) => void; + commands: Record Promise>; + nxCloudTasksRunner: TasksRunner; +} +export async function verifyOrUpdateNxCloudClient( + options: CloudTaskRunnerOptions +): Promise<{ nxCloudClient: NxCloudClient; version: string } | null> { + debugLog('Verifying current cloud bundle'); + const currentBundle = getLatestInstalledRunnerBundle(); + + if (shouldVerifyInstalledRunnerBundle(currentBundle)) { + const axios = createApiAxiosInstance(options); + + let verifyBundleResponse: AxiosResponse; + try { + verifyBundleResponse = await verifyCurrentBundle(axios, currentBundle); + } catch (e: any) { + // Enterprise image compatibility, to be removed + if (e.message === 'Request failed with status code 404' && options.url) { + throw new NxCloudEnterpriseOutdatedError(options.url); + } + + debugLog( + 'Could not verify bundle. Resetting validation timer and using previously installed or default runner. Error: ', + e + ); + writeBundleVerificationLock(); + + if (currentBundle === null) { + throw new NxCloudClientUnavailableError(); + } + + if (currentBundle.version === 'NX_ENTERPRISE_OUTDATED_IMAGE') { + throw new NxCloudEnterpriseOutdatedError(options.url); + } + + const nxCloudClient = require(currentBundle.fullPath); + if (nxCloudClient.commands === undefined) { + throw new NxCloudEnterpriseOutdatedError(options.url); + } + + return { + version: currentBundle.version, + nxCloudClient, + }; + } + + if (verifyBundleResponse.data.valid) { + debugLog('Currently installed bundle is valid'); + writeBundleVerificationLock(); + return { + version: currentBundle.version, + nxCloudClient: require(currentBundle.fullPath), + }; + } + + const { version, url } = verifyBundleResponse.data; + debugLog( + 'Currently installed bundle is invalid, downloading version', + version, + ' from ', + url + ); + + if (version === 'NX_ENTERPRISE_OUTDATED_IMAGE') { + throw new NxCloudEnterpriseOutdatedError(options.url); + } + + const fullPath = await downloadAndExtractClientBundle( + axios, + runnerBundleInstallDirectory, + version, + url + ); + + debugLog('Done: ', fullPath); + + const nxCloudClient = require(fullPath); + + if (nxCloudClient.commands === undefined) { + throw new NxCloudEnterpriseOutdatedError(options.url); + } + return { version, nxCloudClient }; + } + + if (currentBundle === null) { + throw new NxCloudClientUnavailableError(); + } + + debugLog('Done: ', currentBundle.fullPath); + + return { + version: currentBundle.version, + nxCloudClient: require(currentBundle.fullPath), + }; +} +const runnerBundleInstallDirectory = join(cacheDir, 'cloud'); + +function getLatestInstalledRunnerBundle(): CloudBundleInstall | null { + if (!existsSync(runnerBundleInstallDirectory)) { + mkdirSync(runnerBundleInstallDirectory, { recursive: true }); + } + + try { + const installedBundles: CloudBundleInstall[] = readdirSync( + runnerBundleInstallDirectory + ) + .filter((potentialDirectory) => { + return statSync( + join(runnerBundleInstallDirectory, potentialDirectory) + ).isDirectory(); + }) + .map((fileOrDirectory) => ({ + version: fileOrDirectory, + fullPath: join(runnerBundleInstallDirectory, fileOrDirectory), + })); + + if (installedBundles.length === 0) { + // No installed bundles + return null; + } + + return installedBundles[0]; + } catch (e: any) { + console.log('Could not read runner bundle path:', e.message); + return null; + } +} + +function shouldVerifyInstalledRunnerBundle( + currentBundle: CloudBundleInstall | null +): boolean { + if (process.env.NX_CLOUD_FORCE_REVALIDATE === 'true') { + return true; + } + + // No bundle, need to download anyway + if (currentBundle != null) { + debugLog('A local bundle currently exists: ', currentBundle); + const lastVerification = getLatestBundleVerificationTimestamp(); + // Never been verified, need to verify + if (lastVerification != null) { + // If last verification was less than 30 minutes ago, return the current installed bundle + const THIRTY_MINUTES = 30 * 60 * 1000; + if (Date.now() - lastVerification < THIRTY_MINUTES) { + debugLog( + 'Last verification was within the past 30 minutes, will not verify this time' + ); + return false; + } + debugLog( + 'Last verification was more than 30 minutes ago, verifying bundle is still valid' + ); + } + } + return true; +} + +async function verifyCurrentBundle( + axios: AxiosInstance, + currentBundle: CloudBundleInstall | null +): Promise> { + const contentHash = getBundleContentHash(currentBundle); + const queryParams = + currentBundle && contentHash + ? `?${new URLSearchParams({ + version: currentBundle.version, + contentHash: contentHash, + }).toString()}` + : ''; + return axios.get('/nx-cloud/client/verify' + queryParams); +} + +function getLatestBundleVerificationTimestamp(): number | null { + const lockfilePath = join(runnerBundleInstallDirectory, 'verify.lock'); + + if (existsSync(lockfilePath)) { + const timestampAsString = readFileSync(lockfilePath, 'utf-8'); + + let timestampAsNumber: number; + try { + timestampAsNumber = Number(timestampAsString); + return timestampAsNumber; + } catch (e) { + return null; + } + } + return null; +} + +function writeBundleVerificationLock() { + const lockfilePath = join(runnerBundleInstallDirectory, 'verify.lock'); + + writeFileSync(lockfilePath, new Date().getTime().toString(), 'utf-8'); +} + +function getBundleContentHash( + bundle: CloudBundleInstall | null +): string | null { + if (bundle == null) { + return null; + } + + return hashDirectory(bundle.fullPath); +} + +function hashDirectory(dir: string): string { + const files = readdirSync(dir).sort(); + const hashes = files.map((file) => { + const filePath = join(dir, file); + const stat = statSync(filePath); + + // If the current path is a directory, recursively hash the contents + if (stat.isDirectory()) { + return hashDirectory(filePath); + } + + // If it's a file, hash the file contents + const content = readFileSync(filePath); + return createHash('sha256').update(content).digest('hex'); + }); + + // Hash the combined hashes of the directory's contents + const combinedHashes = hashes.sort().join(''); + return createHash('sha256').update(combinedHashes).digest('hex'); +} + +async function downloadAndExtractClientBundle( + axios: AxiosInstance, + runnerBundleInstallDirectory: string, + version: string, + url: string +): Promise { + let resp; + try { + resp = await axios.get(url, { + responseType: 'stream', + } as AxiosRequestConfig); + } catch (e: any) { + console.error('Error while updating Nx Cloud client bundle'); + throw e; + } + + const bundleExtractLocation = join(runnerBundleInstallDirectory, version); + + if (!existsSync(bundleExtractLocation)) { + mkdirSync(bundleExtractLocation); + } + return new Promise((res, rej) => { + const extract = tar.extract(); + extract.on('entry', function (headers, stream, next) { + if (headers.type === 'directory') { + const directoryPath = join(bundleExtractLocation, headers.name); + if (!existsSync(directoryPath)) { + mkdirSync(directoryPath, { recursive: true }); + } + next(); + + stream.resume(); + } else if (headers.type === 'file') { + const outputFilePath = join(bundleExtractLocation, headers.name); + const writeStream = createWriteStream(outputFilePath); + stream.pipe(writeStream); + + stream.on('end', function () { + next(); + }); + + stream.resume(); + } + }); + + extract.on('error', (e) => { + rej(e); + }); + + extract.on('finish', function () { + removeOldClientBundles(version); + writeBundleVerificationLock(); + res(bundleExtractLocation); + }); + + resp.data.pipe(createGunzip()).pipe(extract); + }); +} + +function removeOldClientBundles(currentInstallVersion: string) { + const filesAndFolders = readdirSync(runnerBundleInstallDirectory); + + for (let fileOrFolder of filesAndFolders) { + const fileOrFolderPath = join(runnerBundleInstallDirectory, fileOrFolder); + + if (fileOrFolder !== currentInstallVersion) { + rmSync(fileOrFolderPath, { recursive: true }); + } + } +} diff --git a/packages/nx/src/nx-cloud/utilities/axios.ts b/packages/nx/src/nx-cloud/utilities/axios.ts new file mode 100644 index 0000000000000..7e0f617b89e67 --- /dev/null +++ b/packages/nx/src/nx-cloud/utilities/axios.ts @@ -0,0 +1,39 @@ +import { AxiosRequestConfig } from 'axios'; +import { join } from 'path'; +import { + ACCESS_TOKEN, + NX_CLOUD_NO_TIMEOUTS, + UNLIMITED_TIMEOUT, +} from './environment'; +import { CloudTaskRunnerOptions } from '../nx-cloud-tasks-runner-shell'; + +const axios = require('axios'); + +export function createApiAxiosInstance(options: CloudTaskRunnerOptions) { + let axiosConfigBuilder = (axiosConfig: AxiosRequestConfig) => axiosConfig; + const baseUrl = + process.env.NX_CLOUD_API || options.url || 'https://cloud.nx.app'; + const accessToken = ACCESS_TOKEN ? ACCESS_TOKEN : options.accessToken!; + + if (!accessToken) { + throw new Error( + `Unable to authenticate. Either define accessToken in nx.json or set the NX_CLOUD_ACCESS_TOKEN env variable.` + ); + } + + if (options.customProxyConfigPath) { + const { nxCloudProxyConfig } = require(join( + process.cwd(), + options.customProxyConfigPath + )); + axiosConfigBuilder = nxCloudProxyConfig ?? axiosConfigBuilder; + } + + return axios.create( + axiosConfigBuilder({ + baseURL: baseUrl, + timeout: NX_CLOUD_NO_TIMEOUTS ? UNLIMITED_TIMEOUT : 10000, + headers: { authorization: accessToken }, + }) + ); +} diff --git a/packages/nx/src/nx-cloud/utilities/environment.ts b/packages/nx/src/nx-cloud/utilities/environment.ts new file mode 100644 index 0000000000000..5b27435507751 --- /dev/null +++ b/packages/nx/src/nx-cloud/utilities/environment.ts @@ -0,0 +1,48 @@ +import * as dotenv from 'dotenv'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { isCI } from '../../utils/is-ci'; +import { workspaceRoot } from '../../utils/workspace-root'; + +// Set once +export const UNLIMITED_TIMEOUT = 9999999; +process.env.NX_CLOUD_AGENT_TIMEOUT_MS + ? Number(process.env.NX_CLOUD_AGENT_TIMEOUT_MS) + : 3600000; +// 60 minutes +process.env.NX_CLOUD_ORCHESTRATOR_TIMEOUT_MS + ? Number(process.env.NX_CLOUD_ORCHESTRATOR_TIMEOUT_MS) + : 3600000; +// 60 minutes +process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_AGENT_COUNT + ? Number(process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_AGENT_COUNT) + : null; +process.env.NX_CLOUD_NUMBER_OF_RETRIES + ? Number(process.env.NX_CLOUD_NUMBER_OF_RETRIES) + : isCI() + ? 10 + : 1; +export let ACCESS_TOKEN; +export let NX_CLOUD_NO_TIMEOUTS; + +loadEnvVars(); +function parseEnv() { + try { + const envContents = readFileSync(join(workspaceRoot, 'nx-cloud.env')); + return dotenv.parse(envContents); + } catch (e) { + return {}; + } +} + +function loadEnvVars() { + const parsed = parseEnv(); + ACCESS_TOKEN = + process.env.NX_CLOUD_AUTH_TOKEN || + process.env.NX_CLOUD_ACCESS_TOKEN || + parsed.NX_CLOUD_AUTH_TOKEN || + parsed.NX_CLOUD_ACCESS_TOKEN; + NX_CLOUD_NO_TIMEOUTS = + process.env.NX_CLOUD_NO_TIMEOUTS === 'true' || + parsed.NX_CLOUD_NO_TIMEOUTS === 'true'; +} diff --git a/packages/nx/src/nx-cloud/utilities/get-cloud-options.ts b/packages/nx/src/nx-cloud/utilities/get-cloud-options.ts new file mode 100644 index 0000000000000..3aa652c3b6ae4 --- /dev/null +++ b/packages/nx/src/nx-cloud/utilities/get-cloud-options.ts @@ -0,0 +1,10 @@ +import { CloudTaskRunnerOptions } from '../nx-cloud-tasks-runner-shell'; +import { readNxJson } from '../../config/nx-json'; +import { getRunnerOptions } from '../../tasks-runner/run-command'; + +export function getCloudOptions(): CloudTaskRunnerOptions { + const nxJson = readNxJson(); + + // TODO: The default is not always cloud? But it's not handled at the moment + return getRunnerOptions('default', nxJson, {}, true); +} diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index 455b7e0ddc6db..acdcc9f9e9a84 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -419,6 +419,27 @@ function shouldUseDynamicLifeCycle( return !tasks.find((t) => shouldStreamOutput(t, null)); } +function loadTasksRunner(modulePath: string) { + try { + const maybeTasksRunner = require(modulePath) as + | TasksRunner + | { default: TasksRunner }; + // to support both babel and ts formats + return 'default' in maybeTasksRunner + ? maybeTasksRunner.default + : maybeTasksRunner; + } catch (e) { + if ( + e.code === 'MODULE_NOT_FOUND' && + (modulePath === 'nx-cloud' || modulePath === '@nrwl/nx-cloud') + ) { + return require('../nx-cloud/nx-cloud-tasks-runner-shell') + .nxCloudTasksRunnerShell; + } + throw e; + } +} + export function getRunner( nxArgs: NxArgs, nxJson: NxJsonConfiguration @@ -435,21 +456,21 @@ export function getRunner( const modulePath: string = getTasksRunnerPath(runner, nxJson); - let tasksRunner = require(modulePath); - // to support both babel and ts formats - if (tasksRunner.default) { - tasksRunner = tasksRunner.default; - } + try { + const tasksRunner = loadTasksRunner(modulePath); - return { - tasksRunner, - runnerOptions: getRunnerOptions( - runner, - nxJson, - nxArgs, - modulePath === 'nx-cloud' - ), - }; + return { + tasksRunner, + runnerOptions: getRunnerOptions( + runner, + nxJson, + nxArgs, + modulePath === 'nx-cloud' + ), + }; + } catch { + throw new Error(`Could not find runner configuration for ${runner}`); + } } function getTasksRunnerPath( runner: string, @@ -473,7 +494,7 @@ function getTasksRunnerPath( return isCloudRunner ? 'nx-cloud' : require.resolve('./default-tasks-runner'); } -function getRunnerOptions( +export function getRunnerOptions( runner: string, nxJson: NxJsonConfiguration, nxArgs: NxArgs, diff --git a/packages/workspace/src/generators/new/new.ts b/packages/workspace/src/generators/new/new.ts index 1a52b8e600832..4fd3a77831d45 100644 --- a/packages/workspace/src/generators/new/new.ts +++ b/packages/workspace/src/generators/new/new.ts @@ -1,5 +1,4 @@ import { - addDependenciesToPackageJson, getPackageManagerCommand, installPackagesTask, joinPathFragments, @@ -21,7 +20,6 @@ interface Schema { appName?: string; skipInstall?: boolean; style?: string; - nxCloud?: boolean; preset: string; defaultBase: string; framework?: string; @@ -49,8 +47,6 @@ export async function newGenerator(tree: Tree, opts: Schema) { addPresetDependencies(tree, options); - addCloudDependencies(tree, options); - return async () => { const pmc = getPackageManagerCommand(options.packageManager); if (pmc.preInstall) { @@ -78,9 +74,6 @@ function validateOptions(options: Schema, host: Tree) { ) { throw new Error(`Cannot select a preset when skipInstall is set to true.`); } - if (options.skipInstall && options.nxCloud) { - throw new Error(`Cannot select nxCloud when skipInstall is set to true.`); - } if ( (options.preset === Preset.NodeStandalone || @@ -144,14 +137,3 @@ function normalizeOptions(options: Schema): NormalizedSchema { return normalized as NormalizedSchema; } - -function addCloudDependencies(host: Tree, options: Schema) { - if (options.nxCloud) { - return addDependenciesToPackageJson( - host, - {}, - { 'nx-cloud': 'latest' }, - join(options.directory, 'package.json') - ); - } -} diff --git a/packages/workspace/src/generators/new/schema.json b/packages/workspace/src/generators/new/schema.json index 7e57cf0614754..8be3cf05a6cd2 100644 --- a/packages/workspace/src/generators/new/schema.json +++ b/packages/workspace/src/generators/new/schema.json @@ -47,11 +47,6 @@ "type": "string", "description": "Application name." }, - "nxCloud": { - "description": "Connect the workspace to the free tier of the distributed cache provided by Nx Cloud.", - "type": "boolean", - "default": false - }, "linter": { "description": "The tool to use for running lint checks.", "type": "string",