From 6db9f714a42eb7203754048b484ec485c3cd8a10 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Fri, 8 Sep 2023 15:05:30 -0600 Subject: [PATCH 1/5] feat: add JIT tests --- messages/cli.install.jit.test.md | 7 ++ package.json | 1 + src/commands/cli/install/jit/test.ts | 26 +++++ src/commands/cli/tarballs/smoke.ts | 82 ++-------------- src/jit.ts | 142 +++++++++++++++++++++++++++ yarn.lock | 2 +- 6 files changed, 186 insertions(+), 74 deletions(-) create mode 100644 messages/cli.install.jit.test.md create mode 100644 src/commands/cli/install/jit/test.ts create mode 100644 src/jit.ts diff --git a/messages/cli.install.jit.test.md b/messages/cli.install.jit.test.md new file mode 100644 index 00000000..7992da7b --- /dev/null +++ b/messages/cli.install.jit.test.md @@ -0,0 +1,7 @@ +# summary + +Test that all JIT plugins can be successfully installed. + +# examples + +- <%= config.bin %> <%= command.id %> diff --git a/package.json b/package.json index d91dd117..c9db2723 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "semver": "^7.5.2", "shelljs": "^0.8.4", "standard-version": "^9.0.0", + "strip-ansi": "^6", "tslib": "^2", "yarn-deduplicate": "^3.1.0" }, diff --git a/src/commands/cli/install/jit/test.ts b/src/commands/cli/install/jit/test.ts new file mode 100644 index 00000000..52bf2aec --- /dev/null +++ b/src/commands/cli/install/jit/test.ts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { join } from 'path'; +import { SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { testJITInstall } from '../../../../jit'; + +Messages.importMessagesDirectory(__dirname); +const messages = Messages.loadMessages('@salesforce/plugin-release-management', 'cli.install.jit.test'); + +export default class Test extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly examples = messages.getMessages('examples'); + + public async run(): Promise { + await testJITInstall({ + jsonEnabled: this.jsonEnabled(), + executable: process.platform === 'win32' ? join('bin', 'run.cmd') : join('bin', 'run'), + }); + } +} diff --git a/src/commands/cli/tarballs/smoke.ts b/src/commands/cli/tarballs/smoke.ts index f7cd703c..c6382f70 100644 --- a/src/commands/cli/tarballs/smoke.ts +++ b/src/commands/cli/tarballs/smoke.ts @@ -8,7 +8,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { exec as execSync, ExecException } from 'child_process'; +import { exec as execSync } from 'child_process'; import { promisify } from 'node:util'; import * as chalk from 'chalk'; import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; @@ -17,6 +17,7 @@ import { Duration, parseJson, ThrottledPromiseAll } from '@salesforce/kit'; import { Interfaces } from '@oclif/core'; import { CLI } from '../../../types'; import { PackageJson } from '../../../package'; +import { testJITInstall } from '../../../jit'; const exec = promisify(execSync); @@ -45,11 +46,7 @@ export default class SmokeTest extends SfCommand { public async run(): Promise { this.flags = (await this.parse(SmokeTest)).flags; - const executables = [path.join('tmp', this.flags.cli, 'bin', this.flags.cli)]; - if (this.flags.cli === CLI.SFDX) { - executables.push(path.join('tmp', this.flags.cli, 'bin', CLI.SF)); - } - await Promise.all(executables.map((executable) => this.smokeTest(executable))); + await this.smokeTest(path.join('tmp', this.flags.cli, 'bin', this.flags.cli)); } private async smokeTest(executable: string): Promise { @@ -60,77 +57,16 @@ export default class SmokeTest extends SfCommand { this.testInstall(executable, '@salesforce/plugin-alias', 'latest'), ]); - // Only run JIT tests for the main executable - if (this.flags.cli === CLI.SFDX && !executable.endsWith('sf')) { - await this.testJITInstall(executable); - } + await this.testJITInstall(executable); await this.initializeAllCommands(executable); } private async testJITInstall(executable: string): Promise { - this.styledHeader('Testing JIT installation'); - const fileData = await fs.promises.readFile('package.json', 'utf8'); - const packageJson = parseJson(fileData) as PackageJson; - const jitPlugins = Object.keys(packageJson.oclif?.jitPlugins ?? {}); - if (jitPlugins.length === 0) return; - - const manifestData = await fs.promises.readFile(path.join('tmp', this.flags.cli, 'oclif.manifest.json'), 'utf8'); - const manifest = parseJson(manifestData) as Interfaces.Manifest; - - const commands = Object.values(manifest.commands); - let failed = false; - - const help = async (command: string): Promise => { - try { - await exec(`${executable} ${command} --help`); - return true; - } catch (e) { - return false; - } - }; - - // We have to test these serially in order to avoid issues when running plugin installs concurrently. - for (const plugin of jitPlugins) { - try { - this.log(`Testing JIT install for ${plugin}`); - const firstCommand = commands.find((c) => c.pluginName === plugin); - if (!firstCommand) { - throw new SfError(`Unable to find command for ${plugin}`); - } - - // Test that --help works on JIT commands - const helpResult = await help(firstCommand.id); - this.log(`${executable} ${firstCommand.id} --help ${helpResult ? chalk.green('PASSED') : chalk.red('FAILED')}`); - - this.log(`${executable} ${firstCommand.id}`); - // Test that executing the command will trigger JIT install - // This will likely always fail because we're not providing all the required flags or it depends on some other setup. - // However, this is okay because all we need to verify is that running the command will trigger the JIT install - const { stdout, stderr } = await exec(`${executable} ${firstCommand.id}`, { maxBuffer: 1024 * 1024 * 100 }); - this.log(stdout); - this.log(stderr); - } catch (e) { - const err = e as ExecException; - // @ts-expect-error ExecException type doesn't have a stdout or stderr property - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - this.log(err.stdout); - // @ts-expect-error ExecException type doesn't have a stdout or stderr property - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - this.log(err.stderr); - } finally { - const result = await this.verifyInstall(plugin, executable, true); - if (result) { - this.log(`✅ ${chalk.green(`Verified installation of ${plugin}\n`)}`); - } else { - failed = true; - this.log(`❌ ${chalk.red(`Failed installation of ${plugin}\n`)}`); - } - } - } - - if (failed) { - throw new SfError('Failed JIT installation'); - } + await testJITInstall({ + jsonEnabled: this.jsonEnabled(), + executable, + manifestPath: path.join('tmp', this.flags.cli, 'oclif.manifest.json'), + }); } private async testInstall(executable: string, plugin: string, tag?: string): Promise { diff --git a/src/jit.ts b/src/jit.ts new file mode 100644 index 00000000..22c4c737 --- /dev/null +++ b/src/jit.ts @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +/* eslint-disable no-await-in-loop */ +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { promisify } from 'util'; +import { exec as execSync, ExecException } from 'child_process'; +import { Ux } from '@salesforce/sf-plugins-core'; +import * as chalk from 'chalk'; +import { SfError } from '@salesforce/core'; +import { parseJson } from '@salesforce/kit'; +import { Interfaces } from '@oclif/core'; +import stripAnsi = require('strip-ansi'); +import { PackageJson } from './package'; + +const exec = promisify(execSync); + +type Options = { + jsonEnabled: boolean; + executable: string; + manifestPath?: string; +}; + +export async function testJITInstall(options: Options): Promise { + const { jsonEnabled, executable } = options; + const ux = new Ux({ jsonEnabled }); + + const tmpDir = path.join(os.tmpdir(), 'sf-jit-test'); + // Clear tmp dir before test to ensure that we're starting from a clean slate + await fs.promises.rm(tmpDir, { recursive: true, force: true }); + await fs.promises.mkdir(tmpDir, { recursive: true }); + + const dataDir = path.join(tmpDir, 'data'); + const cacheDir = path.join(tmpDir, 'cache'); + const configDir = path.join(tmpDir, 'config'); + + process.env.SF_DATA_DIR = dataDir; + process.env.SF_CACHE_DIR = cacheDir; + process.env.SF_CONFIG_DIR = configDir; + + await fs.promises.mkdir(dataDir, { recursive: true }); + await fs.promises.mkdir(cacheDir, { recursive: true }); + await fs.promises.mkdir(configDir, { recursive: true }); + + ux.styledHeader('Testing JIT installation'); + ux.log(`SF_DATA_DIR: ${dataDir}`); + ux.log(`SF_CACHE_DIR: ${cacheDir}`); + ux.log(`SF_CONFIG_DIR: ${configDir}`); + + const fileData = await fs.promises.readFile('package.json', 'utf8'); + const packageJson = parseJson(fileData) as PackageJson; + const jitPlugins = Object.keys(packageJson.oclif?.jitPlugins ?? {}); + if (jitPlugins.length === 0) return; + + let manifestData; + try { + manifestData = options.manifestPath + ? await fs.promises.readFile(options.manifestPath, 'utf8') + : await fs.promises.readFile('oclif.manifest.json', 'utf8'); + } catch { + ux.log('No oclif.manifest.json found. Generating one now.'); + await exec('yarn oclif manifest'); + manifestData = await fs.promises.readFile('oclif.manifest.json', 'utf8'); + } + + const manifest = parseJson(manifestData) as Interfaces.Manifest; + + const commands = Object.values(manifest.commands); + + const help = async (command: string): Promise => { + try { + await exec(`${executable} ${command} --help`); + return true; + } catch (e) { + return false; + } + }; + + const verifyInstall = async (plugin: string): Promise => { + const userPjsonRaw = await fs.promises.readFile(path.join(dataDir, 'package.json'), 'utf-8'); + + const userPjson = parseJson(userPjsonRaw) as PackageJson; + return Boolean(userPjson.dependencies?.[plugin]); + }; + + const passedInstalls: string[] = []; + const failedInstalls: string[] = []; + // We have to test these serially in order to avoid issues when running plugin installs concurrently. + for (const plugin of jitPlugins) { + try { + ux.log(`Testing JIT install for ${plugin}`); + const firstCommand = commands.find((c) => c.pluginName === plugin); + if (!firstCommand) { + throw new SfError(`Unable to find command for ${plugin}`); + } + + // Test that --help works on JIT commands + const helpResult = await help(firstCommand.id); + ux.log(`${executable} ${firstCommand.id} --help ${helpResult ? chalk.green('PASSED') : chalk.red('FAILED')}`); + + ux.log(`${executable} ${firstCommand.id}`); + // Test that executing the command will trigger JIT install + // This will likely always fail because we're not providing all the required flags or it depends on some other setup. + // However, this is okay because all we need to verify is that running the command will trigger the JIT install + const { stdout, stderr } = await exec(`${executable} ${firstCommand.id}`); + ux.log(stripAnsi(stdout)); + ux.log(stripAnsi(stderr)); + } catch (e) { + const err = e as ExecException; + // @ts-expect-error ExecException type doesn't have a stdout or stderr property + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + ux.log(stripAnsi(err.stdout)); + // @ts-expect-error ExecException type doesn't have a stdout or stderr property + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + ux.log(stripAnsi(err.stderr)); + } finally { + const result = await verifyInstall(plugin); + if (result) { + ux.log(`✅ ${chalk.green(`Verified installation of ${plugin}\n`)}`); + passedInstalls.push(plugin); + } else { + ux.log(`❌ ${chalk.red(`Failed installation of ${plugin}\n`)}`); + failedInstalls.push(plugin); + } + } + } + + ux.styledHeader('JIT Installation Results'); + ux.log(`Passed (${passedInstalls.length})`); + passedInstalls.forEach((msg) => ux.log(`• ${msg}`)); + if (failedInstalls.length) { + ux.log(); + ux.log(`Failed (${failedInstalls.length})`); + failedInstalls.forEach((msg) => ux.log(`• ${msg}`)); + throw new SfError('Failed JIT installation'); + } +} diff --git a/yarn.lock b/yarn.lock index 7c03a25d..24857a96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8467,7 +8467,7 @@ stringify-package@^1.0.1: resolved "https://registry.yarnpkg.com/stringify-package/-/stringify-package-1.0.1.tgz#e5aa3643e7f74d0f28628b72f3dad5cecfc3ba85" integrity sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg== -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@6.0.1, strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@6.0.1, strip-ansi@^6, strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== From f679d27fde797546e77a273bf28e396a7a844694 Mon Sep 17 00:00:00 2001 From: svc-cli-bot Date: Fri, 8 Sep 2023 21:07:24 +0000 Subject: [PATCH 2/5] chore(release): 4.1.29-dev.0 [skip ci] --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c9db2723..aa6766d7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@salesforce/plugin-release-management", "description": "A plugin for preparing and publishing npm packages", - "version": "4.1.28", + "version": "4.1.29-dev.0", "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "bin": { From bf2ebedb2750b7a6897c328d33a213ff820fe7a8 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Fri, 8 Sep 2023 15:17:19 -0600 Subject: [PATCH 3/5] chore: update snapshot --- command-snapshot.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/command-snapshot.json b/command-snapshot.json index 6430e51c..94b634c0 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -28,6 +28,14 @@ "flagChars": ["c", "p", "r"], "flagAliases": [] }, + { + "command": "cli:install:jit:test", + "plugin": "@salesforce/plugin-release-management", + "flags": ["json"], + "alias": [], + "flagChars": [], + "flagAliases": [] + }, { "command": "cli:install:test", "plugin": "@salesforce/plugin-release-management", From 421bbc4d352e82563b6780f3bfa253bf913f6eb1 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Wed, 13 Sep 2023 13:17:08 -0600 Subject: [PATCH 4/5] feat: remove cli flag from cli:tarballs:smoke --- command-snapshot.json | 4 ++-- messages/cli.tarballs.smoke.json | 1 - src/commands/cli/tarballs/smoke.ts | 14 +++----------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index 94b634c0..52712844 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -101,9 +101,9 @@ { "command": "cli:tarballs:smoke", "plugin": "@salesforce/plugin-release-management", - "flags": ["cli", "json", "verbose"], + "flags": ["json", "verbose"], "alias": [], - "flagChars": ["c"], + "flagChars": [], "flagAliases": [] }, { diff --git a/messages/cli.tarballs.smoke.json b/messages/cli.tarballs.smoke.json index 239f6e09..dd67ca2d 100644 --- a/messages/cli.tarballs.smoke.json +++ b/messages/cli.tarballs.smoke.json @@ -1,6 +1,5 @@ { "description": "smoke tests for the tarballed CLI\n Tests that the CLI and every command can be initialized.", "examples": ["<%= config.bin %> <%= command.id %> --cli sfdx", "<%= config.bin %> <%= command.id %> --cli sf"], - "cliFlag": "the cli to install", "verboseFlag": "show the --help output for each command" } diff --git a/src/commands/cli/tarballs/smoke.ts b/src/commands/cli/tarballs/smoke.ts index c6382f70..e785baa0 100644 --- a/src/commands/cli/tarballs/smoke.ts +++ b/src/commands/cli/tarballs/smoke.ts @@ -15,7 +15,6 @@ import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, SfError } from '@salesforce/core'; import { Duration, parseJson, ThrottledPromiseAll } from '@salesforce/kit'; import { Interfaces } from '@oclif/core'; -import { CLI } from '../../../types'; import { PackageJson } from '../../../package'; import { testJITInstall } from '../../../jit'; @@ -30,13 +29,6 @@ export default class SmokeTest extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static readonly flags = { - cli: Flags.custom({ - options: Object.values(CLI), - })({ - summary: messages.getMessage('cliFlag'), - char: 'c', - required: true, - }), verbose: Flags.boolean({ summary: messages.getMessage('verboseFlag'), }), @@ -46,7 +38,7 @@ export default class SmokeTest extends SfCommand { public async run(): Promise { this.flags = (await this.parse(SmokeTest)).flags; - await this.smokeTest(path.join('tmp', this.flags.cli, 'bin', this.flags.cli)); + await this.smokeTest(path.join('tmp', 'sf', 'bin', 'sf')); } private async smokeTest(executable: string): Promise { @@ -65,7 +57,7 @@ export default class SmokeTest extends SfCommand { await testJITInstall({ jsonEnabled: this.jsonEnabled(), executable, - manifestPath: path.join('tmp', this.flags.cli, 'oclif.manifest.json'), + manifestPath: path.join('tmp', 'sf', 'oclif.manifest.json'), }); } @@ -95,7 +87,7 @@ export default class SmokeTest extends SfCommand { } private async initializeAllCommands(executable: string): Promise { - this.styledHeader(`Initializing help for all ${this.flags.cli} commands`); + this.styledHeader("Initializing help for all 'sf' commands"); // Ran into memory issues when running all commands at once. Now we run them in batches of 10. const throttledPromise = new ThrottledPromiseAll({ concurrency: 10, From bb0626a33365c5a79c9d3c4ac252a47d1eb872db Mon Sep 17 00:00:00 2001 From: Eric Willhoit Date: Wed, 13 Sep 2023 16:12:45 -0500 Subject: [PATCH 5/5] chore: clean up messages --- messages/cli.tarballs.smoke.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/messages/cli.tarballs.smoke.json b/messages/cli.tarballs.smoke.json index dd67ca2d..5c345a3d 100644 --- a/messages/cli.tarballs.smoke.json +++ b/messages/cli.tarballs.smoke.json @@ -1,5 +1,5 @@ { - "description": "smoke tests for the tarballed CLI\n Tests that the CLI and every command can be initialized.", - "examples": ["<%= config.bin %> <%= command.id %> --cli sfdx", "<%= config.bin %> <%= command.id %> --cli sf"], + "description": "smoke tests for the sf CLI\n Tests that the CLI and every command can be initialized.", + "examples": ["<%= config.bin %> <%= command.id %>", "<%= config.bin %> <%= command.id %>"], "verboseFlag": "show the --help output for each command" }