feat: add JIT tests

command-snapshot.json

@@ -28,6 +28,14 @@
     "flagChars": ["c", "p", "r"],
     "flagAliases": []
   },
+  {
+    "command": "cli:install:jit:test",
+    "plugin": "@salesforce/plugin-release-management",
+    "flags": ["json"],
+    "alias": [],
+    "flagChars": [],
+    "flagAliases": []
+  },
   {
     "command": "cli:install:test",
     "plugin": "@salesforce/plugin-release-management",
@@ -93,9 +101,9 @@
   {
     "command": "cli:tarballs:smoke",
     "plugin": "@salesforce/plugin-release-management",
-    "flags": ["cli", "json", "verbose"],
+    "flags": ["json", "verbose"],
     "alias": [],
-    "flagChars": ["c"],
+    "flagChars": [],
     "flagAliases": []
   },
   {

messages/cli.install.jit.test.md

@@ -0,0 +1,7 @@
+# summary
+
+Test that all JIT plugins can be successfully installed.
+
+# examples
+
+- <%= config.bin %> <%= command.id %>

messages/cli.tarballs.smoke.json

@@ -1,6 +1,5 @@
 {
-  "description": "smoke tests for the tarballed CLI\n Tests that the CLI and every command can be initialized.",
-  "examples": ["<%= config.bin %> <%= command.id %> --cli sfdx", "<%= config.bin %> <%= command.id %> --cli sf"],
-  "cliFlag": "the cli to install",
+  "description": "smoke tests for the sf CLI\n Tests that the CLI and every command can be initialized.",
+  "examples": ["<%= config.bin %> <%= command.id %>", "<%= config.bin %> <%= command.id %>"],
   "verboseFlag": "show the --help output for each command"
 }

package.json

@@ -28,6 +28,7 @@
     "semver": "^7.5.2",
     "shelljs": "^0.8.4",
     "standard-version": "^9.0.0",
+    "strip-ansi": "^6",
     "tslib": "^2",
     "yarn-deduplicate": "^3.1.0"
   },

src/commands/cli/install/jit/test.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2023, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+
+import { join } from 'path';
+import { SfCommand } from '@salesforce/sf-plugins-core';
+import { Messages } from '@salesforce/core';
+import { testJITInstall } from '../../../../jit';
+
+Messages.importMessagesDirectory(__dirname);
+const messages = Messages.loadMessages('@salesforce/plugin-release-management', 'cli.install.jit.test');
+
+export default class Test extends SfCommand<void> {
+  public static readonly summary = messages.getMessage('summary');
+  public static readonly examples = messages.getMessages('examples');
+
+  public async run(): Promise<void> {
+    await testJITInstall({
+      jsonEnabled: this.jsonEnabled(),
+      executable: process.platform === 'win32' ? join('bin', 'run.cmd') : join('bin', 'run'),
+    });
+  }
+}

src/commands/cli/tarballs/smoke.ts

@@ -8,15 +8,15 @@
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
-import { exec as execSync, ExecException } from 'child_process';
+import { exec as execSync } from 'child_process';
 import { promisify } from 'node:util';
 import * as chalk from 'chalk';
 import { Flags, SfCommand } from '@salesforce/sf-plugins-core';
 import { Messages, SfError } from '@salesforce/core';
 import { Duration, parseJson, ThrottledPromiseAll } from '@salesforce/kit';
 import { Interfaces } from '@oclif/core';
-import { CLI } from '../../../types';
 import { PackageJson } from '../../../package';
+import { testJITInstall } from '../../../jit';
 
 const exec = promisify(execSync);
 
@@ -29,13 +29,6 @@ export default class SmokeTest extends SfCommand<void> {
 
   public static readonly examples = messages.getMessages('examples');
   public static readonly flags = {
-    cli: Flags.custom<CLI>({
-      options: Object.values(CLI),
-    })({
-      summary: messages.getMessage('cliFlag'),
-      char: 'c',
-      required: true,
-    }),
     verbose: Flags.boolean({
       summary: messages.getMessage('verboseFlag'),
     }),
@@ -45,11 +38,7 @@ export default class SmokeTest extends SfCommand<void> {
 
   public async run(): Promise<void> {
     this.flags = (await this.parse(SmokeTest)).flags;
-    const executables = [path.join('tmp', this.flags.cli, 'bin', this.flags.cli)];
-    if (this.flags.cli === CLI.SFDX) {
-      executables.push(path.join('tmp', this.flags.cli, 'bin', CLI.SF));
-    }
-    await Promise.all(executables.map((executable) => this.smokeTest(executable)));
+    await this.smokeTest(path.join('tmp', 'sf', 'bin', 'sf'));
   }
 
   private async smokeTest(executable: string): Promise<void> {
@@ -60,77 +49,16 @@ export default class SmokeTest extends SfCommand<void> {
       this.testInstall(executable, '@salesforce/plugin-alias', 'latest'),
     ]);
 
-    // Only run JIT tests for the main executable
-    if (this.flags.cli === CLI.SFDX && !executable.endsWith('sf')) {
-      await this.testJITInstall(executable);
-    }
+    await this.testJITInstall(executable);
     await this.initializeAllCommands(executable);
   }
 
   private async testJITInstall(executable: string): Promise<void> {
-    this.styledHeader('Testing JIT installation');
-    const fileData = await fs.promises.readFile('package.json', 'utf8');
-    const packageJson = parseJson(fileData) as PackageJson;
-    const jitPlugins = Object.keys(packageJson.oclif?.jitPlugins ?? {});
-    if (jitPlugins.length === 0) return;
-
-    const manifestData = await fs.promises.readFile(path.join('tmp', this.flags.cli, 'oclif.manifest.json'), 'utf8');
-    const manifest = parseJson(manifestData) as Interfaces.Manifest;
-
-    const commands = Object.values(manifest.commands);
-    let failed = false;
-
-    const help = async (command: string): Promise<boolean> => {
-      try {
-        await exec(`${executable} ${command} --help`);
-        return true;
-      } catch (e) {
-        return false;
-      }
-    };
-
-    // We have to test these serially in order to avoid issues when running plugin installs concurrently.
-    for (const plugin of jitPlugins) {
-      try {
-        this.log(`Testing JIT install for ${plugin}`);
-        const firstCommand = commands.find((c) => c.pluginName === plugin);
-        if (!firstCommand) {
-          throw new SfError(`Unable to find command for ${plugin}`);
-        }
-
-        // Test that --help works on JIT commands
-        const helpResult = await help(firstCommand.id);
-        this.log(`${executable} ${firstCommand.id} --help ${helpResult ? chalk.green('PASSED') : chalk.red('FAILED')}`);
-
-        this.log(`${executable} ${firstCommand.id}`);
-        // Test that executing the command will trigger JIT install
-        // This will likely always fail because we're not providing all the required flags or it depends on some other setup.
-        // However, this is okay because all we need to verify is that running the command will trigger the JIT install
-        const { stdout, stderr } = await exec(`${executable} ${firstCommand.id}`, { maxBuffer: 1024 * 1024 * 100 });
-        this.log(stdout);
-        this.log(stderr);
-      } catch (e) {
-        const err = e as ExecException;
-        // @ts-expect-error ExecException type doesn't have a stdout or stderr property
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        this.log(err.stdout);
-        // @ts-expect-error ExecException type doesn't have a stdout or stderr property
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-        this.log(err.stderr);
-      } finally {
-        const result = await this.verifyInstall(plugin, executable, true);
-        if (result) {
-          this.log(`✅ ${chalk.green(`Verified installation of ${plugin}\n`)}`);
-        } else {
-          failed = true;
-          this.log(`❌ ${chalk.red(`Failed installation of ${plugin}\n`)}`);
-        }
-      }
-    }
-
-    if (failed) {
-      throw new SfError('Failed JIT installation');
-    }
+    await testJITInstall({
+      jsonEnabled: this.jsonEnabled(),
+      executable,
+      manifestPath: path.join('tmp', 'sf', 'oclif.manifest.json'),
+    });
   }
 
   private async testInstall(executable: string, plugin: string, tag?: string): Promise<void> {
@@ -159,7 +87,7 @@ export default class SmokeTest extends SfCommand<void> {
   }
 
   private async initializeAllCommands(executable: string): Promise<void> {
-    this.styledHeader(`Initializing help for all ${this.flags.cli} commands`);
+    this.styledHeader("Initializing help for all 'sf' commands");
     // Ran into memory issues when running all commands at once. Now we run them in batches of 10.
     const throttledPromise = new ThrottledPromiseAll<string, string | void>({
       concurrency: 10,

src/jit.ts

@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2023, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+/* eslint-disable no-await-in-loop */
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { promisify } from 'util';
+import { exec as execSync, ExecException } from 'child_process';
+import { Ux } from '@salesforce/sf-plugins-core';
+import * as chalk from 'chalk';
+import { SfError } from '@salesforce/core';
+import { parseJson } from '@salesforce/kit';
+import { Interfaces } from '@oclif/core';
+import stripAnsi = require('strip-ansi');
+import { PackageJson } from './package';
+
+const exec = promisify(execSync);
+
+type Options = {
+  jsonEnabled: boolean;
+  executable: string;
+  manifestPath?: string;
+};
+
+export async function testJITInstall(options: Options): Promise<void> {
+  const { jsonEnabled, executable } = options;
+  const ux = new Ux({ jsonEnabled });
+
+  const tmpDir = path.join(os.tmpdir(), 'sf-jit-test');
+  // Clear tmp dir before test to ensure that we're starting from a clean slate
+  await fs.promises.rm(tmpDir, { recursive: true, force: true });
+  await fs.promises.mkdir(tmpDir, { recursive: true });
+
+  const dataDir = path.join(tmpDir, 'data');
+  const cacheDir = path.join(tmpDir, 'cache');
+  const configDir = path.join(tmpDir, 'config');
+
+  process.env.SF_DATA_DIR = dataDir;
+  process.env.SF_CACHE_DIR = cacheDir;
+  process.env.SF_CONFIG_DIR = configDir;
+
+  await fs.promises.mkdir(dataDir, { recursive: true });
+  await fs.promises.mkdir(cacheDir, { recursive: true });
+  await fs.promises.mkdir(configDir, { recursive: true });
+
+  ux.styledHeader('Testing JIT installation');
+  ux.log(`SF_DATA_DIR: ${dataDir}`);
+  ux.log(`SF_CACHE_DIR: ${cacheDir}`);
+  ux.log(`SF_CONFIG_DIR: ${configDir}`);
+
+  const fileData = await fs.promises.readFile('package.json', 'utf8');
+  const packageJson = parseJson(fileData) as PackageJson;
+  const jitPlugins = Object.keys(packageJson.oclif?.jitPlugins ?? {});
+  if (jitPlugins.length === 0) return;
+
+  let manifestData;
+  try {
+    manifestData = options.manifestPath
+      ? await fs.promises.readFile(options.manifestPath, 'utf8')
+      : await fs.promises.readFile('oclif.manifest.json', 'utf8');
+  } catch {
+    ux.log('No oclif.manifest.json found. Generating one now.');
+    await exec('yarn oclif manifest');
+    manifestData = await fs.promises.readFile('oclif.manifest.json', 'utf8');
+  }
+
+  const manifest = parseJson(manifestData) as Interfaces.Manifest;
+
+  const commands = Object.values(manifest.commands);
+
+  const help = async (command: string): Promise<boolean> => {
+    try {
+      await exec(`${executable} ${command} --help`);
+      return true;
+    } catch (e) {
+      return false;
+    }
+  };
+
+  const verifyInstall = async (plugin: string): Promise<boolean> => {
+    const userPjsonRaw = await fs.promises.readFile(path.join(dataDir, 'package.json'), 'utf-8');
+
+    const userPjson = parseJson(userPjsonRaw) as PackageJson;
+    return Boolean(userPjson.dependencies?.[plugin]);
+  };
+
+  const passedInstalls: string[] = [];
+  const failedInstalls: string[] = [];
+  // We have to test these serially in order to avoid issues when running plugin installs concurrently.
+  for (const plugin of jitPlugins) {
+    try {
+      ux.log(`Testing JIT install for ${plugin}`);
+      const firstCommand = commands.find((c) => c.pluginName === plugin);
+      if (!firstCommand) {
+        throw new SfError(`Unable to find command for ${plugin}`);
+      }
+
+      // Test that --help works on JIT commands
+      const helpResult = await help(firstCommand.id);
+      ux.log(`${executable} ${firstCommand.id} --help ${helpResult ? chalk.green('PASSED') : chalk.red('FAILED')}`);
+
+      ux.log(`${executable} ${firstCommand.id}`);
+      // Test that executing the command will trigger JIT install
+      // This will likely always fail because we're not providing all the required flags or it depends on some other setup.
+      // However, this is okay because all we need to verify is that running the command will trigger the JIT install
+      const { stdout, stderr } = await exec(`${executable} ${firstCommand.id}`);
+      ux.log(stripAnsi(stdout));
+      ux.log(stripAnsi(stderr));
+    } catch (e) {
+      const err = e as ExecException;
+      // @ts-expect-error ExecException type doesn't have a stdout or stderr property
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      ux.log(stripAnsi(err.stdout));
+      // @ts-expect-error ExecException type doesn't have a stdout or stderr property
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      ux.log(stripAnsi(err.stderr));
+    } finally {
+      const result = await verifyInstall(plugin);
+      if (result) {
+        ux.log(`✅ ${chalk.green(`Verified installation of ${plugin}\n`)}`);
+        passedInstalls.push(plugin);
+      } else {
+        ux.log(`❌ ${chalk.red(`Failed installation of ${plugin}\n`)}`);
+        failedInstalls.push(plugin);
+      }
+    }
+  }
+
+  ux.styledHeader('JIT Installation Results');
+  ux.log(`Passed (${passedInstalls.length})`);
+  passedInstalls.forEach((msg) => ux.log(`• ${msg}`));
+  if (failedInstalls.length) {
+    ux.log();
+    ux.log(`Failed (${failedInstalls.length})`);
+    failedInstalls.forEach((msg) => ux.log(`• ${msg}`));
+    throw new SfError('Failed JIT installation');
+  }
+}

yarn.lock

@@ -8504,7 +8504,7 @@ stringify-package@^1.0.1:
   resolved "https://registry.yarnpkg.com/stringify-package/-/stringify-package-1.0.1.tgz#e5aa3643e7f74d0f28628b72f3dad5cecfc3ba85"
   integrity sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg==
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", [email protected], strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", [email protected], strip-ansi@^6, strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==