From 3186fa4f10dd5586f15182e917b32dbcae79a2f4 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 3 Mar 2021 08:40:05 -0600 Subject: [PATCH 1/2] docs: updated readme (typing, exitCode) --- README.md | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 378a1c9d..0ecd0c79 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,17 @@ Using a different file extension will help separate your unit tests from your NU { "scripts": { "test": "mocha **/*.test.ts", - "test-nut": "mocha **/*.nut.ts" + "test:nuts": "mocha **/*.nut.ts" } } ``` +Because NUTs run much slower than unit tests, you may also want to add some mocha options + +- `--timeout 600000` to give tests time to complete +- `--parallel` to run tests in parallel (requires mocha >= 8) +- `--slow 4500` to set a reasonable value for the color coded output + **See [Samples](./SAMPLES.md) doc for many testkit usecases and sample code** ## Running Commands @@ -52,6 +58,37 @@ const result = execCmd('mycommand --myflag --json'); console.log(result.jsonOutput); ``` +### Simplified Assertions + +You can pass in an expected exit code in the options to avoid repetitively checking for it to easily assert pass/fail. + +```typescript +import { execCmd } from '@salesforce/cli-plugins-testkit'; + +execCmd('force:user:password:generate', { ensureExitCode: 0 }); +``` + +### Simplified Types + +You can set the return type to avoid casting/any/unknown in your tests. + +```typescript +import { execCmd } from '@salesforce/cli-plugins-testkit'; + +const output = execCmd('mycommand --myflag --json').jsonOutput; +``` + +`output` will be of type + +```typescript +{ + status: number; + result: SomeType; +} +``` + +### Other Executables + The executable can then be configured in CI using the `TESTKIT_EXECUTABLE_PATH`. ```bash @@ -64,8 +101,8 @@ npm install sfdx@latest-rc # Target the local sfdx export TESTKIT_EXECUTABLE_PATH=./node_modules/.bin/sfdx -# Run NUT test (requires a test-nut script target in the package.json) -yarn test-nut +# Run NUT test (requires a test:nuts script target in the package.json) +yarn test:nuts ``` You will notice that the executable is not configurable in the `execCmd` method directly. If you need to run other commands not located in your plugin, use shelljs directly. From 8080ad225e83e2ef4a5d4553db8446dd1d009280 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Wed, 3 Mar 2021 08:47:39 -0600 Subject: [PATCH 2/2] docs: typing on samples --- SAMPLES.md | 180 ++++++++++++++++++++++++++--------------------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/SAMPLES.md b/SAMPLES.md index 8c067dec..e899e858 100644 --- a/SAMPLES.md +++ b/SAMPLES.md @@ -1,12 +1,12 @@ -**NOTE:** *This is currently a static doc but will soon be generated with compiled example code so that the doc is alwas up to date.* +**NOTE:** _This is currently a static doc but will soon be generated with compiled example code so that the doc is always up to date._ # Using Testkit’s execCmd function -The execCmd function allows plugin commands to execute with a specific CLI executable, defaulting to the plugin’s ./bin/run . It can automatically ensure a specific exit code and throw an error when that exit code is not returned. Commands can be executed synchronously or asynchronously. All command results exitCode, stdout, and stderr are returned. If the --json flag was provided the results will have the parsed JSON output. Command execution time is provided as a Duration object for easy manipulation. +The execCmd function allows plugin commands to execute with a specific CLI executable, defaulting to the plugin’s ./bin/run . It can automatically ensure a specific exit code and throw an error when that exit code is not returned. Commands can be executed synchronously or asynchronously. All command results exitCode, stdout, and stderr are returned. If the --json flag was provided the results will have the parsed JSON output. Command execution time is provided as a Duration object for easy manipulation. ## Testing plugin commands synchronously -***Usecase: I have a plugin with commands that I want to run within tests synchronously using my plugin’s ./bin/run*** +**_Usecase: I have a plugin with commands that I want to run within tests synchronously using my plugin’s ./bin/run_** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; @@ -21,7 +21,7 @@ describe('Sample NUT', () => { ## Testing plugin commands asynchronously -***Usecase: I have a plugin with commands that I want to run within tests asynchronously.*** +**_Usecase: I have a plugin with commands that I want to run within tests asynchronously._** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; @@ -35,7 +35,7 @@ describe('Sample NUT', () => { ## Testing plugin commands exit code -***Usecase: I have a plugin with commands that I want to run and throw an error if a certain exit code is not returned automatically.*** +**_Usecase: I have a plugin with commands that I want to run and throw an error if a certain exit code is not returned automatically._** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; @@ -49,7 +49,7 @@ describe('Sample NUT', () => { ## Testing plugin commands with a specific SFDX binary -***Usecase: I have a plugin with commands that I want to run with a specific SFDX binary rather than my plugin’s ./bin/run.*** +**_Usecase: I have a plugin with commands that I want to run with a specific SFDX binary rather than my plugin’s ./bin/run._** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; @@ -66,7 +66,7 @@ describe('Sample NUT', () => { ## Testing plugin commands JSON output -***Usecase: I have a plugin with commands that I want to run and have parsed JSON output returned for easy verification.*** +**_Usecase: I have a plugin with commands that I want to run and have parsed JSON output returned for easy verification._** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; @@ -75,24 +75,40 @@ describe('Sample NUT', () => { it('using testkit to ensure exit code 0', () => { // Simply have your command use the --json flag const rv = execCmd('config:list --json'); - expect(rv.jsonOutput.result).deep.equals([{ - "key": "defaultdevhubusername", - "location": "Global", - "value": "myDevHub" - }]) + expect(rv.jsonOutput.result).deep.equals([ + { + key: 'defaultdevhubusername', + location: 'Global', + value: 'myDevHub', + }, + ]); }); }); ``` -## Getting command execution times +## Specifying return types -***Usecase: I want to ensure my plugin commands execute within an acceptable duration range.*** +**_Usecase: I have a plugin with commands that I want to run and have typed JSON output._** ```typescript import { execCmd } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { + it('using testkit with typescript types', () => { + const rv = execCmd('config:list --json'); + // rv is of type JsonResult with property "result" of type "SomeType" + }); +}); +``` + +## Getting command execution times +**_Usecase: I want to ensure my plugin commands execute within an acceptable duration range._** + +```typescript +import { execCmd } from '@salesforce/cli-plugins-testkit'; + +describe('Sample NUT', () => { it('config:list should execute in less than 5 seconds', () => { const t1 = execCmd(`config:list`).execCmdDuration.asMillis(); const t2 = execCmd(`config:list`).execCmdDuration.asMillis(); @@ -109,27 +125,26 @@ A TestSession provides conveniences to testing plugin commands with options to a ## Testing with generated sfdx project -***Usecase: I have a plugin with commands that require a SFDX project but the tests don’t care about the contents of the project.*** +**_Usecase: I have a plugin with commands that require a SFDX project but the tests don’t care about the contents of the project._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { testSession = TestSession.create({ project: { - name: 'MyTestProject' - } + name: 'MyTestProject', + }, }); }); it('using testkit to run sync commands', () => { execCmd('config:list', { ensureExitCode: 0 }); }); - + after(async () => { await testSession?.clean(); }); @@ -138,28 +153,27 @@ describe('Sample NUT', () => { ## Testing with local sfdx project -***Usecase: I have a plugin with commands that require a SFDX project and the test project I want to use is in a local directory within my plugin repo.*** +**_Usecase: I have a plugin with commands that require a SFDX project and the test project I want to use is in a local directory within my plugin repo._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import * as path from 'path'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { testSession = TestSession.create({ project: { - sourceDir: path.join(process.cwd(), 'localTestProj') - } + sourceDir: path.join(process.cwd(), 'localTestProj'), + }, }); }); it('using testkit to run sync commands', () => { execCmd('config:list', { ensureExitCode: 0 }); }); - + after(async () => { await testSession?.clean(); }); @@ -168,27 +182,26 @@ describe('Sample NUT', () => { ## Testing with git cloned sfdx project -***Usecase: I have a plugin with commands that require a SFDX project and the test project I want to use is in a git repo.*** +**_Usecase: I have a plugin with commands that require a SFDX project and the test project I want to use is in a git repo._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { testSession = TestSession.create({ project: { - gitClone: 'https://github.com/trailheadapps/ebikes-lwc.git' - } + gitClone: 'https://github.com/trailheadapps/ebikes-lwc.git', + }, }); }); it('using testkit to run sync commands', () => { execCmd('config:list', { ensureExitCode: 0 }); }); - + after(async () => { await testSession?.clean(); }); @@ -197,13 +210,12 @@ describe('Sample NUT', () => { ## Testing with no sfdx project -***Usecase: I have a plugin with commands that do not require a SFDX project but I want other things from a TestSession such as easy authentication to a devhub, access to a unique ID, homedir stub, etc.*** +**_Usecase: I have a plugin with commands that do not require a SFDX project but I want other things from a TestSession such as easy authentication to a devhub, access to a unique ID, homedir stub, etc._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -213,7 +225,7 @@ describe('Sample NUT', () => { it('using testkit to run sync commands', () => { execCmd(`config:set instanceUrl=${testSession.id}`, { ensureExitCode: 0 }); }); - + after(async () => { await testSession?.clean(); }); @@ -222,28 +234,27 @@ describe('Sample NUT', () => { ## Override the location of a TestSession -***Usecase: I want my tests to control the location of my TestSession.*** +**_Usecase: I want my tests to control the location of my TestSession._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { tmpdir } from 'os'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { testSession = TestSession.create({ // NOTE: you can also override with an env var. // See section on Testkit env vars. - sessionDir: tmpdir() + sessionDir: tmpdir(), }); }); it('using testkit to run sync commands', () => { execCmd(`config:set instanceUrl=${testSession.id}`); }); - + after(async () => { await testSession?.clean(); }); @@ -252,13 +263,12 @@ describe('Sample NUT', () => { ## Cleaning TestSessions -***Usecase: I want to use TestSessions but clean everything when the tests are done running.*** +**_Usecase: I want to use TestSessions but clean everything when the tests are done running._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -268,7 +278,7 @@ describe('Sample NUT', () => { it('using testkit to run sync commands', () => { execCmd(`config:set instanceUrl=${testSession.id}`, { ensureExitCode: 0 }); }); - + // Do this in your tests to keep your plugin clean. after(async () => { await testSession?.clean(); @@ -278,15 +288,14 @@ describe('Sample NUT', () => { ## Archiving TestSessions -***Usecase: I want to use TestSessions but zip the test session directory when the tests are done running.*** +**_Usecase: I want to use TestSessions but zip the test session directory when the tests are done running._** -***NOTE: Must set env var: `TESTKIT_ENABLE_ZIP=true`*** +**_NOTE: Must set env var: `TESTKIT_ENABLE_ZIP=true`_** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -296,7 +305,7 @@ describe('Sample NUT', () => { it('using testkit to run sync commands', () => { execCmd(`config:set instanceUrl=${testSession.id}`, { ensureExitCode: 0 }); }); - + after(async () => { await testSession?.zip(); await testSession?.clean(); @@ -306,15 +315,14 @@ describe('Sample NUT', () => { ## Archiving TestSessions on test failure -***Usecase: I want to use TestSessions but zip the test session directory when the tests fail.*** +**_Usecase: I want to use TestSessions but zip the test session directory when the tests fail._** -***NOTE: Must set env var: `TESTKIT_ENABLE_ZIP=true`*** +**_NOTE: Must set env var: `TESTKIT_ENABLE_ZIP=true`_** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -325,12 +333,12 @@ describe('Sample NUT', () => { execCmd(`config:set instanceUrl=${testSession.id}`, { ensureExitCode: 0 }); }); - afterEach(function() { + afterEach(function () { if (this.currentTest?.state !== 'passed') { await testSession?.zip(); } }); - + after(async () => { await testSession?.clean(); }); @@ -339,14 +347,13 @@ describe('Sample NUT', () => { ## Testing with setup commands -***Usecase: I have a plugin with commands and tests require other commands to run for setup, and I want to reference the command results of the setup commands.*** +**_Usecase: I have a plugin with commands and tests require other commands to run for setup, and I want to reference the command results of the setup commands._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { env } from '@salesforce/kit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -356,12 +363,12 @@ describe('Sample NUT', () => { testSession = TestSession.create({ project: { - name: 'MyTestProject' + name: 'MyTestProject', }, setupCommands: [ `sfdx auth:jwt:grant -i ${cid} -f ${jwtFile} -u ${devhub} -d`, - 'sfdx force:org:create edition=Developer' - ] + 'sfdx force:org:create edition=Developer', + ], }); }); @@ -370,7 +377,7 @@ describe('Sample NUT', () => { const username = testSession.setup[1].result.username; execCmd(`user:create -v ${devhubUsername} -u ${username}`); }); - + after(async () => { await testSession?.clean(); }); @@ -379,14 +386,13 @@ describe('Sample NUT', () => { ## Testing with scratch orgs -***Usecase: I have a plugin with commands that require an org.*** +**_Usecase: I have a plugin with commands that require an org._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { env } from '@salesforce/kit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -396,12 +402,12 @@ describe('Sample NUT', () => { testSession = TestSession.create({ project: { - name: 'MyTestProject' + name: 'MyTestProject', }, setupCommands: [ `sfdx auth:jwt:grant -i ${cid} -f ${jwtFile} -u ${devhub} -d -a devhub`, - 'sfdx force:org:create -f config/project-scratch-def.json' - ] + 'sfdx force:org:create -f config/project-scratch-def.json', + ], }); }); @@ -409,7 +415,7 @@ describe('Sample NUT', () => { const username = testSession.setup[1].result.username; execCmd(`user:create -u ${username}`); }); - + after(async () => { await testSession?.clean(); }); @@ -418,7 +424,7 @@ describe('Sample NUT', () => { ## Testkit Debug output -***Usecase: I want to see all the stuff that the testkit is doing, either because I’m curious or need to troubleshoot.*** +**_Usecase: I want to see all the stuff that the testkit is doing, either because I’m curious or need to troubleshoot._** ```bash # To see all testkit debug output, in your shell @@ -429,40 +435,37 @@ export DEBUG=testkit:project ## Testing with multiple test projects -***Usecase: Some of my plugin command tests require multiple SFDX test projects. How do I do that?*** +**_Usecase: Some of my plugin command tests require multiple SFDX test projects. How do I do that?_** ```typescript import { execCmd, TestSession, TestProject } from '@salesforce/cli-plugins-testkit'; import * as path from 'path'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { testSession = TestSession.create({ project: { - sourceDir: path.join(process.cwd(), 'localTestProj') + sourceDir: path.join(process.cwd(), 'localTestProj'), }, - setupCommands: [ - 'sfdx force:org:create -f config/project-scratch-def.json' - ] + setupCommands: ['sfdx force:org:create -f config/project-scratch-def.json'], }); }); it('using testkit to run sync commands', () => { execCmd('config:list', { ensureExitCode: 0 }); }); - + it('using testkit to run sync commands', () => { // Create another test project and reset the cwd stub const project2 = new TestProject({ - name: 'project2' + name: 'project2', }); testSession.stubCwd(project2.dir); execCmd(`force:source:pull -u ${testSession.orgUsername}`); }); - + after(async () => { await testSession?.clean(); }); @@ -471,9 +474,9 @@ describe('Sample NUT', () => { ## Testing with multiple scratch orgs -***Usecase: Some of my plugin command tests require multiple scratch orgs. How do I do that?*** +**_Usecase: Some of my plugin command tests require multiple scratch orgs. How do I do that?_** -***NOTE: Scratch orgs can take a while to create so you may want to create them in parallel in a global test fixture and refer to them in your tests. There are lots of possibilities though and this example shows a few ways how you might create multiple scratch orgs in a test file.*** +**_NOTE: Scratch orgs can take a while to create so you may want to create them in parallel in a global test fixture and refer to them in your tests. There are lots of possibilities though and this example shows a few ways how you might create multiple scratch orgs in a test file._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; @@ -498,25 +501,25 @@ describe('Sample NUT 1', () => { it('using testkit to run sync commands', () => { execCmd('config:list', { ensureExitCode: 0 }); }); - + it('create a 2nd org and get the username from the json output', () => { // Note that this org will not be deleted for you by TestSession. const rv = execCmd(`org:create -f config/project-scratch-def.json --json`); const org2 = rv.jsonOutput.result.username; execCmd(`force:source:pull -u ${org2}`); }); - + it('create a 3rd org using a username override', () => { const username = `${testSession.id}@scratch.org`; execCmd(`org:create -f config/project-scratch-def.json username=${username}`); execCmd(`force:source:pull -u ${username}`); }); - + it('create a 4th org and rely on defaultusername', () => { execCmd(`org:create -f config/project-scratch-def.json -s`); execCmd(`force:source:pull`); }); - + after(async () => { await testSession?.clean(); }); @@ -537,15 +540,14 @@ describe('Sample NUT 2', () => { ## Changing the process.cwd stub -***Usecase: The TestSession stubs process.cwd() to my SFDX project for me, but what if I want to change it during tests?*** +**_Usecase: The TestSession stubs process.cwd() to my SFDX project for me, but what if I want to change it during tests?_** -***NOTE: you could also change the cwd for one command by overriding in execCmd options.*** +**_NOTE: you could also change the cwd for one command by overriding in execCmd options._** ```typescript import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; describe('Sample NUT', () => { - let testSession: TestSession; before(() => { @@ -555,13 +557,13 @@ describe('Sample NUT', () => { it('using testkit to run sync commands', () => { execCmd('config:get defaultusername'); }); - + it('test a command after changing the process.cwd stub', () => { // Change the stubbed process.cwd dir testSession.stubCwd(__dirname); execCmd('config:get defaultusername'); }); - + after(async () => { await testSession?.clean(); }); @@ -570,7 +572,6 @@ describe('Sample NUT', () => { ## Creating a TestSession with authed devhub (TBD) - # Testkit Environment Variables ```shell @@ -590,7 +591,7 @@ TESTKIT_AUTH_URL ## Reusing orgs during test runs -***Usecase: I want the tests to use a specific scratch org or reuse one from a previous test run.*** +**_Usecase: I want the tests to use a specific scratch org or reuse one from a previous test run._** ```bash export TESTKIT_ORG_USERNAME="test1@scratch.org" @@ -598,7 +599,7 @@ export TESTKIT_ORG_USERNAME="test1@scratch.org" ## Reusing projects during test runs -***Usecase: I want the tests to use a specific SFDX project directory and not copy one into a TestSession dir. Or I kept the test artifacts from a previous run and want to reuse them.*** +**_Usecase: I want the tests to use a specific SFDX project directory and not copy one into a TestSession dir. Or I kept the test artifacts from a previous run and want to reuse them._** ```bash export TESTKIT_PROJECT_DIR="/Users/me/projects/MyTestProject" @@ -606,7 +607,7 @@ export TESTKIT_PROJECT_DIR="/Users/me/projects/MyTestProject" ## Using my actual homedir during test runs -***Usecase: I want to have the tests use my actual home directory.*** +**_Usecase: I want to have the tests use my actual home directory._** ```bash export TESTKIT_HOMEDIR="/Users/me" @@ -614,7 +615,7 @@ export TESTKIT_HOMEDIR="/Users/me" ## Keeping test artifacts (orgs, projects, test sessions) after test runs -***Usecase: I want to keep the test project, scratch org, and test session dir after the tests are done running to troubleshoot.*** +**_Usecase: I want to keep the test project, scratch org, and test session dir after the tests are done running to troubleshoot._** ```bash export TESTKIT_SAVE_ARTIFACTS=true @@ -622,8 +623,7 @@ export TESTKIT_SAVE_ARTIFACTS=true # Testkit Recommendations -1. Clean the TestSession in a code block that always runs (e.g., mocha’s after() ) to keep your plugin clean. You can always choose to zip a project or test session after tests run or on each test failure. -1. Point your CI jobs to different CLI executables using the TESTKIT_EXECUTABLE_PATH env var to ensure your plugin works with the various ways the CLI can be installed. By default it will use your plugin’s ./bin/run but you can define a local or global npm install path or install from public archives. -1. Use a naming pattern for test files that use the testkit. These are not unit tests so we like to refer to them as “NUTs” and have a convention of *.nut.ts so we can run them separately from unit tests. +1. Clean the TestSession in a code block that always runs (e.g., mocha’s after() ) to keep your plugin clean. You can always choose to zip a project or test session after tests run or on each test failure. +1. Point your CI jobs to different CLI executables using the TESTKIT_EXECUTABLE_PATH env var to ensure your plugin works with the various ways the CLI can be installed. By default it will use your plugin’s ./bin/run but you can define a local or global npm install path or install from public archives. +1. Use a naming pattern for test files that use the testkit. These are not unit tests so we like to refer to them as “NUTs” and have a convention of \*.nut.ts so we can run them separately from unit tests. 1. Use SFDX_USE_GENERIC_UNIX_KEYCHAIN=true to prevent keychain issues. -