feat: add verbose flag to bulk delete and upsert (#615)

* feat: add verbose flag to bulk delete and upsert * test: add NUTs * feat: adjust verbose flag code according to code review * Remove default value for boolean flags * Print both Id (if exists) and Sf_Id columns * Only print table if --json is not set * Only download results if either --json or --verbose is set * Adjust, add and format tests * test: fix NUTs * chore: gen snapshot * chore: actually use os.EOL --------- Co-authored-by: Willie Ruemmele <[email protected]>
salesforcecli · Jul 14, 2023 · 1b52226 · 1b52226
1 parent 91a0668
commit 1b52226
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 21 deletions.
diff --git a/command-snapshot.json b/command-snapshot.json
@@ -10,7 +10,7 @@
   {
     "command": "data:delete:bulk",
     "plugin": "@salesforce/plugin-data",
-    "flags": ["api-version", "async", "file", "json", "loglevel", "sobject", "target-org", "wait"],
+    "flags": ["api-version", "async", "file", "json", "loglevel", "sobject", "target-org", "verbose", "wait"],
     "alias": [],
     "flagChars": ["a", "f", "o", "s", "w"],
     "flagAliases": ["apiversion", "csvfile", "sobjecttype", "targetusername", "u"]
@@ -135,7 +135,18 @@
   {
     "command": "data:upsert:bulk",
     "plugin": "@salesforce/plugin-data",
-    "flags": ["api-version", "async", "external-id", "file", "json", "loglevel", "sobject", "target-org", "wait"],
+    "flags": [
+      "api-version",
+      "async",
+      "external-id",
+      "file",
+      "json",
+      "loglevel",
+      "sobject",
+      "target-org",
+      "verbose",
+      "wait"
+    ],
     "alias": [],
     "flagChars": ["a", "f", "i", "o", "s", "w"],
     "flagAliases": ["apiversion", "csvfile", "externalid", "sobjecttype", "targetusername", "u"]

diff --git a/messages/bulk.operation.command.md b/messages/bulk.operation.command.md
@@ -13,3 +13,7 @@ Number of minutes to wait for the command to complete before displaying the resu
 # flags.async.summary
 
 Run the command asynchronously.
+
+# flags.verbose
+
+Print verbose output of failed records if result is available.
diff --git a/package.json b/package.json
@@ -98,6 +98,7 @@
     "prepare": "sf-install",
     "test": "wireit",
     "test:nuts": "nyc mocha \"./test/**/*.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
+    "test:nuts:bulk": "nyc mocha \"./test/**/dataBulk.nut.ts\" --slow 4500 --timeout 600000 --parallel --jobs 20",
     "test:only": "wireit",
     "version": "oclif readme"
   },
@@ -255,4 +256,4 @@
       "output": []
     }
   }
-}
+}
diff --git a/src/bulkOperationCommand.ts b/src/bulkOperationCommand.ts
@@ -10,7 +10,8 @@ import * as os from 'os';
 import { Flags } from '@salesforce/sf-plugins-core';
 import { Duration } from '@salesforce/kit';
 import { Connection, Messages } from '@salesforce/core';
-import { BulkOperation, IngestJobV2, IngestOperation, JobInfoV2 } from 'jsforce/api/bulk';
+import { ux } from '@oclif/core';
+import { BulkOperation, IngestJobV2, IngestJobV2FailedResults, IngestOperation, JobInfoV2 } from 'jsforce/api/bulk';
 import { Schema } from 'jsforce';
 import { orgFlags } from './flags';
 import { BulkDataRequestCache } from './bulkDataRequestCache';
@@ -57,9 +58,11 @@ export abstract class BulkOperationCommand extends BulkBaseCommand {
     async: Flags.boolean({
       char: 'a',
       summary: messages.getMessage('flags.async.summary'),
-      default: false,
       exclusive: ['wait'],
     }),
+    verbose: Flags.boolean({
+      summary: messages.getMessage('flags.verbose'),
+    }),
   };
 
   /**
@@ -93,11 +96,27 @@ export abstract class BulkOperationCommand extends BulkBaseCommand {
     return job.check();
   }
 
+  private static printBulkErrors(failedResults: IngestJobV2FailedResults<Schema>): void {
+    const columns = {
+      id: { header: 'Id' },
+      sfId: { header: 'Sf_Id' },
+      error: { header: 'Error' },
+    };
+    const options = { title: `Bulk Failures [${failedResults.length}]` };
+    ux.log();
+    ux.table(
+      failedResults.map((f) => ({ id: 'Id' in f ? f.Id : '', sfId: f.sf__Id, error: f.sf__Error })),
+      columns,
+      options
+    );
+  }
+
   public async runBulkOperation(
     sobject: string,
     csvFileName: string,
     connection: Connection,
     wait: number,
+    verbose: boolean,
     operation: BulkOperation,
     options?: { extIdField: string }
   ): Promise<BulkResultV2> {
@@ -133,10 +152,20 @@ export abstract class BulkOperationCommand extends BulkBaseCommand {
         }
         this.displayBulkV2Result(jobInfo);
         const result = { jobInfo } as BulkResultV2;
-        if (!isBulkV2RequestDone(jobInfo) || !this.jsonEnabled()) {
+        if (!isBulkV2RequestDone(jobInfo)) {
           return result;
         }
-        result.records = transformResults(await this.job.getAllResults());
+        if (this.jsonEnabled()) {
+          result.records = transformResults(await this.job.getAllResults());
+        }
+        // We only print human readable error outputs if --json is not specified.
+        // The JSON result itself will already contain the error information (see above).
+        else if (verbose) {
+          const records = await this.job.getAllResults();
+          if (records?.failedResults?.length > 0) {
+            BulkOperationCommand.printBulkErrors(records.failedResults);
+          }
+        }
         return result;
       } catch (err) {
         this.spinner.stop();

diff --git a/src/commands/data/delete/bulk.ts b/src/commands/data/delete/bulk.ts
@@ -25,7 +25,7 @@ export default class Delete extends BulkOperationCommand {
 
     await validateSobjectType(flags.sobject, conn);
 
-    return this.runBulkOperation(flags.sobject, flags.file, conn, flags.async ? 0 : flags.wait?.minutes, 'delete');
+    return this.runBulkOperation(flags.sobject, flags.file, conn, flags.async ? 0 : flags.wait?.minutes, flags.verbose, 'delete');
   }
 
   // eslint-disable-next-line class-methods-use-this

diff --git a/src/commands/data/upsert/bulk.ts b/src/commands/data/upsert/bulk.ts
@@ -35,7 +35,7 @@ export default class Upsert extends BulkOperationCommand {
 
     await validateSobjectType(flags.sobject, conn);
 
-    return this.runBulkOperation(flags.sobject, flags.file, conn, flags.async ? 0 : flags.wait?.minutes, 'upsert', {
+    return this.runBulkOperation(flags.sobject, flags.file, conn, flags.async ? 0 : flags.wait?.minutes, flags.verbose, 'upsert', {
       extIdField: flags['external-id'],
     });
   }

diff --git a/test/commands/data/dataBulk.nut.ts b/test/commands/data/dataBulk.nut.ts
@@ -8,13 +8,16 @@ import * as path from 'path';
 import { strict as assert } from 'node:assert/strict';
 import * as fs from 'fs';
 import * as os from 'os';
-import { expect } from 'chai';
+import { expect, config as chaiConfig } from 'chai';
 import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit';
 import { sleep } from '@salesforce/kit';
 import { ensurePlainObject } from '@salesforce/ts-types';
+import { SaveResult } from 'jsforce';
 import { BulkResultV2 } from '../../../src/types';
 import { QueryResult } from './dataSoqlQuery.nut';
 
+chaiConfig.truncateThreshold = 0;
+
 let testSession: TestSession;
 
 /** Verify that the operation completed successfully and results are available before attempting to do stuff with the results */
@@ -24,9 +27,7 @@ const isCompleted = async (cmd: string): Promise<void> => {
     // eslint-disable-next-line no-await-in-loop
     await sleep(2000);
     const result = execCmd<BulkResultV2>(cmd);
-    // eslint-disable-next-line no-console
     if (result.jsonOutput?.status === 0) {
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       if (result.jsonOutput.result.jobInfo.state === 'JobComplete') {
         complete = true;
       }
@@ -42,9 +43,7 @@ const checkBulkResumeJsonResponse = (jobId: string, operation: 'delete' | 'upser
   const statusResponse = execCmd<BulkResultV2>(`data:${operation}:resume --job-id ${jobId} --json`, {
     ensureExitCode: 0,
   }).jsonOutput?.result;
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
   expect(statusResponse?.jobInfo.state).to.equal('JobComplete');
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
   expect(statusResponse?.records?.successfulResults).to.be.an('array').with.lengthOf(10);
 };
 
@@ -55,7 +54,7 @@ const checkBulkResumeJsonResponse = (jobId: string, operation: 'delete' | 'upser
 const checkBulkStatusHumanResponse = (statusCommand: string): void => {
   const statusResponse = execCmd(statusCommand, {
     ensureExitCode: 0,
-  }).shellOutput.stdout.split('\n');
+  }).shellOutput.stdout.split(os.EOL);
   const jobState = statusResponse.find((line) => line.includes('Status'));
   expect(jobState).to.include('Job Complete');
 };
@@ -65,7 +64,6 @@ describe('data:bulk commands', () => {
     testSession = await TestSession.create({
       scratchOrgs: [
         {
-          executable: 'sfdx',
           config: 'config/project-scratch-def.json',
           setDefault: true,
         },
@@ -76,7 +74,7 @@ describe('data:bulk commands', () => {
   });
 
   after(async () => {
-    await testSession?.clean();
+    // await testSession?.clean();
   });
 
   describe('data:bulk verify json and human responses', () => {
@@ -99,6 +97,108 @@ describe('data:bulk commands', () => {
       });
     });
   });
+
+  describe('bulk data commands with --verbose', () => {
+    it('should print table because of --verbose and errors', () => {
+      fs.writeFileSync(path.join(testSession.project.dir, 'data.csv'), `Id${os.EOL}001000000000000AAA`);
+
+      const result = execCmd('data:delete:bulk --sobject Account --file data.csv --wait 10 --verbose', {
+        ensureExitCode: 1,
+      }).shellOutput;
+      // Bulk Failures [1]
+      // ==========================================================================
+      // | Id                 Sf_Id Error
+      // | ────────────────── ───── ───────────────────────────────────────────────
+      // | 001000000000000AAA       MALFORMED_ID:malformed id 001000000000000AAA:--
+
+      expect(result).to.include('Bulk Failures [1]');
+      expect(result).to.include('Id');
+      expect(result).to.include('Sf_Id');
+      expect(result).to.include('Error');
+      expect(result).to.include('INVALID_CROSS_REFERENCE_KEY:invalid cross reference id');
+      // expect(result).to.include('MALFORMED_ID:bad id')
+    });
+
+    it('should not print table because of errors and missing --verbose', () => {
+      fs.writeFileSync(path.join(testSession.project.dir, 'data.csv'), `Id${os.EOL}001000000000000AAA`);
+
+      const result = execCmd('data:delete:bulk --sobject Account --file data.csv --wait 10', { ensureExitCode: 1 })
+        .shellOutput.stdout;
+
+      expect(result).to.not.include('Bulk Failures [1]');
+    });
+
+    it('should not print error table when there are no errors', () => {
+      // insert account
+      const accountId = execCmd<SaveResult>('data:create:record -s Account  --values Name=test --json', {
+        ensureExitCode: 0,
+      }).jsonOutput?.result.id;
+      fs.writeFileSync(path.join(testSession.project.dir, 'account.csv'), `Id${os.EOL}${accountId}`);
+      const result = execCmd('data:delete:bulk --sobject Account --file account.csv --wait 10 --verbose', {
+        ensureExitCode: 0,
+      }).shellOutput.stdout; // eslint-disable-next-line no-console
+      expect(result).to.include('Status Job Complete Records processed 1. Records failed 0.');
+    });
+
+    it('should have information in --json', () => {
+      fs.writeFileSync(path.join(testSession.project.dir, 'data.csv'), `Id${os.EOL}001000000000000AAA`);
+
+      const result = execCmd<BulkResultV2>(
+        'data:delete:bulk --sobject Account --file data.csv --wait 10 --verbose --json',
+        { ensureExitCode: 1 }
+      ).jsonOutput?.result.records;
+      /*
+        {
+          "status": 1,
+          "result": {
+            "jobInfo": {
+              "id": "<ID>",
+              "operation": "delete",
+              "object": "Account",
+              // ...
+            },
+            "records": {
+              "successfulResults": [],
+              "failedResults": [
+                {
+                  "sf__Id": "",
+                  "sf__Error": "MALFORMED_ID:malformed id 001000000000000AAA:--",
+                  "Id": "001000000000000AAA"
+
+              ],
+              "unprocessedRecords": []
+            }
+          },
+          "warnings": []
+        }
+      */
+
+      expect(result?.failedResults[0]).to.have.all.keys('sf__Id', 'sf__Error', 'Id');
+      expect(result?.failedResults[0].sf__Id).to.equal('001000000000000AAA');
+      expect(result?.failedResults[0].sf__Error).to.equal('INVALID_CROSS_REFERENCE_KEY:invalid cross reference id:--');
+      // expect(result?.failedResults[0].sf__Error).to.equal('MALFORMED_ID:bad id       001000000000000AAA:--')
+      expect(result?.failedResults[0].Id).to.equal('001000000000000AAA');
+      expect(result?.successfulResults.length).to.equal(0);
+    });
+
+    it('should print verbose success with json', () => {
+      // insert account
+      const accountId = execCmd<SaveResult>('data:create:record -s Account --values Name=test --json', {
+        ensureExitCode: 0,
+      }).jsonOutput?.result.id;
+      fs.writeFileSync(path.join(testSession.project.dir, 'account.csv'), `Id${os.EOL}${accountId}`);
+
+      const result = execCmd<BulkResultV2>(
+        'data:delete:bulk --sobject Account --file account.csv --wait 10 --verbose --json',
+        { ensureExitCode: 0 }
+      ).jsonOutput?.result.records;
+      expect(result?.successfulResults[0]).to.have.all.keys('sf__Id', 'sf__Created', 'Id');
+      expect(result?.successfulResults[0].sf__Id).to.equal(accountId);
+      expect(result?.successfulResults[0].sf__Created).to.equal('false');
+      expect(result?.successfulResults[0].Id).to.equal(accountId);
+      expect(result?.failedResults.length).to.equal(0);
+    });
+  });
 });
 
 const queryAccountRecords = () => {
@@ -141,18 +241,14 @@ const bulkInsertAccounts = (): BulkResultV2 => {
     'bulkUpsert.csv'
   )} --external-id Id --json --wait 10`;
   const rawResponse = execCmd(cmd);
-  // eslint-disable-next-line no-console
-  console.error(`rawResponse: ${JSON.stringify(rawResponse, null, 2)}`);
   const response: BulkResultV2 | undefined = rawResponse.jsonOutput?.result as BulkResultV2;
   if (response?.records) {
-    /* eslint-disable @typescript-eslint/no-unsafe-member-access, ,@typescript-eslint/no-unsafe-assignment */
     const records = response.records?.successfulResults;
     assert.equal(records?.length, 10);
     const bulkUpsertResult = response.records?.successfulResults[0];
     assert(Object.keys(ensurePlainObject(bulkUpsertResult)).includes('sf__Id'));
     const jobInfo = response.jobInfo;
     assert('id' in jobInfo);
-    /* eslint-enable @typescript-eslint/no-unsafe-member-access, ,@typescript-eslint/no-unsafe-assignment */
   }
   return response;
 };