feat: add retrievetargetdir to source:retrieve

messages/retrieve.json

@@ -11,9 +11,11 @@
     "To retrieve all Apex classes and two specific profiles (one of which has a space in its name):\n   $ sfdx force:source:retrieve -m \"ApexClass, Profile:My Profile, Profile: AnotherProfile\"",
     "To retrieve all metadata components listed in a manifest:\n   $ sfdx force:source:retrieve -x path/to/package.xml",
     "To retrieve metadata from a package or multiple packages:\n   $ sfdx force:source:retrieve -n MyPackageName\n   $ sfdx force:source:retrieve -n \"Package1, PackageName With Spaces, Package3\"",
-    "To retrieve all metadata from a package and specific components that aren’t in the package, specify both -n | --packagenames and one other scoping parameter:\n   $ sfdx force:source:retrieve -n MyPackageName -p path/to/apex/classes\n   $ sfdx force:source:retrieve -n MyPackageName -m ApexClass:MyApexClass\n   $ sfdx force:source:retrieve -n MyPackageName -x path/to/package.xml"
+    "To retrieve all metadata from a package and specific components that aren’t in the package, specify both -n | --packagenames and one other scoping parameter:\n   $ sfdx force:source:retrieve -n MyPackageName -p path/to/apex/classes\n   $ sfdx force:source:retrieve -n MyPackageName -m ApexClass:MyApexClass\n   $ sfdx force:source:retrieve -n MyPackageName -x path/to/package.xml",
+    "To retrieve source files to a given directory instead of the default package directory specified in sfdx-project.json:\n   $ sfdx force:source:retrieve -m \"StandardValueSet:TaskStatus\" -r path/to/unpackaged"
   ],
   "flags": {
+    "retrievetargetdir": "directory root for the retrieved source files",
     "sourcePath": "comma-separated list of source file paths to retrieve",
     "wait": "wait time for command to finish in minutes",
     "manifest": "file path for manifest (package.xml) of components to retrieve",
@@ -24,6 +26,11 @@
     "forceoverwrite": "ignore conflict warnings and overwrite changes to the project"
   },
   "flagsLong": {
+    "retrievetargetdir": [
+      "The root of the directory structure into which the source files are retrieved.",
+      "If the target directory matches one of the package directories in your sfdx-project.json file, the command fails.",
+      "Running the command multiple times with the same target adds new files and overwrites existing files."
+    ],
     "wait": "Number of minutes to wait for the command to complete and display results to the terminal window. If the command continues to run after the wait period, the CLI returns control of the terminal window to you.",
     "manifest": [
       "The complete path for the manifest (package.xml) file that specifies the components to retrieve.",
@@ -52,5 +59,6 @@
   "errorColumn": "PROBLEM",
   "nothingToRetrieve": "Specify a source path, manifest, metadata, or package names to retrieve.",
   "wantsToRetrieveCustomFields": "Because you're retrieving one or more CustomFields, we're also retrieving the CustomObject to which it's associated.",
-  "retrieveWontDelete": "You currently have files deleted in your org.  The retrieve command will NOT delete them from your local project"
+  "retrieveWontDelete": "You currently have files deleted in your org.  The retrieve command will NOT delete them from your local project",
+  "retrieveTargetDirOverlapsPackage": "The retrieve target directory [%s] overlaps one of your package directories.  Specify a different retrieve target directory and try again."
 }

src/commands/force/source/retrieve.ts

@@ -6,7 +6,8 @@
  */
 
 import * as os from 'os';
-import { join } from 'path';
+import { dirname, join, resolve } from 'path';
+import * as fs from 'fs';
 import { flags, FlagsConfig } from '@salesforce/command';
 import { Messages, SfError, SfProject } from '@salesforce/core';
 import { Duration } from '@salesforce/kit';
@@ -19,6 +20,7 @@ import {
   RetrieveResultFormatter,
 } from '../../../formatters/retrieveResultFormatter';
 import { filterConflictsByComponentSet, trackingSetup, updateTracking } from '../../../trackingFunctions';
+import { promisesQueue } from '../../../promiseQueue';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/plugin-source', 'retrieve');
@@ -30,6 +32,12 @@ export class Retrieve extends SourceCommand {
   public static readonly requiresProject = true;
   public static readonly requiresUsername = true;
   public static readonly flagsConfig: FlagsConfig = {
+    retrievetargetdir: flags.directory({
+      char: 'r',
+      description: messages.getMessage('flags.retrievetargetdir'),
+      longDescription: messages.getMessage('flagsLong.retrievetargetdir'),
+      exclusive: ['packagenames', 'sourcepath'],
+    }),
     apiversion: flags.builtin({
       /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
       // @ts-ignore force char override for backward compat
@@ -80,16 +88,24 @@ export class Retrieve extends SourceCommand {
   protected readonly lifecycleEventNames = ['preretrieve', 'postretrieve'];
   protected retrieveResult: RetrieveResult;
   protected tracking: SourceTracking;
+  private resolvedTargetDir: string;
 
   public async run(): Promise<RetrieveCommandResult> {
     await this.preChecks();
     await this.retrieve();
     this.resolveSuccess();
     await this.maybeUpdateTracking();
+    await this.moveResultsForRetrieveTargetDir();
     return this.formatResult();
   }
 
   protected async preChecks(): Promise<void> {
+    if (this.flags.retrievetargetdir) {
+      this.resolvedTargetDir = resolve(this.flags.retrievetargetdir as string);
+      if (this.overlapsPackage()) {
+        throw messages.createError('retrieveTargetDirOverlapsPackage', [this.flags.retrievetargetdir as string]);
+      }
+    }
     // we need something to retrieve
     const retrieveInputs = [this.flags.manifest, this.flags.metadata, this.flags.sourcepath, this.flags.packagenames];
     if (!retrieveInputs.some((x) => x)) {
@@ -115,11 +131,11 @@ export class Retrieve extends SourceCommand {
       sourcepath: this.getFlag<string[]>('sourcepath'),
       manifest: this.flags.manifest && {
         manifestPath: this.getFlag<string>('manifest'),
-        directoryPaths: this.getPackageDirs(),
+        directoryPaths: this.flags.retrievetargetdir ? [] : this.getPackageDirs(),
       },
       metadata: this.flags.metadata && {
         metadataEntries: this.getFlag<string[]>('metadata'),
-        directoryPaths: this.getPackageDirs(),
+        directoryPaths: this.flags.retrievetargetdir ? [] : this.getPackageDirs(),
       },
     });
 
@@ -155,7 +171,7 @@ export class Retrieve extends SourceCommand {
     const mdapiRetrieve = await this.componentSet.retrieve({
       usernameOrConnection: this.org.getUsername(),
       merge: true,
-      output: this.project.getDefaultPackage().fullPath,
+      output: this.getFlag<string>('retrievetargetdir') || this.project.getDefaultPackage().fullPath,
       packageOptions: this.getFlag<string[]>('packagenames'),
     });
 
@@ -220,4 +236,58 @@ export class Retrieve extends SourceCommand {
     });
     return hasCustomField && !hasCustomObject;
   }
+
+  private async moveResultsForRetrieveTargetDir(): Promise<void> {
+    async function mv(src: string): Promise<string[]> {
+      let directories: string[] = [];
+      let files: string[] = [];
+      const srcStat = await fs.promises.stat(src);
+      if (srcStat.isDirectory()) {
+        const contents = await fs.promises.readdir(src, { withFileTypes: true });
+        [directories, files] = contents.reduce<[string[], string[]]>(
+          (acc, dirent) => {
+            if (dirent.isDirectory()) {
+              acc[0].push(dirent.name);
+            } else {
+              acc[1].push(dirent.name);
+            }
+            return acc;
+          },
+          [[], []]
+        );
+
+        directories = directories.map((dir) => join(src, dir));
+      } else {
+        files.push(src);
+      }
+      await promisesQueue(
+        files,
+        async (file: string): Promise<string> => {
+          const dest = join(src.replace(join('main', 'default'), ''), file);
+          const destDir = dirname(dest);
+          await fs.promises.mkdir(destDir, { recursive: true });
+          await fs.promises.rename(join(src, file), dest);
+          return dest;
+        },
+        50
+      );
+      return directories;
+    }
+
+    if (!this.flags.retrievetargetdir) {
+      return;
+    }
+
+    // move contents of 'main/default' to 'retrievetargetdir'
+    await promisesQueue([join(this.resolvedTargetDir, 'main', 'default')], mv, 5, true);
+    // remove 'main/default'
+    await fs.promises.rmdir(join(this.flags.retrievetargetdir as string, 'main'), { recursive: true });
+    this.retrieveResult.getFileResponses().forEach((fileResponse) => {
+      fileResponse.filePath = fileResponse.filePath.replace(join('main', 'default'), '');
+    });
+  }
+
+  private overlapsPackage(): boolean {
+    return !!this.project.getPackageNameFromPath(this.resolvedTargetDir);
+  }
 }

src/promiseQueue.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+import { ensureArray } from '@salesforce/kit';
+
+/**
+ * Function to throttle a list of promises.
+ *
+ * @param sourceQueue - The list of items to process.
+ * @param producer - The function to produce a promise from an item.
+ * @param concurrency - The number of promises to run at a time.
+ * @param queueResults - Whether to queue the results of the promises.
+ */
+export async function promisesQueue<T>(
+  sourceQueue: T[],
+  producer: (T) => Promise<T | T[]>,
+  concurrency: number,
+  queueResults = false
+): Promise<T[]> {
+  const results: T[] = [];
+  let queue = [...sourceQueue];
+  while (queue.length > 0) {
+    const next = queue.slice(0, concurrency);
+    queue = queue.slice(concurrency);
+    // eslint-disable-next-line no-await-in-loop
+    const nextResults = (await Promise.all(ensureArray(next.map(producer))))
+      .flat(1)
+      .filter((val) => val !== undefined) as T[];
+    if (queueResults) {
+      queue.push(...nextResults);
+    }
+    results.push(...nextResults);
+  }
+  return results;
+}

test/commands/source/retrieve.test.ts

@@ -57,9 +57,11 @@ describe('force:source:retrieve', () => {
       await this.init();
       return this.run();
     }
+
     public setOrg(org: Org) {
       this.org = org;
     }
+
     public setProject(project: SfProject) {
       this.project = project;
     }
@@ -73,6 +75,7 @@ describe('force:source:retrieve', () => {
         stubInterface<SfProject>(sandbox, {
           getDefaultPackage: () => ({ fullPath: defaultPackagePath }),
           getUniquePackageDirectories: () => [{ fullPath: defaultPackagePath }],
+          getPackageDirectories: () => [{ fullPath: defaultPackagePath }],
           resolveProjectConfig: resolveProjectConfigStub,
         })
       );
@@ -93,6 +96,7 @@ describe('force:source:retrieve', () => {
     stubMethod(sandbox, UX.prototype, 'stopSpinner');
     stubMethod(sandbox, UX.prototype, 'styledHeader');
     stubMethod(sandbox, UX.prototype, 'table');
+    stubMethod(sandbox, Retrieve.prototype, 'moveResultsForRetrieveTargetDir');
     return cmd.runIt();
   };
 
@@ -165,6 +169,21 @@ describe('force:source:retrieve', () => {
     ensureRetrieveArgs();
     ensureHookArgs();
   });
+  it('should pass along retrievetargetdir', async () => {
+    const sourcepath = ['somepath'];
+    const metadata = ['ApexClass:MyClass'];
+    const result = await runRetrieveCmd(['--retrievetargetdir', sourcepath[0], '--metadata', metadata[0], '--json']);
+    expect(result).to.deep.equal(expectedResults);
+    ensureCreateComponentSetArgs({
+      sourcepath: undefined,
+      metadata: {
+        directoryPaths: [],
+        metadataEntries: ['ApexClass:MyClass'],
+      },
+    });
+    ensureRetrieveArgs({ output: sourcepath[0] });
+    ensureHookArgs();
+  });
 
   it('should pass along metadata', async () => {
     const metadata = ['ApexClass:MyClass'];

test/nuts/seeds/retrieve.retrievetargetdir.seed.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2020, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+
+import { SourceTestkit } from '@salesforce/source-testkit';
+import { JsonMap } from '@salesforce/ts-types';
+import { TEST_REPOS_MAP } from '../testMatrix';
+
+// DO NOT TOUCH. generateNuts.ts will insert these values
+const REPO = TEST_REPOS_MAP.get('%REPO_URL%');
+
+context('Retrieve metadata NUTs [name: %REPO_NAME%]', () => {
+  let testkit: SourceTestkit;
+
+  before(async () => {
+    testkit = await SourceTestkit.create({
+      repository: REPO.gitUrl,
+      nut: __filename,
+    });
+    await testkit.trackGlobs(testkit.packageGlobs);
+    await testkit.deploy({ args: `--sourcepath ${testkit.packageNames.join(',')}` });
+  });
+
+  after(async () => {
+    try {
+      await testkit?.clean();
+    } catch (e) {
+      // if the it fails to clean, don't throw so NUTs will pass
+      // eslint-disable-next-line no-console
+      console.log('Clean Failed: ', e);
+    }
+  });
+
+  describe('--retrievetargetdir flag', () => {
+    for (const testCase of REPO.retrieve.retrievetargetdir) {
+      it(`should retrieve ${testCase.toRetrieve}`, async () => {
+        await testkit.modifyLocalGlobs(testCase.toVerify);
+        await testkit.retrieve({ args: `--retrievetargetdir targetdir --metadata ${testCase.toRetrieve}` });
+        await testkit.expect.filesToBeRetrieved(testCase.toVerify, testCase.toIgnore);
+      });
+    }
+
+    it('should throw an error if the metadata is not valid', async () => {
+      const retrieve = (await testkit.retrieve({ args: '--retrievetargetdir targetdir --metadata DOES_NOT_EXIST', exitCode: 1 })) as JsonMap;
+      testkit.expect.errorToHaveName(retrieve, 'SfError');
+    });
+  });
+});

test/nuts/testMatrix.ts

@@ -83,6 +83,9 @@ const testRepos: RepoConfig[] = [
           toIgnore: ['foo-bar/app/lwc/mycomponent/mycomponent.js-meta.xml'],
         },
       ],
+      retrievetargetdir: [
+        { toRetrieve: 'ApexClass', toVerify: ['targetdir/**/*.cls'] },
+      ],
     },
     convert: {
       sourcepath: [
@@ -193,6 +196,10 @@ const testRepos: RepoConfig[] = [
           ],
         },
       ],
+      retrievetargetdir: [
+        { toRetrieve: 'ApexClass', toVerify: ['targetdir/classes/*'] },
+      ],
+
     },
     convert: {
       sourcepath: [
@@ -264,6 +271,7 @@ export type RepoConfig = {
     metadata: RetrieveTestCase[];
     sourcepath: RetrieveTestCase[];
     manifest: RetrieveTestCase[];
+    retrievetargetdir: RetrieveTestCase[];
   };
   convert: {
     metadata: ConvertTestCase[];