feat: add retrievetargetdir to source:retrieve

messages/retrieve.json

@@ -11,9 +11,11 @@
     "To retrieve all Apex classes and two specific profiles (one of which has a space in its name):\n   $ sfdx force:source:retrieve -m \"ApexClass, Profile:My Profile, Profile: AnotherProfile\"",
     "To retrieve all metadata components listed in a manifest:\n   $ sfdx force:source:retrieve -x path/to/package.xml",
     "To retrieve metadata from a package or multiple packages:\n   $ sfdx force:source:retrieve -n MyPackageName\n   $ sfdx force:source:retrieve -n \"Package1, PackageName With Spaces, Package3\"",
-    "To retrieve all metadata from a package and specific components that aren’t in the package, specify both -n | --packagenames and one other scoping parameter:\n   $ sfdx force:source:retrieve -n MyPackageName -p path/to/apex/classes\n   $ sfdx force:source:retrieve -n MyPackageName -m ApexClass:MyApexClass\n   $ sfdx force:source:retrieve -n MyPackageName -x path/to/package.xml"
+    "To retrieve all metadata from a package and specific components that aren’t in the package, specify both -n | --packagenames and one other scoping parameter:\n   $ sfdx force:source:retrieve -n MyPackageName -p path/to/apex/classes\n   $ sfdx force:source:retrieve -n MyPackageName -m ApexClass:MyApexClass\n   $ sfdx force:source:retrieve -n MyPackageName -x path/to/package.xml",
+    "To retrieve source files to a given directory instead of the default package directory specified in sfdx-project.json:\n   $ sfdx force:source:retrieve -m \"StandardValueSet:TaskStatus\" -r path/to/unpackaged"
   ],
   "flags": {
+    "retrievetargetdir": "directory root for the retrieved source files",
     "sourcePath": "comma-separated list of source file paths to retrieve",
     "wait": "wait time for command to finish in minutes",
     "manifest": "file path for manifest (package.xml) of components to retrieve",
@@ -24,6 +26,11 @@
     "forceoverwrite": "ignore conflict warnings and overwrite changes to the project"
   },
   "flagsLong": {
+    "retrievetargetdir": [
+      "The root of the directory structure into which the source files are retrieved.",
+      "If the target directory matches one of the package directories in your sfdx-project.json file, the retrieved source files are placed in the matching package directory.",
+      "Running the command multiple times with the same target will add new files and overwrite existing files."
+    ],
     "wait": "Number of minutes to wait for the command to complete and display results to the terminal window. If the command continues to run after the wait period, the CLI returns control of the terminal window to you.",
     "manifest": [
       "The complete path for the manifest (package.xml) file that specifies the components to retrieve.",

src/commands/force/source/retrieve.ts

@@ -6,7 +6,8 @@
  */
 
 import * as os from 'os';
-import { join } from 'path';
+import { dirname, join, resolve } from 'path';
+import * as fs from 'fs';
 import { flags, FlagsConfig } from '@salesforce/command';
 import { Messages, SfError, SfProject } from '@salesforce/core';
 import { Duration } from '@salesforce/kit';
@@ -19,6 +20,7 @@ import {
   RetrieveResultFormatter,
 } from '../../../formatters/retrieveResultFormatter';
 import { filterConflictsByComponentSet, trackingSetup, updateTracking } from '../../../trackingFunctions';
+import { promisesQueue } from '../../../promiseQueue';
 
 Messages.importMessagesDirectory(__dirname);
 const messages = Messages.loadMessages('@salesforce/plugin-source', 'retrieve');
@@ -30,6 +32,12 @@ export class Retrieve extends SourceCommand {
   public static readonly requiresProject = true;
   public static readonly requiresUsername = true;
   public static readonly flagsConfig: FlagsConfig = {
+    retrievetargetdir: flags.directory({
+      char: 'r',
+      description: messages.getMessage('flags.retrievetargetdir'),
+      longDescription: messages.getMessage('flagsLong.retrievetargetdir'),
+      exclusive: ['packagenames', 'sourcepath'],
+    }),
     apiversion: flags.builtin({
       /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
       // @ts-ignore force char override for backward compat
@@ -86,6 +94,7 @@ export class Retrieve extends SourceCommand {
     await this.retrieve();
     this.resolveSuccess();
     await this.maybeUpdateTracking();
+    await this.moveResultsForRetrieveTargetDir();
     return this.formatResult();
   }
 
@@ -115,11 +124,11 @@ export class Retrieve extends SourceCommand {
       sourcepath: this.getFlag<string[]>('sourcepath'),
       manifest: this.flags.manifest && {
         manifestPath: this.getFlag<string>('manifest'),
-        directoryPaths: this.getPackageDirs(),
+        directoryPaths: this.flags.retrievetargetdir ? [] : this.getPackageDirs(),
       },
       metadata: this.flags.metadata && {
         metadataEntries: this.getFlag<string[]>('metadata'),
-        directoryPaths: this.getPackageDirs(),
+        directoryPaths: this.flags.retrievetargetdir ? [] : this.getPackageDirs(),
       },
     });
 
@@ -155,7 +164,7 @@ export class Retrieve extends SourceCommand {
     const mdapiRetrieve = await this.componentSet.retrieve({
       usernameOrConnection: this.org.getUsername(),
       merge: true,
-      output: this.project.getDefaultPackage().fullPath,
+      output: this.getFlag<string>('retrievetargetdir') || this.project.getDefaultPackage().fullPath,
       packageOptions: this.getFlag<string[]>('packagenames'),
     });
 
@@ -220,4 +229,56 @@ export class Retrieve extends SourceCommand {
     });
     return hasCustomField && !hasCustomObject;
   }
+
+  private async moveResultsForRetrieveTargetDir(): Promise<void> {
+    async function mv(src: string): Promise<string[]> {
+      let directories: string[] = [];
+      let files: string[] = [];
+      const srcStat = await fs.promises.stat(src);
+      if (srcStat.isDirectory()) {
+        const contents = await fs.promises.readdir(src);
+        directories = contents.filter((c) => fs.statSync(join(src, c)).isDirectory()).map((c) => join(src, c));
+        files = contents.filter((c) => !fs.statSync(join(src, c)).isDirectory());
+      } else {
+        files.push(src);
+      }
+      await promisesQueue(
+        files,
+        async (file: string): Promise<string> => {
+          const dest = join(src.replace(join('main', 'default'), ''), file);
+          const destDir = dirname(dest);
+          await fs.promises.mkdir(destDir, { recursive: true });
+          await fs.promises.rename(join(src, file), dest);
+          return dest;
+        },
+        50
+      );
+      return directories;
+    }
+
+    if (!this.flags.retrievetargetdir) {
+      return;
+    }
+    let overlapsProject = true;
+    const resolvedTargetDir = resolve(this.flags.retrievetargetdir as string);
+    overlapsProject = !!this.project.getPackageDirectories().find((pkgDir) => {
+      if (pkgDir.fullPath) {
+        return pkgDir.fullPath.includes(resolvedTargetDir);
+      }
+      return false;
+    });
+    if (overlapsProject) {
+      return;
+    }
+
+    const moveQueue: string[] = [];
+    // move contents of 'main/default' to 'retrievetargetdir'
+    moveQueue.push(join(resolvedTargetDir, 'main', 'default'));
+    await promisesQueue(moveQueue, mv, 5, true);
+    // remove 'main/default'
+    await fs.promises.rmdir(join(this.flags.retrievetargetdir as string, 'main'), { recursive: true });
+    this.retrieveResult.getFileResponses().forEach((fileResponse) => {
+      fileResponse.filePath = fileResponse.filePath.replace(join('main', 'default'), '');
+    });
+  }
 }

src/promiseQueue.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2022, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+/**
+ * Function to throttle a list of promises.
+ *
+ * @param sourceQueue - The list of items to process.
+ * @param producer - The function to produce a promise from an item.
+ * @param concurrency - The number of promises to run at a time.
+ * @param queueResults - Whether to queue the results of the promises.
+ */
+export async function promisesQueue<T>(
+  sourceQueue: T[],
+  producer: (T) => Promise<T | T[]>,
+  concurrency: number,
+  queueResults = false
+): Promise<T[]> {
+  const results: T[] = [];
+  let queue = [...sourceQueue];
+  while (queue.length > 0) {
+    const next = queue.slice(0, concurrency);
+    queue = queue.slice(concurrency);
+    // eslint-disable-next-line no-await-in-loop
+    const nextResults = (await Promise.all(next.map(producer)))
+      .reduce<T[]>((acc, val) => {
+        if (Array.isArray(val)) {
+          acc.push(...val);
+        } else {
+          acc.push(val);
+        }
+        return acc;
+      }, [])
+      .filter((val) => val !== undefined);
+    if (queueResults) {
+      queue.push(...nextResults);
+    }
+    results.push(...nextResults);
+  }
+  return results;
+}