diff --git a/messages/exportApi.md b/messages/exportApi.md index 5d39e677..0da7c4a4 100644 --- a/messages/exportApi.md +++ b/messages/exportApi.md @@ -23,3 +23,7 @@ Processed %s records from query: %s Query returned more than 200 records. Run the command using the --plan flag instead. Record Count: %s Query: %s + +# noRecordsReturned + +The query for %s returned 0 records. diff --git a/src/commands/data/export/tree.ts b/src/commands/data/export/tree.ts index f8ed4aad..01d28df4 100644 --- a/src/commands/data/export/tree.ts +++ b/src/commands/data/export/tree.ts @@ -11,10 +11,12 @@ import { orgFlags, prefixValidation } from '../../../flags.js'; import { ExportConfig, runExport } from '../../../export.js'; import type { DataPlanPart, SObjectTreeFileContents } from '../../../types.js'; +export type ExportTreeResult = DataPlanPart[] | SObjectTreeFileContents; + Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-data', 'tree.export'); -export default class Export extends SfCommand { +export default class Export extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); @@ -24,6 +26,7 @@ export default class Export extends SfCommand { + public async run(): Promise { const { flags } = await this.parse(Export); if (flags.plan) { this.warn(messages.getMessage('PlanJsonWarning')); @@ -54,7 +57,7 @@ export default class Export extends SfCommand>; /** only used internally, but a more useful structure than the original */ type PlanFile = Omit & { contents: SObjectTreeFileContents; file: string; dir: string }; -export const runExport = async (configInput: ExportConfig): Promise => { - const { outputDir, plan, query, conn, prefix, ux } = validate(configInput); - const logger = Logger.childFromRoot('runExport'); - const { records: recordsFromQuery } = await queryRecords(conn)(query); - const describe = await cacheAllMetadata(conn)(recordsFromQuery); - +export const runExport = async (configInput: ExportConfig): Promise => { + const { outputDir, plan, queries, conn, prefix, ux } = validate(configInput); const refFromIdByType: RefFromIdByType = new Map(); - const flatRecords = recordsFromQuery.flatMap(flattenNestedRecords); - flatRecords.map(buildRefMap(refFromIdByType)); // get a complete map of ID<->Ref - - logger.debug(messages.getMessage('dataExportRecordCount', [flatRecords.length, query])); + const logger = Logger.childFromRoot('runExport'); + const objectRecordMap = new Map(); if (outputDir) { await fs.promises.mkdir(outputDir, { recursive: true }); } + await Promise.all( + queries.map(async (query) => { + const { records: recordsFromQuery } = await queryRecords(conn)(query); + await cacheAllMetadata(conn)(recordsFromQuery); + + objectRecordMap.set( + recordsFromQuery.at(0)?.attributes.type ?? + // try to match the object being queried + new RegExp(/from (\w+)/gim).exec(query)?.at(1) ?? + '', + recordsFromQuery + ); + + // get a complete map of ID<->Ref + const flatRecords = recordsFromQuery.flatMap(flattenNestedRecords).map(buildRefMap(refFromIdByType)); + + logger.debug(messages.getMessage('dataExportRecordCount', [flatRecords.length, query])); + }) + ); + + const allQueryValues = Array.from(objectRecordMap.values()).flat(); + const describe = await cacheAllMetadata(conn)(allQueryValues); + if (plan) { const planMap = reduceByType( - recordsFromQuery + allQueryValues .flatMap(flattenWithChildRelationships(describe)()) .map(addReferenceIdToAttributes(refFromIdByType)) .map(removeChildren) @@ -70,7 +88,7 @@ export const runExport = async (configInput: ExportConfig): Promise ({ sobject, contents: { records }, @@ -80,25 +98,48 @@ export const runExport = async (configInput: ExportConfig): Promise 1 + ? DATA_PLAN_FILENAME_PART + : getPrefixedFileName([...describe.keys(), DATA_PLAN_FILENAME_PART].join('-'), prefix); + await Promise.all([ - ...planFiles.map(writePlanDataFile(ux)), - fs.promises.writeFile(path.join(outputDir ?? '', planName), JSON.stringify(output, null, 4)), + ...contentFiles.map(writePlanDataFile(ux)), + fs.promises.writeFile(path.join(outputDir ?? '', planName), JSON.stringify(planContent, null, 4)), ]); - return output; + return planContent; } else { - if (flatRecords.length > 200) { - // use lifecycle so warnings show up in stdout and in the json - await Lifecycle.getInstance().emitWarning( - messages.getMessage('dataExportRecordCountWarning', [flatRecords.length, query]) - ); - } - const contents = { records: processRecordsForNonPlan(describe)(refFromIdByType)(recordsFromQuery) }; - const filename = path.join(outputDir ?? '', getPrefixedFileName(`${[...describe.keys()].join('-')}.json`, prefix)); - ux.log(`wrote ${flatRecords.length} records to ${filename}`); - fs.writeFileSync(filename, JSON.stringify(contents, null, 4)); - return contents; + const records: SObjectTreeInput[] = []; + objectRecordMap.forEach((basicRecords, query) => { + if (basicRecords.length > 200) { + // use lifecycle so warnings show up in stdout and in the json + void Lifecycle.getInstance().emitWarning( + messages.getMessage('dataExportRecordCountWarning', [basicRecords.length, query]) + ); + } + if (!basicRecords.length) { + // use lifecycle so warnings show up in stdout and in the json + void Lifecycle.getInstance().emitWarning(messages.getMessage('noRecordsReturned', [query])); + } else { + const contents = { records: processRecordsForNonPlan(describe)(refFromIdByType)(basicRecords) }; + // if we have multiple queries, we'll accumulate all the processed records + records.push(...contents.records); + const filename = path.join( + outputDir ?? '', + // if we have multiple queries, name each file according to the object being queried, otherwise, join them together for previous behavior + getPrefixedFileName(`${queries.length > 1 ? query : [...describe.keys()].join('-')}.json`, prefix) + ); + ux.log(`wrote ${basicRecords.length} records to ${filename}`); + fs.writeFileSync(filename, JSON.stringify(contents, null, 4)); + } + }); + + return { + records, + }; } }; @@ -121,7 +162,7 @@ const writePlanDataFile = (ux: Ux) => async (p: PlanFile): Promise => { await fs.promises.writeFile(path.join(p.dir, p.file), JSON.stringify(p.contents, null, 4)); - ux.log(`wrote ${p.contents.records.length} records to ${p.file}`); + ux.log(`wrote ${p.contents.records.length} records to ${path.join(p.dir, p.file)}`); }; // future: use Map.groupBy() when it's available @@ -207,7 +248,8 @@ export const replaceParentReferences = const typeDescribe = ensure(describe.get(record.attributes.type), `Missing describe for ${record.attributes.type}`); const replacedReferences = Object.fromEntries( Object.entries(record) - .filter(isRelationshipFieldFilter(typeDescribe)) // only look at the fields that are references + .filter(isRelationshipFieldFilter(typeDescribe)) + // only look at the fields that are references // We can check describe to see what the type could be. // If it narrows to only 1 type, pass that to refFromId. // If it's polymorphic, don't pass a type because refFromId will need to check all the types. @@ -261,23 +303,29 @@ const addReferenceIdToAttributes = * @param config - The export configuration. */ const validate = (config: ExportConfig): ExportConfig => { - if (!config.query) { + if (!config.queries) { throw new SfError(messages.getMessage('queryNotProvided'), 'queryNotProvided'); } - - const filepath = path.resolve(process.cwd(), config.query); - if (fs.existsSync(filepath)) { - config.query = fs.readFileSync(filepath, 'utf8'); - - if (!config.query) { - throw messages.createError('queryNotProvided'); + const queries: string[] = []; + config.queries.map((q) => { + const filepath = path.resolve(process.cwd(), q); + if (fs.existsSync(filepath)) { + const query = fs.readFileSync(filepath, 'utf8'); + + if (!config.queries) { + throw messages.createError('queryNotProvided'); + } + // we've validate that the passed file is a valid soql statement, continue the iteration based on the assumption + // that q is a soql string, rather than a file name - this combines logic for --query contact.txt and --query "SELECT..." + q = query; } - } - config.query = config.query.trim(); - if (!config.query.toLowerCase().startsWith('select')) { - throw messages.createError('soqlInvalid', [config.query]); - } + if (!q.toLowerCase().startsWith('select')) { + throw messages.createError('soqlInvalid', [q]); + } + queries.push(q); + }); + config.queries = queries; return config; }; diff --git a/test/commands/data/tree/dataTree.nut.ts b/test/commands/data/tree/dataTree.nut.ts index 810c9c06..c8c685f3 100644 --- a/test/commands/data/tree/dataTree.nut.ts +++ b/test/commands/data/tree/dataTree.nut.ts @@ -37,7 +37,7 @@ describe('data:tree commands', () => { it('should error with invalid soql', () => { const result = execCmd( - `data:export:tree --query 'SELECT' --prefix INT --outputdir ${path.join('.', 'export_data')}` + `data:export:tree --query 'SELECT' --prefix INT --output-dir ${path.join('.', 'export_data')}` ); const stdError = getString(result, 'shellOutput.stderr', '').toLowerCase(); const errorKeywords = ['malformed', 'check the soql', 'invalid soql query']; @@ -54,7 +54,7 @@ describe('data:tree commands', () => { }); execCmd( - `data:export:tree --query "${query}" --prefix INT --outputdir ${path.join('.', 'export_data')} --plan --json`, + `data:export:tree --query "${query}" --prefix INT --output-dir ${path.join('.', 'export_data')} --plan --json`, { ensureExitCode: 0 } ); diff --git a/test/commands/data/tree/dataTreeCommonChild.nut.ts b/test/commands/data/tree/dataTreeCommonChild.nut.ts index 9b5ace94..247537f2 100644 --- a/test/commands/data/tree/dataTreeCommonChild.nut.ts +++ b/test/commands/data/tree/dataTreeCommonChild.nut.ts @@ -55,7 +55,7 @@ describe('data:tree commands with a polymorphic whatId (on tasks) shared between ); execCmd( - `data:export:tree --query "${query}" --prefix ${prefix} --outputdir ${path.join( + `data:export:tree --query "${query}" --prefix ${prefix} --output-dir ${path.join( '.', 'export_data' )} --plan --json`, diff --git a/test/commands/data/tree/dataTreeDeep.nut.ts b/test/commands/data/tree/dataTreeDeep.nut.ts index 8a3fdd43..ed8ee358 100644 --- a/test/commands/data/tree/dataTreeDeep.nut.ts +++ b/test/commands/data/tree/dataTreeDeep.nut.ts @@ -37,7 +37,7 @@ describe('data:tree commands with more than 2 levels', () => { it('should error with invalid soql', () => { const result = execCmd( - `data:export:tree --query 'SELECT' --prefix INT --outputdir ${path.join('.', 'export_data')}` + `data:export:tree --query 'SELECT' --prefix INT --output-dir ${path.join('.', 'export_data')}` ); const stdError = getString(result, 'shellOutput.stderr', '').toLowerCase(); const errorKeywords = ['malformed', 'check the soql', 'invalid soql query']; @@ -54,7 +54,7 @@ describe('data:tree commands with more than 2 levels', () => { }); execCmd( - `data:export:tree --query "${query}" --prefix INT --outputdir ${path.join('.', 'export_data')} --plan --json`, + `data:export:tree --query "${query}" --prefix INT --output-dir ${path.join('.', 'export_data')} --plan --json`, { ensureExitCode: 0 } ); diff --git a/test/commands/data/tree/dataTreeDeepBeta.nut.ts b/test/commands/data/tree/dataTreeDeepBeta.nut.ts index a5b14d36..2a9d2ece 100644 --- a/test/commands/data/tree/dataTreeDeepBeta.nut.ts +++ b/test/commands/data/tree/dataTreeDeepBeta.nut.ts @@ -38,7 +38,7 @@ describe('data:tree beta commands with more than 2 levels', () => { it('should error with invalid soql', () => { const result = execCmd( - `data:export:tree --query 'SELECT' --prefix ${prefix} --outputdir ${path.join('.', 'export_data')}` + `data:export:tree --query 'SELECT' --prefix ${prefix} --output-dir ${path.join('.', 'export_data')}` ); const stdError = getString(result, 'shellOutput.stderr', '').toLowerCase(); const errorKeywords = ['malformed', 'check the soql', 'invalid soql query']; @@ -55,7 +55,7 @@ describe('data:tree beta commands with more than 2 levels', () => { }); execCmd( - `data:export:tree --query "${query}" --prefix ${prefix} --outputdir ${path.join( + `data:export:tree --query "${query}" --prefix ${prefix} --output-dir ${path.join( '.', 'export_data' )} --plan --json`, diff --git a/test/commands/data/tree/dataTreeJunction.nut.ts b/test/commands/data/tree/dataTreeJunction.nut.ts index 7a9c13d2..a68a13e5 100644 --- a/test/commands/data/tree/dataTreeJunction.nut.ts +++ b/test/commands/data/tree/dataTreeJunction.nut.ts @@ -8,6 +8,7 @@ import path from 'node:path'; import { expect } from 'chai'; import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { ImportResult } from '../../../../src/api/data/tree/importTypes.js'; +import { ExportTreeResult } from '../../../../src/commands/data/export/tree.js'; describe('data:tree commands', () => { let testSession: TestSession; @@ -50,7 +51,7 @@ describe('data:tree commands', () => { const query = "select Id, Name, (Select Id, FirstName, LastName, (select AccountId, ContactId from AccountContactRoles) from Contacts), (select Id, ContactId, AccountId from AccountContactRelations where Account.Name != 'We Know Everybody') from Account where Name != 'Sample Account for Entitlements'"; - execCmd(`data:export:tree --query "${query}" --outputdir ${path.join('.', 'export_data')} --plan --json`, { + execCmd(`data:export:tree --query "${query}" --output-dir ${path.join('.', 'export_data')} --plan --json`, { ensureExitCode: 0, }); @@ -67,4 +68,24 @@ describe('data:tree commands', () => { ); expect(importResult.jsonOutput?.result.length).to.equal(12); }); + + it('can export -> import junction with multiple queries', async () => { + const exportResult = execCmd( + `data:export:tree --plan --output-dir ${path.join( + '.', + 'junction' + )} --query "select AccountId, ContactId from AccountContactRole" --query "Select Id, AccountId, FirstName, LastName from Contact" --query "select Id, ContactId, AccountId from AccountContactRelation where Account.Name != 'We Know Everybody'" --query "select ID, Name from Account where Name != 'Sample Account for Entitlements'"` + ); + + expect(exportResult.shellOutput.stdout).to.include( + `records to ${path.join('.', 'junction', 'AccountContactRelation.json')}` + ); + expect(exportResult.shellOutput.stdout).to.include(`records to ${path.join('junction', 'Account.json')}`); + expect(exportResult.shellOutput.stdout).to.include(`records to ${path.join('junction', 'Contact.json')}`); + + execCmd( + `data:import:tree --target-org importOrg --plan ${path.join('.', 'junction', 'plan.json')}`, + { ensureExitCode: 0 } + ); + }); }); diff --git a/test/commands/data/tree/dataTreeMoreThan200.nut.ts b/test/commands/data/tree/dataTreeMoreThan200.nut.ts index bd67a0db..8352800d 100644 --- a/test/commands/data/tree/dataTreeMoreThan200.nut.ts +++ b/test/commands/data/tree/dataTreeMoreThan200.nut.ts @@ -50,7 +50,7 @@ describe('data:tree commands with more than 200 records are batches in safe grou expect(importResult.jsonOutput?.result.length).to.equal(265, 'Expected 265 records to be imported'); execCmd( - `data:export:tree --query "${query}" --prefix ${prefix} --outputdir ${path.join( + `data:export:tree --query "${query}" --prefix ${prefix} --output-dir ${path.join( '.', 'export_data' )} --plan --json`, diff --git a/test/commands/data/tree/dataTreeSelfReferencing.nut.ts b/test/commands/data/tree/dataTreeSelfReferencing.nut.ts index 64da6a1e..cc95af31 100644 --- a/test/commands/data/tree/dataTreeSelfReferencing.nut.ts +++ b/test/commands/data/tree/dataTreeSelfReferencing.nut.ts @@ -44,7 +44,7 @@ describe('data:tree commands with records that refer to other records of the sam }); execCmd( - `data:export:tree --query "${query}" --prefix INT --outputdir ${path.join('.', 'export_data')} --plan --json`, + `data:export:tree --query "${query}" --prefix INT --output-dir ${path.join('.', 'export_data')} --plan --json`, { ensureExitCode: 0 } );