Skip to content

Commit

Permalink
Merge pull request #83 from texei/data-lookup-override
Browse files Browse the repository at this point in the history 
Data lookup override
  • Loading branch information
@FabienTaillon
FabienTaillon authored Nov 24, 2021
2 parents 01f042f + e318021 commit f165787
Show file tree
Hide file tree
Showing 7 changed files with 4,629 additions and 4,474 deletions.
40 changes: 21 additions & 19 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Link the plugin: sfdx plugins:link .
<!-- commands -->
* [`sfdx texei:contractstatus:value:add -l <string> -a <string> [-s <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeicontractstatusvalueadd--l-string--a-string--s-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataexport--d-string--o-string--p-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataimport--d-string--a--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataimport--d-string--a--u-string---apiversion-string---verbose---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:data:plan:generate -d <string> -o <string> [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataplangenerate--d-string--o-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:org:contractfieldhistory:fix [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeiorgcontractfieldhistoryfix--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
* [`sfdx texei:org:shape:extract [-d <string>] [-s <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeiorgshapeextract--d-string--s-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
Expand Down Expand Up @@ -72,7 +72,7 @@ EXAMPLE
--targetusername texei
```

_See code: [src/commands/texei/contractstatus/value/add.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/contractstatus/value/add.ts)_
_See code: [src/commands/texei/contractstatus/value/add.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/contractstatus/value/add.ts)_

## `sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -108,15 +108,15 @@ EXAMPLES
sfdx texei:data:export --dataplan ./data/data-plan.json --outputdir ./data --targetusername texei
```

_See code: [src/commands/texei/data/export.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/data/export.ts)_
_See code: [src/commands/texei/data/export.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/data/export.ts)_

## `sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`
## `sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

import objects' data to org

```
USAGE
$ sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--json] [--loglevel
$ sfdx texei:data:import -d <string> [-a] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel
trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]
OPTIONS
Expand All @@ -138,12 +138,14 @@ OPTIONS
--loglevel=(trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL) [default: warn] logging level for
this command invocation
--verbose verbose output of import result
EXAMPLE
$ sfdx texei:data:import --inputdir ./data --targetusername texei-scratch
Data imported!
```

_See code: [src/commands/texei/data/import.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/data/import.ts)_
_See code: [src/commands/texei/data/import.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/data/import.ts)_

## `sfdx texei:data:plan:generate -d <string> -o <string> [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -170,7 +172,7 @@ EXAMPLE
$ sfdx texei:data:plan:generate --objects Account,Contact,MyCustomObject__c --outputdir ./data
```

_See code: [src/commands/texei/data/plan/generate.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/data/plan/generate.ts)_
_See code: [src/commands/texei/data/plan/generate.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/data/plan/generate.ts)_

## `sfdx texei:org:contractfieldhistory:fix [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -198,7 +200,7 @@ EXAMPLE
History tracking fixed.
```

_See code: [src/commands/texei/org/contractfieldhistory/fix.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/org/contractfieldhistory/fix.ts)_
_See code: [src/commands/texei/org/contractfieldhistory/fix.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/org/contractfieldhistory/fix.ts)_

## `sfdx texei:org:shape:extract [-d <string>] [-s <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -233,7 +235,7 @@ EXAMPLE
$ sfdx texei:org:shape:extract -u [email protected]"
```

_See code: [src/commands/texei/org/shape/extract.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/org/shape/extract.ts)_
_See code: [src/commands/texei/org/shape/extract.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/org/shape/extract.ts)_

## `sfdx texei:package:dependencies:install [-k <string>] [-b <string>] [-p <string>] [-s <string>] [-n <string>] [-w <number>] [-r] [-a <string>] [-v <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -296,7 +298,7 @@ EXAMPLE
$ texei:package:dependencies:install -u MyScratchOrg -v MyDevHub -k "1:MyPackage1Key 2: 3:MyPackage3Key" -b "DEV"
```

_See code: [src/commands/texei/package/dependencies/install.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/package/dependencies/install.ts)_
_See code: [src/commands/texei/package/dependencies/install.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/package/dependencies/install.ts)_

## `sfdx texei:profile:clean [-k <string>] [-p <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -330,7 +332,7 @@ EXAMPLES
custom-sfdx-source-folder/main/profiles,source-folder-2/main/profiles/myAdmin.profile-meta.xml
```

_See code: [src/commands/texei/profile/clean.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/profile/clean.ts)_
_See code: [src/commands/texei/profile/clean.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/profile/clean.ts)_

## `sfdx texei:sharedactivities:enable [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -357,7 +359,7 @@ EXAMPLE
$ sfdx texei:sharedactivities:enable
```

_See code: [src/commands/texei/sharedactivities/enable.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/sharedactivities/enable.ts)_
_See code: [src/commands/texei/sharedactivities/enable.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/sharedactivities/enable.ts)_

## `sfdx texei:sharingcalc:recalculate [-s <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -388,7 +390,7 @@ EXAMPLE
Recalculated Sharing Rules
```

_See code: [src/commands/texei/sharingcalc/recalculate.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/sharingcalc/recalculate.ts)_
_See code: [src/commands/texei/sharingcalc/recalculate.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/sharingcalc/recalculate.ts)_

## `sfdx texei:sharingcalc:resume [-s <string>] [-t <number>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -422,7 +424,7 @@ EXAMPLE
Sharing calculations resumed
```

_See code: [src/commands/texei/sharingcalc/resume.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/sharingcalc/resume.ts)_
_See code: [src/commands/texei/sharingcalc/resume.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/sharingcalc/resume.ts)_

## `sfdx texei:sharingcalc:suspend [-s <string>] [-t <number>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -456,7 +458,7 @@ EXAMPLE
Sharing calculations suspended
```

_See code: [src/commands/texei/sharingcalc/suspend.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/sharingcalc/suspend.ts)_
_See code: [src/commands/texei/sharingcalc/suspend.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/sharingcalc/suspend.ts)_

## `sfdx texei:skinnyprofile:retrieve [-t <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -486,7 +488,7 @@ EXAMPLE
$ texei:skinnyprofile:retrieve -u MyScratchOrg
```

_See code: [src/commands/texei/skinnyprofile/retrieve.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/skinnyprofile/retrieve.ts)_
_See code: [src/commands/texei/skinnyprofile/retrieve.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/skinnyprofile/retrieve.ts)_

## `sfdx texei:source:customlabel:replace -l <string> -v <string> [-p <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -510,7 +512,7 @@ EXAMPLE
$ texei:source:customlabel:replace --label GreatSalesforceBlog --value https://blog.texei.com
```

_See code: [src/commands/texei/source/customlabel/replace.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/source/customlabel/replace.ts)_
_See code: [src/commands/texei/source/customlabel/replace.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/source/customlabel/replace.ts)_

## `sfdx texei:source:layouts:cleanorg [-p <string>] [-v <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -543,7 +545,7 @@ EXAMPLES
$ texei:source:layouts:cleanorg --targetusername myScratchOrg --targetdevhubusername myDevHub
```

_See code: [src/commands/texei/source/layouts/cleanorg.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/source/layouts/cleanorg.ts)_
_See code: [src/commands/texei/source/layouts/cleanorg.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/source/layouts/cleanorg.ts)_

## `sfdx texei:user:update [-v <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand Down Expand Up @@ -576,5 +578,5 @@ EXAMPLES
sfdx texei:user:update --values "LanguageLocaleKey=en_US UserPermissionsMarketingUser=true" --json
```

_See code: [src/commands/texei/user/update.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.11.1/src/commands/texei/user/update.ts)_
_See code: [src/commands/texei/user/update.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.12.0/src/commands/texei/user/update.ts)_
<!-- commandsstop -->
3 changes: 2 additions & 1 deletion messages/data-import.json
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"commandDescription": "import objects' data to org",
"inputFlagDescription": "directory with files to import",
"allOrNoneFlagDescription": "any failed records in a call cause all changes for the call to be rolled back"
"allOrNoneFlagDescription": "any failed records in a call cause all changes for the call to be rolled back",
"verbose": "verbose output of import result"
}
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "texei-sfdx-plugin",
"description": "Texeï's plugin for sfdx",
"version": "1.11.1",
"version": "1.12.0",
"author": "Texeï",
"bugs": "https://github.com/https://github.com/texei/texei-sfdx-plugin/issues",
"dependencies": {
Expand All @@ -12,6 +12,7 @@
"@salesforce/core": "^2.1.5",
"bl": "^5.0.0",
"child-process-promise": "^2.2.1",
"csvtojson": "^2.0.10",
"npm-check-updates": "^11.5.1",
"puppeteer": "^9.0.0",
"tslib": "^2.2.0",
Expand Down
85 changes: 74 additions & 11 deletions src/commands/texei/data/export.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ interface DataPlan {
name: string;
label: string;
filters: string;
lookupOverride: {}
excludedFields: Array<string>;
}

Expand All @@ -23,6 +24,7 @@ let conn:Connection;
let objectList:Array<DataPlan>;
let lastReferenceIds: Map<string, number> = new Map<string, number>();
let globallyExcludedFields: Array<string>;
let globallyOverridenLookup: Map<string, string>;

export default class Export extends SfdxCommand {

Expand Down Expand Up @@ -74,6 +76,14 @@ export default class Export extends SfdxCommand {
if (dataPlan.excludedFields) {
globallyExcludedFields = dataPlan.excludedFields;
}
// If there are some global lookup override, add them
if (dataPlan.lookupOverride) {
globallyOverridenLookup = new Map<string, string>();

for (const [key, value] of Object.entries(dataPlan.lookupOverride)) {
globallyOverridenLookup.set(key, value as string);
}
}
}
else {
throw new SfdxError(`Either objects or dataplan flag is mandatory`);
Expand Down Expand Up @@ -101,6 +111,8 @@ export default class Export extends SfdxCommand {
// Query to retrieve creatable sObject fields
let fields = [];
let lookups = [];
let overriddenLookups = [];
let relationshipFields = [];
let userFieldsReference = [];
const describeResult = await conn.sobject(sobject.name).describe();

Expand All @@ -121,17 +133,50 @@ export default class Export extends SfdxCommand {

if (field.createable && !fieldsToExclude.includes(field.name)) {

fields.push(field.name);
// If it's a lookup, also add it to the lookup list, to be replaced later
// Excluding OwnerId as we are not importing users anyway
if (field.referenceTo && field.referenceTo.length > 0 && field.name != 'OwnerId' && field.name != 'RecordTypeId') {

// If User is queried, use the reference, otherwise use the Scratch Org User
if (!objectList.find(x => x.name === 'User') && field.referenceTo.includes('User')) {
userFieldsReference.push(field.name);
// If it's a lookup field and it's overridden at the field level, use the override
if (sobject.lookupOverride
&& sobject.lookupOverride[field.name]) {

this.debug(`Field found in override: ${field.name}`);

if (!(field.referenceTo && field.referenceTo.length > 0)) {
throw new SfdxError(`Field ${field.name} is listed in lookupOverride but isn't a lookup field`);
}
else {
overriddenLookups.push(field.relationshipName);
sobject.lookupOverride[field.name]?.split(',')?.forEach(relationshipField => {
relationshipFields.push(`${field.relationshipName}.${relationshipField}`);
});
}
}
else {
// If it's a lookup, also add it to the lookup list, to be replaced later
// Excluding OwnerId as we are not importing users anyway
if (field.referenceTo && field.referenceTo.length > 0 && field.name != 'OwnerId' && field.name != 'RecordTypeId') {

if (globallyOverridenLookup?.get(field.referenceTo[0])) {
this.debug(`FOUND ${field.name}: ${globallyOverridenLookup.get(field.referenceTo[0])}`);

// If it's a lookup field and it's overridden at the field level, use the override
overriddenLookups.push(field.relationshipName);
globallyOverridenLookup.get(field.referenceTo[0])?.split(',')?.forEach(relationshipField => {
relationshipFields.push(`${field.relationshipName}.${relationshipField}`);
});
}
else {
fields.push(field.name);

// If User is queried, use the reference, otherwise use the Scratch Org User
if (!objectList.find(x => x.name === 'User') && field.referenceTo.includes('User')) {
userFieldsReference.push(field.name);
}
else {
lookups.push(field.name);
}
}
}
else {
lookups.push(field.name);
fields.push(field.name);
}
}
}
Expand All @@ -151,6 +196,11 @@ export default class Export extends SfdxCommand {
fields.push('IsStandard');
}

// Adding relationship fields from lookupOverride, if any
if (relationshipFields) {
fields = fields.concat(relationshipFields);
}

// Query to get sObject data
const recordQuery = `SELECT Id, ${fields.join()}
FROM ${sobject.name}
Expand All @@ -164,14 +214,14 @@ export default class Export extends SfdxCommand {
const recordResults = (await conn.autoFetchQuery(recordQuery, options)).records;

// Replace Lookup Ids + Record Type Ids by references
await this.cleanJsonRecord(sobject, sObjectLabel, recordResults, recordIdsMap, lookups, userFieldsReference);
await this.cleanJsonRecord(sobject, sObjectLabel, recordResults, recordIdsMap, lookups, overriddenLookups, userFieldsReference);

return recordResults;
}

// Clean JSON to have the same output format as force:data:tree:export
// Main difference: RecordTypeId is replaced by DeveloperName
private async cleanJsonRecord(sobject: DataPlan, objectLabel: string, records, recordIdsMap, lookups: Array<string>, userFieldsReference: Array<string>,) {
private async cleanJsonRecord(sobject: DataPlan, objectLabel: string, records, recordIdsMap, lookups: Array<string>, overriddenLookups: Array<string>, userFieldsReference: Array<string>,) {

let refId = 0;
// If this object was already exported before, start the numbering after the last one already used
Expand Down Expand Up @@ -210,6 +260,19 @@ export default class Export extends SfdxCommand {
record[lookup] = recordIdsMap.get(record[lookup]);
}

// Replace overridden lookups
for (const lookup of overriddenLookups) {
if (record[lookup]) {
// If lookup isn't empty, remove useless information
// Keeping "type" so we don't have to do a describe to know what is the related sObject at import
delete record[lookup].attributes.url;
}
else {
// Remove empty lookup relationship field
delete record[lookup];
}
}

// Replace RecordTypeId with DeveloperName, to replace later with newly generated Id
if (record.RecordTypeId && record.RecordType) {
record.RecordTypeId = record.RecordType.DeveloperName;
Expand Down
Loading

0 comments on commit f165787

Please sign in to comment.