Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support for queryAll in regular and bulk #602

Merged
merged 4 commits into from
Jun 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
378 changes: 79 additions & 299 deletions CHANGELOG.md

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions command-snapshot.json
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
"command": "data:query",
"plugin": "@salesforce/plugin-data",
"flags": [
"all-rows",
"api-version",
"async",
"bulk",
Expand Down
2 changes: 1 addition & 1 deletion messages/bulkv2.upsert.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ See "Prepare CSV Files" in the Bulk API Developer Guide for details on formattin

- Bulk upsert records to a custom object in an org with alias my-scratch and wait 5 minutes for the command to complete:

<%= config.bin %> <%= command.id %> --sobject MyObject__c --file files/file.csv --external-id MyField__c --wait 5 --target-org my-scratch
<%= config.bin %> <%= command.id %> --sobject MyObject**c --file files/file.csv --external-id MyField**c --wait 5 --target-org my-scratch

# flags.externalid

Expand Down
4 changes: 4 additions & 0 deletions messages/soql.query.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ Use Bulk API 2.0 to run the query.

Use Bulk API 2.0, but don't wait for the job to complete.

# flags.all-rows.summary

Include deleted records. By default, deleted records are not returned.

# flags.wait

Time to wait for the command to finish, in minutes.
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
},
"dependencies": {
"@oclif/core": "^2.8.7",
"@salesforce/core": "^4.0.1",
"@salesforce/core": "^4.3.3",
"@salesforce/kit": "^3.0.3",
"@salesforce/sf-plugins-core": "^3.1.1",
"@salesforce/ts-types": "^2.0.3",
Expand Down Expand Up @@ -255,4 +255,4 @@
"output": []
}
}
}
}
24 changes: 17 additions & 7 deletions src/commands/data/query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ export class DataSoqlQueryCommand extends SfCommand<unknown> {
dependsOn: ['bulk'],
exclusive: ['wait'],
}),
'all-rows': Flags.boolean({
summary: messages.getMessage('flags.all-rows.summary'),
}),
'result-format': resultFormatFlag,
perflog: perflogFlag,
};
Expand Down Expand Up @@ -115,7 +118,8 @@ export class DataSoqlQueryCommand extends SfCommand<unknown> {
flags['use-tooling-api'] ? conn.tooling : conn,
queryString,
this.logger,
this.configAggregator.getInfo('org-max-query-limit').value as number
this.configAggregator.getInfo('org-max-query-limit').value as number,
flags['all-rows']
);
if (!this.jsonEnabled()) {
displayResults({ ...queryResult }, flags['result-format'] as FormatTypes);
Expand All @@ -132,11 +136,16 @@ export class DataSoqlQueryCommand extends SfCommand<unknown> {
* @param query
* @param timeout
*/
private async runBulkSoqlQuery(connection: Connection, query: string, timeout: Duration): Promise<SoqlQueryResult> {
private async runBulkSoqlQuery(
connection: Connection,
query: string,
timeout: Duration,
allRows: boolean | undefined = false
): Promise<SoqlQueryResult> {
connection.bulk2.pollTimeout = timeout.milliseconds ?? Duration.minutes(5).milliseconds;
let res: Record[];
try {
res = (await connection.bulk2.query(query)) ?? [];
res = (await connection.bulk2.query(query, allRows ? { scanAll: true } : {})) ?? [];
return transformBulkResults(res, query);
} catch (e) {
const err = e as Error & { jobId: string };
Expand All @@ -155,15 +164,16 @@ export class DataSoqlQueryCommand extends SfCommand<unknown> {
connection: Connection | Connection['tooling'],
query: string,
logger: Logger,
maxFetch: number | undefined
maxFetch: number | undefined,
allRows: boolean | undefined = false
): Promise<SoqlQueryResult> {
logger.debug('running query');

const options = {
const result = await connection.query(query, {
autoFetch: true,
maxFetch: maxFetch ?? 50_000,
};
const result = await connection.query(query, options);
scanAll: allRows,
});
if (result.records.length && result.totalSize > result.records.length) {
this.warn(
`The query result is missing ${
Expand Down
34 changes: 8 additions & 26 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -900,10 +900,10 @@
strip-ansi "6.0.1"
ts-retry-promise "^0.7.0"

"@salesforce/core@^4.0.1", "@salesforce/core@^4.1.0", "@salesforce/core@^4.1.3", "@salesforce/core@^4.3.1":
version "4.3.1"
resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-4.3.1.tgz#d4f16ce7a89e90d456a9407e5994f155b7d05134"
integrity sha512-mqlVWVQiy24sdHR3DvBlSlg3QDCqVj4AIhcgvcVSTu3BtKMLaR2C6himtTdKIH/Fl6wLkxPcy+LufQ8J3+HBEw==
"@salesforce/core@^4.0.1", "@salesforce/core@^4.1.0", "@salesforce/core@^4.1.3", "@salesforce/core@^4.3.1", "@salesforce/core@^4.3.3":
version "4.3.3"
resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-4.3.3.tgz#1b64227413e43323f2f6b6a6ee0331239c1df468"
integrity sha512-oCKx9K+sM1rqooWU/nkf/uZVm7KAJparsVA04oA34//yE0wWeyhUXvcXYtp13WVp24XExxjR7q1Vpu7x4LOuHw==
dependencies:
"@salesforce/bunyan" "^2.0.0"
"@salesforce/kit" "^3.0.3"
Expand All @@ -916,7 +916,7 @@
faye "^1.4.0"
form-data "^4.0.0"
js2xmlparser "^4.0.1"
jsforce "^2.0.0-beta.24"
jsforce "^2.0.0-beta.25"
jsonwebtoken "9.0.0"
jszip "3.10.1"
proper-lockfile "^4.1.2"
Expand Down Expand Up @@ -1302,12 +1302,7 @@
dependencies:
"@types/node" "*"

"@types/semver@^7.3.12":
version "7.3.13"
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91"
integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==

"@types/semver@^7.5.0":
"@types/semver@^7.3.12", "@types/semver@^7.5.0":
version "7.5.0"
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.0.tgz#591c1ce3a702c45ee15f47a42ade72c2fd78978a"
integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==
Expand Down Expand Up @@ -4341,7 +4336,7 @@ jsesc@^2.5.1:
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==

jsforce@^2.0.0-beta.24, jsforce@^2.0.0-beta.27:
jsforce@^2.0.0-beta.25, jsforce@^2.0.0-beta.27:
version "2.0.0-beta.27"
resolved "https://registry.yarnpkg.com/jsforce/-/jsforce-2.0.0-beta.27.tgz#ba822b18b77bea4529491060a9b07bc1009735ac"
integrity sha512-d9dDWWeHwayRKPo8FJBAIUyk8sNXGSHwdUjR6al3yK0YKci27Jc1XfNaQTxEAuymHQJVaCb1gxTKqmA4uznFdQ==
Expand Down Expand Up @@ -5905,20 +5900,7 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.4.0, readable-stre
string_decoder "^1.1.1"
util-deprecate "^1.0.1"

readable-stream@^2.0.2, readable-stream@^2.3.5:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.3"
isarray "~1.0.0"
process-nextick-args "~2.0.0"
safe-buffer "~5.1.1"
string_decoder "~1.1.1"
util-deprecate "~1.0.1"

readable-stream@~2.3.6:
readable-stream@^2.0.2, readable-stream@^2.3.5, readable-stream@~2.3.6:
version "2.3.8"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
Expand Down