Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow setting maxFileAdd via env var #295

Merged
merged 11 commits into from
Jan 3, 2023
Merged

Allow setting maxFileAdd via env var #295

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
3032066
fix: testing lowering maxFileAdd
iowillhoit Dec 26, 2022
7d3f898
chore(release): 2.2.16-dev.1 [skip ci]
svc-cli-bot Dec 26, 2022
060b821
fix: maxFileAdd env var
iowillhoit Dec 28, 2022
766bb69
Merge branch 'prerelease/max-file-add-test' of github.com:forcedotcom…
iowillhoit Dec 28, 2022
792f5e3
chore(release): 2.2.16-dev.2 [skip ci]
svc-cli-bot Dec 28, 2022
8e3d011
fix: better error message
iowillhoit Dec 28, 2022
8d2f54a
chore(release): 2.2.16-dev.3 [skip ci]
svc-cli-bot Dec 28, 2022
093490f
Update src/shared/localShadowRepo.ts
iowillhoit Jan 3, 2023
b1dc011
chore: version
iowillhoit Jan 3, 2023
b7f8802
chore: dep bumps
iowillhoit Jan 3, 2023
18f526d
refactor: remove nested ternary
iowillhoit Jan 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/onPushToMain.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,17 @@ name: version, tag and github release

on:
push:
branches: [main]

branches:
- main
- prerelease/*
tags-ignore:
- '*'
jobs:
release:
uses: salesforcecli/github-workflows/.github/workflows/githubRelease.yml@main
secrets: inherit
with:
prerelease: ${{ github.ref_name != 'main' }}

# most repos won't use this
# depends on previous job to avoid git collisions, not for any functionality reason
Expand Down
15 changes: 13 additions & 2 deletions .github/workflows/onRelease.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: publish

on:
release:
types: [released]
types: [published]
# support manual release in case something goes wrong and needs to be repeated or tested
workflow_dispatch:
inputs:
Expand All @@ -11,9 +11,20 @@ on:
type: string
required: true
jobs:
getDistTag:
outputs:
tag: ${{ steps.distTag.outputs.tag }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.release.tag_name || inputs.tag }}
- uses: salesforcecli/github-workflows/.github/actions/getPreReleaseTag@main
id: distTag
npm:
uses: salesforcecli/github-workflows/.github/workflows/npmPublish.yml@main
needs: [getDistTag]
with:
tag: latest
tag: ${{ needs.getDistTag.outputs.tag || 'latest' }}
githubTag: ${{ github.event.release.tag_name || inputs.tag }}
secrets: inherit
27 changes: 27 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,30 @@
## [2.2.16-dev.3](https://github.com/forcedotcom/source-tracking/compare/2.2.16-dev.2...2.2.16-dev.3) (2022-12-28)


### Bug Fixes

* better error message ([8e3d011](https://github.com/forcedotcom/source-tracking/commit/8e3d0110c1994fd9389c6896d1a1eef415d87462))



## [2.2.16-dev.2](https://github.com/forcedotcom/source-tracking/compare/2.2.16-dev.1...2.2.16-dev.2) (2022-12-28)


### Bug Fixes

* maxFileAdd env var ([060b821](https://github.com/forcedotcom/source-tracking/commit/060b821587ebc10a41919426617e70045ba6db65))



## [2.2.16-dev.1](https://github.com/forcedotcom/source-tracking/compare/2.2.15...2.2.16-dev.1) (2022-12-26)


### Bug Fixes

* testing lowering maxFileAdd ([3032066](https://github.com/forcedotcom/source-tracking/commit/303206643bfa3ef5b6c48f18d4250f3462130da3))



## [2.2.15](https://github.com/forcedotcom/source-tracking/compare/2.2.14...2.2.15) (2022-12-14)


Expand Down
2 changes: 1 addition & 1 deletion LICENSE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Copyright (c) 2022, Salesforce.com, Inc.
Copyright (c) 2023, Salesforce.com, Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Expand Down
8 changes: 4 additions & 4 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@
"/oclif.manifest.json"
],
"dependencies": {
"@salesforce/core": "^3.32.8",
"@salesforce/kit": "^1.8.0",
"@salesforce/source-deploy-retrieve": "^7.5.16",
"@salesforce/core": "^3.32.12",
"@salesforce/kit": "^1.8.1",
"@salesforce/source-deploy-retrieve": "^7.5.22",
"graceful-fs": "^4.2.10",
"isomorphic-git": "1.17.0",
"ts-retry-promise": "^0.7.0"
Expand Down Expand Up @@ -84,4 +84,4 @@
"publishConfig": {
"access": "public"
}
}
}
17 changes: 14 additions & 3 deletions src/shared/localShadowRepo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ import * as path from 'path';
import * as os from 'os';
import * as fs from 'graceful-fs';
import { NamedPackageDir, Logger, SfError } from '@salesforce/core';
import { env } from '@salesforce/kit';
import * as git from 'isomorphic-git';
import { chunkArray, isLwcLocalOnlyTest, pathIsInFolder } from './functions';

/** returns the full path to where we store the shadow repo */
const getGitDir = (orgId: string, projectPath: string, useSfdxTrackingFiles = false): string => path.join(projectPath, useSfdxTrackingFiles ? '.sfdx' : '.sf', 'orgs', orgId, 'localSourceTracking');
const getGitDir = (orgId: string, projectPath: string, useSfdxTrackingFiles = false): string =>
path.join(projectPath, useSfdxTrackingFiles ? '.sfdx' : '.sf', 'orgs', orgId, 'localSourceTracking');

// filenames were normalized when read from isogit
const toFilenames = (rows: StatusRow[]): string[] => rows.map((row) => row[FILE]);
Expand Down Expand Up @@ -59,7 +61,9 @@ export class ShadowRepo {
this.projectPath = options.projectPath;
this.packageDirs = options.packageDirs;
this.isWindows = os.type() === 'Windows_NT';
this.maxFileAdd = this.isWindows ? 8000 : 15000;

const batchSize = env.getNumber('SFDX_SOURCE_TRACKING_BATCH_SIZE');
this.maxFileAdd = batchSize ? batchSize : this.isWindows ? 8000 : 15000;
Copy link
Contributor

@mshanemc mshanemc Jan 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rather than nested ternary, you could do something like

Suggested change
this.maxFileAdd = batchSize ? batchSize : this.isWindows ? 8000 : 15000;
this.maxFileAdd = batchSize ?? this.isWindows ? 8000 : 15000;

or just

this.maxFileAdd = env.getNumber('SFDX_SOURCE_TRACKING_BATCH_SIZE') ?? this.isWindows ? 8000 : 15000

or use the "default" 2nd prop of env,

this.maxFileAdd = env.getNumber('SFDX_SOURCE_TRACKING_BATCH_SIZE',  this.isWindows ? 8000 : 15000)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea, I generally don't like nested ternaries but I like the combination of a nullish coalescing and a ternary less. Mainly because it is not immediately obvious what is being compared. The code you suggested would not work correctly, it would compare the nullish first and use that result for the ternary. If you want to combine the two operators, you'd need to wrap your ternary in parentheses.

// resolves 'tern-true' 👎
console.log('foo' ?? true ? 'tern-true' : 'tern-false');

// resolves 'foo'
console.log('foo' ?? (true ? 'tern-true' : 'tern-false'));

// resolves ""
console.log('' ?? (true ? 'tern-true' : 'tern-false'));

// resolves "tern-true"
console.log(undefined ?? (true ? 'tern-true' : 'tern-false'));

// resolves "tern-false"
console.log(undefined ?? (false ? 'tern-true' : 'tern-false'));

https://jsfiddle.net/hxjzaq5c/1/

I do like the idea of using the default for getNumber, I will switch to that. Thanks for the suggestion

}

// think of singleton behavior but unique to the projectPath
Expand Down Expand Up @@ -243,7 +247,14 @@ export class ShadowRepo {
} catch (e) {
if (e instanceof git.Errors.MultipleGitError) {
this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
const error = new SfError(e.message, e.name, [], 1);
const error = new SfError(
e.message,
e.name,
[
`One potential reason you're getting this error is that the number of files that source tracking is batching exceeds your user-specific file limits. Increase your hard file limit in the same session by executing 'ulimit -Hn ${this.maxFileAdd}'. Or set the 'SFDX_SOURCE_TRACKING_BATCH_SIZE' environment variable to a value lower than the output of 'ulimit -Hn'.\nNote: Don't set this environment variable too close to the upper limit or your system will still hit it. If you continue to get the error, lower the value of the environment variable even more.`,
],
1
);
error.setData(e.errors);
throw error;
}
Expand Down
80 changes: 67 additions & 13 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@
strip-ansi "6.0.1"
ts-retry-promise "^0.7.0"

"@salesforce/core@^3.30.9", "@salesforce/core@^3.32.8":
"@salesforce/core@^3.30.9":
version "3.32.9"
resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.32.9.tgz#bb655f7cdfc5b97538421147add20e8b84a364ff"
integrity sha512-dpvatNXEtxuT0/8dW/iO6KHTVGO38rUVfZ5HgPRoOOfKt0bq0KSi3PrVj1LyA7iyE7wCzliXY4vTdLhqYtZY6g==
Expand All @@ -696,6 +696,29 @@
jsonwebtoken "8.5.1"
ts-retry-promise "^0.7.0"

"@salesforce/core@^3.32.12":
version "3.32.12"
resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.32.12.tgz#853cc5b6a5f95d4896b2d34a40a6042ef9aa6d2c"
integrity sha512-27rqSiQWul7b/OkJs19FYDv2M/S4oI4efiGv+6sR7UWv7D7CG1P+0XpgLS3d9xRYF30h98n6VQr4W2a+BWFRvA==
dependencies:
"@salesforce/bunyan" "^2.0.0"
"@salesforce/kit" "^1.8.0"
"@salesforce/schemas" "^1.4.0"
"@salesforce/ts-types" "^1.5.21"
"@types/graceful-fs" "^4.1.5"
"@types/semver" "^7.3.13"
ajv "^8.11.2"
archiver "^5.3.0"
change-case "^4.1.2"
debug "^3.2.7"
faye "^1.4.0"
form-data "^4.0.0"
graceful-fs "^4.2.9"
js2xmlparser "^4.0.1"
jsforce "^2.0.0-beta.19"
jsonwebtoken "9.0.0"
ts-retry-promise "^0.7.0"

"@salesforce/dev-config@^3.0.0", "@salesforce/dev-config@^3.1.0":
version "3.1.0"
resolved "https://registry.yarnpkg.com/@salesforce/dev-config/-/dev-config-3.1.0.tgz#8eb5b35860ff60d1c1dc3fd9329b01a28475d5b9"
Expand Down Expand Up @@ -749,6 +772,15 @@
shx "^0.3.3"
tslib "^2.2.0"

"@salesforce/kit@^1.8.1":
version "1.8.1"
resolved "https://registry.yarnpkg.com/@salesforce/kit/-/kit-1.8.1.tgz#289ca9f4094157c1deaa77cc72f295fdc71f043f"
integrity sha512-fCMKh7yWiWtl9C2OXamwkzxD7r6MddrsxKAvlMYZTrTeVG/wxrV03NIhARVkT0k12CsmWNYv83VSPobBAufiIA==
dependencies:
"@salesforce/ts-types" "^1.7.1"
shx "^0.3.3"
tslib "^2.2.0"

"@salesforce/prettier-config@^0.0.2":
version "0.0.2"
resolved "https://registry.yarnpkg.com/@salesforce/prettier-config/-/prettier-config-0.0.2.tgz#ded39bf7cb75238edc9db6dd093649111350f8bc"
Expand All @@ -759,21 +791,21 @@
resolved "https://registry.yarnpkg.com/@salesforce/schemas/-/schemas-1.4.0.tgz#7dff427c8059895d8108176047aee600703c82d6"
integrity sha512-BJ25uphssN42Zy6kksheFHMTLiR98AAHe/Wxnv0T4dYxtrEbUjSXVAGKZqfewJPFXA4xB5gxC+rQZtfz6xKCFg==

"@salesforce/source-deploy-retrieve@^7.5.16":
version "7.5.16"
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-7.5.16.tgz#7abcb1228e7c6de2ddc18ecdb593e2628dc36465"
integrity sha512-pmhnZJnNkF6w88Z5DHDWFrBi63Ny5yGUp2aHTl6WH6LCX7l3+cguSFIQKuTUj1ZO6omqkasuWJE6OPpcoEfoHg==
"@salesforce/source-deploy-retrieve@^7.5.22":
version "7.5.22"
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-7.5.22.tgz#6825f75c6a1d762149f9e330683182d8f3003110"
integrity sha512-+ZDCj5zULWs6lj2B978XKt8pj1ondHoxqSh7O3UpyyjAppCnrQGGrduoAzycjrSVmGRXkwrrTybLzA8zil9zsw==
dependencies:
"@salesforce/core" "^3.32.8"
"@salesforce/core" "^3.32.12"
"@salesforce/kit" "^1.8.0"
"@salesforce/ts-types" "^1.7.1"
archiver "^5.3.1"
fast-xml-parser "^3.21.1"
got "^11.8.5"
got "^11.8.6"
graceful-fs "^4.2.10"
ignore "^5.2.0"
ignore "^5.2.4"
mime "2.6.0"
minimatch "^5.1.0"
minimatch "^5.1.2"
proxy-agent "^5.0.0"
proxy-from-env "^1.1.0"
unzipper "0.10.11"
Expand Down Expand Up @@ -3130,10 +3162,10 @@ globby@^11.0.1, globby@^11.1.0:
merge2 "^1.4.1"
slash "^3.0.0"

got@^11.8.5:
version "11.8.5"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.5.tgz#ce77d045136de56e8f024bebb82ea349bc730046"
integrity sha512-o0Je4NvQObAuZPHLFoRSkdG2lTgtcynqymzg2Vupdx6PorhaT5MCbIyXG6d4D94kk8ZG57QeosgdiqfJWhEhlQ==
got@^11.8.6:
version "11.8.6"
resolved "https://registry.yarnpkg.com/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a"
integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==
dependencies:
"@sindresorhus/is" "^4.0.0"
"@szmarczak/http-timer" "^4.0.5"
Expand Down Expand Up @@ -3346,6 +3378,11 @@ ignore@^5.1.1, ignore@^5.1.4, ignore@^5.2.0:
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a"
integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==

ignore@^5.2.4:
version "5.2.4"
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324"
integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==

import-fresh@^3.0.0, import-fresh@^3.2.1:
version "3.3.0"
resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"
Expand Down Expand Up @@ -3864,6 +3901,16 @@ [email protected]:
ms "^2.1.1"
semver "^5.6.0"

[email protected]:
version "9.0.0"
resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
dependencies:
jws "^3.2.2"
lodash "^4.17.21"
ms "^2.1.1"
semver "^7.3.8"

just-extend@^4.0.2:
version "4.2.1"
resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
Expand Down Expand Up @@ -4222,6 +4269,13 @@ minimatch@^5.0.1, minimatch@^5.1.0:
dependencies:
brace-expansion "^2.0.1"

minimatch@^5.1.2:
version "5.1.2"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.2.tgz#0939d7d6f0898acbd1508abe534d1929368a8fff"
integrity sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg==
dependencies:
brace-expansion "^2.0.1"

[email protected]:
version "4.1.0"
resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"
Expand Down