Allow setting maxFileAdd via env var

.github/workflows/onPushToMain.yml

@@ -3,12 +3,17 @@ name: version, tag and github release
 
 on:
   push:
-    branches: [main]
-
+    branches:
+      - main
+      - prerelease/*
+    tags-ignore:
+      - '*'
 jobs:
   release:
     uses: salesforcecli/github-workflows/.github/workflows/githubRelease.yml@main
     secrets: inherit
+    with:
+      prerelease: ${{ github.ref_name != 'main' }}
 
   # most repos won't use this
   # depends on previous job to avoid git collisions, not for any functionality reason

.github/workflows/onRelease.yml

@@ -2,7 +2,7 @@ name: publish
 
 on:
   release:
-    types: [released]
+    types: [published]
   # support manual release in case something goes wrong and needs to be repeated or tested
   workflow_dispatch:
     inputs:
@@ -11,9 +11,20 @@ on:
         type: string
         required: true
 jobs:
+  getDistTag:
+    outputs:
+      tag: ${{ steps.distTag.outputs.tag }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.release.tag_name || inputs.tag  }}
+      - uses: salesforcecli/github-workflows/.github/actions/getPreReleaseTag@main
+        id: distTag
   npm:
     uses: salesforcecli/github-workflows/.github/workflows/npmPublish.yml@main
+    needs: [getDistTag]
     with:
-      tag: latest
+      tag: ${{ needs.getDistTag.outputs.tag || 'latest' }}
       githubTag: ${{ github.event.release.tag_name || inputs.tag }}
     secrets: inherit

CHANGELOG.md

@@ -1,3 +1,30 @@
+## [2.2.16-dev.3](https://github.com/forcedotcom/source-tracking/compare/2.2.16-dev.2...2.2.16-dev.3) (2022-12-28)
+
+
+### Bug Fixes
+
+* better error message ([8e3d011](https://github.com/forcedotcom/source-tracking/commit/8e3d0110c1994fd9389c6896d1a1eef415d87462))
+
+
+
+## [2.2.16-dev.2](https://github.com/forcedotcom/source-tracking/compare/2.2.16-dev.1...2.2.16-dev.2) (2022-12-28)
+
+
+### Bug Fixes
+
+* maxFileAdd env var ([060b821](https://github.com/forcedotcom/source-tracking/commit/060b821587ebc10a41919426617e70045ba6db65))
+
+
+
+## [2.2.16-dev.1](https://github.com/forcedotcom/source-tracking/compare/2.2.15...2.2.16-dev.1) (2022-12-26)
+
+
+### Bug Fixes
+
+* testing lowering maxFileAdd ([3032066](https://github.com/forcedotcom/source-tracking/commit/303206643bfa3ef5b6c48f18d4250f3462130da3))
+
+
+
 ## [2.2.15](https://github.com/forcedotcom/source-tracking/compare/2.2.14...2.2.15) (2022-12-14)
 
 

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2022, Salesforce.com, Inc.
+Copyright (c) 2023, Salesforce.com, Inc.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

package.json

@@ -43,9 +43,9 @@
     "/oclif.manifest.json"
   ],
   "dependencies": {
-    "@salesforce/core": "^3.32.8",
-    "@salesforce/kit": "^1.8.0",
-    "@salesforce/source-deploy-retrieve": "^7.5.16",
+    "@salesforce/core": "^3.32.12",
+    "@salesforce/kit": "^1.8.1",
+    "@salesforce/source-deploy-retrieve": "^7.5.22",
     "graceful-fs": "^4.2.10",
     "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.7.0"
@@ -84,4 +84,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

src/shared/localShadowRepo.ts

@@ -9,11 +9,13 @@ import * as path from 'path';
 import * as os from 'os';
 import * as fs from 'graceful-fs';
 import { NamedPackageDir, Logger, SfError } from '@salesforce/core';
+import { env } from '@salesforce/kit';
 import * as git from 'isomorphic-git';
 import { chunkArray, isLwcLocalOnlyTest, pathIsInFolder } from './functions';
 
 /** returns the full path to where we store the shadow repo */
-const getGitDir = (orgId: string, projectPath: string, useSfdxTrackingFiles = false): string => path.join(projectPath, useSfdxTrackingFiles ? '.sfdx' : '.sf', 'orgs', orgId, 'localSourceTracking');
+const getGitDir = (orgId: string, projectPath: string, useSfdxTrackingFiles = false): string =>
+  path.join(projectPath, useSfdxTrackingFiles ? '.sfdx' : '.sf', 'orgs', orgId, 'localSourceTracking');
 
 // filenames were normalized when read from isogit
 const toFilenames = (rows: StatusRow[]): string[] => rows.map((row) => row[FILE]);
@@ -59,7 +61,8 @@ export class ShadowRepo {
     this.projectPath = options.projectPath;
     this.packageDirs = options.packageDirs;
     this.isWindows = os.type() === 'Windows_NT';
-    this.maxFileAdd = this.isWindows ? 8000 : 15000;
+
+    this.maxFileAdd = env.getNumber('SFDX_SOURCE_TRACKING_BATCH_SIZE', this.isWindows ? 8000 : 15000);
   }
 
   // think of singleton behavior but unique to the projectPath
@@ -243,7 +246,14 @@ export class ShadowRepo {
         } catch (e) {
           if (e instanceof git.Errors.MultipleGitError) {
             this.logger.error('multiple errors on git.add', e.errors.slice(0, 5));
-            const error = new SfError(e.message, e.name, [], 1);
+            const error = new SfError(
+              e.message,
+              e.name,
+              [
+                `One potential reason you're getting this error is that the number of files that source tracking is batching exceeds your user-specific file limits. Increase your hard file limit in the same session by executing 'ulimit -Hn ${this.maxFileAdd}'.  Or set the 'SFDX_SOURCE_TRACKING_BATCH_SIZE' environment variable to a value lower than the output of 'ulimit -Hn'.\nNote: Don't set this environment variable too close to the upper limit or your system will still hit it. If you continue to get the error, lower the value of the environment variable even more.`,
+              ],
+              1
+            );
             error.setData(e.errors);
             throw error;
           }

yarn.lock

@@ -673,7 +673,7 @@
     strip-ansi "6.0.1"
     ts-retry-promise "^0.7.0"
 
-"@salesforce/core@^3.30.9", "@salesforce/core@^3.32.8":
+"@salesforce/core@^3.30.9":
   version "3.32.9"
   resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.32.9.tgz#bb655f7cdfc5b97538421147add20e8b84a364ff"
   integrity sha512-dpvatNXEtxuT0/8dW/iO6KHTVGO38rUVfZ5HgPRoOOfKt0bq0KSi3PrVj1LyA7iyE7wCzliXY4vTdLhqYtZY6g==
@@ -696,6 +696,29 @@
     jsonwebtoken "8.5.1"
     ts-retry-promise "^0.7.0"
 
+"@salesforce/core@^3.32.12":
+  version "3.32.12"
+  resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-3.32.12.tgz#853cc5b6a5f95d4896b2d34a40a6042ef9aa6d2c"
+  integrity sha512-27rqSiQWul7b/OkJs19FYDv2M/S4oI4efiGv+6sR7UWv7D7CG1P+0XpgLS3d9xRYF30h98n6VQr4W2a+BWFRvA==
+  dependencies:
+    "@salesforce/bunyan" "^2.0.0"
+    "@salesforce/kit" "^1.8.0"
+    "@salesforce/schemas" "^1.4.0"
+    "@salesforce/ts-types" "^1.5.21"
+    "@types/graceful-fs" "^4.1.5"
+    "@types/semver" "^7.3.13"
+    ajv "^8.11.2"
+    archiver "^5.3.0"
+    change-case "^4.1.2"
+    debug "^3.2.7"
+    faye "^1.4.0"
+    form-data "^4.0.0"
+    graceful-fs "^4.2.9"
+    js2xmlparser "^4.0.1"
+    jsforce "^2.0.0-beta.19"
+    jsonwebtoken "9.0.0"
+    ts-retry-promise "^0.7.0"
+
 "@salesforce/dev-config@^3.0.0", "@salesforce/dev-config@^3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@salesforce/dev-config/-/dev-config-3.1.0.tgz#8eb5b35860ff60d1c1dc3fd9329b01a28475d5b9"
@@ -749,6 +772,15 @@
     shx "^0.3.3"
     tslib "^2.2.0"
 
+"@salesforce/kit@^1.8.1":
+  version "1.8.1"
+  resolved "https://registry.yarnpkg.com/@salesforce/kit/-/kit-1.8.1.tgz#289ca9f4094157c1deaa77cc72f295fdc71f043f"
+  integrity sha512-fCMKh7yWiWtl9C2OXamwkzxD7r6MddrsxKAvlMYZTrTeVG/wxrV03NIhARVkT0k12CsmWNYv83VSPobBAufiIA==
+  dependencies:
+    "@salesforce/ts-types" "^1.7.1"
+    shx "^0.3.3"
+    tslib "^2.2.0"
+
 "@salesforce/prettier-config@^0.0.2":
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/@salesforce/prettier-config/-/prettier-config-0.0.2.tgz#ded39bf7cb75238edc9db6dd093649111350f8bc"
@@ -759,21 +791,21 @@
   resolved "https://registry.yarnpkg.com/@salesforce/schemas/-/schemas-1.4.0.tgz#7dff427c8059895d8108176047aee600703c82d6"
   integrity sha512-BJ25uphssN42Zy6kksheFHMTLiR98AAHe/Wxnv0T4dYxtrEbUjSXVAGKZqfewJPFXA4xB5gxC+rQZtfz6xKCFg==
 
-"@salesforce/source-deploy-retrieve@^7.5.16":
-  version "7.5.16"
-  resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-7.5.16.tgz#7abcb1228e7c6de2ddc18ecdb593e2628dc36465"
-  integrity sha512-pmhnZJnNkF6w88Z5DHDWFrBi63Ny5yGUp2aHTl6WH6LCX7l3+cguSFIQKuTUj1ZO6omqkasuWJE6OPpcoEfoHg==
+"@salesforce/source-deploy-retrieve@^7.5.22":
+  version "7.5.22"
+  resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-7.5.22.tgz#6825f75c6a1d762149f9e330683182d8f3003110"
+  integrity sha512-+ZDCj5zULWs6lj2B978XKt8pj1ondHoxqSh7O3UpyyjAppCnrQGGrduoAzycjrSVmGRXkwrrTybLzA8zil9zsw==
   dependencies:
-    "@salesforce/core" "^3.32.8"
+    "@salesforce/core" "^3.32.12"
     "@salesforce/kit" "^1.8.0"
     "@salesforce/ts-types" "^1.7.1"
     archiver "^5.3.1"
     fast-xml-parser "^3.21.1"
-    got "^11.8.5"
+    got "^11.8.6"
     graceful-fs "^4.2.10"
-    ignore "^5.2.0"
+    ignore "^5.2.4"
     mime "2.6.0"
-    minimatch "^5.1.0"
+    minimatch "^5.1.2"
     proxy-agent "^5.0.0"
     proxy-from-env "^1.1.0"
     unzipper "0.10.11"
@@ -3130,10 +3162,10 @@ globby@^11.0.1, globby@^11.1.0:
     merge2 "^1.4.1"
     slash "^3.0.0"
 
-got@^11.8.5:
-  version "11.8.5"
-  resolved "https://registry.yarnpkg.com/got/-/got-11.8.5.tgz#ce77d045136de56e8f024bebb82ea349bc730046"
-  integrity sha512-o0Je4NvQObAuZPHLFoRSkdG2lTgtcynqymzg2Vupdx6PorhaT5MCbIyXG6d4D94kk8ZG57QeosgdiqfJWhEhlQ==
+got@^11.8.6:
+  version "11.8.6"
+  resolved "https://registry.yarnpkg.com/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a"
+  integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==
   dependencies:
     "@sindresorhus/is" "^4.0.0"
     "@szmarczak/http-timer" "^4.0.5"
@@ -3346,6 +3378,11 @@ ignore@^5.1.1, ignore@^5.1.4, ignore@^5.2.0:
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a"
   integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==
 
+ignore@^5.2.4:
+  version "5.2.4"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324"
+  integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==
+
 import-fresh@^3.0.0, import-fresh@^3.2.1:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"
@@ -3864,6 +3901,16 @@ [email protected]:
     ms "^2.1.1"
     semver "^5.6.0"
 
+[email protected]:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz#d0faf9ba1cc3a56255fe49c0961a67e520c1926d"
+  integrity sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==
+  dependencies:
+    jws "^3.2.2"
+    lodash "^4.17.21"
+    ms "^2.1.1"
+    semver "^7.3.8"
+
 just-extend@^4.0.2:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
@@ -4222,6 +4269,13 @@ minimatch@^5.0.1, minimatch@^5.1.0:
   dependencies:
     brace-expansion "^2.0.1"
 
+minimatch@^5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.2.tgz#0939d7d6f0898acbd1508abe534d1929368a8fff"
+  integrity sha512-bNH9mmM9qsJ2X4r2Nat1B//1dJVcn3+iBLa3IgqJ7EbGaDNepL9QSHOxN4ng33s52VMMhhIfgCYDk3C4ZmlDAg==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 [email protected]:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619"