From fe9a06f2009d7bc1bc2327d637c68e9810a1e5e1 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 8 Oct 2022 01:02:08 +0200 Subject: [PATCH 01/11] Enable `cml comment create --publish` by default (#1208) * Enable `cml comment create --publish` by default * Fix test --- bin/cml/comment/create.js | 1 + bin/cml/comment/create.test.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cml/comment/create.js b/bin/cml/comment/create.js index 59ea2a60d..8036dca6f 100644 --- a/bin/cml/comment/create.js +++ b/bin/cml/comment/create.js @@ -24,6 +24,7 @@ exports.options = kebabcaseKeys({ }, publish: { type: 'boolean', + default: true, description: 'Upload any local images found in the Markdown report' }, publishUrl: { diff --git a/bin/cml/comment/create.test.js b/bin/cml/comment/create.test.js index a7a8a28ec..e9245a437 100644 --- a/bin/cml/comment/create.test.js +++ b/bin/cml/comment/create.test.js @@ -23,7 +23,7 @@ describe('Comment integration tests', () => { --commit-sha, --head-sha Commit SHA linked to this comment [string] [default: \\"HEAD\\"] --publish Upload any local images found in the Markdown report - [boolean] + [boolean] [default: true] --publish-url Self-hosted image server URL [string] [default: \\"https://asset.cml.dev\\"] --watch Watch for changes and automatically update the From b1707f3e66b823f67f60c315cbed53510fe6ba2b Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 8 Oct 2022 06:28:40 +0200 Subject: [PATCH 02/11] Fix release workflows, again (#1214) * Use fallback false value on test-deploy workflow * Replace `softprops/action-gh-release` with `gh` --- .github/workflows/test-deploy.yml | 38 ++++++++----------------------- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 05333b1ef..6f4fb3434 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -102,42 +102,24 @@ jobs: registry-url: https://registry.npmjs.org - run: npm install - run: - npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} + npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} || ${{ + inputs.release && 'true' || 'false' }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - name: install ldid - run: | + - run: | sudo apt install --yes libplist-dev git clone --branch v2.1.5 git://git.saurik.com/ldid.git sudo g++ -pipe -o /usr/bin/ldid ldid/ldid.cpp -I. -x c ldid/{lookup2.c,sha1.h} -lplist -lcrypto - - id: build - name: build - run: | - echo ::set-output name=tag::${GITHUB_REF#refs/tags/} + - run: | cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc npx --yes pkg --no-bytecode --public-packages "*" --public package.json - rm assets/magic.mgc - for cmd in '' runner publish pr; do - build/cml-linux-x64 $cmd --version - done + for cmd in '' runner publish pr; do build/cml-linux-x64 $cmd --version; done cp build/cml-linux{-x64,} cp build/cml-macos{-x64,} - - uses: softprops/action-gh-release@v1 - if: inputs.release - with: - files: | - build/cml-alpine-arm64 - build/cml-alpine-x64 - build/cml-linux-arm64 - build/cml-linuxstatic-arm64 - build/cml-linuxstatic-x64 - build/cml-linux-x64 - build/cml-linux - build/cml-macos-arm64 - build/cml-macos-x64 - build/cml-macos - build/cml-win-arm64.exe - build/cml-win-x64.exe + - if: inputs.release + run: + find build -type f | xargs gh release upload $(basename ${{ + github.head_ref }}) env: GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} images: @@ -145,4 +127,4 @@ jobs: secrets: inherit uses: ./.github/workflows/images.yml with: - release: ${{ inputs.release }} + release: ${{ inputs.release || false }} diff --git a/package-lock.json b/package-lock.json index 0068608d3..a0ab015f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dvcorg/cml", - "version": "0.18.1", + "version": "0.18.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@dvcorg/cml", - "version": "0.18.1", + "version": "0.18.0", "license": "Apache-2.0", "dependencies": { "@actions/core": "^1.9.1", diff --git a/package.json b/package.json index 39853c0c8..1b37f4033 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dvcorg/cml", - "version": "0.18.1", + "version": "0.18.0", "description": "

", "author": { "name": "Iterative Inc", From 2b47d0c6b4b18f93488c8561cb2178adcc8dd941 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sat, 8 Oct 2022 06:34:28 +0200 Subject: [PATCH 03/11] Fast-forward version number to v0.18.3 (#1215) 18.0.3 --- .github/workflows/test-deploy.yml | 4 +--- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 6f4fb3434..e3fb49862 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -101,9 +101,7 @@ jobs: with: registry-url: https://registry.npmjs.org - run: npm install - - run: - npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} || ${{ - inputs.release && 'true' || 'false' }} + - run: npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | diff --git a/package-lock.json b/package-lock.json index a0ab015f4..3364c9edb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dvcorg/cml", - "version": "0.18.0", + "version": "18.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@dvcorg/cml", - "version": "0.18.0", + "version": "18.0.3", "license": "Apache-2.0", "dependencies": { "@actions/core": "^1.9.1", diff --git a/package.json b/package.json index 1b37f4033..ad7623eb7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dvcorg/cml", - "version": "0.18.0", + "version": "18.0.3", "description": "

", "author": { "name": "Iterative Inc", From c83044eec0cfae6dfc0d96d1cf714ebe2565680b Mon Sep 17 00:00:00 2001 From: "Olivaw[bot]" <64868532+iterative-olivaw@users.noreply.github.com> Date: Sat, 8 Oct 2022 06:43:13 +0200 Subject: [PATCH 04/11] 18.0.4 (#1216) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3364c9edb..f03f441a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dvcorg/cml", - "version": "18.0.3", + "version": "18.0.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@dvcorg/cml", - "version": "18.0.3", + "version": "18.0.4", "license": "Apache-2.0", "dependencies": { "@actions/core": "^1.9.1", diff --git a/package.json b/package.json index ad7623eb7..0ba606977 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dvcorg/cml", - "version": "18.0.3", + "version": "18.0.4", "description": "

", "author": { "name": "Iterative Inc", From c97e5481fb91932dc137da94de0a189d54bdc694 Mon Sep 17 00:00:00 2001 From: Daniel Barnes Date: Sat, 8 Oct 2022 15:07:48 -0700 Subject: [PATCH 05/11] CodeQL recommendations (#1218) * url parsing based on codeql results * Update src/drivers/github.js * debug * lazy debugging * bad ternary? * lazy debugging * Revert things This reverts commit df90c98f220d7bad08a4313277bc77ec8a08401a. This reverts commit f11c8ae959cf73276bc07c5a641b2c5eab5d2bdd. This reverts commit 39d0a456180be3908d15b899b1c946fb3b1e5d13. * Cleanup --- bin/cml/asset/publish.js | 4 ++-- src/cml.js | 7 ++++--- src/drivers/github.js | 5 ++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/cml/asset/publish.js b/bin/cml/asset/publish.js index 208deb2cd..d545c0fe2 100644 --- a/bin/cml/asset/publish.js +++ b/bin/cml/asset/publish.js @@ -15,8 +15,8 @@ exports.handler = async (opts) => { opts.native = true; } - const { file, repo, native, asset: path } = opts; - const cml = new CML({ ...opts, repo: native ? repo : 'cml' }); + const { file, asset: path } = opts; + const cml = new CML({ ...opts }); const output = await cml.publish({ ...opts, path }); if (!file) console.log(output); diff --git a/src/cml.js b/src/cml.js index 8a755e26b..b303942d8 100755 --- a/src/cml.js +++ b/src/cml.js @@ -69,9 +69,10 @@ const inferToken = () => { const inferDriver = (opts = {}) => { const { repo } = opts; if (repo) { - if (repo.includes('github.com')) return GITHUB; - if (repo.includes('gitlab.com')) return GITLAB; - if (/bitbucket\.(com|org)/.test(repo)) return BB; + const url = new URL(repo); + if (url.hostname === 'github.com') return GITHUB; + if (url.hostname === 'gitlab.com') return GITLAB; + if (/bitbucket\.(com|org)/.test(url.hostname)) return BB; } if (GITHUB_REPOSITORY) return GITHUB; diff --git a/src/drivers/github.js b/src/drivers/github.js index a4ab4fd81..5362f59e5 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -66,11 +66,10 @@ const octokit = (token, repo) => { onAbuseLimit: throttleHandler } }; - - if (!repo.includes('github.com')) { + const { host, hostname } = new url.URL(repo); + if (hostname !== 'github.com') { // GitHub Enterprise, use the: repo URL host + '/api/v3' - as baseURL // as per: https://developer.github.com/enterprise/v3/enterprise-admin/#endpoint-urls - const { host } = new url.URL(repo); octokitOptions.baseUrl = `https://${host}/api/v3`; } From c9ecc0c63ca4d6ea79c6ec55f753f933c1c4202f Mon Sep 17 00:00:00 2001 From: Vedant Pandya Date: Sun, 9 Oct 2022 03:47:11 +0530 Subject: [PATCH 06/11] Create codeql-analysis.yml (#1096) * Create codeql-analysis.yml * Update .github/workflows/codeql-analysis.yml Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> Co-authored-by: Daniel Barnes --- .github/workflows/codeql-analysis.yml | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..a5cbd1b2e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,29 @@ +name: CodeQL + +on: + pull_request: + paths-ignore: + - '**.md' + - assets/** + schedule: + - cron: '0 0 * * *' # everyday @ 0000 UTC + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v3 + - uses: github/codeql-action/init@v2 + with: + languages: javascript + - uses: github/codeql-action/autobuild@v2 + - uses: github/codeql-action/analyze@v2 From e9546a2ccc596cedc34173e2267af1e23a35394c Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Mon, 10 Oct 2022 04:35:22 +0100 Subject: [PATCH 07/11] fix driver detection (#1209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix driver detection - fixes #1066 * remove unneeded variable * 🤖 Co-authored-by: Daniel Barnes Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- bin/cml/runner/launch.js | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/bin/cml/runner/launch.js b/bin/cml/runner/launch.js index 669db00b2..8b568db12 100755 --- a/bin/cml/runner/launch.js +++ b/bin/cml/runner/launch.js @@ -338,16 +338,8 @@ const run = async (opts) => { process.on(signal, () => shutdown({ ...opts, reason: signal })); }); - const { - driver, - workdir, - cloud, - labels, - name, - reuse, - reuseIdle, - dockerVolumes - } = opts; + const { workdir, cloud, labels, name, reuse, reuseIdle, dockerVolumes } = + opts; await cml.repoTokenCheck(); @@ -375,7 +367,7 @@ const run = async (opts) => { } if (reuseIdle) { - if (driver === 'bitbucket') { + if (cml.driver === 'bitbucket') { throw new Error( 'cml runner flag --reuse-idle is unsupported by bitbucket' ); @@ -396,7 +388,7 @@ const run = async (opts) => { if (dockerVolumes.length && cml.driver !== 'gitlab') winston.warn('Parameters --docker-volumes is only supported in gitlab'); - if (driver === 'github') + if (cml.driver === 'github') winston.warn( 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' ); From 35d137d1dffec620f37bd70e96644bb30141e14c Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Tue, 11 Oct 2022 17:18:21 +0200 Subject: [PATCH 08/11] 0.18.5 (#1220) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f03f441a3..e7926f321 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dvcorg/cml", - "version": "18.0.4", + "version": "0.18.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@dvcorg/cml", - "version": "18.0.4", + "version": "0.18.4", "license": "Apache-2.0", "dependencies": { "@actions/core": "^1.9.1", diff --git a/package.json b/package.json index 0ba606977..229f2f4a4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dvcorg/cml", - "version": "18.0.4", + "version": "0.18.4", "description": "

", "author": { "name": "Iterative Inc", From c1ec211b4bec618ff0aba9230f38e9588ed1e4d3 Mon Sep 17 00:00:00 2001 From: "Olivaw[bot]" <64868532+iterative-olivaw@users.noreply.github.com> Date: Tue, 11 Oct 2022 17:24:39 +0200 Subject: [PATCH 09/11] 0.18.5 (#1221) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index e7926f321..b12751de9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dvcorg/cml", - "version": "0.18.4", + "version": "0.18.5", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@dvcorg/cml", - "version": "0.18.4", + "version": "0.18.5", "license": "Apache-2.0", "dependencies": { "@actions/core": "^1.9.1", diff --git a/package.json b/package.json index 229f2f4a4..73d2b48ec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dvcorg/cml", - "version": "0.18.4", + "version": "0.18.5", "description": "

", "author": { "name": "Iterative Inc", From 188cde19cba9310943eabcb90dbca0a4b4aed9dd Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Wed, 12 Oct 2022 00:36:29 +0200 Subject: [PATCH 10/11] Enhance telemetry data and exclude `tensorboard` tracebacks (#1172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enrich analytics with container information and options * Add reminder * Avoid stack trace leak on Tensorboard * Make the code self-mindblowing * Pin `is-docker` in `package.json` 🙀 * Update tests * Fix double `.options` * Simplify container * Fix tests, again * Simplify telemetry options * Fix tests, again * Restore name/full differentiation * Revert telemetryData * Fix legacy commands * Surely not meant to be here * Apply suggestions from code review Co-authored-by: Domas Monkus * Unpack yargs in function body * fixup! Unpack yargs in function body * Add a Helpful Comment™ * Fix merge * Apply suggestions from code review Co-authored-by: Domas Monkus --- Dockerfile | 2 ++ bin/cml.js | 28 +++++++++++++++++++++---- bin/cml/asset/publish.js | 9 ++++++-- bin/cml/check/create.js | 6 +++++- bin/cml/comment/create.js | 15 +++++++++---- bin/cml/pr.js | 1 + bin/cml/pr/create.js | 6 +++++- bin/cml/repo/prepare.js | 6 +++++- bin/cml/runner.js | 1 + bin/cml/runner/launch.js | 21 +++++++++++++------ bin/cml/tensorboard/connect.e2e.test.js | 2 +- bin/cml/tensorboard/connect.js | 17 +++++++++------ bin/cml/workflow/rerun.js | 6 +++++- package-lock.json | 20 ++++++++++++++++++ package.json | 1 + src/analytics.e2e.test.js | 5 +++-- src/analytics.js | 9 ++++++-- 17 files changed, 124 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9cc787155..807f18ec9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,6 +116,8 @@ WORKDIR ${CML_RUNNER_PATH} # SET SPECIFIC ENVIRONMENT VARIABLES ENV IN_DOCKER=1 ENV RUNNER_ALLOW_RUNASROOT=1 +# Environment variable used by cml to detect it's been installed using the docker image. +ENV _CML_CONTAINER_IMAGE=true # DEFINE ENTRY POINT AND COMMAND # Smart entrypoint understands commands like `bash` or `/bin/sh` but defaults to `cml`; diff --git a/bin/cml.js b/bin/cml.js index 836767332..80df965e1 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -48,7 +48,6 @@ const setupOpts = (opts) => { const { markdownfile } = opts; opts.markdownFile = markdownfile; - opts.cmlCommand = opts._[0]; opts.cml = new CML(opts); }; @@ -76,9 +75,29 @@ const setupLogger = (opts) => { }); }; -const setupTelemetry = async (opts) => { - const { cml, cmlCommand: action } = opts; - opts.telemetryEvent = await jitsuEventPayload({ action, cml }); +const setupTelemetry = async (opts, yargs) => { + const { cml, _: command } = opts; + + const options = {}; + for (const [name, option] of Object.entries(opts.options)) { + // Skip options with default values (i.e. not explicitly set by users) + if (opts[name] && !yargs.parsed.defaulted[name]) { + switch (option.telemetryData) { + case 'name': + options[name] = null; + break; + case 'full': + options[name] = opts[name]; + break; + } + } + } + + opts.telemetryEvent = await jitsuEventPayload({ + action: command.join(':'), + extra: { options }, + cml + }); }; const runPlugin = async ({ $0: executable, command }) => { @@ -99,6 +118,7 @@ const handleError = (message, error) => { (async () => { setupLogger({ log: 'debug' }); + try { await yargs .env('CML') diff --git a/bin/cml/asset/publish.js b/bin/cml/asset/publish.js index d545c0fe2..09fce8e20 100644 --- a/bin/cml/asset/publish.js +++ b/bin/cml/asset/publish.js @@ -23,7 +23,11 @@ exports.handler = async (opts) => { else await fs.writeFile(file, output); }; -exports.builder = (yargs) => yargs.env('CML_ASSET').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_ASSET') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ url: { @@ -51,7 +55,8 @@ exports.options = kebabcaseKeys({ }, rmWatermark: { type: 'boolean', - description: 'Avoid CML watermark.' + description: 'Avoid CML watermark.', + telemetryData: 'name' }, mimeType: { type: 'string', diff --git a/bin/cml/check/create.js b/bin/cml/check/create.js index 43637d7b3..89ebcea9b 100755 --- a/bin/cml/check/create.js +++ b/bin/cml/check/create.js @@ -10,7 +10,11 @@ exports.handler = async (opts) => { await cml.checkCreate({ ...opts, report }); }; -exports.builder = (yargs) => yargs.env('CML_CHECK').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_CHECK') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ token: { diff --git a/bin/cml/comment/create.js b/bin/cml/comment/create.js index 8036dca6f..b325b45d0 100644 --- a/bin/cml/comment/create.js +++ b/bin/cml/comment/create.js @@ -8,7 +8,11 @@ exports.handler = async (opts) => { console.log(await cml.commentCreate(opts)); }; -exports.builder = (yargs) => yargs.env('CML_COMMENT').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_COMMENT') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ pr: { @@ -30,7 +34,8 @@ exports.options = kebabcaseKeys({ publishUrl: { type: 'string', default: 'https://asset.cml.dev', - description: 'Self-hosted image server URL' + description: 'Self-hosted image server URL', + telemetryData: 'name' }, watch: { type: 'boolean', @@ -44,7 +49,8 @@ exports.options = kebabcaseKeys({ native: { type: 'boolean', description: - "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub" + "Uses driver's native capabilities to upload assets instead of CML's storage; not available on GitHub", + telemetryData: 'name' }, update: { type: 'boolean', @@ -55,6 +61,7 @@ exports.options = kebabcaseKeys({ rmWatermark: { type: 'boolean', description: - 'Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others' + 'Avoid watermark; CML needs a watermark to be able to distinguish CML comments from others', + telemetryData: 'name' } }); diff --git a/bin/cml/pr.js b/bin/cml/pr.js index 65930cbfb..4e9f6b3a3 100644 --- a/bin/cml/pr.js +++ b/bin/cml/pr.js @@ -16,5 +16,6 @@ exports.builder = (yargs) => ]) ) ) + .option('options', { default: options, hidden: true }) .check(({ globpath }) => globpath) .strict(); diff --git a/bin/cml/pr/create.js b/bin/cml/pr/create.js index 3e8274104..dc98eda9c 100755 --- a/bin/cml/pr/create.js +++ b/bin/cml/pr/create.js @@ -15,7 +15,11 @@ exports.handler = async (opts) => { console.log(link); }; -exports.builder = (yargs) => yargs.env('CML_PR').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_PR') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ md: { diff --git a/bin/cml/repo/prepare.js b/bin/cml/repo/prepare.js index b3218f788..302f75c5c 100644 --- a/bin/cml/repo/prepare.js +++ b/bin/cml/repo/prepare.js @@ -10,7 +10,11 @@ exports.handler = async (opts) => { await cml.ci(opts); }; -exports.builder = (yargs) => yargs.env('CML_REPO').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_REPO') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ unshallow: { diff --git a/bin/cml/runner.js b/bin/cml/runner.js index f294d4369..c97d0a0c5 100644 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -16,5 +16,6 @@ exports.builder = (yargs) => ]) ) ) + .option('options', { default: options, hidden: true }) .check(() => process.argv.some((arg) => arg.startsWith('-'))) .strict(); diff --git a/bin/cml/runner/launch.js b/bin/cml/runner/launch.js index 8b568db12..bc9c2e5af 100755 --- a/bin/cml/runner/launch.js +++ b/bin/cml/runner/launch.js @@ -418,7 +418,11 @@ exports.handler = async (opts) => { } }; -exports.builder = (yargs) => yargs.env('CML_RUNNER').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_RUNNER') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ labels: { @@ -454,13 +458,15 @@ exports.options = kebabcaseKeys({ type: 'boolean', conflicts: ['single', 'reuseIdle'], description: - "Don't launch a new runner if an existing one has the same name or overlapping labels" + "Don't launch a new runner if an existing one has the same name or overlapping labels", + telemetryData: 'name' }, reuseIdle: { type: 'boolean', conflicts: ['reuse', 'single'], description: - "Creates a new runner only if the matching labels don't exist or are already busy" + "Creates a new runner only if the matching labels don't exist or are already busy", + telemetryData: 'name' }, workdir: { type: 'string', @@ -476,7 +482,8 @@ exports.options = kebabcaseKeys({ cloud: { type: 'string', choices: ['aws', 'azure', 'gcp', 'kubernetes'], - description: 'Cloud to deploy the runner' + description: 'Cloud to deploy the runner', + telemetryData: 'full' }, cloudRegion: { type: 'string', @@ -529,12 +536,14 @@ exports.options = kebabcaseKeys({ type: 'number', default: -1, description: - 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' + 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price', + telemetryData: 'name' }, cloudStartupScript: { type: 'string', description: - 'Run the provided Base64-encoded Linux shell script during the instance initialization' + 'Run the provided Base64-encoded Linux shell script during the instance initialization', + telemetryData: 'name' }, cloudAwsSecurityGroup: { type: 'string', diff --git a/bin/cml/tensorboard/connect.e2e.test.js b/bin/cml/tensorboard/connect.e2e.test.js index f4dec0840..1b3f0b40e 100644 --- a/bin/cml/tensorboard/connect.e2e.test.js +++ b/bin/cml/tensorboard/connect.e2e.test.js @@ -34,7 +34,7 @@ describe('tbLink', () => { error = err; } - expect(error.message).toBe(`Tensorboard took too long. ${message}`); + expect(error.message).toBe(`Tensorboard took too long`); }); test('valid url is returned', async () => { diff --git a/bin/cml/tensorboard/connect.js b/bin/cml/tensorboard/connect.js index 809d50809..7e0b073f3 100644 --- a/bin/cml/tensorboard/connect.js +++ b/bin/cml/tensorboard/connect.js @@ -3,6 +3,7 @@ const kebabcaseKeys = require('kebabcase-keys'); const { spawn } = require('child_process'); const { homedir } = require('os'); const tempy = require('tempy'); +const winston = require('winston'); const { exec, watermarkUri, sleep } = require('../../../src/utils'); @@ -28,8 +29,8 @@ const tbLink = async (opts = {}) => { chrono = chrono + chronoStep; } - const error = await fs.readFile(stderror, 'utf8'); - throw new Error(`Tensorboard took too long. ${error}`); + winston.error(await fs.readFile(stderror, 'utf8')); + throw new Error(`Tensorboard took too long`); }; const launchAndWaitLink = async (opts = {}) => { @@ -51,8 +52,8 @@ const launchAndWaitLink = async (opts = {}) => { proc.unref(); proc.on('exit', async (code, signal) => { if (code || signal) { - const error = await fs.readFile(stderrPath, 'utf8'); - throw new Error(`Tensorboard failed with error: ${error}`); + winston.error(await fs.readFile(stderrPath, 'utf8')); + throw new Error(`Tensorboard failed with error ${code || signal}`); } }); @@ -95,7 +96,10 @@ exports.handler = async (opts) => { }; exports.builder = (yargs) => - yargs.env('CML_TENSORBOARD').options(exports.options); + yargs + .env('CML_TENSORBOARD') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ credentials: { @@ -136,6 +140,7 @@ exports.options = kebabcaseKeys({ }, rmWatermark: { type: 'boolean', - description: 'Avoid CML watermark' + description: 'Avoid CML watermark', + telemetryData: 'name' } }); diff --git a/bin/cml/workflow/rerun.js b/bin/cml/workflow/rerun.js index 18c5f0936..40639434f 100644 --- a/bin/cml/workflow/rerun.js +++ b/bin/cml/workflow/rerun.js @@ -8,7 +8,11 @@ exports.handler = async (opts) => { await cml.pipelineRerun(opts); }; -exports.builder = (yargs) => yargs.env('CML_WORKFLOW').options(exports.options); +exports.builder = (yargs) => + yargs + .env('CML_WORKFLOW') + .option('options', { default: exports.options, hidden: true }) + .options(exports.options); exports.options = kebabcaseKeys({ id: { diff --git a/package-lock.json b/package-lock.json index b12751de9..41a5ee36b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "git-url-parse": "^13.1.0", "globby": "^11.0.4", "https-proxy-agent": "^5.0.1", + "is-docker": "^2.2.1", "js-base64": "^3.7.2", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", @@ -3967,6 +3968,20 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "license": "MIT", @@ -10137,6 +10152,11 @@ "is-decimal": { "version": "1.0.4" }, + "is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==" + }, "is-extglob": { "version": "2.1.1" }, diff --git a/package.json b/package.json index 73d2b48ec..48f03c75d 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "git-url-parse": "^13.1.0", "globby": "^11.0.4", "https-proxy-agent": "^5.0.1", + "is-docker": "2.2.1", "js-base64": "^3.7.2", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", diff --git a/src/analytics.e2e.test.js b/src/analytics.e2e.test.js index 1a632892b..365bba3ac 100644 --- a/src/analytics.e2e.test.js +++ b/src/analytics.e2e.test.js @@ -8,8 +8,9 @@ describe('analytics tests', () => { const cml = new CML({ repo: REPO, token: TOKEN }); const action = 'test'; const cloud = 'azure'; + const container = 'cml'; const more = { one: 1, two: 2 }; - const extra = { cloud, ...more }; + const extra = { cloud, container, ...more }; const error = 'Ouch!'; const regex = /\d+\.\d+\.\d+/; @@ -25,7 +26,7 @@ describe('analytics tests', () => { expect(pl.backend).toBe(cloud); expect(pl.error).toBe(error); expect(Object.keys(pl.extra).sort()).toEqual( - ['ci'].concat(Object.keys(more)).sort() + ['ci', 'container'].concat(Object.keys(more)).sort() ); if (isCI()) { diff --git a/src/analytics.js b/src/analytics.js index 239651569..bb810f3be 100644 --- a/src/analytics.js +++ b/src/analytics.js @@ -9,6 +9,7 @@ const { scrypt } = require('crypto'); const { v4: uuidv4, v5: uuidv5, parse } = require('uuid'); const { userConfigDir } = require('appdirs'); const winston = require('winston'); +const isDocker = require('is-docker'); const { version: VERSION } = require('../package.json'); const { exec, fileExists, getos } = require('./utils'); @@ -150,7 +151,6 @@ const jitsuEventPayload = async ({ } = {}) => { try { const { cloud: backend = '', ...extraRest } = extra; - extraRest.ci = guessCI(); const osname = OS(); let { release = os.release() } = await getos(); @@ -171,7 +171,12 @@ const jitsuEventPayload = async ({ os_version: release, backend, error, - extra: extraRest + extra: { + ...extraRest, + ci: guessCI(), + container: + process.env._CML_CONTAINER_IMAGE === 'true' ? 'cml' : isDocker() + } }; } catch (err) { return {}; From 926f62c1e47115f76eb1200fd4050da4a07aa749 Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Wed, 12 Oct 2022 01:07:18 +0200 Subject: [PATCH 11/11] Install `ldid` with `brew` (#1217) * Install `ldid` with `conda` * Apply suggestions from code review * Apply suggestions from code review * Use `brew` instead of `conda` * Add helpful comment --- .github/workflows/test-deploy.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index e3fb49862..a9e9b90e2 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -104,10 +104,9 @@ jobs: - run: npm ${{ inputs.release && 'publish' || 'publish --dry-run' }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - run: | - sudo apt install --yes libplist-dev - git clone --branch v2.1.5 git://git.saurik.com/ldid.git - sudo g++ -pipe -o /usr/bin/ldid ldid/ldid.cpp -I. -x c ldid/{lookup2.c,sha1.h} -lplist -lcrypto + # Step required "thanks" to https://github.com/actions/runner-images/issues/6283 + - uses: Homebrew/actions/setup-homebrew@29b2973f18e5aca5443f4b3d87f68eda7a27b22b + - run: brew install ldid - run: | cp node_modules/@npcz/magic/dist/magic.mgc assets/magic.mgc npx --yes pkg --no-bytecode --public-packages "*" --public package.json