From a257e84fbdf85544c3adeb229b73b9f8ff0c72ad Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 29 May 2022 18:31:00 +0200 Subject: [PATCH 1/8] Tensorboard tests fix protofuf downgrade (#1031) * Tensorboard PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION * export * protobuf * alter in tb section --- .github/workflows/test-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 62737cdd9..1557c856e 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -36,7 +36,9 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 - - run: pip install tensorboard + - run: | + pip install tensorboard + pip install -I protobuf==3.20.1 - run: npm ci - run: npm run test env: From eeebc435ea435f23b95d44611c1e5a5e5122fb5d Mon Sep 17 00:00:00 2001 From: Helio Machado <0x2b3bfa0+git@googlemail.com> Date: Sun, 29 May 2022 18:37:26 +0200 Subject: [PATCH 2/8] Remove inaccurate Bitbucket Server mention (#1024) * Remove inaccurate Bitbucket Server mention * Apply suggestions from code review Co-authored-by: Casper da Costa-Luis Co-authored-by: Casper da Costa-Luis --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index adccb3298..fedd8f629 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,7 @@ requirement. ### Bitbucket Please see our docs on -[CML with Bitbucket Cloud](https://github.com/iterative/cml/wiki/CML-with-Bitbucket-Cloud). -_Bitbucket Server support estimated to arrive by mid 2021._ +[CML with Bitbucket Cloud](https://cml.dev/doc/usage?tab=Bitbucket). ### GitHub From 02529750f0176d7484f0d1adcf8596e531340d11 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 29 May 2022 18:50:21 +0200 Subject: [PATCH 3/8] Removed tfFile (#1029) Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- bin/cml/runner.js | 65 ++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index f83a7e751..531866607 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -125,47 +125,42 @@ const runCloud = async (opts) => { cloudStartupScript: startupScript, cloudAwsSecurityGroup: awsSecurityGroup, cloudAwsSubnet: awsSubnet, - tfFile, workdir } = opts; + if (gpu === 'tesla') + winston.warn( + 'GPU model "tesla" has been deprecated; please use "v100" instead.' + ); + const tfPath = workdir; const tfMainPath = join(tfPath, 'main.tf'); - let tpl; - if (tfFile) { - tpl = await fs.writeFile(tfMainPath, await fs.readFile(tfFile)); - } else { - if (gpu === 'tesla') - winston.warn( - 'GPU model "tesla" has been deprecated; please use "v100" instead.' - ); - tpl = tf.iterativeCmlRunnerTpl({ - tpiVersion, - repo, - token, - driver, - labels, - cmlVersion, - idleTimeout, - name, - single, - cloud, - region, - type, - permissionSet, - metadata, - gpu: gpu === 'tesla' ? 'v100' : gpu, - hddSize, - sshPrivate, - spot, - spotPrice, - startupScript, - awsSecurityGroup, - awsSubnet, - dockerVolumes - }); - } + const tpl = tf.iterativeCmlRunnerTpl({ + tpiVersion, + repo, + token, + driver, + labels, + cmlVersion, + idleTimeout, + name, + single, + cloud, + region, + type, + permissionSet, + metadata, + gpu: gpu === 'tesla' ? 'v100' : gpu, + hddSize, + sshPrivate, + spot, + spotPrice, + startupScript, + awsSecurityGroup, + awsSubnet, + dockerVolumes + }); await fs.writeFile(tfMainPath, tpl); await tf.init({ dir: tfPath }); From 3c9e3b5eb29fc54ba551d520d597c3b003d12ad6 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 29 May 2022 18:58:42 +0200 Subject: [PATCH 4/8] Runner faster termination (#1021) * Runner faster termination * delayed destroy * single check * Remove unnecessary Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- bin/cml/runner.js | 9 ++++----- src/cml.js | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 531866607..2812e07a5 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -37,8 +37,8 @@ const shutdown = async (opts) => { try { winston.info(`Unregistering runner ${name}...`); - RUNNER && RUNNER.kill('SIGINT'); await cml.unregisterRunner({ name }); + RUNNER && RUNNER.kill('SIGINT'); winston.info('\tSuccess'); } catch (err) { winston.error(`\tFailed: ${err.message}`); @@ -69,6 +69,9 @@ const shutdown = async (opts) => { const destroyTerraform = async () => { if (!tfResource) return; + winston.info(`Waiting ${destroyDelay} seconds to destroy`); + await sleep(destroyDelay); + try { winston.debug(await tf.destroy({ dir: tfPath })); } catch (err) { @@ -82,9 +85,6 @@ const shutdown = async (opts) => { winston.info('runner status', { reason, status: 'terminated' }); } - winston.info(`waiting ${destroyDelay} seconds before exiting...`); - await sleep(destroyDelay); - if (!cloud) { try { await unregisterRunner(); @@ -427,7 +427,6 @@ exports.handler = async (opts) => { await run(opts); } catch (error) { await shutdown({ ...opts, error }); - throw error; } }; diff --git a/src/cml.js b/src/cml.js index 060900622..76f6698f9 100755 --- a/src/cml.js +++ b/src/cml.js @@ -339,7 +339,9 @@ class CML { } async unregisterRunner(opts = {}) { - const { id: runnerId } = await this.runnerByName(opts); + const { id: runnerId } = (await this.runnerByName(opts)) || {}; + if (!runnerId) throw new Error(`Runner not found`); + return await getDriver(this).unregisterRunner({ runnerId, ...opts }); } From 50fcff57cdbc9217fad45c227f8de523c9f08b11 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 29 May 2022 18:00:29 +0100 Subject: [PATCH 5/8] fix: upgrade https-proxy-agent from 5.0.0 to 5.0.1 (#998) Snyk has created this PR to upgrade https-proxy-agent from 5.0.0 to 5.0.1. See this package in npm: https://www.npmjs.com/package/https-proxy-agent See this project in Snyk: https://app.snyk.io/org/casperdcl/project/58d0fcac-fe96-4a3b-b587-5fa62067cfa1?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- package-lock.json | 16 ++++++++-------- package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index cce6df9b3..e5a4d21dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "fs-extra": "^9.1.0", "git-url-parse": "^11.6.0", "globby": "^11.0.4", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.1", "js-base64": "^3.7.2", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", @@ -68,7 +68,7 @@ "transparent-proxy": "^1.8.7" }, "engines": { - "node": "<=16.x" + "node": ">=16.0.0" } }, "node_modules/@actions/core": { @@ -3935,9 +3935,9 @@ } }, "node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "dependencies": { "agent-base": "6", "debug": "4" @@ -11122,9 +11122,9 @@ } }, "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "requires": { "agent-base": "6", "debug": "4" diff --git a/package.json b/package.json index 2e10443f3..eac09a75b 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "fs-extra": "^9.1.0", "git-url-parse": "^11.6.0", "globby": "^11.0.4", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.1", "js-base64": "^3.7.2", "kebabcase-keys": "^1.0.0", "node-fetch": "^2.6.5", From d9bd91ddcd5a8f16599052d12cf99ea9786d3801 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 29 May 2022 18:00:50 +0100 Subject: [PATCH 6/8] fix: upgrade simple-git from 3.6.0 to 3.7.0 (#999) Snyk has created this PR to upgrade simple-git from 3.6.0 to 3.7.0. See this package in npm: https://www.npmjs.com/package/simple-git See this project in Snyk: https://app.snyk.io/org/casperdcl/project/58d0fcac-fe96-4a3b-b587-5fa62067cfa1?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e5a4d21dd..fc8a18343 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ "proxy-agent": "^5.0.0", "pseudoexec": "^0.2.0", "semver": "^7.3.5", - "simple-git": "^3.5.0", + "simple-git": "^3.7.0", "strip-ansi": "^6.0.1", "strip-url-auth": "^1.0.1", "tar": "^6.1.11", @@ -6933,9 +6933,9 @@ "dev": true }, "node_modules/simple-git": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.6.0.tgz", - "integrity": "sha512-2e+4QhOVO59GeLsHgwSMKNrSKCnuACeA/gMNrLCYR8ID9qwm4hViVt4WsODcUGjx//KDv6GMLC6Hs/MeosgXxg==", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.0.tgz", + "integrity": "sha512-O9HlI83ywqkYqnr7Wh3CqKNNrMkfjzpKQSGtJAhk7+H5P+lAxHBTIPgu/eO/0D9pMciepgs433p0d5S+NYv5Jg==", "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", @@ -13365,9 +13365,9 @@ "dev": true }, "simple-git": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.6.0.tgz", - "integrity": "sha512-2e+4QhOVO59GeLsHgwSMKNrSKCnuACeA/gMNrLCYR8ID9qwm4hViVt4WsODcUGjx//KDv6GMLC6Hs/MeosgXxg==", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.0.tgz", + "integrity": "sha512-O9HlI83ywqkYqnr7Wh3CqKNNrMkfjzpKQSGtJAhk7+H5P+lAxHBTIPgu/eO/0D9pMciepgs433p0d5S+NYv5Jg==", "requires": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", diff --git a/package.json b/package.json index eac09a75b..a99bae461 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "proxy-agent": "^5.0.0", "pseudoexec": "^0.2.0", "semver": "^7.3.5", - "simple-git": "^3.5.0", + "simple-git": "^3.7.0", "strip-ansi": "^6.0.1", "strip-url-auth": "^1.0.1", "tar": "^6.1.11", From 204c518c3189494b06b7ddcab4f804ca21062561 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 29 May 2022 18:14:24 +0100 Subject: [PATCH 7/8] [Snyk] Upgrade @actions/core from 1.6.0 to 1.7.0 (#1007) * fix: upgrade @actions/core from 1.6.0 to 1.7.0 Snyk has created this PR to upgrade @actions/core from 1.6.0 to 1.7.0. See this package in npm: https://www.npmjs.com/package/@actions/core See this project in Snyk: https://app.snyk.io/org/casperdcl/project/58d0fcac-fe96-4a3b-b587-5fa62067cfa1?utm_source=github&utm_medium=referral&page=upgrade-pr * Woof! Co-authored-by: snyk-bot Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- package-lock.json | 36 +++++++++++++++++++++++++++--------- package.json | 2 +- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index fc8a18343..8ffdae70f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.16.0", "license": "Apache-2.0", "dependencies": { - "@actions/core": "^1.6.0", + "@actions/core": "^1.8.2", "@actions/github": "^4.0.0", "@npcz/magic": "^1.3.12", "@octokit/core": "^3.5.1", @@ -72,11 +72,19 @@ } }, "node_modules/@actions/core": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", - "integrity": "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", + "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", "dependencies": { - "@actions/http-client": "^1.0.11" + "@actions/http-client": "^2.0.1" + } + }, + "node_modules/@actions/core/node_modules/@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "dependencies": { + "tunnel": "^0.0.6" } }, "node_modules/@actions/github": { @@ -8118,11 +8126,21 @@ }, "dependencies": { "@actions/core": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", - "integrity": "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw==", + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", + "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", "requires": { - "@actions/http-client": "^1.0.11" + "@actions/http-client": "^2.0.1" + }, + "dependencies": { + "@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "requires": { + "tunnel": "^0.0.6" + } + } } }, "@actions/github": { diff --git a/package.json b/package.json index a99bae461..cd7f53e42 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "colors": "1.4.0" }, "dependencies": { - "@actions/core": "^1.6.0", + "@actions/core": "^1.8.2", "@actions/github": "^4.0.0", "@npcz/magic": "^1.3.12", "@octokit/core": "^3.5.1", From 95019fb57df5ceb40c5d44c050050b91d0b4635d Mon Sep 17 00:00:00 2001 From: Snyk bot Date: Sun, 29 May 2022 22:41:47 +0300 Subject: [PATCH 8/8] fix: upgrade simple-git from 3.7.0 to 3.7.1 (#1033) Snyk has created this PR to upgrade simple-git from 3.7.0 to 3.7.1. See this package in npm: https://www.npmjs.com/package/simple-git See this project in Snyk: https://app.snyk.io/org/casperdcl/project/58d0fcac-fe96-4a3b-b587-5fa62067cfa1?utm_source=github&utm_medium=referral&page=upgrade-pr --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8ffdae70f..76e993430 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ "proxy-agent": "^5.0.0", "pseudoexec": "^0.2.0", "semver": "^7.3.5", - "simple-git": "^3.7.0", + "simple-git": "^3.7.1", "strip-ansi": "^6.0.1", "strip-url-auth": "^1.0.1", "tar": "^6.1.11", @@ -6941,9 +6941,9 @@ "dev": true }, "node_modules/simple-git": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.0.tgz", - "integrity": "sha512-O9HlI83ywqkYqnr7Wh3CqKNNrMkfjzpKQSGtJAhk7+H5P+lAxHBTIPgu/eO/0D9pMciepgs433p0d5S+NYv5Jg==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.1.tgz", + "integrity": "sha512-+Osjtsumbtew2y9to0pOYjNzSIr4NkKGBg7Po5SUtjQhaJf2QBmiTX/9E9cv9rmc7oUiSGFIB9e7ys5ibnT9+A==", "dependencies": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", @@ -13383,9 +13383,9 @@ "dev": true }, "simple-git": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.0.tgz", - "integrity": "sha512-O9HlI83ywqkYqnr7Wh3CqKNNrMkfjzpKQSGtJAhk7+H5P+lAxHBTIPgu/eO/0D9pMciepgs433p0d5S+NYv5Jg==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.7.1.tgz", + "integrity": "sha512-+Osjtsumbtew2y9to0pOYjNzSIr4NkKGBg7Po5SUtjQhaJf2QBmiTX/9E9cv9rmc7oUiSGFIB9e7ys5ibnT9+A==", "requires": { "@kwsites/file-exists": "^1.1.1", "@kwsites/promise-deferred": "^1.1.1", diff --git a/package.json b/package.json index cd7f53e42..0b118ed3e 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "proxy-agent": "^5.0.0", "pseudoexec": "^0.2.0", "semver": "^7.3.5", - "simple-git": "^3.7.0", + "simple-git": "^3.7.1", "strip-ansi": "^6.0.1", "strip-url-auth": "^1.0.1", "tar": "^6.1.11",