diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 9d7d9b5d9..446354a6a 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -65,10 +65,10 @@ jobs: TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy TEST_GITLAB_SHA: f8b8b49a253243830ef59a7f090eb887157b2b67 TEST_GITLAB_ISSUE: 1 - TEST_BBCLOUD_TOKEN: ${{ secrets.TEST_BBCLOUD_TOKEN }} - TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy - TEST_BBCLOUD_SHA: b511535a89f76d3d311b1c15e3e712b15c0b94e3 - TEST_BBCLOUD_ISSUE: 1 + TEST_BITBUCKET_TOKEN: ${{ secrets.TEST_BITBUCKET_TOKEN }} + TEST_BITBUCKET_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy + TEST_BITBUCKET_SHA: b511535a89f76d3d311b1c15e3e712b15c0b94e3 + TEST_BITBUCKET_ISSUE: 1 test-os: needs: authorize name: test-${{ matrix.system }} diff --git a/bin/cml.js b/bin/cml.js index 4d8358cd5..5c15e15c8 100755 --- a/bin/cml.js +++ b/bin/cml.js @@ -5,7 +5,7 @@ const { pseudoexec } = require('pseudoexec'); const kebabcaseKeys = require('kebabcase-keys'); const which = require('which'); -const winston = require('winston'); +const { logger, setupLogger } = require('../src/logger'); const yargs = require('yargs'); const CML = require('../src/cml').default; @@ -72,30 +72,6 @@ const setupOpts = (opts) => { opts.cml = new CML(opts); }; -const setupLogger = (opts) => { - const { log: level } = opts; - - winston.configure({ - format: process.stdout.isTTY - ? winston.format.combine( - winston.format.colorize({ all: true }), - winston.format.simple() - ) - : winston.format.combine( - winston.format.errors({ stack: true }), - winston.format.json() - ), - transports: [ - new winston.transports.Console({ - stderrLevels: Object.keys(winston.config.npm.levels), - handleExceptions: true, - handleRejections: true, - level - }) - ] - }); -}; - const setupTelemetry = async (opts, yargs) => { const { cml, _: command } = opts; @@ -201,7 +177,7 @@ const handleError = (message, error) => { const event = { ...telemetryEvent, error: err.message }; await send({ event }); } - winston.error(err); + logger.error(err); process.exit(1); } })(); diff --git a/bin/cml.test.js b/bin/cml.test.js index 06d19cedc..b41e6a9bd 100644 --- a/bin/cml.test.js +++ b/bin/cml.test.js @@ -13,7 +13,6 @@ describe('command-line interface tests', () => { cml.js comment Manage comments cml.js pr Manage pull requests cml.js runner Manage self-hosted (cloud & on-premise) CI runners - cml.js tensorboard Manage tensorboard.dev connections cml.js workflow Manage CI workflows cml.js ci Prepare Git repository for CML operations diff --git a/bin/cml/asset/publish.js b/bin/cml/asset/publish.js index fa07e165d..ea1f49994 100644 --- a/bin/cml/asset/publish.js +++ b/bin/cml/asset/publish.js @@ -1,6 +1,6 @@ const fs = require('fs').promises; const kebabcaseKeys = require('kebabcase-keys'); -const winston = require('winston'); +const { logger } = require('../../../src/logger'); const { CML } = require('../../../src/cml'); @@ -12,7 +12,7 @@ exports.description = `${DESCRIPTION}\n${DOCSURL}`; exports.handler = async (opts) => { if (opts.gitlabUploads) { - winston.warn( + logger.warn( '--gitlab-uploads will be deprecated soon, use --native instead' ); opts.native = true; diff --git a/bin/cml/comment/create.e2e.test.js b/bin/cml/comment/create.e2e.test.js index 1951bc4c3..c37073898 100644 --- a/bin/cml/comment/create.e2e.test.js +++ b/bin/cml/comment/create.e2e.test.js @@ -33,18 +33,4 @@ describe('Comment integration tests', () => { path ); }); - - test('cml send-comment to current repo', async () => { - const report = `## Test Comment`; - - await fs.writeFile(path, report); - await exec('node', './bin/cml.js', 'send-comment', path); - }); - - test('cml send-comment --publish to current repo', async () => { - const report = `## Test Comment\n![](assets/logo.png)`; - - await fs.writeFile(path, report); - await exec('node', './bin/cml.js', 'send-comment', '--publish', path); - }); }); diff --git a/bin/cml/runner/launch.js b/bin/cml/runner/launch.js index b548680fb..9dd501291 100755 --- a/bin/cml/runner/launch.js +++ b/bin/cml/runner/launch.js @@ -4,7 +4,7 @@ const fs = require('fs').promises; const net = require('net'); const kebabcaseKeys = require('kebabcase-keys'); const timestring = require('timestring'); -const winston = require('winston'); +const { logger } = require('../../../src/logger'); const { exec, randid, sleep } = require('../../../src/utils'); const tf = require('../../../src/terraform'); @@ -29,26 +29,26 @@ const shutdown = async (opts) => { if (!RUNNER) return true; try { - winston.info(`Unregistering runner ${name}...`); + logger.info(`Unregistering runner ${name}...`); await cml.unregisterRunner({ name }); } catch (err) { if (err.message.includes('is still running a job')) { - winston.warn(`\tCancelling shutdown: ${err.message}`); + logger.warn(`\tCancelling shutdown: ${err.message}`); return false; } - winston.error(`\tFailed: ${err.message}`); + logger.error(`\tFailed: ${err.message}`); } RUNNER.kill('SIGINT'); - winston.info('\tSuccess'); + logger.info('\tSuccess'); return true; }; const retryWorkflows = async () => { try { if (!noRetry && RUNNER_JOBS_RUNNING.length > 0) { - winston.info(`Still pending jobs, retrying workflow...`); + logger.info(`Still pending jobs, retrying workflow...`); await Promise.all( RUNNER_JOBS_RUNNING.map( @@ -58,14 +58,14 @@ const shutdown = async (opts) => { ); } } catch (err) { - winston.error(err); + logger.error(err); } }; const destroyLeo = async () => { if (!tfResource) return; - winston.info(`Waiting ${destroyDelay} seconds to destroy`); + logger.info(`Waiting ${destroyDelay} seconds to destroy`); await sleep(destroyDelay); const { cloud, id, region } = JSON.parse( @@ -83,7 +83,7 @@ const shutdown = async (opts) => { id ); } catch (err) { - winston.error(`\tFailed destroying with LEO: ${err.message}`); + logger.error(`\tFailed destroying with LEO: ${err.message}`); } }; @@ -97,7 +97,7 @@ const shutdown = async (opts) => { clearInterval(watcher); await retryWorkflows(); } catch (err) { - winston.error(`Error connecting the SCM: ${err.message}`); + logger.error(`Error connecting the SCM: ${err.message}`); } } @@ -105,13 +105,13 @@ const shutdown = async (opts) => { if (error) throw error; - winston.info('runner status', { reason, status: 'terminated' }); + logger.info('runner status', { reason, status: 'terminated' }); process.exit(0); }; const runCloud = async (opts) => { const runTerraform = async (opts) => { - winston.info('Terraform apply...'); + logger.info('Terraform apply...'); const { token, repo, driver } = cml; const { @@ -143,7 +143,7 @@ const runCloud = async (opts) => { await tf.checkMinVersion(); if (gpu === 'tesla') - winston.warn( + logger.warn( 'GPU model "tesla" has been deprecated; please use "v100" instead.' ); @@ -189,7 +189,7 @@ const runCloud = async (opts) => { return tfstate; }; - winston.info('Deploying cloud runner plan...'); + logger.info('Deploying cloud runner plan...'); const tfstate = await runTerraform(opts); const { resources } = tfstate; for (const resource of resources) { @@ -221,14 +221,14 @@ const runCloud = async (opts) => { timeouts: attributes.timeouts, kubernetesNodeSelector: attributes.kubernetes_node_selector }; - winston.info(JSON.stringify(nonSensitiveValues)); + logger.info(JSON.stringify(nonSensitiveValues)); } } } }; const runLocal = async (opts) => { - winston.info(`Launching ${cml.driver} runner`); + logger.info(`Launching ${cml.driver} runner`); const { workdir, name, @@ -265,10 +265,10 @@ const runLocal = async (opts) => { if (process.platform === 'linux') { const acpiSock = net.connect('/var/run/acpid.socket'); acpiSock.on('connect', () => { - winston.info('Connected to acpid service.'); + logger.info('Connected to acpid service.'); }); acpiSock.on('error', (err) => { - winston.warn( + logger.warn( `Error connecting to ACPI socket: ${err.message}. The acpid.service helps with instance termination detection.` ); }); @@ -283,10 +283,10 @@ const runLocal = async (opts) => { const dataHandler = ({ cloudSpot }) => async (data) => { - winston.debug(data.toString()); + logger.debug(data.toString()); const logs = await cml.parseRunnerLog({ data, name, cloudSpot }); for (const log of logs) { - winston.info('runner status', log); + logger.info('runner status', log); if (log.status === 'job_started') { const { job: id, pipeline, date } = log; @@ -380,7 +380,7 @@ const run = async (opts) => { throw new Error( `Runner name ${name} is already in use. Please change the name or terminate the existing runner.` ); - winston.info(`Reusing existing runner named ${name}...`); + logger.info(`Reusing existing runner named ${name}...`); return; } @@ -390,9 +390,7 @@ const run = async (opts) => { (runner) => runner.online ) ) { - winston.info( - `Reusing existing online runners with the ${labels} labels...` - ); + logger.info(`Reusing existing online runners with the ${labels} labels...`); return; } @@ -402,7 +400,7 @@ const run = async (opts) => { 'cml runner flag --reuse-idle is unsupported by bitbucket' ); } - winston.info( + logger.info( `Checking for existing idle runner matching labels: ${labels}.` ); const currentRunners = await cml.runnersByLabels({ labels, runners }); @@ -410,25 +408,25 @@ const run = async (opts) => { (runner) => runner.online && !runner.busy ); if (availableRunner) { - winston.info('Found matching idle runner.', availableRunner); + logger.info('Found matching idle runner.', availableRunner); return; } } if (dockerVolumes.length && cml.driver !== 'gitlab') - winston.warn('Parameters --docker-volumes is only supported in gitlab'); + logger.warn('Parameters --docker-volumes is only supported in gitlab'); if (cml.driver === 'github') - winston.warn( + logger.warn( 'Github Actions timeout has been updated from 72h to 35 days. Update your workflow accordingly to be able to restart it automatically.' ); if (RUNNER_NAME) - winston.warn( + logger.warn( 'ignoring RUNNER_NAME environment variable, use CML_RUNNER_NAME or --name instead' ); - winston.info(`Preparing workdir ${workdir}...`); + logger.info(`Preparing workdir ${workdir}...`); await fs.mkdir(workdir, { recursive: true }); await fs.chmod(workdir, '766'); diff --git a/bin/cml/tensorboard.js b/bin/cml/tensorboard.js index 851141bfc..06dbbf38b 100644 --- a/bin/cml/tensorboard.js +++ b/bin/cml/tensorboard.js @@ -1,5 +1,5 @@ exports.command = 'tensorboard'; -exports.description = 'Manage tensorboard.dev connections'; +exports.description = false; exports.builder = (yargs) => yargs .options({ diff --git a/bin/cml/tensorboard/connect.e2e.test.js b/bin/cml/tensorboard/connect.e2e.test.js index 6ac4b96b8..21b457bfc 100644 --- a/bin/cml/tensorboard/connect.e2e.test.js +++ b/bin/cml/tensorboard/connect.e2e.test.js @@ -19,7 +19,7 @@ const rmTbDevExperiment = async (tbOutput) => { }; describe('tbLink', () => { - test('timeout without result throws exception', async () => { + test.skip('timeout without result throws exception', async () => { const stdout = tempy.file({ extension: 'log' }); const stderror = tempy.file({ extension: 'log' }); const message = 'there is an error'; @@ -37,7 +37,7 @@ describe('tbLink', () => { expect(error.message).toBe(`Tensorboard took too long`); }); - test('valid url is returned', async () => { + test.skip('valid url is returned', async () => { const stdout = tempy.file({ extension: 'log' }); const stderror = tempy.file({ extension: 'log' }); const message = 'https://iterative.ai'; @@ -51,7 +51,7 @@ describe('tbLink', () => { }); describe('CML e2e', () => { - test('cml tensorboard-dev --md returns md and after command TB is still up', async () => { + test.skip('cml tensorboard-dev --md returns md and after command TB is still up', async () => { const name = 'My experiment'; const desc = 'Test experiment'; const title = 'go to the experiment'; @@ -80,7 +80,7 @@ describe('CML e2e', () => { expect(output.includes('cml=tb')).toBe(true); }); - test('cml tensorboard-dev invalid creds', async () => { + test.skip('cml tensorboard-dev invalid creds', async () => { try { await exec( 'node', diff --git a/bin/cml/tensorboard/connect.js b/bin/cml/tensorboard/connect.js index 8d6aa787b..fea2beff1 100644 --- a/bin/cml/tensorboard/connect.js +++ b/bin/cml/tensorboard/connect.js @@ -3,7 +3,7 @@ const kebabcaseKeys = require('kebabcase-keys'); const { spawn } = require('child_process'); const { homedir } = require('os'); const tempy = require('tempy'); -const winston = require('winston'); +const { logger } = require('../../../src/logger'); const { exec, watermarkUri, sleep } = require('../../../src/utils'); @@ -29,7 +29,7 @@ const tbLink = async (opts = {}) => { chrono = chrono + chronoStep; } - winston.error(await fs.readFile(stderror, 'utf8')); + logger.error(await fs.readFile(stderror, 'utf8')); throw new Error(`Tensorboard took too long`); }; @@ -52,7 +52,7 @@ const launchAndWaitLink = async (opts = {}) => { proc.unref(); proc.on('exit', async (code, signal) => { if (code || signal) { - winston.error(await fs.readFile(stderrPath, 'utf8')); + logger.error(await fs.readFile(stderrPath, 'utf8')); throw new Error(`Tensorboard failed with error ${code || signal}`); } }); @@ -81,6 +81,11 @@ exports.command = 'connect'; exports.description = `${DESCRIPTION}\n${DOCSURL}`; exports.handler = async (opts) => { + if (new Date() > new Date('2024-01-01')) { + logger.error('TensorBoard.dev has been shut down as of January 1, 2024'); + return; + } + const { file, credentials, name, description } = opts; const path = `${homedir()}/.config/tensorboard/credentials`; diff --git a/bin/legacy/deprecation.js b/bin/legacy/deprecation.js index c13e44f43..93802b038 100644 --- a/bin/legacy/deprecation.js +++ b/bin/legacy/deprecation.js @@ -1,4 +1,4 @@ -const winston = require('winston'); +const { logger } = require('../../src/logger'); // addDeprecationNotice adds middleware to the yargs chain to display a deprecation notice. const addDeprecationNotice = (opts = {}) => { @@ -13,7 +13,7 @@ const deprecationNotice = (opts, notice) => { if (driver.warn) { driver.warn(notice); } else { - winston.warn(notice); + logger.warn(notice); } }; diff --git a/package.json b/package.json index 5d19b1802..53256d05f 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "test": "jest --forceExit", "test:unit": "jest --testPathIgnorePatterns='e2e.test.js$' --forceExit", "test:e2e": "jest --testNamePattern='e2e.test.js$' --forceExit", - "do_snapshots": "jest --updateSnapshot" + "snapshot": "jest --updateSnapshot" }, "husky": { "hooks": { diff --git a/src/analytics.js b/src/analytics.js index 6fd3a3711..1d06a3d1b 100644 --- a/src/analytics.js +++ b/src/analytics.js @@ -8,7 +8,7 @@ const { promisify } = require('util'); const { scrypt } = require('crypto'); const { v4: uuidv4, v5: uuidv5, parse } = require('uuid'); const { userConfigDir } = require('appdirs'); -const winston = require('winston'); +const { logger } = require('./logger'); const isDocker = require('is-docker'); const { version: VERSION } = require('../package.json'); @@ -18,7 +18,7 @@ const { ITERATIVE_ANALYTICS_ENDPOINT = 'https://telemetry.cml.dev/api/v1/s2s/event?ip_policy=strict', ITERATIVE_ANALYTICS_TOKEN = 's2s.jtyjusrpsww4k9b76rrjri.bl62fbzrb7nd9n6vn5bpqt', ITERATIVE_DO_NOT_TRACK, - + CODESPACES, GITHUB_SERVER_URL, GITHUB_REPOSITORY_OWNER, GITHUB_ACTOR, @@ -52,7 +52,7 @@ const deterministic = async (data) => { }; const guessCI = () => { - if (GITHUB_SERVER_URL) return 'github'; + if (GITHUB_SERVER_URL && !CODESPACES) return 'github'; if (CI_SERVER_URL) return 'gitlab'; if (BITBUCKET_WORKSPACE) return 'bitbucket'; if (TF_BUILD) return 'azure'; @@ -133,7 +133,7 @@ const userId = async ({ cml } = {}) => { return id; } catch (err) { - winston.debug(`userId failure: ${err.message}`); + logger.debug(`userId failure: ${err.message}`); } }; @@ -223,7 +223,7 @@ const send = async ({ }); clearInterval(id); } catch (err) { - winston.debug(`Send analytics failed: ${err.message}`); + logger.debug(`Send analytics failed: ${err.message}`); } }; diff --git a/src/cml.js b/src/cml.js index fe971896d..95df80442 100644 --- a/src/cml.js +++ b/src/cml.js @@ -6,7 +6,7 @@ const git = require('simple-git')('./'); const path = require('path'); const fs = require('fs').promises; const chokidar = require('chokidar'); -const winston = require('winston'); +const { logger } = require('./logger'); const remark = require('remark'); const visit = require('unist-util-visit'); @@ -138,7 +138,7 @@ class CML { try { return await exec('git', 'rev-parse', ref); } catch (err) { - winston.warn( + logger.warn( 'Failed to obtain SHA. Perhaps not in the correct git folder' ); } @@ -224,7 +224,7 @@ class CML { if (node.url && !isWatermark(node)) { // Check for embedded images from dvclive if (node.url.startsWith('data:image/')) { - winston.debug( + logger.debug( `found already embedded image, head: ${node.url.slice(0, 25)}` ); const encodedData = node.url.slice(node.url.indexOf(',') + 1); @@ -253,7 +253,7 @@ class CML { }); } catch (err) { if (err.code === 'ENOENT') - winston.debug(`file not found: ${node.url} (${absolutePath})`); + logger.debug(`file not found: ${node.url} (${absolutePath})`); else throw err; } } @@ -281,7 +281,7 @@ class CML { if (lock) return; lock = true; try { - winston.info(`watcher event: ${event} ${path}`); + logger.info(`watcher event: ${event} ${path}`); await this.commentCreate({ ...opts, update: update || !first, @@ -291,12 +291,12 @@ class CML { await fs.unlink(triggerFile); } } catch (err) { - winston.warn(err); + logger.warn(err); } first = false; lock = false; }); - winston.info('watching for file changes...'); + logger.info('watching for file changes...'); await waitForever(); } @@ -426,7 +426,7 @@ class CML { const env = {}; const sensitive = [ '_CML_RUNNER_SENSITIVE_ENV', - ...process.env._CML_RUNNER_SENSITIVE_ENV.split(':') + ...(process.env._CML_RUNNER_SENSITIVE_ENV || '').split(':') ]; for (const variable in process.env) if (!sensitive.includes(variable)) env[variable] = process.env[variable]; @@ -560,7 +560,7 @@ class CML { const { files } = await git.status(); if (!files.length && globs.length) { - winston.warn('No changed files matched by glob path. Nothing to do.'); + logger.warn('No changed files matched by glob path. Nothing to do.'); return; } @@ -575,7 +575,7 @@ class CML { ); if (!paths.length && globs.length) { - winston.warn('Input files are not affected. Nothing to do.'); + logger.warn('Input files are not affected. Nothing to do.'); return; } @@ -596,7 +596,7 @@ class CML { target = targetBranch; } catch (error) { - winston.error('The target branch does not exist.'); + logger.error('The target branch does not exist.'); process.exit(1); } } @@ -669,7 +669,7 @@ Automated commits for ${this.repo}/commit/${sha} created by CML. } logError(e) { - winston.error(e.message); + logger.error(e.message); } } diff --git a/src/commenttarget.js b/src/commenttarget.js index fd527216c..64484675e 100644 --- a/src/commenttarget.js +++ b/src/commenttarget.js @@ -1,4 +1,4 @@ -const winston = require('winston'); +const { logger } = require('./logger'); const SEPARATOR = '/'; @@ -22,14 +22,14 @@ async function parseCommentTarget(opts = {}) { let commitPr; switch (commentTarget.toLowerCase()) { case 'commit': - winston.debug(`Comment target "commit" mapped to "commit/${drv.sha}"`); + logger.debug(`Comment target "commit" mapped to "commit/${drv.sha}"`); return { target: 'commit', commitSha: drv.sha }; case 'pr': case 'auto': // Determine PR id from forge env vars (if we're in a PR context). prNumber = drv.pr; if (prNumber) { - winston.debug( + logger.debug( `Comment target "${commentTarget}" mapped to "pr/${prNumber}"` ); return { target: 'pr', prNumber: prNumber }; @@ -39,14 +39,14 @@ async function parseCommentTarget(opts = {}) { [commitPr = {}] = await drv.commitPrs({ commitSha: drv.sha }); if (commitPr.url) { [prNumber] = commitPr.url.split('/').slice(-1); - winston.debug( + logger.debug( `Comment target "${commentTarget}" mapped to "pr/${prNumber}" based on commit "${drv.sha}"` ); return { target: 'pr', prNumber }; } // If target is 'auto', fallback to issuing commit comments. if (commentTarget === 'auto') { - winston.debug( + logger.debug( `Comment target "${commentTarget}" mapped to "commit/${drv.sha}"` ); return { target: 'commit', commitSha: drv.sha }; diff --git a/src/drivers/bitbucket_cloud.e2e.test.js b/src/drivers/bitbucket_cloud.e2e.test.js index 7715b5913..60cd615b0 100644 --- a/src/drivers/bitbucket_cloud.e2e.test.js +++ b/src/drivers/bitbucket_cloud.e2e.test.js @@ -1,9 +1,9 @@ const BitbucketCloud = require('./bitbucket_cloud'); const { - TEST_BBCLOUD_TOKEN: TOKEN, - TEST_BBCLOUD_REPO: REPO, - TEST_BBCLOUD_SHA: SHA, - TEST_BBCLOUD_ISSUE: ISSUE = 1 + TEST_BITBUCKET_TOKEN: TOKEN, + TEST_BITBUCKET_REPO: REPO, + TEST_BITBUCKET_SHA: SHA, + TEST_BITBUCKET_ISSUE: ISSUE = 1 } = process.env; describe('Non Enviromental tests', () => { diff --git a/src/drivers/bitbucket_cloud.js b/src/drivers/bitbucket_cloud.js index 76680da79..c54e3f907 100644 --- a/src/drivers/bitbucket_cloud.js +++ b/src/drivers/bitbucket_cloud.js @@ -1,10 +1,10 @@ const crypto = require('crypto'); const fetch = require('node-fetch'); -const winston = require('winston'); const { URL } = require('url'); const { spawn } = require('child_process'); const FormData = require('form-data'); const ProxyAgent = require('proxy-agent'); +const { logger } = require('../logger'); const { fetchUploadData, exec, gpuPresent, sleep } = require('../utils'); @@ -168,7 +168,7 @@ class BitbucketCloud { const { projectPath } = this; const { workdir, name, labels, env } = opts; - winston.warn( + logger.warn( `Bitbucket runner is working under /tmp folder and not under ${workdir} as expected` ); @@ -320,7 +320,7 @@ class BitbucketCloud { } async prAutoMerge({ pullRequestId, mergeMode, mergeMessage }) { - winston.warn( + logger.warn( 'Auto-merge is unsupported by Bitbucket Cloud; see https://jira.atlassian.com/browse/BCLOUD-14286. Trying to merge immediately...' ); const { projectPath } = this; @@ -420,7 +420,7 @@ class BitbucketCloud { const { projectPath } = this; if (!id && jobId) - winston.warn('BitBucket Cloud does not support pipelineRerun by jobId!'); + logger.warn('BitBucket Cloud does not support pipelineRerun by jobId!'); const { target } = await this.request({ endpoint: `/repositories/${projectPath}/pipelines/${id}`, @@ -435,7 +435,7 @@ class BitbucketCloud { } async pipelineJobs(opts = {}) { - winston.warn('BitBucket Cloud does not support pipelineJobs yet!'); + logger.warn('BitBucket Cloud does not support pipelineJobs yet!'); return []; } @@ -527,7 +527,7 @@ class BitbucketCloud { headers['Content-Type'] = 'application/json'; const requestUrl = url || `${api}${endpoint}`; - winston.debug( + logger.debug( `Bitbucket API request, method: ${method}, url: "${requestUrl}"` ); const response = await fetch(requestUrl, { @@ -542,7 +542,7 @@ class BitbucketCloud { : await response.text(); if (!response.ok) { - winston.debug(`Response status is ${response.status}`); + logger.debug(`Response status is ${response.status}`); // Attempt to get additional context. We have observed two different error schemas // from BitBucket API responses: `{"error": {"message": "Error message"}}` and // `{"error": "Error message"}`, apart from plain text responses like `Bad Request`. @@ -558,7 +558,7 @@ class BitbucketCloud { } warn(message) { - winston.warn(message); + logger.warn(message); } } diff --git a/src/drivers/github.js b/src/drivers/github.js index ad5e88c13..d0ca8f727 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -12,7 +12,7 @@ const tar = require('tar'); const ProxyAgent = require('proxy-agent'); const { download, exec, sleep } = require('../utils'); -const winston = require('winston'); +const { logger } = require('../logger'); const CHECK_TITLE = 'CML Report'; process.env.RUNNER_ALLOW_RUNASROOT = 1; @@ -55,7 +55,7 @@ const octokit = (token, repo, log) => { const throttleHandler = (reason, offset) => async (retryAfter, options) => { if (options.request.retryCount <= 5) { - winston.info( + logger.info( `Retrying because of ${reason} in ${retryAfter + offset} seconds` ); await new Promise((resolve) => setTimeout(resolve, offset * 1000)); @@ -184,9 +184,9 @@ class Github { const warning = 'This command only works inside a Github runner or a Github app.'; - if (!CI || TPI_TASK) winston.warn(warning); + if (!CI || TPI_TASK) logger.warn(warning); if (GITHUB_TOKEN && GITHUB_TOKEN !== this.token) - winston.warn( + logger.warn( `Your token is different than the GITHUB_TOKEN, this command does not work with PAT. ${warning}` ); @@ -209,7 +209,7 @@ class Github { async runnerToken() { const { owner, repo } = ownerRepo({ uri: this.repo }); - const { actions } = octokit(this.token, this.repo, winston); + const { actions } = octokit(this.token, this.repo, logger); if (typeof repo !== 'undefined') { const { @@ -238,7 +238,7 @@ class Github { async unregisterRunner(opts) { const { runnerId } = opts; const { owner, repo } = ownerRepo({ uri: this.repo }); - const { actions } = octokit(this.token, this.repo, winston); + const { actions } = octokit(this.token, this.repo, logger); if (typeof repo !== 'undefined') { await actions.deleteSelfHostedRunnerFromRepo({ @@ -309,7 +309,7 @@ class Github { async runners(opts = {}) { const { owner, repo } = ownerRepo({ uri: this.repo }); - const { paginate, actions } = octokit(this.token, this.repo, winston); + const { paginate, actions } = octokit(this.token, this.repo, logger); let runners; if (typeof repo === 'undefined') { @@ -331,7 +331,7 @@ class Github { async runnerById(opts = {}) { const { id } = opts; const { owner, repo } = ownerRepo({ uri: this.repo }); - const { actions } = octokit(this.token, this.repo, winston); + const { actions } = octokit(this.token, this.repo, logger); if (typeof repo === 'undefined') { const { data: runner } = await actions.getSelfHostedRunnerForOrg({ @@ -353,7 +353,7 @@ class Github { async runnerJob({ runnerId, status = 'queued' } = {}) { const { owner, repo } = ownerRepo({ uri: this.repo }); - const octokitClient = octokit(this.token, this.repo, winston); + const octokitClient = octokit(this.token, this.repo, logger); if (status === 'running') status = 'in_progress'; @@ -532,18 +532,18 @@ class Github { try { if (await this.isProtected({ branch: base })) { - winston.warn( + logger.warn( `Failed to enable auto-merge: Enable the feature in your repository settings: ${settingsUrl}#merge_types_auto_merge. Trying to merge immediately...` ); } else { - winston.warn( + logger.warn( `Failed to enable auto-merge: Set up branch protection and add "required status checks" for branch '${base}': ${settingsUrl}/branches. Trying to merge immediately...` ); } } catch (err) { if (!err.message.includes('Resource not accessible by integration')) throw err; - winston.warn( + logger.warn( `Failed to enable auto-merge. Trying to merge immediately...` ); } @@ -687,7 +687,7 @@ class Github { async pipelineRerun({ id = GITHUB_RUN_ID, jobId } = {}) { const { owner, repo } = ownerRepo({ uri: this.repo }); - const { actions } = octokit(this.token, this.repo, winston); + const { actions } = octokit(this.token, this.repo, logger); if (!id && jobId) { ({ @@ -736,7 +736,7 @@ class Github { async pipelineJobs(opts = {}) { const { jobs: runnerJobs } = opts; const { owner, repo } = ownerRepo({ uri: this.repo }); - const { actions } = octokit(this.token, this.repo, winston); + const { actions } = octokit(this.token, this.repo, logger); const jobs = await Promise.all( runnerJobs.map(async (job) => { diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 3360e39c7..f6f0e471f 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -7,7 +7,7 @@ const fse = require('fs-extra'); const { resolve } = require('path'); const ProxyAgent = require('proxy-agent'); const { backOff } = require('exponential-backoff'); -const winston = require('winston'); +const { logger } = require('../logger'); const { fetchUploadData, download, gpuPresent } = require('../utils'); @@ -144,11 +144,21 @@ class Gitlab { return { uri: `${repo}${url}`, mime, size }; } - async runnerToken() { + async runnerToken(body) { const projectPath = await this.projectPath(); - const endpoint = `/projects/${projectPath}`; + const legacyEndpoint = `/projects/${projectPath}`; + const endpoint = `/user/runners`; - const { runners_token: runnersToken } = await this.request({ endpoint }); + const { id, runners_token: runnersToken } = await this.request({ + endpoint: legacyEndpoint + }); + + if (runnersToken === null) { + if (!body) body = new URLSearchParams(); + body.append('project_id', id); + body.append('runner_type', 'project_type'); + return (await this.request({ endpoint, method: 'POST', body })).token; + } return runnersToken; } @@ -156,16 +166,18 @@ class Gitlab { async registerRunner(opts = {}) { const { tags, name } = opts; - const token = await this.runnerToken(); const endpoint = `/runners`; const body = new URLSearchParams(); body.append('description', name); body.append('tag_list', tags); - body.append('token', token); body.append('locked', 'true'); body.append('run_untagged', 'true'); body.append('access_level', 'not_protected'); + const token = await this.runnerToken(new URLSearchParams(body)); + if (token.startsWith('glrt-')) return { token }; + + body.append('token', token); return await this.request({ endpoint, method: 'POST', body }); } @@ -330,7 +342,7 @@ class Gitlab { }) ); } catch ({ message }) { - winston.warn( + logger.warn( `Failed to enable auto-merge: ${message}. Trying to merge immediately...` ); body.set('merge_when_pipeline_succeeds', false); @@ -560,7 +572,7 @@ class Gitlab { } if (!url) throw new Error('Gitlab API endpoint not found'); - winston.debug(`Gitlab API request, method: ${method}, url: "${url}"`); + logger.debug(`Gitlab API request, method: ${method}, url: "${url}"`); const headers = { 'PRIVATE-TOKEN': token, Accept: 'application/json' }; const response = await fetch(url, { @@ -570,7 +582,7 @@ class Gitlab { agent: new ProxyAgent() }); if (!response.ok) { - winston.debug(`Response status is ${response.status}`); + logger.debug(`Response status is ${response.status}`); throw new Error(response.statusText); } if (raw) return response; @@ -579,7 +591,7 @@ class Gitlab { } warn(message) { - winston.warn(message); + logger.warn(message); } } diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 000000000..e1e7a5c7c --- /dev/null +++ b/src/logger.js @@ -0,0 +1,32 @@ +const logger = require('winston'); + +const setupLogger = (opts) => { + const { log: level, silent } = opts; + + logger.configure({ + format: process.stdout.isTTY + ? logger.format.combine( + logger.format.colorize({ all: true }), + logger.format.simple() + ) + : logger.format.combine( + logger.format.errors({ stack: true }), + logger.format.json() + ), + transports: [ + new logger.transports.Console({ + stderrLevels: Object.keys(logger.config.npm.levels), + handleExceptions: true, + handleRejections: true, + level, + silent + }) + ] + }); +}; + +if (typeof jest !== 'undefined') { + setupLogger({ log: 'debug', silent: true }); +} + +module.exports = { logger, setupLogger }; diff --git a/src/terraform.js b/src/terraform.js index 344438f4d..11776a5aa 100644 --- a/src/terraform.js +++ b/src/terraform.js @@ -1,6 +1,6 @@ const fs = require('fs').promises; const { ltr } = require('semver'); -const winston = require('winston'); +const { logger } = require('./logger'); const { exec, tfCapture } = require('./utils'); const MIN_TF_VER = '0.14.0'; @@ -123,7 +123,7 @@ const iterativeCmlRunnerTpl = (opts = {}) => { } } }; - winston.debug(`terraform data: ${JSON.stringify(tfObj)}`); + logger.debug(`terraform data: ${JSON.stringify(tfObj)}`); return tfObj; }; diff --git a/src/terraform.test.js b/src/terraform.test.js index c165cdd04..6240a3753 100644 --- a/src/terraform.test.js +++ b/src/terraform.test.js @@ -1,18 +1,6 @@ -const winston = require('winston'); const { iterativeCmlRunnerTpl } = require('./terraform'); describe('Terraform tests', () => { - beforeAll(() => { - winston.configure({ - transports: [ - new winston.transports.Console({ - level: 'error', - handleExceptions: true - }) - ] - }); - }); - test('default options', async () => { const output = iterativeCmlRunnerTpl({}); expect(JSON.stringify(output, null, 2)).toMatchInlineSnapshot(` diff --git a/src/utils.js b/src/utils.js index 7bf891412..2fbed55e4 100644 --- a/src/utils.js +++ b/src/utils.js @@ -5,7 +5,7 @@ const fetch = require('node-fetch'); const ProxyAgent = require('proxy-agent'); const NodeSSH = require('node-ssh').NodeSSH; const stripAnsi = require('strip-ansi'); -const winston = require('winston'); +const { logger } = require('./logger'); const uuid = require('uuid'); const getOS = require('getos'); @@ -212,12 +212,12 @@ const tfCapture = async (command, args = [], options = {}) => { try { const { '@level': level, '@message': message } = JSON.parse(line); if (level === 'error') { - winston.error(`terraform error: ${message}`); + logger.error(`terraform error: ${message}`); } else { - winston.info(message); + logger.info(message); } } catch (err) { - winston.info(line); + logger.info(line); } }; buf.toString('utf8').split('\n').forEach(parse);