Unregister GitHub runner

.github/workflows/publish.yml

@@ -4,6 +4,12 @@ on: [push, pull_request]
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+  TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} 
+  TEST_GITHUB_REPO: https://github.com/iterative/cml_qa_tests_dummy
+  TEST_GITHUB_SHA: 62edc8b3f46a60b3fe1e5c08fd3e0046d350ee29
+  TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }}
+  TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy
+  TEST_GITLAB_SHA: c4c13286e78dc252dd2611f31a755f10d343fbd4
 
 jobs:
   test_and_deploy:

bin/cml-cloud-runner-entrypoint.js

@@ -1,40 +1,40 @@
 #!/usr/bin/env node
 
 const { spawn } = require('child_process');
+
 const { exec, randid } = require('../src/utils');
-const { URL } = require('url');
+const CML = require('../src/cml');
 
 const {
   DOCKER_MACHINE, // DEPRECATED
 
   RUNNER_PATH,
-  RUNNER_REPO,
   RUNNER_IDLE_TIMEOUT = 5 * 60,
   RUNNER_LABELS = 'cml',
   RUNNER_NAME = randid(),
   RUNNER_EXECUTOR = 'shell',
   RUNNER_RUNTIME = '',
-  RUNNER_IMAGE = 'dvcorg/cml:latest'
+  RUNNER_IMAGE = 'dvcorg/cml:latest',
+
+  RUNNER_DRIVER,
+  RUNNER_REPO,
+  repo_token
 } = process.env;
 
-const { protocol, host, pathname } = new URL(RUNNER_REPO);
+const cml = new CML({
+  driver: RUNNER_DRIVER,
+  repo: RUNNER_REPO,
+  token: repo_token
+});
+const IS_GITHUB = cml.driver === 'github';
+const { protocol, host } = new URL(RUNNER_REPO);
 const RUNNER_REPO_ORIGIN = `${protocol}//${host}`;
-process.env.CI_API_V4_URL = `${RUNNER_REPO_ORIGIN}/api/v4/`;
-process.env.GITHUB_REPOSITORY = process.env.CI_PROJECT_PATH = pathname.substring(
-  1,
-  pathname.length - (pathname.endsWith('/') ? 1 : 0)
-);
 
-const IS_GITHUB = RUNNER_REPO_ORIGIN === 'https://github.com';
 let TIMEOUT_TIMER = 0;
 let JOB_RUNNING = false;
 let RUNNER_TOKEN;
 let GITLAB_CI_TOKEN;
 
-const { get_runner_token, register_runner } = IS_GITHUB
-  ? require('../src/github')
-  : require('../src/gitlab');
-
 const shutdown_docker_machine = async () => {
   console.log('Shutting down docker machine');
   try {
@@ -54,7 +54,6 @@ const shutdown_host = async () => {
       );
     } catch (err) {
       console.log(`Failed destroying terraform: ${err.message}`);
-      // shutdown_host();
     }
   } catch (err) {
     console.log(err.message);
@@ -67,11 +66,7 @@ const shutdown = async (error) => {
 
     try {
       if (IS_GITHUB) {
-        console.log(
-          await exec(
-            `${RUNNER_PATH}/config.sh remove --token "${RUNNER_TOKEN}"`
-          )
-        );
+        await cml.unregister_runner({ name: RUNNER_NAME });
       } else {
         console.log(await exec(`gitlab-runner verify --delete`));
         console.log(
@@ -80,7 +75,9 @@ const shutdown = async (error) => {
           )
         );
       }
-    } catch (err) {}
+    } catch (err) {
+      console.log(err);
+    }
 
     await shutdown_docker_machine();
     await shutdown_host();
@@ -98,7 +95,7 @@ process.on('SIGTERM', shutdown);
 process.on('SIGINT', shutdown);
 process.on('SIGQUIT', shutdown);
 const run = async () => {
-  RUNNER_TOKEN = await get_runner_token();
+  RUNNER_TOKEN = await cml.runner_token();
 
   if (!RUNNER_TOKEN) {
     throw new Error(
@@ -124,16 +121,16 @@ const run = async () => {
     command = `${RUNNER_PATH}/run.sh`;
   } else {
     console.log('Registering Gitlab runner');
-    const runner = await register_runner({
+    const runner = await cml.register_runner({
       tags: RUNNER_LABELS,
-      token: RUNNER_TOKEN
+      runner_token: RUNNER_TOKEN
     });
 
     GITLAB_CI_TOKEN = runner.token;
 
     command = `gitlab-runner --log-format="json" run-single \
-      --url "${RUNNER_REPO_ORIGIN}" \
       --token "${runner.token}" \
+      --url "${RUNNER_REPO_ORIGIN}" \
       --executor "${RUNNER_EXECUTOR}" \
       --docker-runtime "${RUNNER_RUNTIME}" \
       --docker-image "${RUNNER_IMAGE}" \

bin/cml-cloud-runner.js

@@ -5,16 +5,18 @@ const NodeSSH = require('node-ssh').NodeSSH;
 const fss = require('fs');
 const fs = fss.promises;
 const path = require('path');
+
 const {
   exec,
   sleep,
   ssh_public_from_private_rsa,
   parse_param_newline
 } = require('../src/utils');
 
-const { handle_error, repo: REPO, token: TOKEN } = process.env.GITHUB_ACTIONS
-  ? require('../src/github')
-  : require('../src/gitlab');
+const CML = require('../src/cml');
+
+let REPO;
+let TOKEN;
 
 const TF_FOLDER = '.cml';
 const TF_NO_LOCAL = '.nolocal';
@@ -55,7 +57,8 @@ const setup_runners = async (opts) => {
     name: runner_name,
     image = 'dvcorg/cml:latest',
     'rsa-private-key': rsa_private_key,
-    attached
+    attached,
+    driver
   } = opts;
 
   const tf_path = path.join(TF_FOLDER, 'main.tf');
@@ -107,6 +110,7 @@ const setup_runners = async (opts) => {
       -v $(pwd)/main.tf:/main.tf \
       -e "repo_token=${repo_token}" \
       -e "RUNNER_REPO=${runner_repo}" \
+      -e "RUNNER_DRIVER=${driver}" \
       ${runner_labels ? `-e "RUNNER_LABELS=${runner_labels}"` : ''} \
       ${
         runner_idle_timeout
@@ -208,9 +212,13 @@ const shutdown = async () => {
 };
 
 const run = async (opts) => {
+  const cml = new CML(opts);
+  REPO = cml.env_repo();
+  TOKEN = cml.env_token();
+
   try {
     const terraform_state = await run_terraform(opts);
-    await setup_runners({ terraform_state, ...opts });
+    await setup_runners({ terraform_state, ...opts, driver: cml.driver });
     await sleep(20);
   } catch (err) {
     await destroy_terraform({});
@@ -267,5 +275,21 @@ const argv = yargs
   .boolean('attached')
   .describe('attached', 'Runs the runner in the foreground.')
   .coerce('rsa-private-key', parse_param_newline)
+  .default('repo')
+  .describe(
+    'repo',
+    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+  )
+  .default('token')
+  .describe(
+    'token',
+    'Personal access token to be used. If not specified in extracted from ENV repo_token or GITLAB_TOKEN.'
+  )
+  .default('driver')
+  .choices('driver', ['github', 'gitlab'])
+  .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h').argv;
-run(argv).catch((e) => handle_error(e));
+run(argv).catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

bin/cml-cloud-runner.test.js

@@ -13,8 +13,8 @@ describe('CML e2e', () => {
         --version          Show version number                               [boolean]
         --repo-token       Repository token. Defaults to workflow env variable
                            repo_token.
-        --repo             Repository to register with. Tries to guess from workflow
-                           env variables.
+        --repo             Specifies the repo to be used. If not specified is
+                           extracted from the CI ENV.
         --labels           Comma delimited runner labels. Defaults to cml
         --idle-timeout     Time in seconds for the runner to be waiting for jobs
                            before shutting down. Defaults to 5 min
@@ -29,6 +29,10 @@ describe('CML e2e', () => {
         --rsa-private-key  Your private RSA SHH key. If not provided will be generated
                            by the tf provider.                           [default: \\"\\"]
         --attached         Runs the runner in the foreground.                [boolean]
+        --token            Personal access token to be used. If not specified in
+                           extracted from ENV repo_token or GITLAB_TOKEN.
+        --driver           If not specify it infers it from the ENV.
+                                                         [choices: \\"github\\", \\"gitlab\\"]
         -h                 Show help                                         [boolean]"
     `);
   });

bin/cml-publish.js

@@ -6,33 +6,18 @@ console.log = console.error;
 const fs = require('fs').promises;
 const pipe_args = require('../src/pipe-args');
 const yargs = require('yargs');
-const { publish_file } = require('../src/report');
 
-const { GITHUB_ACTIONS } = process.env;
-
-const { handle_error } = GITHUB_ACTIONS
-  ? require('../src/github')
-  : require('../src/gitlab');
+const CML = require('../src/cml');
 
 const run = async (opts) => {
-  const { data, file } = opts;
-  let { 'gitlab-uploads': gitlab_uploads } = opts;
-  const path = opts._[0];
+  const { data, file, 'gitlab-uploads': gitlab_uploads } = opts;
 
+  const path = opts._[0];
   let buffer;
   if (data) buffer = Buffer.from(data, 'binary');
 
-  if (GITHUB_ACTIONS && gitlab_uploads) {
-    console.error(`
-    *********************************************
-    * gitlab-uploads option is only for gitlab! *
-    * *******************************************
-    `);
-
-    gitlab_uploads = false;
-  }
-
-  const output = await publish_file({ buffer, path, gitlab_uploads, ...opts });
+  const cml = new CML(opts);
+  const output = await cml.publish({ buffer, path, gitlab_uploads, ...opts });
 
   if (!file) print(output);
   else await fs.writeFile(file, output);
@@ -42,6 +27,7 @@ pipe_args.load('binary');
 const data = pipe_args.piped_arg();
 const argv = yargs
   .usage(`Usage: $0 <path to file>`)
+  .describe('md', 'Output in markdown format [title || name](url).')
   .boolean('md')
   .describe('md', 'Output in markdown format [title || name](url).')
   .default('title')
@@ -52,12 +38,35 @@ const argv = yargs
     'gitlab-uploads',
     'Uses GitLab uploads instead of CML storage. Use GitLab uploads to get around CML size limitations for hosting artifacts persistently. Only available for GitLab CI.'
   )
+  .deprecateOption('gitlab-uploads', 'Use native instead')
+  .boolean('native')
+  .describe(
+    'native',
+    "Uses driver's native capabilities to upload assets instead of CML's backend."
+  )
   .default('file')
   .describe(
     'file',
     'Append the output to the given file. Create it if does not exist.'
   )
   .alias('file', 'f')
+  .default('repo')
+  .describe(
+    'repo',
+    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+  )
+  .default('token')
+  .describe(
+    'token',
+    'Personal access token to be used. If not specified in extracted from ENV repo_token or GITLAB_TOKEN.'
+  )
+  .default('driver')
+  .choices('driver', ['github', 'gitlab'])
+  .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h')
   .demand(data ? 0 : 1).argv;
-run({ ...argv, data }).catch((e) => handle_error(e));
+
+run({ ...argv, data }).catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

bin/cml-publish.test.js

@@ -17,9 +17,17 @@ describe('CML e2e', () => {
         --gitlab-uploads  Uses GitLab uploads instead of CML storage. Use GitLab
                           uploads to get around CML size limitations for hosting
                           artifacts persistently. Only available for GitLab CI.
-                                                                             [boolean]
+                                            [deprecated: Use native instead] [boolean]
+        --native          Uses driver's native capabilities to upload assets instead
+                          of CML's backend.                                  [boolean]
         --file, -f        Append the output to the given file. Create it if does not
                           exist.
+        --repo            Specifies the repo to be used. If not specified is extracted
+                          from the CI ENV.
+        --token           Personal access token to be used. If not specified in
+                          extracted from ENV repo_token or GITLAB_TOKEN.
+        --driver          If not specify it infers it from the ENV.
+                                                         [choices: \\"github\\", \\"gitlab\\"]
         -h                Show help                                          [boolean]"
     `);
   });
@@ -86,4 +94,14 @@ describe('CML e2e', () => {
     expect(fs.existsSync(file)).toBe(true);
     await fs.promises.unlink(file);
   });
+
+  test('cml-publish assets/test.svg in Gitlab storage', async () => {
+    const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env;
+
+    const output = await exec(
+      `echo none | node ./bin/cml-publish.js --repo=${repo} --token=${token} --gitlab-uploads assets/test.svg`
+    );
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
 });

bin/cml-send-comment.js

@@ -5,16 +5,15 @@ console.log = console.error;
 const fs = require('fs').promises;
 const yargs = require('yargs');
 
-const { head_sha: HEAD_SHA, handle_error, comment } = process.env.GITHUB_ACTIONS
-  ? require('../src/github')
-  : require('../src/gitlab');
+const CML = require('../src/cml');
 
 const run = async (opts) => {
   const { 'commit-sha': sha, 'head-sha': head_sha } = opts;
   const path = opts._[0];
   const report = await fs.readFile(path, 'utf-8');
 
-  await comment({ commit_sha: sha || head_sha || HEAD_SHA, report });
+  const cml = new CML(opts);
+  await cml.comment_create({ report, commit_sha: sha || head_sha });
 };
 
 const argv = yargs
@@ -25,8 +24,25 @@ const argv = yargs
     'Commit SHA linked to this comment. Defaults to HEAD.'
   )
   .default('head-sha')
-  .describe('head-sha', 'Commit SHA linked to this comment. Defaults to HEAD')
+  .describe('head-sha', 'Commit SHA linked to this comment. Defaults to HEAD.')
   .deprecateOption('head-sha', 'Use commit-sha instead')
+  .default('repo')
+  .describe(
+    'repo',
+    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+  )
+  .default('token')
+  .describe(
+    'token',
+    'Personal access token to be used. If not specified in extracted from ENV repo_token.'
+  )
+  .default('driver')
+  .choices('driver', ['github', 'gitlab'])
+  .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h')
   .demand(1).argv;
-run(argv).catch((e) => handle_error(e));
+
+run(argv).catch((e) => {
+  console.error(e);
+  process.exit(1);
+});