CML contexts (#311)

* [wip] CML contexts * octokit on demand * Cml contexts (#314) * token * clean env before tests * print env * yaml * yaml * yaml * yaml * check * fix repo env * fix repo env * fix repo env * fix repo env * check name * env github tests * env github tests * log check * log check * no check tessts * enviromental tests * workflow eenv * gitlab uploads and github_token * snapshots * docker entrypooint * docker entrypoint register_runner * update templates * token repo * token repo * specify driver * Cml contexts sha fix (#315) * token * clean env before tests * print env * yaml * yaml * yaml * yaml * check * fix repo env * fix repo env * fix repo env * fix repo env * check name * env github tests * env github tests * log check * log check * no check tessts * enviromental tests * workflow eenv * gitlab uploads and github_token * snapshots * log env * log env * update also github.context * No log * point to iterative repos * point to iterative repos without slash * point to iterative ai * test slash * test slash * fix strip * removal of strip * refactor * slash * minor refactor * no console * Cml contexts sha (#326) * git sha * commit_sha * log * log gh sha * log gh sha * show origin * remote as repo * remote as repo * tests * remove one test not working on gitlab * space * publish backend * no banckend in publish * fix small review comments * test template * test template * adjust cml-cloud-runner * bump version * snapshot * snapshot
iterative · Oct 29, 2020 · 6f9c79b · 6f9c79b · github-actions · Oct 29, 2020
1 parent 436a1e7
commit 6f9c79b
Show file tree

Hide file tree

Showing 23 changed files with 847 additions and 404 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -4,6 +4,12 @@ on: [push, pull_request]
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+  TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} 
+  TEST_GITHUB_REPO: https://github.com/iterative/cml_qa_tests_dummy
+  TEST_GITHUB_SHA: 62edc8b3f46a60b3fe1e5c08fd3e0046d350ee29
+  TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }}
+  TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy
+  TEST_GITLAB_SHA: c4c13286e78dc252dd2611f31a755f10d343fbd4
 
 jobs:
   test_and_deploy:

diff --git a/bin/cml-cloud-runner-entrypoint.js b/bin/cml-cloud-runner-entrypoint.js
@@ -1,40 +1,40 @@
 #!/usr/bin/env node
 
 const { spawn } = require('child_process');
+
 const { exec, randid } = require('../src/utils');
-const { URL } = require('url');
+const CML = require('../src/cml');
 
 const {
   DOCKER_MACHINE, // DEPRECATED
 
   RUNNER_PATH,
-  RUNNER_REPO,
   RUNNER_IDLE_TIMEOUT = 5 * 60,
   RUNNER_LABELS = 'cml',
   RUNNER_NAME = randid(),
   RUNNER_EXECUTOR = 'shell',
   RUNNER_RUNTIME = '',
-  RUNNER_IMAGE = 'dvcorg/cml:latest'
+  RUNNER_IMAGE = 'dvcorg/cml:latest',
+
+  RUNNER_DRIVER,
+  RUNNER_REPO,
+  repo_token
 } = process.env;
 
-const { protocol, host, pathname } = new URL(RUNNER_REPO);
+const cml = new CML({
+  driver: RUNNER_DRIVER,
+  repo: RUNNER_REPO,
+  token: repo_token
+});
+const IS_GITHUB = cml.driver === 'github';
+const { protocol, host } = new URL(RUNNER_REPO);
 const RUNNER_REPO_ORIGIN = `${protocol}//${host}`;
-process.env.CI_API_V4_URL = `${RUNNER_REPO_ORIGIN}/api/v4/`;
-process.env.GITHUB_REPOSITORY = process.env.CI_PROJECT_PATH = pathname.substring(
-  1,
-  pathname.length - (pathname.endsWith('/') ? 1 : 0)
-);
 
-const IS_GITHUB = RUNNER_REPO_ORIGIN === 'https://github.com';
 let TIMEOUT_TIMER = 0;
 let JOB_RUNNING = false;
 let RUNNER_TOKEN;
 let GITLAB_CI_TOKEN;
 
-const { get_runner_token, register_runner } = IS_GITHUB
-  ? require('../src/github')
-  : require('../src/gitlab');
-
 const shutdown_docker_machine = async () => {
   console.log('Shutting down docker machine');
   try {
@@ -54,7 +54,6 @@ const shutdown_host = async () => {
       );
     } catch (err) {
       console.log(`Failed destroying terraform: ${err.message}`);
-      // shutdown_host();
     }
   } catch (err) {
     console.log(err.message);
@@ -98,7 +97,7 @@ process.on('SIGTERM', shutdown);
 process.on('SIGINT', shutdown);
 process.on('SIGQUIT', shutdown);
 const run = async () => {
-  RUNNER_TOKEN = await get_runner_token();
+  RUNNER_TOKEN = await cml.runner_token();
 
   if (!RUNNER_TOKEN) {
     throw new Error(
@@ -124,16 +123,16 @@ const run = async () => {
     command = `${RUNNER_PATH}/run.sh`;
   } else {
     console.log('Registering Gitlab runner');
-    const runner = await register_runner({
+    const runner = await cml.register_runner({
       tags: RUNNER_LABELS,
-      token: RUNNER_TOKEN
+      runner_token: RUNNER_TOKEN
     });
 
     GITLAB_CI_TOKEN = runner.token;
 
     command = `gitlab-runner --log-format="json" run-single \
-      --url "${RUNNER_REPO_ORIGIN}" \
       --token "${runner.token}" \
+      --url "${RUNNER_REPO_ORIGIN}" \
       --executor "${RUNNER_EXECUTOR}" \
       --docker-runtime "${RUNNER_RUNTIME}" \
       --docker-image "${RUNNER_IMAGE}" \

diff --git a/bin/cml-cloud-runner.js b/bin/cml-cloud-runner.js
@@ -5,16 +5,16 @@ const NodeSSH = require('node-ssh').NodeSSH;
 const fss = require('fs');
 const fs = fss.promises;
 const path = require('path');
+
 const {
   exec,
   sleep,
   ssh_public_from_private_rsa,
   parse_param_newline
 } = require('../src/utils');
 
-const { handle_error, repo: REPO, token: TOKEN } = process.env.GITHUB_ACTIONS
-  ? require('../src/github')
-  : require('../src/gitlab');
+const CML = require('../src/cml');
+let cml;
 
 const TF_FOLDER = '.cml';
 const TF_NO_LOCAL = '.nolocal';
@@ -45,11 +45,10 @@ const ssh_connect = async (opts) => {
 };
 
 const setup_runners = async (opts) => {
+  const { repo, token, driver } = cml;
   const {
     terraform_state,
     username = 'ubuntu',
-    'repo-token': repo_token = TOKEN,
-    repo: runner_repo = REPO,
     labels: runner_labels,
     'idle-timeout': runner_idle_timeout,
     name: runner_name,
@@ -61,12 +60,12 @@ const setup_runners = async (opts) => {
   const tf_path = path.join(TF_FOLDER, 'main.tf');
   const tfstate_path = path.join(TF_FOLDER, 'terraform.tfstate');
 
-  if (!repo_token)
+  if (!token)
     throw new Error(
       'Repository token not set. Your repo_token is not available!'
     );
 
-  if (!runner_repo)
+  if (!repo)
     throw new Error(
       'Repo not set. Your repo must be set to register the runner!'
     );
@@ -105,8 +104,9 @@ const setup_runners = async (opts) => {
       -e AWS_ACCESS_KEY_ID=${process.env.AWS_ACCESS_KEY_ID} \
       -v $(pwd)/terraform.tfstate:/terraform.tfstate \
       -v $(pwd)/main.tf:/main.tf \
-      -e "repo_token=${repo_token}" \
-      -e "RUNNER_REPO=${runner_repo}" \
+      -e "repo_token=${token}" \
+      -e "RUNNER_REPO=${repo}" \
+      -e "RUNNER_DRIVER=${driver}" \
       ${runner_labels ? `-e "RUNNER_LABELS=${runner_labels}"` : ''} \
       ${
         runner_idle_timeout
@@ -208,6 +208,8 @@ const shutdown = async () => {
 };
 
 const run = async (opts) => {
+  cml = new CML(opts);
+
   try {
     const terraform_state = await run_terraform(opts);
     await setup_runners({ terraform_state, ...opts });
@@ -225,16 +227,6 @@ process.on('SIGQUIT', shutdown);
 
 const argv = yargs
   .usage(`Usage: $0`)
-  .default('repo-token')
-  .describe(
-    'repo-token',
-    'Repository token. Defaults to workflow env variable repo_token.'
-  )
-  .default('repo')
-  .describe(
-    'repo',
-    'Repository to register with. Tries to guess from workflow env variables.'
-  )
   .default('labels')
   .describe('labels', 'Comma delimited runner labels. Defaults to cml')
   .default('idle-timeout')
@@ -267,5 +259,21 @@ const argv = yargs
   .boolean('attached')
   .describe('attached', 'Runs the runner in the foreground.')
   .coerce('rsa-private-key', parse_param_newline)
+  .default('repo')
+  .describe(
+    'repo',
+    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+  )
+  .default('token')
+  .describe(
+    'token',
+    'Personal access token to be used. If not specified in extracted from ENV repo_token or GITLAB_TOKEN.'
+  )
+  .default('driver')
+  .choices('driver', ['github', 'gitlab'])
+  .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h').argv;
-run(argv).catch((e) => handle_error(e));
+run(argv).catch((e) => {
+  console.error(e);
+  process.exit(1);
+});
diff --git a/bin/cml-cloud-runner.test.js b/bin/cml-cloud-runner.test.js
@@ -11,10 +11,6 @@ describe('CML e2e', () => {
 
       Options:
         --version          Show version number                               [boolean]
-        --repo-token       Repository token. Defaults to workflow env variable
-                           repo_token.
-        --repo             Repository to register with. Tries to guess from workflow
-                           env variables.
         --labels           Comma delimited runner labels. Defaults to cml
         --idle-timeout     Time in seconds for the runner to be waiting for jobs
                            before shutting down. Defaults to 5 min
@@ -29,6 +25,12 @@ describe('CML e2e', () => {
         --rsa-private-key  Your private RSA SHH key. If not provided will be generated
                            by the tf provider.                           [default: \\"\\"]
         --attached         Runs the runner in the foreground.                [boolean]
+        --repo             Specifies the repo to be used. If not specified is
+                           extracted from the CI ENV.
+        --token            Personal access token to be used. If not specified in
+                           extracted from ENV repo_token or GITLAB_TOKEN.
+        --driver           If not specify it infers it from the ENV.
+                                                         [choices: \\"github\\", \\"gitlab\\"]
         -h                 Show help                                         [boolean]"
     `);
   });

diff --git a/bin/cml-publish.js b/bin/cml-publish.js
@@ -6,33 +6,18 @@ console.log = console.error;
 const fs = require('fs').promises;
 const pipe_args = require('../src/pipe-args');
 const yargs = require('yargs');
-const { publish_file } = require('../src/report');
 
-const { GITHUB_ACTIONS } = process.env;
-
-const { handle_error } = GITHUB_ACTIONS
-  ? require('../src/github')
-  : require('../src/gitlab');
+const CML = require('../src/cml');
 
 const run = async (opts) => {
-  const { data, file } = opts;
-  let { 'gitlab-uploads': gitlab_uploads } = opts;
-  const path = opts._[0];
+  const { data, file, 'gitlab-uploads': gitlab_uploads } = opts;
 
+  const path = opts._[0];
   let buffer;
   if (data) buffer = Buffer.from(data, 'binary');
 
-  if (GITHUB_ACTIONS && gitlab_uploads) {
-    console.error(`
-    *********************************************
-    * gitlab-uploads option is only for gitlab! *
-    * *******************************************
-    `);
-
-    gitlab_uploads = false;
-  }
-
-  const output = await publish_file({ buffer, path, gitlab_uploads, ...opts });
+  const cml = new CML(opts);
+  const output = await cml.publish({ buffer, path, gitlab_uploads, ...opts });
 
   if (!file) print(output);
   else await fs.writeFile(file, output);
@@ -42,6 +27,7 @@ pipe_args.load('binary');
 const data = pipe_args.piped_arg();
 const argv = yargs
   .usage(`Usage: $0 <path to file>`)
+  .describe('md', 'Output in markdown format [title || name](url).')
   .boolean('md')
   .describe('md', 'Output in markdown format [title || name](url).')
   .default('title')
@@ -52,12 +38,35 @@ const argv = yargs
     'gitlab-uploads',
     'Uses GitLab uploads instead of CML storage. Use GitLab uploads to get around CML size limitations for hosting artifacts persistently. Only available for GitLab CI.'
   )
+  .deprecateOption('gitlab-uploads', 'Use --native instead')
+  .boolean('native')
+  .describe(
+    'native',
+    "Uses driver's native capabilities to upload assets instead of CML's storage."
+  )
   .default('file')
   .describe(
     'file',
     'Append the output to the given file. Create it if does not exist.'
   )
   .alias('file', 'f')
+  .default('repo')
+  .describe(
+    'repo',
+    'Specifies the repo to be used. If not specified is extracted from the CI ENV.'
+  )
+  .default('token')
+  .describe(
+    'token',
+    'Personal access token to be used. If not specified in extracted from ENV repo_token or GITLAB_TOKEN.'
+  )
+  .default('driver')
+  .choices('driver', ['github', 'gitlab'])
+  .describe('driver', 'If not specify it infers it from the ENV.')
   .help('h')
   .demand(data ? 0 : 1).argv;
-run({ ...argv, data }).catch((e) => handle_error(e));
+
+run({ ...argv, data }).catch((e) => {
+  console.error(e);
+  process.exit(1);
+});
diff --git a/bin/cml-publish.test.js b/bin/cml-publish.test.js
@@ -17,9 +17,17 @@ describe('CML e2e', () => {
         --gitlab-uploads  Uses GitLab uploads instead of CML storage. Use GitLab
                           uploads to get around CML size limitations for hosting
                           artifacts persistently. Only available for GitLab CI.
-                                                                             [boolean]
+                                          [deprecated: Use --native instead] [boolean]
+        --native          Uses driver's native capabilities to upload assets instead
+                          of CML's storage.                                  [boolean]
         --file, -f        Append the output to the given file. Create it if does not
                           exist.
+        --repo            Specifies the repo to be used. If not specified is extracted
+                          from the CI ENV.
+        --token           Personal access token to be used. If not specified in
+                          extracted from ENV repo_token or GITLAB_TOKEN.
+        --driver          If not specify it infers it from the ENV.
+                                                         [choices: \\"github\\", \\"gitlab\\"]
         -h                Show help                                          [boolean]"
     `);
   });
@@ -86,4 +94,14 @@ describe('CML e2e', () => {
     expect(fs.existsSync(file)).toBe(true);
     await fs.promises.unlink(file);
   });
+
+  test('cml-publish assets/test.svg in Gitlab storage', async () => {
+    const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env;
+
+    const output = await exec(
+      `echo none | node ./bin/cml-publish.js --repo=${repo} --token=${token} --gitlab-uploads assets/test.svg`
+    );
+
+    expect(output.startsWith('https://')).toBe(true);
+  });
 });