Proxy support

Dockerfile

@@ -39,7 +39,7 @@ ENV LC_ALL="en_US.UTF-8"
 # INSTALL NODE, GIT & GO
 RUN add-apt-repository ppa:git-core/ppa --yes \
  && add-apt-repository ppa:longsleep/golang-backports --yes \
- && curl --location https://deb.nodesource.com/setup_12.x | bash \
+ && curl --location https://deb.nodesource.com/setup_14.x | bash \
  && apt-get update \
  && apt-get install --yes git golang-go nodejs \
  && apt-get clean \

README.md

@@ -540,6 +540,11 @@ compute resources as secrets. In the above example, `AWS_ACCESS_KEY_ID` and
 Please see our docs about
 [configuring cloud storage providers](#configuring-cloud-storage-providers).
 
+#### Proxy support
+
+CML support proxy via known environment variables `http_proxy` and
+`https_proxy`.
+
 #### On-premise (Local) Runners
 
 This means using on-premise machines as self-hosted runners. The `cml-runner`

bin/cml-tensorboard-dev.js

@@ -64,6 +64,7 @@ const run = async (opts) => {
 
   // reads stdout every 5 secs to find the tb uri
   setInterval(async () => {
+    console.log('1sec');
     const stdoutData = await fs.readFile(stdoutPath, 'utf8');
     const regex = /(https?:\/\/[^\s]+)/;
     const matches = stdoutData.match(regex);
@@ -85,6 +86,7 @@ const run = async (opts) => {
 
   // waits 1 min before dies
   setTimeout(async () => {
+    console.log('here');
     closeFd(stdoutFd) && closeFd(stderrFd);
     console.error(await fs.readFile(stderrPath, 'utf8'));
     throw new Error('Tensorboard took too long! Canceled.');

bin/cml-tensorboard-dev.test.js

@@ -43,10 +43,14 @@ describe('CML e2e', () => {
     `);
   });
 
-  test('cml-tensorboard-dev.js --md returns md and after command TB is still up', async () => {
+  test.skip('cml-tensorboard-dev.js --md returns md and after command TB is still up', async () => {
     const name = 'My experiment';
     const desc = 'Test experiment';
     const title = 'go to the experiment';
+    console.log(`node ./bin/cml-tensorboard-dev.js --credentials '${CREDENTIALS}' \
+    --md --title '${title}' \
+    --logdir logs --name '${name}' --description '${desc}'`);
+
     const output = await exec(
       `node ./bin/cml-tensorboard-dev.js --credentials '${CREDENTIALS}' \
         --md --title '${title}' \

package.json

@@ -68,6 +68,7 @@
     "fs-extra": "^9.0.1",
     "git-url-parse": "^11.4.0",
     "globby": "^11.0.3",
+    "https-proxy-agent": "^5.0.0",
     "js-base64": "^3.5.2",
     "mmmagic": "^0.5.3",
     "node-fetch": "^2.6.0",
@@ -92,8 +93,13 @@
     "husky": "^4.2.3",
     "jest": "^24.9.0",
     "lint-staged": "^10.0.8",
-    "prettier": "^2.1.1"
+    "prettier": "^2.1.1",
+    "transparent-proxy": "^1.8.5"
   },
   "description": "<p align=\"center\">   <img src=\"https://static.iterative.ai/img/cml/title_strip_trim.png\" width=400> </p>",
-  "homepage": "https://github.com/iterative/cml#readme"
+  "homepage": "https://github.com/iterative/cml#readme",
+  "jest": {
+    "globalSetup": "./tests/setup.js",
+    "globalTeardown": "./tests/teardown.js"
+  }
 }

src/cml.test.js

@@ -1,6 +1,7 @@
 const CML = require('../src/cml').default;
 
 jest.setTimeout(60000);
+
 describe('Github tests', () => {
   const OLD_ENV = process.env;
 

src/drivers/bitbucket_cloud.js

@@ -1,6 +1,8 @@
 const fetch = require('node-fetch');
 const { URL } = require('url');
 
+const { proxyAgent } = require('../utils');
+
 const { BITBUCKET_COMMIT, BITBUCKET_BRANCH } = process.env;
 class BitbucketCloud {
   constructor(opts = {}) {
@@ -181,10 +183,12 @@ class BitbucketCloud {
       'Content-Type': 'application/json',
       Authorization: 'Basic ' + `${token}`
     };
+
     const response = await fetch(url || `${api}${endpoint}`, {
       method,
       headers,
-      body
+      body,
+      agent: proxyAgent()
     });
 
     if (response.status > 300) {

src/drivers/bitbucket_cloud.test.js

@@ -1,33 +1,40 @@
 jest.setTimeout(120000);
+
 const BitbucketCloud = require('./bitbucket_cloud');
 const {
   TEST_BBCLOUD_TOKEN: TOKEN,
   TEST_BBCLOUD_REPO: REPO,
   TEST_BBCLOUD_SHA: SHA
 } = process.env;
+
 describe('Non Enviromental tests', () => {
   const client = new BitbucketCloud({ repo: REPO, token: TOKEN });
+
   test('test repo and token', async () => {
     expect(client.repo).toBe(REPO);
     expect(client.token).toBe(TOKEN);
   });
+
   test('Comment', async () => {
     const report = '## Test comment';
     const commitSha = SHA;
 
     await client.commentCreate({ report, commitSha });
   });
+
   test('Check', async () => {
     await expect(client.checkCreate()).rejects.toThrow(
       'Bitbucket Cloud does not support check!'
     );
   });
+
   test('Publish', async () => {
     const path = `${__dirname}/../../assets/logo.png`;
     await expect(client.upload({ path })).rejects.toThrow(
       'Bitbucket Cloud does not support upload!'
     );
   });
+
   test('Runner token', async () => {
     await expect(client.runnerToken()).rejects.toThrow(
       'Bitbucket Cloud does not support runnerToken!'

src/drivers/github.js

@@ -8,7 +8,7 @@ const { Octokit } = require('@octokit/rest');
 const { throttling } = require('@octokit/plugin-throttling');
 const tar = require('tar');
 
-const { download, exec } = require('../utils');
+const { download, exec, proxyAgent } = require('../utils');
 
 const CHECK_TITLE = 'CML Report';
 process.env.RUNNER_ALLOW_RUNASROOT = 1;
@@ -51,6 +51,7 @@ const octokit = (token, repo) => {
     }
   };
   const octokitOptions = {
+    request: { agent: proxyAgent() },
     auth: token,
     throttle: {
       onRateLimit: throttleHandler,

src/drivers/github.test.js

@@ -1,5 +1,4 @@
 jest.setTimeout(80000);
-
 const GithubClient = require('./github');
 
 const {

src/drivers/gitlab.js

@@ -6,7 +6,7 @@ const fs = require('fs').promises;
 const fse = require('fs-extra');
 const { resolve } = require('path');
 
-const { fetchUploadData, download, exec } = require('../utils');
+const { fetchUploadData, download, exec, proxyAgent } = require('../utils');
 
 const {
   IN_DOCKER,
@@ -271,7 +271,12 @@ class Gitlab {
     if (!url) throw new Error('Gitlab API endpoint not found');
 
     const headers = { 'PRIVATE-TOKEN': token, Accept: 'application/json' };
-    const response = await fetch(url, { method, headers, body });
+    const response = await fetch(url, {
+      method,
+      headers,
+      body,
+      agent: proxyAgent()
+    });
 
     if (response.status > 300) throw new Error(response.statusText);
     if (raw) return response;

src/utils.js

@@ -1,4 +1,5 @@
 const fetch = require('node-fetch');
+const HttpsProxyAgent = require('https-proxy-agent');
 const fs = require('fs');
 const PATH = require('path');
 const mmm = require('mmmagic');
@@ -165,6 +166,14 @@ const sshConnection = async (opts) => {
   return ssh;
 };
 
+const proxyAgent = (opts = {}) => {
+  const { https_proxy: httpsProxy, http_proxy: httpProxy } = process.env;
+  const { url = httpsProxy || httpProxy } = opts;
+
+  if (!url) return;
+  return new HttpsProxyAgent(url);
+};
+
 exports.exec = exec;
 exports.fetchUploadData = fetchUploadData;
 exports.upload = upload;
@@ -175,3 +184,4 @@ exports.sshPublicFromPrivateRsa = sshPublicFromPrivateRsa;
 exports.watermarkUri = watermarkUri;
 exports.download = download;
 exports.sshConnection = sshConnection;
+exports.proxyAgent = proxyAgent;

src/utils.test.js

@@ -1,4 +1,9 @@
-const { exec, upload, sshPublicFromPrivateRsa } = require('./utils');
+const {
+  exec,
+  upload,
+  sshPublicFromPrivateRsa,
+  proxyAgent
+} = require('./utils');
 
 describe('exec tests', () => {
   test('exec is await and outputs hello', async () => {
@@ -44,4 +49,19 @@ describe('Other tests', () => {
       'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwMFO1qTq68Uao1TU6W+911SWZN+mce+OTodS5mne0vMiWm7AFJuaxMKzL1NGeXOAUY+oeTAE7NCyi56mqEo3R8kOvwYbIQKmz27lzL0ZniI9UkYkE9aFslvTgou6cqcJO6S8GMNnJLLXbmX37zGUgZyuTAuot885WHqKyW0Pg/zplBNIPBRA+yhQG8z17dj8VwixoE7KgRiNK9CN5Yz/2TRlsHLan9LH9vHSK477bj+jPtbII4V5ZP5Du/70Seb1fe3648DVpg4mjDmLrQkW/rtNanKMVLw3bD9VroeLe3PW91nLq+noOuljPETtxxrhGagE28U6igPTmIQYczrpz ';
     expect(publicKey).toBe(expected);
   });
+
+  describe('proxyAgent', () => {
+    const { http_proxy, https_proxy } = process.env;
+    process.env.http_proxy = 'https://www.example1.com';
+    expect(proxyAgent().proxy.host).toBe('www.example1.com');
+
+    // https_proxy overrides http_proxy
+    process.env.https_proxy = 'https://www.example2.com';
+    expect(proxyAgent().proxy.host).toBe('www.example2.com');
+
+    const url = 'https://www.example.com';
+    expect(proxyAgent({ url: `${url}` }).proxy.host).toBe('www.example.com');
+
+    process.env = { ...process.env, http_proxy, https_proxy };
+  });
 });

tests/proxy.js

@@ -0,0 +1,22 @@
+const ProxyServer = require('transparent-proxy');
+
+const PORT = 9093;
+const startProxy = () => {
+  if (global.proxyTestsServer) return;
+
+  global.proxyTestsServer = new ProxyServer();
+  global.proxyTestsServer.listen(PORT, '0.0.0.0', () =>
+    console.log(`Proxy listening on port ${PORT}`)
+  );
+};
+const stopProxy = () => {
+  console.log('Teardown Jest. Stoping Proxy...');
+  global.proxyTestsServer.close();
+  global.proxyTestsServer.unref();
+};
+
+module.exports = {
+  startProxy,
+  stopProxy,
+  PORT
+};

tests/setup.js

@@ -0,0 +1,5 @@
+const { startProxy, PORT } = require('./proxy');
+module.exports = () => {
+  process.env.http_proxy = `http://localhost:${PORT}`;
+  startProxy();
+};

tests/teardown.js

@@ -0,0 +1,4 @@
+const { stopProxy } = require('./proxy');
+module.exports = () => {
+  stopProxy();
+};