fix: issues with running HTTPS requests through HTTP proxy

This adds and configured `global-agent`
(https://github.com/gajus/global-agent) and replaces Axios with `got` 
(https://github.com/sindresorhus/got) which, since Axios has a
long-standing issue with routing HTTPS requests through HTTPS proxies.

garden-service/package.json

@@ -64,9 +64,12 @@
     "fs-extra": "^8.1.0",
     "get-port": "^5.1.0",
     "glob": "^7.1.6",
+    "global-agent": "^2.1.8",
+    "got": "^10.5.7",
     "gray-matter": "^4.0.2",
     "has-ansi": "^4.0.0",
     "hasha": "^5.1.0",
+    "http-status-codes": "^1.4.0",
     "humanize-string": "^2.1.0",
     "indent-string": "^4.0.0",
     "inquirer": "^7.0.1",
@@ -143,7 +146,9 @@
     "@types/dockerode": "^2.5.21",
     "@types/fs-extra": "^8.0.1",
     "@types/glob": "^7.1.1",
+    "@types/global-agent": "^2.1.0",
     "@types/google-cloud__kms": "^1.5.0",
+    "@types/got": "^9.6.9",
     "@types/hapi__joi": "^16.0.5",
     "@types/has-ansi": "^3.0.0",
     "@types/inquirer": "6.5.0",

garden-service/src/cli/helpers.ts

@@ -6,7 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import axios from "axios"
 import chalk from "chalk"
 import ci = require("ci-info")
 import { pathExists } from "fs-extra"
@@ -23,6 +22,7 @@ import { LogEntry } from "../logger/log-entry"
 import { STATIC_DIR, VERSION_CHECK_URL } from "../constants"
 import { printWarningMessage } from "../logger/util"
 import { GlobalConfigStore, globalConfigKeys } from "../config-store"
+import { got, GotResponse } from "../util/http"
 
 // Parameter types T which map between the Parameter<T> class and the Sywac cli library.
 // In case we add types that aren't supported natively by Sywac, see: http://sywac.io/docs/sync-config.html#custom
@@ -236,7 +236,7 @@ export async function checkForUpdates(config: GlobalConfigStore, logger: LogEntr
       headers["X-ci-name"] = ci.name
     }
 
-    const res = await axios.get(`${VERSION_CHECK_URL}?${qs.stringify(query)}`, { headers })
+    const res = await got(`${VERSION_CHECK_URL}?${qs.stringify(query)}`, { headers }).json<GotResponse<any>>()
     const configObj = await config.get()
     const showMessage =
       configObj.lastVersionCheck &&
@@ -246,8 +246,8 @@ export async function checkForUpdates(config: GlobalConfigStore, logger: LogEntr
 
     // we check again for lastVersionCheck because in the first run it doesn't exist
     if (showMessage || !configObj.lastVersionCheck) {
-      if (res.data.status === "OUTDATED") {
-        printWarningMessage(logger, res.data.message)
+      if (res.body.status === "OUTDATED") {
+        printWarningMessage(logger, res.body.message)
         await config.set([globalConfigKeys.lastVersionCheck], { lastRun: new Date() })
       }
     }

garden-service/src/commands/call.ts

@@ -7,17 +7,17 @@
  */
 
 import { parse, resolve } from "url"
-import Axios from "axios"
 import chalk from "chalk"
-import { isObject } from "util"
+import { getStatusText } from "http-status-codes"
 import { Command, CommandResult, CommandParams, StringParameter } from "./base"
 import { splitFirst } from "../util/util"
 import { ParameterError, RuntimeError } from "../exceptions"
-import { find, includes, pick } from "lodash"
+import { find, includes } from "lodash"
 import { ServiceIngress, getIngressUrl } from "../types/service"
-import dedent = require("dedent")
+import { dedent } from "../util/string"
 import { printHeader } from "../logger/util"
 import { emptyRuntimeContext } from "../runtime-context"
+import { got, GotResponse } from "../util/http"
 
 const callArgs = {
   serviceAndPath: new StringParameter({
@@ -28,6 +28,18 @@ const callArgs = {
 
 type Args = typeof callArgs
 
+interface CallResult {
+  serviceName: string
+  path: string
+  url: string
+  response: {
+    status: number
+    statusText: string
+    headers: GotResponse["headers"]
+    data: string | object
+  }
+}
+
 export class CallCommand extends Command<Args> {
   name = "call"
   help = "Call a service ingress endpoint."
@@ -46,7 +58,7 @@ export class CallCommand extends Command<Args> {
 
   arguments = callArgs
 
-  async action({ garden, log, headerLog, args }: CommandParams<Args>): Promise<CommandResult> {
+  async action({ garden, log, headerLog, args }: CommandParams<Args>): Promise<CommandResult<CallResult>> {
     printHeader(headerLog, "Call", "telephone_receiver")
 
     let [serviceName, path] = splitFirst(args.serviceAndPath, "/")
@@ -147,37 +159,55 @@ export class CallCommand extends Command<Args> {
     // this is to accept self-signed certs
     process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"
 
-    const req = Axios({
+    const req = got({
       method,
       url,
       headers: { host },
     })
 
     // TODO: add verbose and debug logging (request/response headers etc.)
-    let res
+    let res: GotResponse<string>
+    let statusText = ""
 
     try {
       res = await req
       entry.setSuccess()
-      log.info(chalk.green(`${res.status} ${res.statusText}\n`))
+      statusText = getStatusText(res.statusCode)
+      log.info(chalk.green(`${res.statusCode} ${statusText}\n`))
     } catch (err) {
       res = err.response
       entry.setError()
-      const error = res ? `${res.status} ${res.statusText}` : err.message
+      statusText = getStatusText(res.statusCode)
+      const error = res ? `${res.statusCode} ${statusText}` : err.message
       log.info(chalk.red(error + "\n"))
       return {}
     }
 
-    const resStr = isObject(res.data) ? JSON.stringify(res.data, null, 2) : res.data
+    let output: string | object = res.body
+
+    if (res.headers["content-type"] === "application/json") {
+      try {
+        output = JSON.parse(res.body)
+      } catch (err) {
+        throw new RuntimeError(`Got content-type=application/json but could not parse output as JSON`, {
+          response: res,
+        })
+      }
+    }
 
-    res.data && log.info(chalk.white(resStr))
+    res.body && log.info(chalk.white(res.body))
 
     return {
       result: {
         serviceName,
         path,
         url,
-        response: pick(res, ["status", "statusText", "headers", "data"]),
+        response: {
+          data: output,
+          headers: res.headers,
+          status: res.statusCode,
+          statusText,
+        },
       },
     }
   }

garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts

@@ -126,12 +126,12 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)
 
   while (nextUrl) {
     const res = await queryRegistry(ctx, log, nextUrl)
-    repositories.push(...res.data.repositories)
+    repositories.push(...res.body.repositories)
 
     // Paginate
-    const linkHeader = res.headers["Link"]
+    const linkHeader = <string | undefined>res.headers["Link"]
     if (linkHeader) {
-      nextUrl = linkHeader.match(/<(.*)>/)[1]
+      nextUrl = linkHeader.match(/<(.*)>/)![1]
     } else {
       nextUrl = ""
     }
@@ -144,13 +144,13 @@ async function getImagesInRegistry(ctx: KubernetesPluginContext, log: LogEntry)
 
     while (nextUrl) {
       const res = await queryRegistry(ctx, log, nextUrl)
-      if (res.data.tags) {
-        images.push(...res.data.tags.map((tag: string) => `${repo}:${tag}`))
+      if (res.body.tags) {
+        images.push(...res.body.tags.map((tag: string) => `${repo}:${tag}`))
       }
       // Paginate
-      const linkHeader = res.headers["link"]
+      const linkHeader = <string | undefined>res.headers["link"]
       if (linkHeader) {
-        nextUrl = linkHeader.match(/<(.*)>/)[1]
+        nextUrl = linkHeader.match(/<(.*)>/)![1]
       } else {
         nextUrl = ""
       }

garden-service/src/plugins/kubernetes/container/util.ts

@@ -11,20 +11,15 @@ import { getPortForward } from "../port-forward"
 import { CLUSTER_REGISTRY_DEPLOYMENT_NAME, CLUSTER_REGISTRY_PORT } from "../constants"
 import { LogEntry } from "../../../logger/log-entry"
 import { KubernetesPluginContext } from "../config"
-import axios, { AxiosRequestConfig } from "axios"
 import { getSystemNamespace } from "../namespace"
+import { got, GotOptions, GotResponse } from "../../../util/http"
 
-export async function queryRegistry(
-  ctx: KubernetesPluginContext,
-  log: LogEntry,
-  path: string,
-  opts: AxiosRequestConfig = {}
-) {
+export async function queryRegistry(ctx: KubernetesPluginContext, log: LogEntry, path: string, opts?: GotOptions) {
   const registryFwd = await getRegistryPortForward(ctx, log)
   const baseUrl = `http://localhost:${registryFwd.localPort}/v2/`
   const url = resolve(baseUrl, path)
 
-  return axios({ url, ...opts })
+  return got(url, opts).json<GotResponse<any>>()
 }
 
 export async function getRegistryPortForward(ctx: KubernetesPluginContext, log: LogEntry) {

garden-service/src/util/ext-tools.ts

@@ -11,7 +11,6 @@ import { pathExists, createWriteStream, ensureDir, chmod, remove, move } from "f
 import { ConfigurationError, ParameterError, GardenBaseError } from "../exceptions"
 import { join, dirname, basename, sep } from "path"
 import { hashString, exec } from "./util"
-import Axios from "axios"
 import tar from "tar"
 import { SupportedPlatform, GARDEN_GLOBAL_PATH } from "../constants"
 import { LogEntry } from "../logger/log-entry"
@@ -21,6 +20,7 @@ import uuid from "uuid"
 import crossSpawn from "cross-spawn"
 import { spawn } from "./util"
 import { Writable } from "stream"
+import got from "got/dist/source"
 const AsyncLock = require("async-lock")
 
 const toolsPath = join(GARDEN_GLOBAL_PATH, "tools")
@@ -137,20 +137,19 @@ export class Library {
   }
 
   protected async fetch(tmpPath: string, log: LogEntry) {
-    const response = await Axios({
+    const response = got.stream({
       method: "GET",
       url: this.spec.url,
-      responseType: "stream",
     })
 
     // compute the sha256 checksum
     const hash = createHash("sha256")
     hash.setEncoding("hex")
-    response.data.pipe(hash)
+    response.pipe(hash)
 
     return new Promise((resolve, reject) => {
-      response.data.on("error", (err) => {
-        log.setError(`Failed fetching ${response.request.url}`)
+      response.on("error", (err) => {
+        log.setError(`Failed fetching ${this.spec.url}`)
         reject(err)
       })
 
@@ -176,8 +175,8 @@ export class Library {
 
       if (!this.spec.extract) {
         const targetExecutable = join(tmpPath, ...this.targetSubpath)
-        response.data.pipe(createWriteStream(targetExecutable))
-        response.data.on("end", () => resolve())
+        response.pipe(createWriteStream(targetExecutable))
+        response.on("end", () => resolve())
       } else {
         const format = this.spec.extract.format
         let extractor: Writable
@@ -201,7 +200,7 @@ export class Library {
           return
         }
 
-        response.data.pipe(extractor)
+        response.pipe(extractor)
 
         extractor.on("error", (err) => {
           log.setError(`Failed extracting ${format} archive ${this.spec.url}`)

garden-service/src/util/http.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2018-2020 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import _got, { Response } from "got"
+import { bootstrap } from "global-agent"
+import { OptionsOfDefaultResponseBody } from "got/dist/source/create"
+
+// Handle proxy environment settings
+// (see https://github.com/gajus/global-agent#what-is-the-reason-global-agentbootstrap-does-not-use-http_proxy)
+bootstrap({
+  environmentVariableNamespace: "",
+  forceGlobalAgent: true,
+})
+
+// Exporting from here to make sure the global-agent bootstrap is executed, and for convenience as well
+export const got = _got
+export type GotOptions = OptionsOfDefaultResponseBody
+export type GotResponse<T = unknown> = Response<T>

garden-service/test/unit/src/commands/call.ts

@@ -129,11 +129,11 @@ describe("commands.call", () => {
       opts: withDefaultGlobalOpts({}),
     })
 
-    expect(result.url).to.equal("http://service-a.test-project-b.local.app.garden:32000/path-a")
-    expect(result.serviceName).to.equal("service-a")
-    expect(result.path).to.equal("/path-a")
-    expect(result.response.status).to.equal(200)
-    expect(result.response.data).to.equal("bla")
+    expect(result!.url).to.equal("http://service-a.test-project-b.local.app.garden:32000/path-a")
+    expect(result!.serviceName).to.equal("service-a")
+    expect(result!.path).to.equal("/path-a")
+    expect(result!.response.status).to.equal(200)
+    expect(result!.response.data).to.equal("bla")
   })
 
   it("should default to the path '/' if that is exposed if no path is requested", async () => {
@@ -154,11 +154,11 @@ describe("commands.call", () => {
       opts: withDefaultGlobalOpts({}),
     })
 
-    expect(result.url).to.equal("http://service-a.test-project-b.local.app.garden:32000/path-a")
-    expect(result.serviceName).to.equal("service-a")
-    expect(result.path).to.equal("/path-a")
-    expect(result.response.status).to.equal(200)
-    expect(result.response.data).to.equal("bla")
+    expect(result!.url).to.equal("http://service-a.test-project-b.local.app.garden:32000/path-a")
+    expect(result!.serviceName).to.equal("service-a")
+    expect(result!.path).to.equal("/path-a")
+    expect(result!.response.status).to.equal(200)
+    expect(result!.response.data).to.equal("bla")
   })
 
   it("should otherwise use the first defined ingress if no path is requested", async () => {
@@ -179,11 +179,11 @@ describe("commands.call", () => {
       opts: withDefaultGlobalOpts({}),
     })
 
-    expect(result.url).to.equal("http://service-b.test-project-b.local.app.garden:32000/")
-    expect(result.serviceName).to.equal("service-b")
-    expect(result.path).to.equal("/")
-    expect(result.response.status).to.equal(200)
-    expect(result.response.data).to.equal("bla")
+    expect(result!.url).to.equal("http://service-b.test-project-b.local.app.garden:32000/")
+    expect(result!.serviceName).to.equal("service-b")
+    expect(result!.path).to.equal("/")
+    expect(result!.response.status).to.equal(200)
+    expect(result!.response.data).to.equal("bla")
   })
 
   it("should use the linkUrl if provided", async () => {
@@ -204,11 +204,11 @@ describe("commands.call", () => {
       opts: withDefaultGlobalOpts({}),
     })
 
-    expect(result.url).to.equal("https://www.example.com")
-    expect(result.serviceName).to.equal("service-a")
-    expect(result.path).to.equal("/")
-    expect(result.response.status).to.equal(200)
-    expect(result.response.data).to.equal("bla")
+    expect(result!.url).to.equal("https://www.example.com")
+    expect(result!.serviceName).to.equal("service-a")
+    expect(result!.path).to.equal("/")
+    expect(result!.response.status).to.equal(200)
+    expect(result!.response.data).to.equal("bla")
   })
 
   it("should return the path for linkUrl", async () => {
@@ -229,11 +229,11 @@ describe("commands.call", () => {
       opts: withDefaultGlobalOpts({}),
     })
 
-    expect(result.url).to.equal("https://www.example.com/hello")
-    expect(result.serviceName).to.equal("service-b")
-    expect(result.path).to.equal("/hello")
-    expect(result.response.status).to.equal(200)
-    expect(result.response.data).to.equal("bla")
+    expect(result!.url).to.equal("https://www.example.com/hello")
+    expect(result!.serviceName).to.equal("service-b")
+    expect(result!.path).to.equal("/hello")
+    expect(result!.response.status).to.equal(200)
+    expect(result!.response.data).to.equal("bla")
   })
 
   it("should error if service isn't running", async () => {