From b293fe2cf20153e43b51a83f9dd7b759afcf2782 Mon Sep 17 00:00:00 2001 From: swist Date: Thu, 5 Mar 2020 16:44:19 +0000 Subject: [PATCH] improvement: allow setting cred helpers in ImagePullSecrets This enables setting credHelper key for docker builders (kaniko/dd) This means we can use docker repositories which have rolling credentials such as ECR. We run this on ECR + kaniko --- garden-service/src/plugins/kubernetes/init.ts | 109 +++++----- .../src/plugins/kubernetes/namespace.ts | 7 +- .../data/test-projects/container/garden.yml | 1 + .../plugins/kubernetes/container/container.ts | 14 ++ .../kubernetes/container/deployment.ts | 40 +++- .../test/unit/src/plugins/kubernetes/init.ts | 186 ++++++++++++++++++ 6 files changed, 305 insertions(+), 52 deletions(-) create mode 100644 garden-service/test/unit/src/plugins/kubernetes/init.ts diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index 831be46f2e..d48ad9eeac 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -14,7 +14,7 @@ import { getMetadataNamespace, getSystemNamespace, } from "./namespace" -import { KubernetesPluginContext, KubernetesConfig, KubernetesProvider } from "./config" +import { KubernetesPluginContext, KubernetesConfig, KubernetesProvider, ProviderSecretRef } from "./config" import { checkTillerStatus, migrateToHelm3 } from "./helm/tiller" import { prepareSystemServices, getSystemServiceStatus, getSystemGarden, systemNamespaceUpToDate } from "./system" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" @@ -33,7 +33,6 @@ import { import { ConfigurationError } from "../../exceptions" import Bluebird from "bluebird" import { readSecret } from "./secrets" -import { extend } from "lodash" import { dockerAuthSecretName, dockerAuthSecretKey } from "./constants" import { V1Secret } from "@kubernetes/client-node" import { KubernetesResource } from "./types" @@ -465,76 +464,90 @@ export function getRegistryHostname(config: KubernetesConfig) { const systemNamespace = config.gardenSystemNamespace return `garden-docker-registry.${systemNamespace}.svc.cluster.local` } -async function prepareDockerAuth( - api: KubeApi, - provider: KubernetesProvider, - log: LogEntry -): Promise> { - // Read all configured imagePullSecrets and combine into a docker config file to use in the in-cluster builders. - const auths: { [name: string]: any } = {} - await Bluebird.map(provider.config.imagePullSecrets, async (secretRef) => { - const secret = await readSecret(api, secretRef) - - if (secret.type !== dockerAuthSecretType) { - throw new ConfigurationError( - dedent` +interface DockerConfigJson { + experimental: string + auths: { [registry: string]: { [key: string]: string } } + credHelpers: { [registry: string]: any } +} +export async function buildDockerAuthConfig( + imagePullSecrets: ProviderSecretRef[], + api: KubeApi +): Promise { + return Bluebird.reduce( + imagePullSecrets, + async (accumulator, secretRef) => { + const secret = await readSecret(api, secretRef) + if (secret.type !== dockerAuthSecretType) { + throw new ConfigurationError( + dedent` Configured imagePullSecret '${secret.metadata.name}' does not appear to be a valid registry secret, because it does not have \`type: ${dockerAuthSecretType}\`. ${dockerAuthDocsLink} `, - { secretRef } - ) - } + { secretRef } + ) + } - // Decode the secret - const encoded = secret.data && secret.data![dockerAuthSecretKey] + // Decode the secret + const encoded = secret.data && secret.data![dockerAuthSecretKey] - if (!encoded) { - throw new ConfigurationError( - dedent` + if (!encoded) { + throw new ConfigurationError( + dedent` Configured imagePullSecret '${secret.metadata.name}' does not appear to be a valid registry secret, because it does not contain a ${dockerAuthSecretKey} key. ${dockerAuthDocsLink} `, - { secretRef } - ) - } + { secretRef } + ) + } - let decoded: any + let decoded: any - try { - decoded = JSON.parse(Buffer.from(encoded, "base64").toString()) - } catch (err) { - throw new ConfigurationError( - dedent` + try { + decoded = JSON.parse(Buffer.from(encoded, "base64").toString()) + } catch (err) { + throw new ConfigurationError( + dedent` Could not parse configured imagePullSecret '${secret.metadata.name}' as a JSON docker authentication file: ${err.message}. ${dockerAuthDocsLink} `, - { secretRef } - ) - } - - if (!decoded.auths) { - throw new ConfigurationError( - dedent` + { secretRef } + ) + } + if (!decoded.auths && !decoded.credHelpers) { + throw new ConfigurationError( + dedent` Could not parse configured imagePullSecret '${secret.metadata.name}' as a valid docker authentication file, - because it is missing an "auths" key. + because it is missing an "auths", "credHelpers" key. ${dockerAuthDocsLink} `, - { secretRef } - ) - } + { secretRef } + ) + } + return { + ...accumulator, + auths: { ...accumulator.auths, ...decoded.auths }, + credHelpers: { ...accumulator.credHelpers, ...decoded.credHelpers }, + } + }, + { experimental: "enabled", auths: {}, credHelpers: {} } + ) +} - extend(auths, decoded.auths) - }) +export async function prepareDockerAuth( + api: KubeApi, + provider: KubernetesProvider, + log: LogEntry +): Promise> { + // Read all configured imagePullSecrets and combine into a docker config file to use in the in-cluster builders. + const config = await buildDockerAuthConfig(provider.config.imagePullSecrets, api) // Enabling experimental features, in order to support advanced registry querying - const config = { auths, experimental: "enabled" } - // Store the config as a Secret (overwriting if necessary) - const systemNamespace = await getSystemNamespace(provider, log) + const systemNamespace = await getSystemNamespace(provider, log, api) return { apiVersion: "v1", diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index ee9f14ee56..da13526e2d 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -97,10 +97,11 @@ export async function getNamespace({ return namespace } -export async function getSystemNamespace(provider: KubernetesProvider, log: LogEntry): Promise { +export async function getSystemNamespace(provider: KubernetesProvider, log: LogEntry, api?: KubeApi): Promise { const namespace = provider.config.gardenSystemNamespace - - const api = await KubeApi.factory(log, provider) + if (!api) { + api = await KubeApi.factory(log, provider) + } await ensureNamespace(api, namespace) return namespace diff --git a/garden-service/test/data/test-projects/container/garden.yml b/garden-service/test/data/test-projects/container/garden.yml index b3e329247b..46d2c401a5 100644 --- a/garden-service/test/data/test-projects/container/garden.yml +++ b/garden-service/test/data/test-projects/container/garden.yml @@ -24,6 +24,7 @@ providers: imagePullSecrets: # Note: We populate this secret in the test code - name: test-docker-auth + - name: test-cred-helper-auth - <<: *clusterDocker environments: [cluster-docker-buildkit] clusterDocker: diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/container.ts b/garden-service/test/integ/src/plugins/kubernetes/container/container.ts index d82e35d1e0..17dfda2064 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/container/container.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/container/container.ts @@ -71,6 +71,20 @@ export async function getContainerTestGarden(environmentName: string = defaultEn } await api.upsert({ kind: "Secret", namespace: "default", obj: authSecret, log: garden.log }) } + + const credentialHelperAuth: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: "test-cred-helper-auth", + namespace: "default", + }, + stringData: { + ".dockerconfigjson": JSON.stringify({ credHelpers: {}, experimental: "enabled" }), + }, + } + await api.upsert({ kind: "Secret", namespace: "default", obj: credentialHelperAuth, log: garden.log }) } const provider = await garden.resolveProvider("local-kubernetes") diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts index 62cf3cf41b..be54e3c121 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts @@ -110,7 +110,7 @@ describe("kubernetes container deployment handlers", () => { }) }) - it("should copy and reference imagePullSecrets", async () => { + it("should copy and reference imagePullSecrets with docker basic auth", async () => { const service = await graph.getService("simple-service") const secretName = "test-docker-auth" @@ -148,6 +148,44 @@ describe("kubernetes container deployment handlers", () => { expect(resource.spec.template.spec.imagePullSecrets).to.eql([{ name: secretName }]) }) + it("should copy and reference imagePullSecrets with docker credential helper", async () => { + const service = await graph.getService("simple-service") + const secretName = "test-cred-helper-auth" + + const authSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: secretName, + namespace: "default", + }, + stringData: { + ".dockerconfigjson": JSON.stringify({ credHelpers: {} }), + }, + } + await api.upsert({ kind: "Secret", namespace: "default", obj: authSecret, log: garden.log }) + + const namespace = garden.projectName + const _provider = cloneDeep(provider) + _provider.config.imagePullSecrets = [{ name: secretName, namespace: "default" }] + + const resource = await createWorkloadManifest({ + api, + provider: _provider, + service, + runtimeContext: emptyRuntimeContext, + namespace, + enableHotReload: false, + log: garden.log, + production: false, + }) + + const copiedSecret = await api.core.readNamespacedSecret(secretName, namespace) + expect(copiedSecret).to.exist + expect(resource.spec.template.spec.imagePullSecrets).to.eql([{ name: secretName }]) + }) + it("should correctly mount a referenced PVC module", async () => { const service = await graph.getService("volume-reference") const namespace = garden.projectName diff --git a/garden-service/test/unit/src/plugins/kubernetes/init.ts b/garden-service/test/unit/src/plugins/kubernetes/init.ts new file mode 100644 index 0000000000..6d8c30157f --- /dev/null +++ b/garden-service/test/unit/src/plugins/kubernetes/init.ts @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { expect } from "chai" +import { resolve, join } from "path" +import td from "testdouble" +import { Garden } from "../../../../../src/garden" +import { prepareDockerAuth } from "../../../../../src/plugins/kubernetes/init" +import { dockerAuthSecretKey } from "../../../../../src/plugins/kubernetes/constants" +import { ConfigurationError } from "../../../../../src/exceptions" +import { + KubernetesProvider, + KubernetesConfig, + defaultResources, + defaultStorage, +} from "../../../../../src/plugins/kubernetes/config" +import { gardenPlugin } from "../../../../../src/plugins/container/container" +import { defaultSystemNamespace } from "../../../../../src/plugins/kubernetes/system" +import { KubeApi } from "../../../../../src/plugins/kubernetes/api" +import { dataDir, makeTestGarden, expectError } from "../../../../helpers" +import { KubernetesResource } from "../../../../../src/plugins/kubernetes/types" +import { V1Secret } from "@kubernetes/client-node" + +const basicConfig: KubernetesConfig = { + name: "kubernetes", + buildMode: "local-docker", + context: "my-cluster", + defaultHostname: "my.domain.com", + deploymentRegistry: { + hostname: "foo.garden", + port: 5000, + namespace: "boo", + }, + forceSsl: false, + gardenSystemNamespace: defaultSystemNamespace, + imagePullSecrets: [ + { + name: "test-docker-auth", + namespace: "default", + }, + { + name: "test-cred-helper-auth", + namespace: "default", + }, + ], + ingressClass: "nginx", + ingressHttpPort: 80, + ingressHttpsPort: 443, + resources: defaultResources, + storage: defaultStorage, + registryProxyTolerations: [], + tlsCertificates: [], + _systemServices: [], +} + +const basicProvider: KubernetesProvider = { + name: "kubernetes", + config: basicConfig, + dependencies: [], + moduleConfigs: [], + status: { ready: true, outputs: {} }, +} + +const dockerSimpleAuthSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: "test-docker-auth", + namespace: "default", + }, + data: { + ".dockerconfigjson": Buffer.from( + JSON.stringify({ auths: { myDockerRepo: "simple-auth" }, experimental: "enabled" }) + ).toString("base64"), + }, +} + +const dockerCredentialHelperSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: "test-cred-helper-auth", + namespace: "default", + }, + data: { + ".dockerconfigjson": Buffer.from( + JSON.stringify({ credHelpers: { myDockerRepo: "ecr-helper" }, experimental: "enabled" }) + ).toString("base64"), + }, +} +const kubeConfigEnvVar = process.env.KUBECONFIG +describe("prepareDockerAuth", () => { + const projectRoot = resolve(dataDir, "test-project-container") + let garden: Garden + let api: KubeApi + before(() => { + process.env.KUBECONFIG = join(projectRoot, "kubeconfig.yml") + }) + + after(() => { + if (kubeConfigEnvVar) { + process.env.KUBECONFIG = kubeConfigEnvVar + } else { + delete process.env.KUBECONFIG + } + }) + + function jsonLoadBase64(data: string) { + return JSON.parse(Buffer.from(data, "base64").toString()) + } + + beforeEach(async () => { + garden = await makeTestGarden(projectRoot, { plugins: [gardenPlugin] }) + api = await KubeApi.factory(garden.log, basicProvider) + }) + describe("when simple login or cred helpers are present", () => { + beforeEach(async () => { + const core = td.replace(api, "core") + td.when(core.listNamespace()).thenResolve({ + items: [{ status: { phase: "Active" }, metadata: { name: "default" } }], + }) + td.when(core.readNamespacedSecret("test-docker-auth", "default")).thenResolve(dockerSimpleAuthSecret) + td.when(core.readNamespacedSecret("test-cred-helper-auth", "default")).thenResolve(dockerCredentialHelperSecret) + td.replace(api, "upsert") + }) + it("should merge both", async () => { + const res = await prepareDockerAuth(api, basicProvider, garden.log) + const dockerAuth = jsonLoadBase64(res.data![dockerAuthSecretKey]) + expect(dockerAuth).to.haveOwnProperty("auths") + expect(dockerAuth.auths.myDockerRepo).to.equal("simple-auth") + expect(dockerAuth).to.haveOwnProperty("credHelpers") + expect(dockerAuth.credHelpers.myDockerRepo).to.equal("ecr-helper") + }) + }) + describe("when both simple login and cred helpers are missing", () => { + beforeEach(async () => { + const core = td.replace(api, "core") + const emptyDockerSimpleAuthSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: "test-docker-auth", + namespace: "default", + }, + data: { + ".dockerconfigjson": Buffer.from(JSON.stringify({ experimental: "enabled" })).toString("base64"), + }, + } + + const emptyDockerCredentialHelperSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: "test-cred-helper-auth", + namespace: "default", + }, + data: { + ".dockerconfigjson": Buffer.from(JSON.stringify({ experimental: "enabled" })).toString("base64"), + }, + } + td.when(core.listNamespace()).thenResolve({ + items: [{ status: { phase: "Active" }, metadata: { name: "default" } }], + }) + td.when(core.readNamespacedSecret("test-docker-auth", "default")).thenResolve(emptyDockerSimpleAuthSecret) + td.when(core.readNamespacedSecret("test-cred-helper-auth", "default")).thenResolve( + emptyDockerCredentialHelperSecret + ) + td.replace(api, "upsert") + }) + it("should fail when both are missing", async () => { + await expectError( + () => prepareDockerAuth(api, basicProvider, garden.log), + (e) => expect(e).to.be.instanceof(ConfigurationError) + ) + }) + }) +})