From 86174f9d56b89040519d08aa970085af0ae003d9 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Thu, 6 Feb 2020 22:39:22 +0100 Subject: [PATCH] fix(k8s): imagePullSecrets weren't copied to the project namespace The documented and expected behavior was to copy secrets referenced in the `imagePullSecrets` field in the `kubernetes` provider config. --- .../kubernetes/container/deployment.ts | 10 +- .../src/plugins/kubernetes/container/logs.ts | 7 +- .../src/plugins/kubernetes/hot-reload.ts | 7 +- .../src/plugins/kubernetes/secrets.ts | 21 ++- .../kubernetes/container/deployment.ts | 139 ++++++++++++++++++ .../test/integ/src/plugins/kubernetes/util.ts | 5 +- 6 files changed, 179 insertions(+), 10 deletions(-) create mode 100644 garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index d19907f12f..54a4bbd636 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -32,6 +32,7 @@ import { gardenAnnotationKey } from "../../../util/string" import { RuntimeContext } from "../../../runtime-context" import { resolve } from "path" import { killPortForwards } from "../port-forward" +import { ensureSecret, prepareImagePullSecrets } from "../secrets" export const DEFAULT_CPU_REQUEST = "10m" export const DEFAULT_MEMORY_REQUEST = "64Mi" @@ -207,7 +208,8 @@ export async function createContainerManifests( const namespace = await getAppNamespace(k8sCtx, log, provider) const api = await KubeApi.factory(log, provider) const ingresses = await createIngressResources(api, provider, namespace, service, log) - const workload = await createWorkloadResource({ + const workload = await createWorkloadManifest({ + api, provider, service, runtimeContext, @@ -231,6 +233,7 @@ export async function createContainerManifests( } interface CreateDeploymentParams { + api: KubeApi provider: KubernetesProvider service: ContainerService runtimeContext: RuntimeContext @@ -240,7 +243,8 @@ interface CreateDeploymentParams { production: boolean } -export async function createWorkloadResource({ +export async function createWorkloadManifest({ + api, provider, service, runtimeContext, @@ -372,7 +376,7 @@ export async function createWorkloadResource({ if (provider.config.imagePullSecrets.length > 0) { // add any configured imagePullSecrets - deployment.spec.template.spec.imagePullSecrets = provider.config.imagePullSecrets.map((s) => ({ name: s.name })) + deployment.spec.template.spec.imagePullSecrets = await prepareImagePullSecrets({ api, provider, namespace, log }) } // this is important for status checks to work correctly, because how K8s normalizes resources diff --git a/garden-service/src/plugins/kubernetes/container/logs.ts b/garden-service/src/plugins/kubernetes/container/logs.ts index fb05eb8dd8..9295169740 100644 --- a/garden-service/src/plugins/kubernetes/container/logs.ts +++ b/garden-service/src/plugins/kubernetes/container/logs.ts @@ -11,17 +11,20 @@ import { ContainerModule } from "../../container/config" import { getAppNamespace } from "../namespace" import { getAllLogs } from "../logs" import { KubernetesPluginContext } from "../config" -import { createWorkloadResource } from "./deployment" +import { createWorkloadManifest } from "./deployment" import { emptyRuntimeContext } from "../../../runtime-context" +import { KubeApi } from "../api" export async function getServiceLogs(params: GetServiceLogsParams) { const { ctx, log, service } = params const k8sCtx = ctx const provider = k8sCtx.provider const namespace = await getAppNamespace(k8sCtx, log, provider) + const api = await KubeApi.factory(log, provider) const resources = [ - await createWorkloadResource({ + await createWorkloadManifest({ + api, provider, service, // No need for the proper context here diff --git a/garden-service/src/plugins/kubernetes/hot-reload.ts b/garden-service/src/plugins/kubernetes/hot-reload.ts index 1dbcd30515..c97469a477 100644 --- a/garden-service/src/plugins/kubernetes/hot-reload.ts +++ b/garden-service/src/plugins/kubernetes/hot-reload.ts @@ -25,10 +25,11 @@ import { KubernetesPluginContext } from "./config" import { HotReloadServiceParams, HotReloadServiceResult } from "../../types/plugin/service/hotReloadService" import { KubernetesResource, KubernetesWorkload, KubernetesList } from "./types" import { normalizeLocalRsyncPath } from "../../util/fs" -import { createWorkloadResource } from "./container/deployment" +import { createWorkloadManifest } from "./container/deployment" import { kubectl } from "./kubectl" import { labelSelectorToString } from "./util" import { exec } from "../../util/util" +import { KubeApi } from "./api" export const RSYNC_PORT_NAME = "garden-rsync" @@ -184,9 +185,11 @@ export async function hotReloadContainer({ const k8sCtx = ctx as KubernetesPluginContext const provider = k8sCtx.provider const namespace = await getAppNamespace(k8sCtx, log, provider) + const api = await KubeApi.factory(log, provider) // Find the currently deployed workload by labels - const manifest = await createWorkloadResource({ + const manifest = await createWorkloadManifest({ + api, provider, service, runtimeContext: { envVars: {}, dependencies: [] }, diff --git a/garden-service/src/plugins/kubernetes/secrets.ts b/garden-service/src/plugins/kubernetes/secrets.ts index 50a9260e98..f55589a65d 100644 --- a/garden-service/src/plugins/kubernetes/secrets.ts +++ b/garden-service/src/plugins/kubernetes/secrets.ts @@ -7,7 +7,7 @@ */ import { KubeApi } from "./api" -import { ProviderSecretRef, KubernetesPluginContext } from "./config" +import { ProviderSecretRef, KubernetesPluginContext, KubernetesProvider } from "./config" import { ConfigurationError } from "../../exceptions" import { getMetadataNamespace } from "./namespace" import { GetSecretParams } from "../../types/plugin/provider/getSecret" @@ -122,3 +122,22 @@ export async function ensureSecret(api: KubeApi, secretRef: ProviderSecretRef, t await api.upsert({ kind: "Secret", namespace: targetNamespace, obj: secret, log }) } + +/** + * Prepare references to imagePullSecrets for use in Pod specs, and ensure they have been copied to the target + * namespace. + */ +export async function prepareImagePullSecrets({ + api, + provider, + namespace, + log, +}: { + api: KubeApi + provider: KubernetesProvider + namespace: string + log: LogEntry +}) { + await Promise.all(provider.config.imagePullSecrets.map((s) => ensureSecret(api, s, namespace, log))) + return provider.config.imagePullSecrets.map((s) => ({ name: s.name })) +} diff --git a/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts new file mode 100644 index 0000000000..8ad0c6e0d9 --- /dev/null +++ b/garden-service/test/integ/src/plugins/kubernetes/container/deployment.ts @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { getDataDir, makeTestGarden } from "../../../../../helpers" +import { expect } from "chai" +import { Garden } from "../../../../../../src/garden" +import { ConfigGraph } from "../../../../../../src/config-graph" +import { emptyRuntimeContext } from "../../../../../../src/runtime-context" +import { KubeApi } from "../../../../../../src/plugins/kubernetes/api" +import { createWorkloadManifest } from "../../../../../../src/plugins/kubernetes/container/deployment" +import { KubernetesProvider } from "../../../../../../src/plugins/kubernetes/config" +import { ensureSecret } from "../../../../../../src/plugins/kubernetes/secrets" +import { V1Secret } from "@kubernetes/client-node" +import { KubernetesResource } from "../../../../../../src/plugins/kubernetes/types" +import { cloneDeep } from "lodash" + +describe("kubernetes container deployment handlers", () => { + let garden: Garden + let graph: ConfigGraph + let provider: KubernetesProvider + let api: KubeApi + + before(async () => { + const root = getDataDir("test-projects", "container") + garden = await makeTestGarden(root) + graph = await garden.getConfigGraph(garden.log) + provider = await garden.resolveProvider("local-kubernetes") + api = await KubeApi.factory(garden.log, provider) + }) + + after(async () => { + await garden.close() + }) + + describe("createWorkloadManifest", () => { + it("should create a basic Deployment resource", async () => { + const service = await graph.getService("simple-service") + + const resource = await createWorkloadManifest({ + api, + provider, + service, + runtimeContext: emptyRuntimeContext, + namespace: garden.projectName, + enableHotReload: false, + log: garden.log, + production: false, + }) + + const version = service.module.version.versionString + + expect(resource).to.eql({ + kind: "Deployment", + apiVersion: "apps/v1", + metadata: { + name: "simple-service-" + version, + annotations: { "garden.io/configured.replicas": "1" }, + namespace: "container", + labels: { "module": "simple-service", "service": "simple-service", "garden.io/version": version }, + }, + spec: { + selector: { matchLabels: { "service": "simple-service", "garden.io/version": version } }, + template: { + metadata: { + labels: { "module": "simple-service", "service": "simple-service", "garden.io/version": version }, + }, + spec: { + containers: [ + { + name: "simple-service", + image: "simple-service:" + version, + env: [ + { name: "POD_NAME", valueFrom: { fieldRef: { fieldPath: "metadata.name" } } }, + { name: "POD_NAMESPACE", valueFrom: { fieldRef: { fieldPath: "metadata.namespace" } } }, + { name: "POD_IP", valueFrom: { fieldRef: { fieldPath: "status.podIP" } } }, + { name: "POD_SERVICE_ACCOUNT", valueFrom: { fieldRef: { fieldPath: "spec.serviceAccountName" } } }, + ], + ports: [{ name: "http", protocol: "TCP", containerPort: 8080 }], + resources: { requests: { cpu: "10m", memory: "64Mi" }, limits: { cpu: "1", memory: "1Gi" } }, + imagePullPolicy: "IfNotPresent", + securityContext: { allowPrivilegeEscalation: false }, + }, + ], + restartPolicy: "Always", + terminationGracePeriodSeconds: 5, + dnsPolicy: "ClusterFirst", + }, + }, + replicas: 1, + strategy: { type: "RollingUpdate", rollingUpdate: { maxUnavailable: 1, maxSurge: 1 } }, + revisionHistoryLimit: 3, + }, + }) + }) + + it("should copy and reference imagePullSecrets", async () => { + const service = await graph.getService("simple-service") + const secretName = "test-docker-auth" + + const authSecret: KubernetesResource = { + apiVersion: "v1", + kind: "Secret", + type: "kubernetes.io/dockerconfigjson", + metadata: { + name: secretName, + namespace: "default", + }, + stringData: { + ".dockerconfigjson": JSON.stringify({ auths: {} }), + }, + } + await api.upsert({ kind: "Secret", namespace: "default", obj: authSecret, log: garden.log }) + + const namespace = garden.projectName + const _provider = cloneDeep(provider) + _provider.config.imagePullSecrets = [{ name: secretName, namespace: "default" }] + + const resource = await createWorkloadManifest({ + api, + provider: _provider, + service, + runtimeContext: emptyRuntimeContext, + namespace, + enableHotReload: false, + log: garden.log, + production: false, + }) + + const copiedSecret = await api.core.readNamespacedSecret(secretName, namespace) + expect(copiedSecret).to.exist + expect(resource.spec.template.spec.imagePullSecrets).to.eql([{ name: secretName }]) + }) + }) +}) diff --git a/garden-service/test/integ/src/plugins/kubernetes/util.ts b/garden-service/test/integ/src/plugins/kubernetes/util.ts index feb40721cd..4390553ae7 100644 --- a/garden-service/test/integ/src/plugins/kubernetes/util.ts +++ b/garden-service/test/integ/src/plugins/kubernetes/util.ts @@ -21,7 +21,7 @@ import { findServiceResource, getResourceContainer, } from "../../../../../src/plugins/kubernetes/util" -import { createWorkloadResource } from "../../../../../src/plugins/kubernetes/container/deployment" +import { createWorkloadManifest } from "../../../../../src/plugins/kubernetes/container/deployment" import { emptyRuntimeContext } from "../../../../../src/runtime-context" import { PluginContext } from "../../../../../src/plugin-context" import { getHelmTestGarden } from "./helm/common" @@ -91,7 +91,8 @@ describe("util", () => { service, }) - const resource = await createWorkloadResource({ + const resource = await createWorkloadManifest({ + api, provider, service, runtimeContext: emptyRuntimeContext,