From e6d84e164af2d6b3a9a49d2fefd1f7c9fb624a5d Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Thu, 12 Apr 2018 18:54:09 +0200 Subject: [PATCH] refactor: split Kubernetes plugin into more modules --- src/plugins/kubernetes/api.ts | 61 ++ src/plugins/kubernetes/deployment.ts | 50 +- src/plugins/kubernetes/index.ts | 577 ++---------------- src/plugins/kubernetes/ingress.ts | 10 + src/plugins/kubernetes/kubectl.ts | 15 + src/plugins/kubernetes/modules.ts | 30 + src/plugins/kubernetes/namespace.ts | 48 ++ src/plugins/kubernetes/status.ts | 241 ++++++++ src/plugins/kubernetes/system-global.ts | 156 +++++ .../system-global/default-backend}/garden.yml | 0 .../ingress-controller}/garden.yml | 0 .../kubernetes-dashboard}/dashboard.yml | 8 +- .../kubernetes-dashboard}/garden.yml | 3 +- 13 files changed, 677 insertions(+), 522 deletions(-) create mode 100644 src/plugins/kubernetes/api.ts create mode 100644 src/plugins/kubernetes/modules.ts create mode 100644 src/plugins/kubernetes/namespace.ts create mode 100644 src/plugins/kubernetes/status.ts create mode 100644 src/plugins/kubernetes/system-global.ts rename static/{garden-default-backend => kubernetes/system-global/default-backend}/garden.yml (100%) rename static/{garden-ingress-controller => kubernetes/system-global/ingress-controller}/garden.yml (100%) rename static/{garden-dashboard => kubernetes/system-global/kubernetes-dashboard}/dashboard.yml (97%) rename static/{garden-dashboard => kubernetes/system-global/kubernetes-dashboard}/garden.yml (65%) diff --git a/src/plugins/kubernetes/api.ts b/src/plugins/kubernetes/api.ts new file mode 100644 index 0000000000..8b43491f78 --- /dev/null +++ b/src/plugins/kubernetes/api.ts @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import * as K8s from "kubernetes-client" + +import { DEFAULT_CONTEXT } from "./kubectl" + +const cachedParams = {} + +function getParams(namespace?: string) { + let params = cachedParams[namespace || ""] + + if (!params) { + const config = K8s.config.loadKubeconfig() + params = K8s.config.fromKubeconfig(config, DEFAULT_CONTEXT) + + params.promises = true + params.namespace = namespace + + cachedParams[namespace || ""] = params + } + + return params +} + +export function coreApi(namespace?: string): any { + return new K8s.Core(getParams(namespace)) +} + +export function extensionsApi(namespace?: string): any { + return new K8s.Extensions(getParams(namespace)) +} + +export async function apiPostOrPut(api: any, name: string, body: object) { + try { + await api.post(body) + } catch (err) { + if (err.code === 409) { + await api(name).put(body) + } else { + throw err + } + } +} + +export async function apiGetOrNull(api: any, name: string) { + try { + return await api(name).get() + } catch (err) { + if (err.code === 404) { + return null + } else { + throw err + } + } +} diff --git a/src/plugins/kubernetes/deployment.ts b/src/plugins/kubernetes/deployment.ts index e0b1508651..d8a1a29541 100644 --- a/src/plugins/kubernetes/deployment.ts +++ b/src/plugins/kubernetes/deployment.ts @@ -6,9 +6,28 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { ContainerService, ContainerServiceConfig } from "../container" +import { DeployServiceParams } from "../../types/plugin" +import { + ContainerModule, + ContainerService, + ContainerServiceConfig, +} from "../container" import { toPairs, extend } from "lodash" -import { ServiceContext } from "../../types/service" +import { + ServiceContext, + ServiceStatus, +} from "../../types/service" +import { + createIngress, + getServiceHostname, +} from "./ingress" +import { apply } from "./kubectl" +import { getAppNamespace } from "./namespace" +import { createServices } from "./service" +import { + checkDeploymentStatus, + waitForDeployment, +} from "./status" const DEFAULT_CPU_REQUEST = 0.01 const DEFAULT_CPU_LIMIT = 0.5 @@ -21,6 +40,33 @@ interface KubeEnvVar { valueFrom?: { fieldRef: { fieldPath: string } } } +export async function deployService( + { ctx, service, env, serviceContext, exposePorts = false, logEntry }: DeployServiceParams, +): Promise { + const namespace = getAppNamespace(ctx, env) + + const deployment = await createDeployment(service, serviceContext, exposePorts) + await apply(deployment, { namespace }) + + // TODO: automatically clean up Services and Ingresses if they should no longer exist + + const kubeservices = await createServices(service, exposePorts) + + for (let kubeservice of kubeservices) { + await apply(kubeservice, { namespace }) + } + + const ingress = await createIngress(service, getServiceHostname(ctx, service)) + + if (ingress !== null) { + await apply(ingress, { namespace }) + } + + await waitForDeployment({ ctx, service, logEntry, env }) + + return checkDeploymentStatus({ ctx, service, env }) +} + export async function createDeployment( service: ContainerService, serviceContext: ServiceContext, exposePorts: boolean, ) { diff --git a/src/plugins/kubernetes/index.ts b/src/plugins/kubernetes/index.ts index 9fd37476ee..a1b9490033 100644 --- a/src/plugins/kubernetes/index.ts +++ b/src/plugins/kubernetes/index.ts @@ -6,9 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import * as Docker from "dockerode" -import { Memoize } from "typescript-memoize" -import * as K8s from "kubernetes-client" import { DeploymentError, NotFoundError } from "../../exceptions" import { ConfigureEnvironmentParams, DeleteConfigParams, @@ -22,31 +19,38 @@ import { TestModuleParams, TestResult, } from "../../types/plugin" import { - ContainerModule, ContainerService, ServiceEndpointSpec, + ContainerModule, } from "../container" import { values, every, map, extend } from "lodash" -import { Environment } from "../../types/common" -import { deserializeKeys, serializeKeys, sleep, splitFirst } from "../../util" -import { Service, ServiceProtocol, ServiceStatus } from "../../types/service" -import { join } from "path" -import { createServices } from "./service" -import { createIngress } from "./ingress" -import { createDeployment } from "./deployment" -import { DEFAULT_CONTEXT, Kubectl, KUBECTL_DEFAULT_TIMEOUT } from "./kubectl" -import { DEFAULT_TEST_TIMEOUT, STATIC_DIR } from "../../constants" -import { LogEntry } from "../../logger" -import { GardenContext } from "../../context" +import { deserializeKeys, serializeKeys, splitFirst } from "../../util" +import { ServiceStatus } from "../../types/service" +import { + apiGetOrNull, + apiPostOrPut, + coreApi, +} from "./api" +import { + createNamespace, + getAppNamespace, + getMetadataNamespace, +} from "./namespace" +import { + deployService, +} from "./deployment" +import { + kubectl, +} from "./kubectl" +import { DEFAULT_TEST_TIMEOUT } from "../../constants" import * as split from "split" import moment = require("moment") -import { EntryStyle, LogSymbolType } from "../../logger/types" - -const GARDEN_SYSTEM_NAMESPACE = "garden-system" - -const ingressControllerModulePath = join(STATIC_DIR, "garden-ingress-controller") -const defaultBackendModulePath = join(STATIC_DIR, "garden-default-backend") -const dashboardModulePath = join(STATIC_DIR, "garden-dashboard") -const dashboardSpecPath = join(dashboardModulePath, "dashboard.yml") -const localIngressPort = 32000 +import { EntryStyle } from "../../logger/types" +import { + checkDeploymentStatus, +} from "./status" +import { + configureGlobalSystem, + getGlobalSystemStatus, +} from "./system-global" export class KubernetesProvider implements Plugin { name = "kubernetes" @@ -61,7 +65,7 @@ export class KubernetesProvider implements Plugin { async getEnvironmentStatus({ ctx, env }: GetEnvironmentStatusParams) { try { // TODO: use API instead of kubectl (I just couldn't find which API call to make) - await this.kubectl().call(["version"]) + await kubectl().call(["version"]) } catch (err) { // TODO: catch error properly if (err.output) { @@ -70,52 +74,26 @@ export class KubernetesProvider implements Plugin { throw err } - const gardenEnv = this.getSystemEnv(env) - - const ingressControllerService = await this.getIngressControllerService(ctx) - const defaultBackendService = await this.getDefaultBackendService(ctx) - const dashboardService = await this.getDashboardService(ctx) - - const ingressControllerStatus = await this.checkDeploymentStatus({ - ctx, - service: ingressControllerService, - env: gardenEnv, - }) - const defaultBackendStatus = await this.checkDeploymentStatus({ - ctx, - service: defaultBackendService, - env: gardenEnv, - }) - const dashboardStatus = await this.checkDeploymentStatus({ - ctx, - service: dashboardService, - env: gardenEnv, - }) + const globalSystemStatus = await getGlobalSystemStatus(ctx, env) const statusDetail = { systemNamespaceReady: false, namespaceReady: false, metadataNamespaceReady: false, - dashboardReady: dashboardStatus.state === "ready", - ingressControllerReady: ingressControllerStatus.state === "ready", - defaultBackendReady: defaultBackendStatus.state === "ready", + ...globalSystemStatus, } - const metadataNamespace = this.getMetadataNamespaceName(ctx) - const namespacesStatus = await this.coreApi().namespaces().get() + const metadataNamespace = getMetadataNamespace(ctx) + const namespacesStatus = await coreApi().namespaces().get() for (const n of namespacesStatus.items) { - if (n.metadata.name === this.getNamespaceName(ctx, env) && n.status.phase === "Active") { + if (n.metadata.name === getAppNamespace(ctx, env) && n.status.phase === "Active") { statusDetail.namespaceReady = true } if (n.metadata.name === metadataNamespace && n.status.phase === "Active") { statusDetail.metadataNamespaceReady = true } - - if (n.metadata.name === GARDEN_SYSTEM_NAMESPACE && n.status.phase === "Active") { - statusDetail.systemNamespaceReady = true - } } let configured = every(values(statusDetail)) @@ -126,105 +104,44 @@ export class KubernetesProvider implements Plugin { } } - async configureEnvironment({ ctx, env, logEntry }: ConfigureEnvironmentParams) { + async configureEnvironment(params: ConfigureEnvironmentParams) { // TODO: use Helm 3 when it's released instead of this custom/manual stuff + const { ctx, env, logEntry } = params const status = await this.getEnvironmentStatus({ ctx, env }) if (status.configured) { return } - if (!status.detail.systemNamespaceReady) { - logEntry && logEntry.setState({ section: "kubernetes", msg: `Creating garden system namespace` }) - await this.coreApi().namespaces.post({ - body: { - apiVersion: "v1", - kind: "Namespace", - metadata: { - name: GARDEN_SYSTEM_NAMESPACE, - annotations: { - "garden.io/generated": "true", - }, - }, - }, - }) - } + await configureGlobalSystem(params, status) if (!status.detail.namespaceReady) { - const ns = this.getNamespaceName(ctx, env) + const ns = getAppNamespace(ctx, env) logEntry && logEntry.setState({ section: "kubernetes", msg: `Creating namespace ${ns}` }) - await this.coreApi().namespaces.post({ - body: { - apiVersion: "v1", - kind: "Namespace", - metadata: { - name: ns, - annotations: { - "garden.io/generated": "true", - }, - }, - }, - }) + await createNamespace(ns) } if (!status.detail.metadataNamespaceReady) { - const ns = this.getMetadataNamespaceName(ctx) + const ns = getMetadataNamespace(ctx) logEntry && logEntry.setState({ section: "kubernetes", msg: `Creating namespace ${ns}` }) - await this.coreApi().namespaces.post({ - body: { - apiVersion: "v1", - kind: "Namespace", - metadata: { - name: ns, - annotations: { - "garden.io/generated": "true", - }, - }, - }, - }) - } - - if (!status.detail.dashboardReady) { - logEntry && logEntry.setState({ section: "kubernetes", msg: `Configuring dashboard` }) - // TODO: deploy this as a service - await this.kubectl(GARDEN_SYSTEM_NAMESPACE).call(["apply", "-f", dashboardSpecPath]) - } - - if (!status.detail.ingressControllerReady) { - logEntry && logEntry.setState({ section: "kubernetes", msg: `Configuring ingress controller` }) - const gardenEnv = this.getSystemEnv(env) - await this.deployService({ - ctx, - service: await this.getDefaultBackendService(ctx), - serviceContext: { envVars: {}, dependencies: {} }, - env: gardenEnv, - logEntry, - }) - await this.deployService({ - ctx, - service: await this.getIngressControllerService(ctx), - serviceContext: { envVars: {}, dependencies: {} }, - env: gardenEnv, - exposePorts: true, - logEntry, - }) + await createNamespace(ns) } } async getServiceStatus({ ctx, service }: GetServiceStatusParams): Promise { // TODO: hash and compare all the configuration files (otherwise internal changes don't get deployed) - return await this.checkDeploymentStatus({ ctx, service }) + return await checkDeploymentStatus({ ctx, service }) } async destroyEnvironment({ ctx, env }: DestroyEnvironmentParams) { - const namespace = this.getNamespaceName(ctx, env) + const namespace = getAppNamespace(ctx, env) const entry = ctx.log.info({ section: "kubernetes", msg: `Deleting namespace ${namespace}`, entryStyle: EntryStyle.activity, }) try { - await this.coreApi().namespace(namespace).delete(namespace) + await coreApi().namespace(namespace).delete(namespace) entry.setSuccess("Finished") } catch (err) { entry.setError(err.message) @@ -232,31 +149,8 @@ export class KubernetesProvider implements Plugin { } } - async deployService( - { ctx, service, env, serviceContext, exposePorts = false, logEntry }: DeployServiceParams, - ) { - const namespace = this.getNamespaceName(ctx, env) - - const deployment = await createDeployment(service, serviceContext, exposePorts) - await this.apply(deployment, { namespace }) - - // TODO: automatically clean up Services and Ingresses if they should no longer exist - - const kubeservices = await createServices(service, exposePorts) - - for (let kubeservice of kubeservices) { - await this.apply(kubeservice, { namespace }) - } - - const ingress = await createIngress(service, this.getServiceHostname(ctx, service)) - - if (ingress !== null) { - await this.apply(ingress, { namespace }) - } - - await this.waitForDeployment({ ctx, service, logEntry, env }) - - return this.getServiceStatus({ ctx, service, env }) + async deployService(params: DeployServiceParams) { + return deployService(params) } async getServiceOutputs({ service }: GetServiceOutputsParams) { @@ -267,7 +161,7 @@ export class KubernetesProvider implements Plugin { async execInService({ ctx, service, env, command }: ExecInServiceParams) { const status = await this.getServiceStatus({ ctx, service, env }) - const namespace = this.getNamespaceName(ctx, env) + const namespace = getAppNamespace(ctx, env) // TODO: this check should probably live outside of the plugin if (!status.state || status.state !== "ready") { @@ -278,7 +172,7 @@ export class KubernetesProvider implements Plugin { } // get a running pod - let res = await this.coreApi(namespace).namespaces.pods.get({ + let res = await coreApi(namespace).namespaces.pods.get({ qs: { labelSelector: `service=${service.name}`, }, @@ -293,7 +187,7 @@ export class KubernetesProvider implements Plugin { } // exec in the pod via kubectl - res = await this.kubectl(namespace).tty(["exec", "-it", pod.metadata.name, "--", ...command]) + res = await kubectl(namespace).tty(["exec", "-it", pod.metadata.name, "--", ...command]) return { code: res.code, output: res.output } } @@ -327,7 +221,7 @@ export class KubernetesProvider implements Plugin { const startedAt = new Date() const timeout = testSpec.timeout || DEFAULT_TEST_TIMEOUT - const res = await this.kubectl(this.getNamespaceName(ctx)).tty(kubecmd, { ignoreError: true, timeout }) + const res = await kubectl(getAppNamespace(ctx)).tty(kubecmd, { ignoreError: true, timeout }) const testResult: TestResult = { version, @@ -337,7 +231,7 @@ export class KubernetesProvider implements Plugin { output: res.output, } - const ns = this.getMetadataNamespaceName(ctx) + const ns = getMetadataNamespace(ctx) const resultKey = `test-result--${module.name}--${version}` const body = { body: { @@ -354,15 +248,15 @@ export class KubernetesProvider implements Plugin { }, } - await apiPostOrPut(this.coreApi(ns).namespaces.configmaps, resultKey, body) + await apiPostOrPut(coreApi(ns).namespaces.configmaps, resultKey, body) return testResult } async getTestResult({ ctx, module, version }: GetTestResultParams) { - const ns = this.getMetadataNamespaceName(ctx) + const ns = getMetadataNamespace(ctx) const resultKey = getTestResultKey(module, version) - const res = await apiGetOrNull(this.coreApi(ns).namespaces.configmaps, resultKey) + const res = await apiGetOrNull(coreApi(ns).namespaces.configmaps, resultKey) return res && deserializeKeys(res.data) } @@ -375,7 +269,7 @@ export class KubernetesProvider implements Plugin { kubectlArgs.push("--follow") } - const proc = this.kubectl(this.getNamespaceName(ctx)).spawn(kubectlArgs) + const proc = kubectl(getAppNamespace(ctx)).spawn(kubectlArgs) proc.stdout .pipe(split()) @@ -400,14 +294,14 @@ export class KubernetesProvider implements Plugin { } async getConfig({ ctx, key }: GetConfigParams) { - const ns = this.getMetadataNamespaceName(ctx) - const res = await apiGetOrNull(this.coreApi(ns).namespaces.secrets, key.join(".")) + const ns = getMetadataNamespace(ctx) + const res = await apiGetOrNull(coreApi(ns).namespaces.secrets, key.join(".")) return res && Buffer.from(res.data.value, "base64").toString() } async setConfig({ ctx, key, value }: SetConfigParams) { // we store configuration in a separate metadata namespace, so that configs aren't cleared when wiping the namespace - const ns = this.getMetadataNamespaceName(ctx) + const ns = getMetadataNamespace(ctx) const body = { body: { apiVersion: "v1", @@ -423,13 +317,13 @@ export class KubernetesProvider implements Plugin { }, } - await apiPostOrPut(this.coreApi(ns).namespaces.secrets, key.join("."), body) + await apiPostOrPut(coreApi(ns).namespaces.secrets, key.join("."), body) } async deleteConfig({ ctx, key }: DeleteConfigParams) { - const ns = this.getMetadataNamespaceName(ctx) + const ns = getMetadataNamespace(ctx) try { - await this.coreApi(ns).namespaces.secrets(key.join(".")).delete() + await coreApi(ns).namespaces.secrets(key.join(".")).delete() } catch (err) { if (err.code === 404) { return { found: false } @@ -441,357 +335,8 @@ export class KubernetesProvider implements Plugin { } //endregion - - //=========================================================================== - //region Internal helpers - //=========================================================================== - - private getNamespaceName(ctx: GardenContext, env?: Environment) { - const currentEnv = env || ctx.getEnvironment() - if (currentEnv.namespace === GARDEN_SYSTEM_NAMESPACE) { - return currentEnv.namespace - } - return `garden--${ctx.projectName}--${currentEnv.namespace}` - } - - private getMetadataNamespaceName(ctx: GardenContext) { - const env = ctx.getEnvironment() - return `garden-metadata--${ctx.projectName}--${env.namespace}` - } - - private async getIngressControllerService(ctx: GardenContext) { - const module = await ctx.resolveModule(ingressControllerModulePath) - - return ContainerService.factory(ctx, module, "ingress-controller") - } - - private async getDefaultBackendService(ctx: GardenContext) { - const module = await ctx.resolveModule(defaultBackendModulePath) - - return ContainerService.factory(ctx, module, "default-backend") - } - - private async getDashboardService(ctx: GardenContext) { - // TODO: implement raw kubernetes module load this module the same way as the ones above - const module = new ContainerModule(ctx, { - version: "0", - name: "garden-dashboard", - type: "container", - path: dashboardModulePath, - services: { - "kubernetes-dashboard": { - daemon: false, - dependencies: [], - endpoints: [], - ports: {}, - volumes: [], - }, - }, - variables: {}, - build: { dependencies: [] }, - test: {}, - }) - - return Service.factory(ctx, module, "kubernetes-dashboard") - } - - protected getProjectHostname() { - // TODO: for remote Garden environments, this will depend on the configured project - // TODO: make configurable for the generic kubernetes plugin - return "local.app.garden" - } - - protected getServiceHostname(ctx: GardenContext, service: ContainerService) { - return `${service.name}.${ctx.projectName}.${this.getProjectHostname()}` - } - - async checkDeploymentStatus( - { ctx, service, resourceVersion, env }: - { ctx: GardenContext, service: ContainerService, resourceVersion?: number, env?: Environment }, - ): Promise { - const type = service.config.daemon ? "daemonsets" : "deployments" - const hostname = this.getServiceHostname(ctx, service) - - const namespace = this.getNamespaceName(ctx, env) - - const endpoints = service.config.endpoints.map((e: ServiceEndpointSpec) => { - // TODO: this should be HTTPS, once we've set up TLS termination at the ingress controller level - const protocol: ServiceProtocol = "http" - - return { - protocol, - hostname, - port: localIngressPort, - url: `${protocol}://${hostname}:${localIngressPort}`, - paths: e.paths, - } - }) - - const out: ServiceStatus = { - endpoints, - runningReplicas: 0, - detail: { resourceVersion }, - } - - let statusRes - let status - - try { - statusRes = await this.extensionsApi(namespace).namespaces[type](service.name).get() - } catch (err) { - if (err.code === 404) { - // service is not running - return out - } else { - throw err - } - } - - status = statusRes.status - - if (!resourceVersion) { - resourceVersion = out.detail.resourceVersion = parseInt(statusRes.metadata.resourceVersion, 10) - } - - out.version = statusRes.metadata.annotations["garden.io/version"] - - // TODO: try to come up with something more efficient. may need to wait for newer k8s version. - // note: the resourceVersion parameter does not appear to work... - const eventsRes = await this.coreApi(namespace).namespaces.events.get() - - // const eventsRes = await this.kubeApi( - // "GET", - // [ - // "apis", apiSection, "v1beta1", - // "watch", - // "namespaces", namespace, - // type + "s", service.fullName, - // ], - // { resourceVersion, watch: "false" }, - // ) - - // look for errors and warnings in the events for the service, abort if we find any - const events = eventsRes.items - - for (let event of events) { - const eventVersion = parseInt(event.metadata.resourceVersion, 10) - - if ( - eventVersion <= resourceVersion || - (!event.metadata.name.startsWith(service.name + ".") && !event.metadata.name.startsWith(service.name + "-")) - ) { - continue - } - - if (eventVersion > resourceVersion) { - out.detail.resourceVersion = eventVersion - } - - if (event.type === "Warning" || event.type === "Error") { - if (event.reason === "Unhealthy") { - // still waiting on readiness probe - continue - } - out.state = "unhealthy" - out.lastError = `${event.reason} - ${event.message}` - return out - } - - let message = event.message - - if (event.reason === event.reason.toUpperCase()) { - // some events like ingress events are formatted this way - message = `${event.reason} ${message}` - } - - if (message) { - out.detail.lastMessage = message - } - } - - // See `https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/rollout_status.go` for a reference - // for this logic. - let available = 0 - out.state = "ready" - let statusMsg = "" - - if (statusRes.metadata.generation > status.observedGeneration) { - statusMsg = `Waiting for spec update to be observed...` - out.state = "deploying" - } else if (service.config.daemon) { - const desired = status.desiredNumberScheduled || 0 - const updated = status.updatedNumberScheduled || 0 - available = status.numberAvailable || 0 - - if (updated < desired) { - statusMsg = `${updated} out of ${desired} new pods updated...` - out.state = "deploying" - } else if (available < desired) { - statusMsg = `${available} out of ${desired} updated pods available...` - out.state = "deploying" - } - } else { - const desired = 1 // TODO: service.count[env.name] || 1 - const updated = status.updatedReplicas || 0 - const replicas = status.replicas || 0 - available = status.availableReplicas || 0 - - if (updated < desired) { - statusMsg = `Waiting for rollout: ${updated} out of ${desired} new replicas updated...` - out.state = "deploying" - } else if (replicas > updated) { - statusMsg = `Waiting for rollout: ${replicas - updated} old replicas pending termination...` - out.state = "deploying" - } else if (available < updated) { - statusMsg = `Waiting for rollout: ${available} out of ${updated} updated replicas available...` - out.state = "deploying" - } - } - - out.runningReplicas = available - out.lastMessage = statusMsg - - return out - } - - async waitForDeployment( - { ctx, service, logEntry, env }: - { ctx: GardenContext, service: ContainerService, logEntry?: LogEntry, env?: Environment }, - ) { - // NOTE: using `kubectl rollout status` here didn't pan out, since it just times out when errors occur. - let loops = 0 - let resourceVersion - let lastMessage - let lastDetailMessage - const startTime = new Date().getTime() - - logEntry && ctx.log.verbose({ - symbol: LogSymbolType.info, - section: service.name, - msg: `Waiting for service to be ready...`, - }) - - while (true) { - await sleep(2000 + 1000 * loops) - - const status = await this.checkDeploymentStatus({ ctx, service, resourceVersion, env }) - - if (status.lastError) { - throw new DeploymentError(`Error deploying ${service.name}: ${status.lastError}`, { - serviceName: service.name, - status, - }) - } - - if (status.detail.lastMessage && status.detail.lastMessage !== lastDetailMessage) { - lastDetailMessage = status.detail.lastMessage - logEntry && ctx.log.verbose({ - symbol: LogSymbolType.info, - section: service.name, - msg: status.detail.lastMessage, - }) - } - - if (status.lastMessage && status.lastMessage !== lastMessage) { - lastMessage = status.lastMessage - logEntry && ctx.log.verbose({ - symbol: LogSymbolType.info, - section: service.name, - msg: status.lastMessage, - }) - } - - if (status.state === "ready") { - break - } - - resourceVersion = status.detail.resourceVersion - - const now = new Date().getTime() - - if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { - throw new Error(`Timed out waiting for ${service.name} to deploy`) - } - } - - logEntry && ctx.log.verbose({ symbol: LogSymbolType.info, section: service.name, msg: `Service deployed` }) - } - - // sadly the TS definitions are no good for this one - @Memoize() - protected coreApi(namespace?: string): any { - const config = K8s.config.loadKubeconfig() - const params: any = K8s.config.fromKubeconfig(config, DEFAULT_CONTEXT) - - params.promises = true - params.namespace = namespace - - return new K8s.Core(params) - } - - @Memoize() - protected extensionsApi(namespace?: string): any { - const config = K8s.config.loadKubeconfig() - const params: any = K8s.config.fromKubeconfig(config, DEFAULT_CONTEXT) - - params.promises = true - params.namespace = namespace - - return new K8s.Extensions(params) - } - - @Memoize() - public kubectl(namespace?: string) { - return new Kubectl({ context: DEFAULT_CONTEXT, namespace }) - } - - @Memoize() - protected getDocker() { - return new Docker() - } - - protected async apply(obj: any, { force = false, namespace }: { force?: boolean, namespace?: string } = {}) { - const data = Buffer.from(JSON.stringify(obj)) - - let args = ["apply"] - force && args.push("--force") - args.push("-f") - args.push("-") - - await this.kubectl(namespace).call(args, { data }) - } - - private getSystemEnv(env: Environment): Environment { - return { name: env.name, namespace: GARDEN_SYSTEM_NAMESPACE, config: { providers: {} } } - } - - //endregion } function getTestResultKey(module: ContainerModule, version: string) { return `test-result--${module.name}--${version}` } - -async function apiPostOrPut(api: any, name: string, body: object) { - try { - await api.post(body) - } catch (err) { - if (err.code === 409) { - await api(name).put(body) - } else { - throw err - } - } -} - -async function apiGetOrNull(api: any, name: string) { - try { - return await api(name).get() - } catch (err) { - if (err.code === 404) { - return null - } else { - throw err - } - } -} diff --git a/src/plugins/kubernetes/ingress.ts b/src/plugins/kubernetes/ingress.ts index 4dd25f9cee..06b02f7ebb 100644 --- a/src/plugins/kubernetes/ingress.ts +++ b/src/plugins/kubernetes/ingress.ts @@ -6,6 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { GardenContext } from "../../context" import { ContainerService } from "../container" export async function createIngress(service: ContainerService, externalHostname: string) { @@ -52,3 +53,12 @@ export async function createIngress(service: ContainerService, externalHostname: }, } } + +export function getProjectHostname() { + // TODO: make configurable + return "local.app.garden" +} + +export function getServiceHostname(ctx: GardenContext, service: ContainerService) { + return `${service.name}.${ctx.projectName}.${getProjectHostname()}` +} diff --git a/src/plugins/kubernetes/kubectl.ts b/src/plugins/kubernetes/kubectl.ts index 7affd2e5af..274bfdb647 100644 --- a/src/plugins/kubernetes/kubectl.ts +++ b/src/plugins/kubernetes/kubectl.ts @@ -144,3 +144,18 @@ export class Kubectl { return ops.concat(args) } } + +export function kubectl(namespace?: string) { + return new Kubectl({ context: DEFAULT_CONTEXT, namespace }) +} + +export async function apply(obj: any, { force = false, namespace }: { force?: boolean, namespace?: string } = {}) { + const data = Buffer.from(JSON.stringify(obj)) + + let args = ["apply"] + force && args.push("--force") + args.push("-f") + args.push("-") + + await kubectl(namespace).call(args, { data }) +} diff --git a/src/plugins/kubernetes/modules.ts b/src/plugins/kubernetes/modules.ts new file mode 100644 index 0000000000..a58ab75a9a --- /dev/null +++ b/src/plugins/kubernetes/modules.ts @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import * as Joi from "joi" +import { identifierRegex } from "../../types/common" +import { + baseServiceSchema, + Module, + ModuleConfig, +} from "../../types/module" +import { ServiceConfig } from "../../types/service" + +export interface KubernetesRawServiceConfig extends ServiceConfig { + specs: string[] +} + +export interface KubernetesRawModuleConfig extends ModuleConfig { } + +export const k8sRawServicesSchema = Joi.object() + .pattern(identifierRegex, baseServiceSchema.keys({ + specs: Joi.array().items(Joi.string()).required(), + })) + .default(() => ({}), "{}") + +export class KubernetesRawModule extends Module { } diff --git a/src/plugins/kubernetes/namespace.ts b/src/plugins/kubernetes/namespace.ts new file mode 100644 index 0000000000..e7c10bc98c --- /dev/null +++ b/src/plugins/kubernetes/namespace.ts @@ -0,0 +1,48 @@ +import { GardenContext } from "../../context" +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { Environment } from "../../types/common" +import { + apiGetOrNull, + coreApi, +} from "./api" +import { GARDEN_GLOBAL_SYSTEM_NAMESPACE } from "./system-global" + +export async function namespaceReady(namespace: string) { + const ns = await apiGetOrNull(coreApi().namespaces, namespace) + return ns && ns.status.phase === "Active" +} + +export async function createNamespace(namespace: string) { + await coreApi().namespaces.post({ + body: { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: namespace, + annotations: { + "garden.io/generated": "true", + }, + }, + }, + }) +} + +export function getAppNamespace(ctx: GardenContext, env?: Environment) { + const currentEnv = env || ctx.getEnvironment() + if (currentEnv.namespace === GARDEN_GLOBAL_SYSTEM_NAMESPACE) { + return currentEnv.namespace + } + return `garden--${ctx.projectName}--${currentEnv.namespace}` +} + +export function getMetadataNamespace(ctx: GardenContext) { + const env = ctx.getEnvironment() + return `garden-metadata--${ctx.projectName}--${env.namespace}` +} diff --git a/src/plugins/kubernetes/status.ts b/src/plugins/kubernetes/status.ts new file mode 100644 index 0000000000..a61379645f --- /dev/null +++ b/src/plugins/kubernetes/status.ts @@ -0,0 +1,241 @@ +import { GardenContext } from "../../context" +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { DeploymentError } from "../../exceptions" +import { LogEntry } from "../../logger" +import { LogSymbolType } from "../../logger/types" +import { Environment } from "../../types/common" +import { + ServiceProtocol, + ServiceStatus, +} from "../../types/service" +import { sleep } from "../../util" +import { + ContainerService, + ServiceEndpointSpec, +} from "../container" +import { + coreApi, + extensionsApi, +} from "./api" +import { getServiceHostname } from "./ingress" +import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl" +import { getAppNamespace } from "./namespace" +import { localIngressPort } from "./system-global" + +export async function checkDeploymentStatus( + { ctx, service, resourceVersion, env }: + { ctx: GardenContext, service: ContainerService, resourceVersion?: number, env?: Environment }, +): Promise { + const type = service.config.daemon ? "daemonsets" : "deployments" + const hostname = getServiceHostname(ctx, service) + const namespace = getAppNamespace(ctx, env) + + const endpoints = service.config.endpoints.map((e: ServiceEndpointSpec) => { + // TODO: this should be HTTPS, once we've set up TLS termination at the ingress controller level + const protocol: ServiceProtocol = "http" + + return { + protocol, + hostname, + port: localIngressPort, + url: `${protocol}://${hostname}:${localIngressPort}`, + paths: e.paths, + } + }) + + const out: ServiceStatus = { + endpoints, + runningReplicas: 0, + detail: { resourceVersion }, + } + + let statusRes + let status + + try { + statusRes = await extensionsApi(namespace).namespaces[type](service.name).get() + } catch (err) { + if (err.code === 404) { + // service is not running + return out + } else { + throw err + } + } + + status = statusRes.status + + if (!resourceVersion) { + resourceVersion = out.detail.resourceVersion = parseInt(statusRes.metadata.resourceVersion, 10) + } + + out.version = statusRes.metadata.annotations["garden.io/version"] + + // TODO: try to come up with something more efficient. may need to wait for newer k8s version. + // note: the resourceVersion parameter does not appear to work... + const eventsRes = await coreApi(namespace).namespaces.events.get() + + // const eventsRes = await this.kubeApi( + // "GET", + // [ + // "apis", apiSection, "v1beta1", + // "watch", + // "namespaces", namespace, + // type + "s", service.fullName, + // ], + // { resourceVersion, watch: "false" }, + // ) + + // look for errors and warnings in the events for the service, abort if we find any + const events = eventsRes.items + + for (let event of events) { + const eventVersion = parseInt(event.metadata.resourceVersion, 10) + + if ( + eventVersion <= resourceVersion || + (!event.metadata.name.startsWith(service.name + ".") && !event.metadata.name.startsWith(service.name + "-")) + ) { + continue + } + + if (eventVersion > resourceVersion) { + out.detail.resourceVersion = eventVersion + } + + if (event.type === "Warning" || event.type === "Error") { + if (event.reason === "Unhealthy") { + // still waiting on readiness probe + continue + } + out.state = "unhealthy" + out.lastError = `${event.reason} - ${event.message}` + return out + } + + let message = event.message + + if (event.reason === event.reason.toUpperCase()) { + // some events like ingress events are formatted this way + message = `${event.reason} ${message}` + } + + if (message) { + out.detail.lastMessage = message + } + } + + // See `https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/rollout_status.go` for a reference + // for this logic. + let available = 0 + out.state = "ready" + let statusMsg = "" + + if (statusRes.metadata.generation > status.observedGeneration) { + statusMsg = `Waiting for spec update to be observed...` + out.state = "deploying" + } else if (service.config.daemon) { + const desired = status.desiredNumberScheduled || 0 + const updated = status.updatedNumberScheduled || 0 + available = status.numberAvailable || 0 + + if (updated < desired) { + statusMsg = `${updated} out of ${desired} new pods updated...` + out.state = "deploying" + } else if (available < desired) { + statusMsg = `${available} out of ${desired} updated pods available...` + out.state = "deploying" + } + } else { + const desired = 1 // TODO: service.count[env.name] || 1 + const updated = status.updatedReplicas || 0 + const replicas = status.replicas || 0 + available = status.availableReplicas || 0 + + if (updated < desired) { + statusMsg = `Waiting for rollout: ${updated} out of ${desired} new replicas updated...` + out.state = "deploying" + } else if (replicas > updated) { + statusMsg = `Waiting for rollout: ${replicas - updated} old replicas pending termination...` + out.state = "deploying" + } else if (available < updated) { + statusMsg = `Waiting for rollout: ${available} out of ${updated} updated replicas available...` + out.state = "deploying" + } + } + + out.runningReplicas = available + out.lastMessage = statusMsg + + return out +} + +export async function waitForDeployment( + { ctx, service, logEntry, env }: + { ctx: GardenContext, service: ContainerService, logEntry?: LogEntry, env?: Environment }, +) { + // NOTE: using `kubectl rollout status` here didn't pan out, since it just times out when errors occur. + let loops = 0 + let resourceVersion = undefined + let lastMessage + let lastDetailMessage + const startTime = new Date().getTime() + + logEntry && ctx.log.verbose({ + symbol: LogSymbolType.info, + section: service.name, + msg: `Waiting for service to be ready...`, + }) + + while (true) { + await sleep(2000 + 1000 * loops) + + const status = await checkDeploymentStatus({ ctx, service, resourceVersion, env }) + + if (status.lastError) { + throw new DeploymentError(`Error deploying ${service.name}: ${status.lastError}`, { + serviceName: service.name, + status, + }) + } + + if (status.detail.lastMessage && (!lastDetailMessage || status.detail.lastMessage !== lastDetailMessage)) { + lastDetailMessage = status.detail.lastMessage + logEntry && ctx.log.verbose({ + symbol: LogSymbolType.info, + section: service.name, + msg: status.detail.lastMessage, + }) + } + + if (status.lastMessage && (!lastMessage && status.lastMessage !== lastMessage)) { + lastMessage = status.lastMessage + logEntry && ctx.log.verbose({ + symbol: LogSymbolType.info, + section: service.name, + msg: status.lastMessage, + }) + } + + if (status.state === "ready") { + break + } + + resourceVersion = status.detail.resourceVersion + + const now = new Date().getTime() + + if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { + throw new Error(`Timed out waiting for ${service.name} to deploy`) + } + } + + logEntry && ctx.log.verbose({ symbol: LogSymbolType.info, section: service.name, msg: `Service deployed` }) +} diff --git a/src/plugins/kubernetes/system-global.ts b/src/plugins/kubernetes/system-global.ts new file mode 100644 index 0000000000..a9c8782888 --- /dev/null +++ b/src/plugins/kubernetes/system-global.ts @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { join } from "path" +import { STATIC_DIR } from "../../constants" +import { GardenContext } from "../../context" +import { Environment } from "../../types/common" +import { + ConfigureEnvironmentParams, + EnvironmentStatus, +} from "../../types/plugin" +import { Service } from "../../types/service" +import { + ContainerModule, + ContainerService, +} from "../container" +import { deployService } from "./deployment" +import { kubectl } from "./kubectl" +import { checkDeploymentStatus } from "./status" +import { + createNamespace, + namespaceReady, +} from "./namespace" + +export const GARDEN_GLOBAL_SYSTEM_NAMESPACE = "garden-system" + +const globalSystemProjectPath = join(STATIC_DIR, "kubernetes", "system-global") +const ingressControllerModulePath = join(globalSystemProjectPath, "ingress-controller") +const defaultBackendModulePath = join(globalSystemProjectPath, "default-backend") +const dashboardModulePath = join(globalSystemProjectPath, "kubernetes-dashboard") +const dashboardSpecPath = join(dashboardModulePath, "dashboard.yml") + +export const localIngressPort = 32000 + +export async function getGlobalSystemStatus(ctx: GardenContext, env: Environment) { + const gardenEnv = getSystemEnv(env) + + const systemNamespaceReady = namespaceReady(GARDEN_GLOBAL_SYSTEM_NAMESPACE) + + if (!systemNamespaceReady) { + return { + systemNamespaceReady, + dashboardReady: false, + ingressControllerReady: false, + defaultBackendReady: false, + } + } + + const ingressControllerService = await getIngressControllerService(ctx) + const defaultBackendService = await getDefaultBackendService(ctx) + const dashboardService = await getDashboardService(ctx) + + const ingressControllerStatus = await checkDeploymentStatus({ + ctx, + service: ingressControllerService, + env: gardenEnv, + }) + const defaultBackendStatus = await checkDeploymentStatus({ + ctx, + service: defaultBackendService, + env: gardenEnv, + }) + const dashboardStatus = await checkDeploymentStatus({ + ctx, + service: dashboardService, + env: gardenEnv, + }) + + return { + systemNamespaceReady, + dashboardReady: dashboardStatus.state === "ready", + ingressControllerReady: ingressControllerStatus.state === "ready", + defaultBackendReady: defaultBackendStatus.state === "ready", + } +} + +export async function configureGlobalSystem( + { ctx, env, logEntry }: ConfigureEnvironmentParams, status: EnvironmentStatus, +) { + if (!status.detail.systemNamespaceReady) { + logEntry && logEntry.setState({ section: "kubernetes", msg: `Creating garden system namespace` }) + await createNamespace(GARDEN_GLOBAL_SYSTEM_NAMESPACE) + } + + if (!status.detail.dashboardReady) { + logEntry && logEntry.setState({ section: "kubernetes", msg: `Configuring dashboard` }) + // TODO: deploy this as a service + await kubectl(GARDEN_GLOBAL_SYSTEM_NAMESPACE).call(["apply", "-f", dashboardSpecPath]) + } + + if (!status.detail.ingressControllerReady) { + logEntry && logEntry.setState({ section: "kubernetes", msg: `Configuring ingress controller` }) + const gardenEnv = getSystemEnv(env) + + await deployService({ + ctx, + service: await getDefaultBackendService(ctx), + serviceContext: { envVars: {}, dependencies: {} }, + env: gardenEnv, + logEntry, + }) + await deployService({ + ctx, + service: await getIngressControllerService(ctx), + serviceContext: { envVars: {}, dependencies: {} }, + env: gardenEnv, + exposePorts: true, + logEntry, + }) + } +} + +function getSystemEnv(env: Environment): Environment { + return { name: env.name, namespace: GARDEN_GLOBAL_SYSTEM_NAMESPACE, config: { providers: {} } } +} + +async function getIngressControllerService(ctx: GardenContext) { + const module = await ctx.resolveModule(ingressControllerModulePath) + + return ContainerService.factory(ctx, module, "ingress-controller") +} + +async function getDefaultBackendService(ctx: GardenContext) { + const module = await ctx.resolveModule(defaultBackendModulePath) + + return ContainerService.factory(ctx, module, "default-backend") +} + +async function getDashboardService(ctx: GardenContext) { + // TODO: implement raw kubernetes module load this module the same way as the ones above + const module = new ContainerModule(ctx, { + version: "0", + name: "garden-dashboard", + type: "container", + path: dashboardModulePath, + services: { + "kubernetes-dashboard": { + daemon: false, + dependencies: [], + endpoints: [], + ports: {}, + volumes: [], + }, + }, + variables: {}, + build: { dependencies: [] }, + test: {}, + }) + + return Service.factory(ctx, module, "kubernetes-dashboard") +} diff --git a/static/garden-default-backend/garden.yml b/static/kubernetes/system-global/default-backend/garden.yml similarity index 100% rename from static/garden-default-backend/garden.yml rename to static/kubernetes/system-global/default-backend/garden.yml diff --git a/static/garden-ingress-controller/garden.yml b/static/kubernetes/system-global/ingress-controller/garden.yml similarity index 100% rename from static/garden-ingress-controller/garden.yml rename to static/kubernetes/system-global/ingress-controller/garden.yml diff --git a/static/garden-dashboard/dashboard.yml b/static/kubernetes/system-global/kubernetes-dashboard/dashboard.yml similarity index 97% rename from static/garden-dashboard/dashboard.yml rename to static/kubernetes/system-global/kubernetes-dashboard/dashboard.yml index 0a9e2908b5..b05adb0c12 100644 --- a/static/garden-dashboard/dashboard.yml +++ b/static/kubernetes/system-global/kubernetes-dashboard/dashboard.yml @@ -45,6 +45,8 @@ metadata: kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: + labels: + k8s-app: kubernetes-dashboard name: kubernetes-dashboard-minimal namespace: garden-system rules: @@ -80,6 +82,8 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: + labels: + k8s-app: kubernetes-dashboard name: kubernetes-dashboard-minimal namespace: garden-system roleRef: @@ -157,13 +161,13 @@ apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard - name: kubernetes-dashboard + name: dashboard namespace: garden-system spec: type: NodePort ports: - port: 443 - nodePort: 32001 + nodePort: 3200 targetPort: 8443 selector: k8s-app: kubernetes-dashboard diff --git a/static/garden-dashboard/garden.yml b/static/kubernetes/system-global/kubernetes-dashboard/garden.yml similarity index 65% rename from static/garden-dashboard/garden.yml rename to static/kubernetes/system-global/kubernetes-dashboard/garden.yml index f2a8d9ca48..b456491cd4 100644 --- a/static/garden-dashboard/garden.yml +++ b/static/kubernetes/system-global/kubernetes-dashboard/garden.yml @@ -1,8 +1,7 @@ module: description: Kubernetes dashboard configuration name: k8s-dashboard - # TODO: add support for raw kubernetes specs as services - type: kubernetes + type: kubernetes-raw services: kubernetes-dashboard: specs: [dashboard.yml]