diff --git a/garden-service/src/logger/renderers.ts b/garden-service/src/logger/renderers.ts index ca7ebf0d6e..2d128efa8d 100644 --- a/garden-service/src/logger/renderers.ts +++ b/garden-service/src/logger/renderers.ts @@ -42,7 +42,7 @@ const cliPadEnd = (s: string, width: number): string => { const truncateSection = (s: string) => cliTruncate(s, SECTION_PREFIX_WIDTH) const sectionStyle = (s: string) => chalk.cyan.italic(cliPadEnd(truncateSection(s), SECTION_PREFIX_WIDTH)) export const msgStyle = (s: string) => hasAnsi(s) ? s : chalk.gray(s) -export const errorStyle = chalk.red +export const errorStyle = (s: string) => hasAnsi(s) ? s : chalk.red(s) /*** RENDER HELPERS ***/ function insertVal(out: string[], idx: number, toRender: Function | string, renderArgs: any[]): string[] { diff --git a/garden-service/src/plugins/kubernetes/api.ts b/garden-service/src/plugins/kubernetes/api.ts index d909c1fe2c..07d83ca0a9 100644 --- a/garden-service/src/plugins/kubernetes/api.ts +++ b/garden-service/src/plugins/kubernetes/api.ts @@ -295,7 +295,8 @@ export class KubeApi { ? `${basePath}/namespaces/${namespace}/${resource.name}/${name}` : `${basePath}/${resource.name}/${name}` - return this.request(log, apiPath) + const res = await this.request(log, apiPath) + return res.body } async upsert( diff --git a/garden-service/src/plugins/kubernetes/container/deployment.ts b/garden-service/src/plugins/kubernetes/container/deployment.ts index 4febf887fa..37844b422d 100644 --- a/garden-service/src/plugins/kubernetes/container/deployment.ts +++ b/garden-service/src/plugins/kubernetes/container/deployment.ts @@ -11,7 +11,7 @@ import { RuntimeContext, Service, ServiceStatus } from "../../../types/service" import { ContainerModule, ContainerService } from "../../container/config" import { createIngressResources } from "./ingress" import { createServiceResources } from "./service" -import { waitForResources } from "../status" +import { waitForResources } from "../status/status" import { apply, deleteObjectsByLabel } from "../kubectl" import { getAppNamespace } from "../namespace" import { PluginContext } from "../../../plugin-context" diff --git a/garden-service/src/plugins/kubernetes/container/status.ts b/garden-service/src/plugins/kubernetes/container/status.ts index 3f82472c2c..149a4246ef 100644 --- a/garden-service/src/plugins/kubernetes/container/status.ts +++ b/garden-service/src/plugins/kubernetes/container/status.ts @@ -16,7 +16,7 @@ import { sleep } from "../../../util/util" import { GetServiceStatusParams } from "../../../types/plugin/service/getServiceStatus" import { ContainerModule } from "../../container/config" import { KubeApi } from "../api" -import { compareDeployedObjects } from "../status" +import { compareDeployedObjects } from "../status/status" import { getIngresses } from "./ingress" import { getAppNamespace } from "../namespace" import { KubernetesPluginContext } from "../config" diff --git a/garden-service/src/plugins/kubernetes/helm/deployment.ts b/garden-service/src/plugins/kubernetes/helm/deployment.ts index 1d27ffd575..4766083853 100644 --- a/garden-service/src/plugins/kubernetes/helm/deployment.ts +++ b/garden-service/src/plugins/kubernetes/helm/deployment.ts @@ -8,7 +8,7 @@ import { ServiceStatus } from "../../../types/service" import { getAppNamespace } from "../namespace" -import { waitForResources } from "../status" +import { waitForResources } from "../status/status" import { helm } from "./helm-cli" import { HelmModule } from "./config" import { diff --git a/garden-service/src/plugins/kubernetes/helm/status.ts b/garden-service/src/plugins/kubernetes/helm/status.ts index 15cdebc373..9c751a5abd 100644 --- a/garden-service/src/plugins/kubernetes/helm/status.ts +++ b/garden-service/src/plugins/kubernetes/helm/status.ts @@ -9,7 +9,7 @@ import { ServiceStatus, ServiceState } from "../../../types/service" import { GetServiceStatusParams } from "../../../types/plugin/service/getServiceStatus" import { getExecModuleBuildStatus } from "../../exec" -import { compareDeployedObjects } from "../status" +import { compareDeployedObjects } from "../status/status" import { KubeApi } from "../api" import { getAppNamespace } from "../namespace" import { LogEntry } from "../../../logger/log-entry" diff --git a/garden-service/src/plugins/kubernetes/helm/tiller.ts b/garden-service/src/plugins/kubernetes/helm/tiller.ts index 0483debb93..8c80a02d51 100644 --- a/garden-service/src/plugins/kubernetes/helm/tiller.ts +++ b/garden-service/src/plugins/kubernetes/helm/tiller.ts @@ -13,7 +13,7 @@ import { helm } from "./helm-cli" import { safeLoadAll } from "js-yaml" import { KubeApi } from "../api" import { getAppNamespace } from "../namespace" -import { checkResourceStatuses, waitForResources } from "../status" +import { checkResourceStatuses, waitForResources } from "../status/status" import { combineStates } from "../../../types/service" import { apply } from "../kubectl" import { KubernetesProvider } from "../config" diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 5c0d21ddce..5b3161d1fe 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -17,7 +17,7 @@ import { getNamespace, getAppNamespace } from "../namespace" import { KubernetesPluginContext } from "../config" import { KubernetesResource } from "../types" import { ServiceStatus } from "../../../types/service" -import { compareDeployedObjects, waitForResources } from "../status" +import { compareDeployedObjects, waitForResources } from "../status/status" import { KubeApi } from "../api" import { ModuleAndRuntimeActions } from "../../../types/plugin/plugin" import { getAllLogs } from "../logs" diff --git a/garden-service/src/plugins/kubernetes/status.ts b/garden-service/src/plugins/kubernetes/status.ts deleted file mode 100644 index da9f05a0f1..0000000000 --- a/garden-service/src/plugins/kubernetes/status.ts +++ /dev/null @@ -1,665 +0,0 @@ -/* - * Copyright (C) 2018 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import { diffString } from "json-diff" -import { DeploymentError } from "../../exceptions" -import { PluginContext } from "../../plugin-context" -import { ServiceState, combineStates } from "../../types/service" -import { sleep, encodeYamlMulti, deepMap } from "../../util/util" -import { KubeApi, KubernetesError } from "./api" -import { KUBECTL_DEFAULT_TIMEOUT, kubectl } from "./kubectl" -import { getAppNamespace } from "./namespace" -import * as Bluebird from "bluebird" -import { KubernetesResource } from "./types" -import { - V1Pod, - V1Deployment, - V1DaemonSet, - V1DaemonSetStatus, - V1StatefulSetStatus, - V1StatefulSet, - V1StatefulSetSpec, - V1DeploymentStatus, -} from "@kubernetes/client-node" -import { some, zip, isArray, isPlainObject, pickBy, mapValues, flatten } from "lodash" -import { KubernetesProvider, KubernetesPluginContext } from "./config" -import { isSubset } from "../../util/is-subset" -import { LogEntry } from "../../logger/log-entry" -import { V1ReplicationController, V1ReplicaSet } from "@kubernetes/client-node" -import dedent = require("dedent") -import { getWorkloadPods, getPods } from "./util" - -interface WorkloadStatus { - state: ServiceState - obj: KubernetesResource - lastMessage?: string - lastError?: string - warning?: true - resourceVersion?: number - logs?: string -} - -type Workload = KubernetesResource - -interface ObjHandler { - (api: KubeApi, namespace: string, obj: KubernetesResource, log: LogEntry, resourceVersion?: number) - : Promise -} - -const podLogLines = 20 - -// Handlers to check the rollout status for K8s objects where that applies. -// Using https://github.com/kubernetes/helm/blob/master/pkg/kube/wait.go as a reference here. -const objHandlers: { [kind: string]: ObjHandler } = { - DaemonSet: checkWorkloadStatus, - Deployment: checkWorkloadStatus, - StatefulSet: checkWorkloadStatus, - - PersistentVolumeClaim: async (api, namespace, obj) => { - const res = await api.core.readNamespacedPersistentVolumeClaim(obj.metadata.name, namespace) - const state: ServiceState = res.status.phase === "Bound" ? "ready" : "deploying" - return { state, obj } - }, - - Pod: async (api, namespace, obj) => { - const res = await api.core.readNamespacedPod(obj.metadata.name, namespace) - return checkPodStatus(obj, [res]) - }, - - ReplicaSet: async (api, namespace, obj) => { - return checkPodStatus(obj, await getPods( - api, namespace, (>obj).spec.selector!.matchLabels!), - ) - }, - - ReplicationController: async (api, namespace, obj) => { - return checkPodStatus(obj, await getPods( - api, namespace, (>obj).spec.selector), - ) - }, - - Service: async (api, namespace, obj) => { - if (obj.spec.type === "ExternalName") { - return { state: "ready", obj } - } - - const status = await api.core.readNamespacedService(obj.metadata.name, namespace) - - if (obj.spec.clusterIP !== "None" && status.spec.clusterIP === "") { - return { state: "deploying", obj } - } - - if (obj.spec.type === "LoadBalancer" && !status.status.loadBalancer!.ingress) { - return { state: "deploying", obj } - } - - return { state: "ready", obj } - }, -} - -async function checkPodStatus(obj: KubernetesResource, pods: KubernetesResource[]): Promise { - for (const pod of pods) { - // TODO: detect unhealthy state (currently we just time out) - const ready = some(pod.status!.conditions!.map(c => c.type === "ready")) - if (!ready) { - return { state: "deploying", obj } - } - } - - return { state: "ready", obj } -} - -/** - * Check the rollout status for the given Deployment, DaemonSet or StatefulSet. - * - * NOTE: This mostly replicates the logic in `kubectl rollout status`. Using that directly here - * didn't pan out, since it doesn't look for events and just times out when errors occur during rollout. - */ -export async function checkWorkloadStatus( - api: KubeApi, namespace: string, obj: KubernetesResource, log: LogEntry, resourceVersion?: number, -): Promise { - const out: WorkloadStatus = { - state: "unhealthy", - obj, - resourceVersion, - } - - let statusRes: Workload - - try { - statusRes = (await api.readBySpec(namespace, obj, log)).body - } catch (err) { - if (err.code && err.code === 404) { - // service is not running - out.lastError = `Could not find ${obj.kind} ${obj.metadata.name}` - return out - } else { - throw err - } - } - - if (!resourceVersion) { - resourceVersion = out.resourceVersion = parseInt(statusRes.metadata.resourceVersion!, 10) - } - - // TODO: try to come up with something more efficient. may need to wait for newer k8s version. - // note: the resourceVersion parameter does not appear to work... - const eventsRes = await api.core.listNamespacedEvent( - namespace, - true, - ) - - // look for errors and warnings in the events for the service, abort if we find any fatal errors - const events = eventsRes.items - - for (let event of events) { - const eventVersion = parseInt(event.involvedObject.resourceVersion || "0", 10) - - if ( - !(event.involvedObject.kind === obj.kind && event.involvedObject.name === obj.metadata.name) - && - !event.metadata.name!.startsWith(obj.metadata.name + ".") - && - !event.metadata.name!.startsWith(obj.metadata.name + "-") - ) { - continue - } - - // TODO: this isn't working right - if (eventVersion !== 0 && eventVersion <= resourceVersion) { - continue - } - - if ( - event.type === "Error" || - event.type === "Failed" || - (event.type === "Warning" && ( - event.message!.includes("CrashLoopBackOff") || - event.message!.includes("ImagePullBackOff") || - event.reason === "BackOff" - )) - ) { - out.state = "unhealthy" - out.lastError = `${event.reason} - ${event.message}` - - // TODO: fetch logs for the pods in the deployment - if (event.involvedObject.kind === "Pod") { - const logs = await getPodLogs(api, namespace, [event.involvedObject!.name!]) - - if (logs) { - out.logs = dedent` - - kubectl -n ${namespace} --context=${api.context} logs ${event.involvedObject.name} - - ` + logs - } - } else { - out.logs = await getWorkloadLogs(api, namespace, statusRes) - } - - return out - } else if (event.type === "Warning") { - out.warning = true - } - - let message = event.message - - if (event.reason === event.reason!.toUpperCase()) { - // some events like ingress events are formatted this way - message = `${event.reason} ${message}` - } - - if (message) { - out.lastMessage = message - } - } - - // See `https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/rollout_status.go` for a reference - // for this logic. - out.state = "ready" - let statusMsg = "" - - if (statusRes.metadata.generation! > statusRes.status!.observedGeneration!) { - statusMsg = `Waiting for spec update to be observed...` - out.state = "deploying" - } else if (obj.kind === "DaemonSet") { - const status = statusRes.status - - const desired = status.desiredNumberScheduled || 0 - const updated = status.updatedNumberScheduled || 0 - const available = status.numberAvailable || 0 - - if (updated < desired) { - statusMsg = `Waiting for rollout: ${updated} out of ${desired} new pods updated...` - out.state = "deploying" - } else if (available < desired) { - statusMsg = `Waiting for rollout: ${available} out of ${desired} updated pods available...` - out.state = "deploying" - } - } else if (obj.kind === "StatefulSet") { - const status = statusRes.status - const statusSpec = >statusRes.spec - - const replicas = status.replicas || 0 - const updated = status.updatedReplicas || 0 - const ready = status.readyReplicas || 0 - - if (replicas && ready < replicas) { - statusMsg = `Waiting for rollout: ${ready} out of ${replicas} new pods updated...` - out.state = "deploying" - } else if (statusSpec.updateStrategy.type === "RollingUpdate" && statusSpec.updateStrategy.rollingUpdate) { - if (replicas && statusSpec.updateStrategy.rollingUpdate.partition) { - const desired = replicas - statusSpec.updateStrategy.rollingUpdate.partition - if (updated < desired) { - statusMsg = - `Waiting for partitioned roll out to finish: ${updated} out of ${desired} new pods have been updated...` - out.state = "deploying" - } - } - } else if (status.updateRevision !== status.currentRevision) { - statusMsg = `Waiting for rolling update to complete...` - out.state = "deploying" - } - } else { - const status = statusRes.status - - const desired = status.replicas || 0 - const updated = status.updatedReplicas || 0 - const replicas = status.replicas || 0 - const available = status.availableReplicas || 0 - - if (updated < desired) { - statusMsg = `Waiting for rollout: ${updated} out of ${desired} new replicas updated...` - out.state = "deploying" - } else if (replicas > updated) { - statusMsg = `Waiting for rollout: ${replicas - updated} old replicas pending termination...` - out.state = "deploying" - } else if (available < updated) { - statusMsg = `Waiting for rollout: ${available} out of ${updated} updated replicas available...` - out.state = "deploying" - } - } - - if (!out.lastMessage) { - out.lastMessage = statusMsg - } - - // Catch timeout conditions here - if (out.state !== "ready") { - for (const condition of statusRes.status!.conditions || []) { - if (condition.status === "False" && condition.reason === "ProgressDeadlineExceeded") { - out.state = "unhealthy" - out.lastError = `${condition.reason} - ${condition.message}` - out.logs = await getWorkloadLogs(api, namespace, statusRes) - break - } - } - } - - return out -} - -/** - * Check if the specified Kubernetes objects are deployed and fully rolled out - */ -export async function checkResourceStatuses( - api: KubeApi, namespace: string, resources: KubernetesResource[], log: LogEntry, prevStatuses?: WorkloadStatus[], -): Promise { - return Bluebird.map(resources, async (obj, i) => { - return checkResourceStatus(api, namespace, obj, log, prevStatuses && prevStatuses[i]) - }) -} - -export async function checkResourceStatus( - api: KubeApi, namespace: string, resource: KubernetesResource, log: LogEntry, prevStatus?: WorkloadStatus, -) { - const handler = objHandlers[resource.kind] - - if (resource.metadata.namespace) { - namespace = resource.metadata.namespace - } - - let status: WorkloadStatus - if (handler) { - try { - status = await handler(api, namespace, resource, log, prevStatus && prevStatus.resourceVersion) - } catch (err) { - // We handle 404s specifically since this might be invoked before some objects are deployed - if (err.code === 404) { - status = { state: "missing", obj: resource } - } else { - throw err - } - } - } else { - // if there is no explicit handler to check the status, we assume there's no rollout phase to wait for - status = { state: "ready", obj: resource } - } - - return status -} - -interface WaitParams { - ctx: PluginContext, - provider: KubernetesProvider, - serviceName: string, - resources: KubernetesResource[], - log: LogEntry, -} - -/** - * Wait until the rollout is complete for each of the given Kubernetes objects - */ -export async function waitForResources({ ctx, provider, serviceName, resources: objects, log }: WaitParams) { - let loops = 0 - let lastMessage - const startTime = new Date().getTime() - - const statusLine = log.info({ - symbol: "info", - section: serviceName, - msg: `Waiting for service to be ready...`, - }) - - const api = await KubeApi.factory(log, provider.config.context) - const namespace = await getAppNamespace(ctx, log, provider) - let prevStatuses: WorkloadStatus[] = objects.map((obj) => ({ - state: "unknown", - obj, - })) - - while (true) { - await sleep(2000 + 1000 * loops) - loops += 1 - - const statuses = await checkResourceStatuses(api, namespace, objects, log, prevStatuses) - - for (const status of statuses) { - if (status.lastError) { - let msg = `Error deploying ${serviceName}: ${status.lastError}` - - if (status.logs !== undefined) { - msg += "\n\nLogs:\n\n" + status.logs - } - - throw new DeploymentError(msg, { - serviceName, - status, - }) - } - - if (status.lastMessage && (!lastMessage || status.lastMessage !== lastMessage)) { - lastMessage = status.lastMessage - const symbol = status.warning === true ? "warning" : "info" - statusLine.setState({ - symbol, - msg: `${status.obj.kind}/${status.obj.metadata.name}: ${status.lastMessage}`, - }) - } - } - - prevStatuses = statuses - - if (combineStates(statuses.map(s => s.state)) === "ready") { - break - } - - const now = new Date().getTime() - - if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { - throw new DeploymentError(`Timed out waiting for ${serviceName} to deploy`, { statuses }) - } - } - - statusLine.setState({ symbol: "info", section: serviceName, msg: `Service deployed` }) -} - -interface ComparisonResult { - state: ServiceState - remoteObjects: KubernetesResource[] -} - -/** - * Check if each of the given Kubernetes objects matches what's installed in the cluster - */ -export async function compareDeployedObjects( - ctx: KubernetesPluginContext, api: KubeApi, namespace: string, resources: KubernetesResource[], log: LogEntry, - skipDiff: boolean, -): Promise { - // Unroll any `List` resource types - resources = flatten(resources.map((r: any) => r.apiVersion === "v1" && r.kind === "List" ? r.items : [r])) - - // Check if any resources are missing from the cluster. - const maybeDeployedObjects = await Bluebird.map(resources, obj => getDeployedObject(ctx, ctx.provider, obj, log)) - const deployedObjects = maybeDeployedObjects.filter(o => o !== null) - - const result: ComparisonResult = { - state: "unknown", - remoteObjects: deployedObjects.filter(o => o !== null), - } - - const logDescription = (obj: KubernetesResource) => `${obj.kind}/${obj.metadata.name}` - - const missingObjectNames = zip(resources, maybeDeployedObjects) - .filter(([_, deployed]) => !deployed) - .map(([obj, _]) => logDescription(obj!)) - - if (missingObjectNames.length === resources.length) { - // All resources missing. - log.verbose(`All resources missing from cluster`) - result.state = "missing" - return result - } else if (missingObjectNames.length > 0) { - // One or more objects missing. - log.verbose(`Resource(s) ${missingObjectNames.join(", ")} missing from cluster`) - result.state = "outdated" - return result - } - - // From here, the state can only be "ready" or "outdated", so we proceed to compare the old & new specs. - - // TODO: The skipDiff parameter is a temporary workaround until we finish implementing diffing in a more reliable way. - if (!skipDiff) { - // First we try using `kubectl diff`, to avoid potential normalization issues (i.e. false negatives). This errors - // with exit code 1 if there is a mismatch, but may also fail with the same exit code for a number of other reasons, - // including the cluster not supporting dry-runs, certain CRDs not supporting dry-runs etc. - const yamlResources = await encodeYamlMulti(resources) - const context = ctx.provider.config.context - - try { - await kubectl.exec({ log, context, namespace, args: ["diff", "-f", "-"], input: Buffer.from(yamlResources) }) - - // If the commands exits succesfully, the check was successful and the diff is empty. - log.verbose(`kubectl diff indicates all resources match the deployed resources.`) - result.state = "ready" - return result - } catch (err) { - // Exited with non-zero code. Check for error messages on stderr. If one is there, the command was unable to - // complete the check, so we fall back to our own mechanism. Otherwise the command worked, but one or more - // resources are missing or outdated. - if (err.stderr && err.stderr.trim() !== "exit status 1") { - log.verbose(`kubectl diff failed: ${err.message}\n${err.stderr}`) - } else { - log.verbose(`kubectl diff indicates one or more resources are outdated.`) - log.silly(err.stdout) - result.state = "outdated" - return result - } - } - } - - // Using kubectl diff didn't work, so we fall back to our own comparison check, which works in _most_ cases, - // but doesn't exhaustively handle normalization issues. - log.verbose(`Getting currently deployed resources...`) - - const deployedObjectStatuses: WorkloadStatus[] = await Bluebird.map( - deployedObjects, - async (obj) => checkResourceStatus(api, namespace, obj, log, undefined)) - - const deployedStates = deployedObjectStatuses.map(s => s.state) - if (deployedStates.find(s => s !== "ready")) { - - const descriptions = zip(deployedObjects, deployedStates) - .filter(([_, s]) => s !== "ready") - .map(([o, s]) => `${logDescription(o!)}: "${s}"`).join("\n") - - log.silly(dedent` - Resource(s) with non-ready status found in the cluster: - - ${descriptions}` + "\n") - - result.state = combineStates(deployedStates) - return result - } - - log.verbose(`Comparing expected and deployed resources...`) - - for (let [newSpec, existingSpec] of zip(resources, deployedObjects) as KubernetesResource[][]) { - // to avoid normalization issues, we convert all numeric values to strings and then compare - newSpec = deepMap(newSpec, v => typeof v === "number" ? v.toString() : v) - existingSpec = deepMap(existingSpec, v => typeof v === "number" ? v.toString() : v) - - // the API version may implicitly change when deploying - existingSpec.apiVersion = newSpec.apiVersion - - // the namespace property is silently dropped when added to non-namespaced - if (newSpec.metadata.namespace && existingSpec.metadata.namespace === undefined) { - delete newSpec.metadata.namespace - } - - if (!existingSpec.metadata.annotations) { - existingSpec.metadata.annotations = {} - } - - // handle auto-filled properties (this is a bit of a design issue in the K8s API) - if (newSpec.kind === "Service" && newSpec.spec.clusterIP === "") { - delete newSpec.spec.clusterIP - } - - // NOTE: this approach won't fly in the long run, but hopefully we can climb out of this mess when - // `kubectl diff` is ready, or server-side apply/diff is ready - if (newSpec.kind === "DaemonSet" || newSpec.kind === "Deployment" || newSpec.kind == "StatefulSet") { - // handle properties that are omitted in the response because they have the default value - // (another design issue in the K8s API) - if (newSpec.spec.minReadySeconds === 0) { - delete newSpec.spec.minReadySeconds - } - if (newSpec.spec.template && newSpec.spec.template.spec && newSpec.spec.template.spec.hostNetwork === false) { - delete newSpec.spec.template.spec.hostNetwork - } - } - - // clean null values - newSpec = removeNull(newSpec) - - if (!isSubset(existingSpec, newSpec)) { - if (newSpec) { - log.verbose(`Resource ${newSpec.metadata.name} is not a superset of deployed resource:`) - log.verbose(diffString(existingSpec, newSpec)) - } - // console.log(JSON.stringify(obj, null, 4)) - // console.log(JSON.stringify(existingSpec, null, 4)) - // console.log("----------------------------------------------------") - // throw new Error("bla") - result.state = "outdated" - return result - } - } - - log.verbose(`All resources match. Environment is ready.`) - - result.state = "ready" - return result -} - -async function getDeployedObject( - ctx: PluginContext, provider: KubernetesProvider, obj: KubernetesResource, log: LogEntry, -): Promise { - const api = await KubeApi.factory(log, provider.config.context) - const namespace = obj.metadata.namespace || await getAppNamespace(ctx, log, provider) - - try { - const res = await api.readBySpec(namespace, obj, log) - return res.body - } catch (err) { - if (err.code === 404) { - return null - } else { - throw err - } - } -} - -/** - * Recursively removes all null value properties from objects - */ -function removeNull(value: T | Iterable): T | Iterable | { [K in keyof T]: T[K] } { - if (isArray(value)) { - return value.map(removeNull) - } else if (isPlainObject(value)) { - return <{ [K in keyof T]: T[K] }>mapValues(pickBy(value, v => v !== null), removeNull) - } else { - return value - } -} - -/** - * Get a formatted list of log tails for each of the specified pods. Used for debugging and error logs. - */ -async function getPodLogs(api: KubeApi, namespace: string, podNames: string[]): Promise { - const allLogs = await Bluebird.map(podNames, async (name) => { - let containerName: string | undefined - - try { - const podRes = await api.core.readNamespacedPod(name, namespace) - const containerNames = podRes.spec.containers.map(c => c.name) - if (containerNames.length > 1) { - containerName = containerNames.filter(n => !n.match(/garden-/))[0] || containerNames[0] - } else { - containerName = containerNames[0] - } - } catch (err) { - if (err.code === 404) { - return "" - } else { - throw err - } - } - - // Putting 5000 bytes as a length limit in addition to the line limit, just as a precaution in case someone - // accidentally logs a binary file or something. - try { - const log = await api.core.readNamespacedPodLog( - name, namespace, containerName, false, 5000, undefined, false, undefined, podLogLines, - ) - return log ? `****** ${name} ******\n${log}` : "" - } catch (err) { - if (err instanceof KubernetesError && err.message.includes("waiting to start")) { - return "" - } else { - throw err - } - } - }) - return allLogs.filter(l => l !== "").join("\n\n") -} - -async function getWorkloadLogs(api: KubeApi, namespace: string, obj: Workload) { - const pods = await getWorkloadPods(api, namespace, obj) - const logs = await getPodLogs(api, namespace, pods.map(pod => pod.metadata.name)) - - if (logs) { - return dedent` - - kubectl -n ${namespace} --context=${api.context} logs ${obj.kind.toLowerCase()}/${obj.metadata.name} - - ` + logs - } else { - return undefined - } -} diff --git a/garden-service/src/plugins/kubernetes/status/events.ts b/garden-service/src/plugins/kubernetes/status/events.ts new file mode 100644 index 0000000000..463ea23e3b --- /dev/null +++ b/garden-service/src/plugins/kubernetes/status/events.ts @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { KubeApi } from "../api" +import { KubernetesResource } from "../types" + +export async function getResourceEvents(api: KubeApi, resource: KubernetesResource, minVersion?: number) { + const fieldSelector = + `involvedObject.apiVersion=${resource.apiVersion},` + + `involvedObject.kind=${resource.kind},` + + `involvedObject.name=${resource.metadata.name}` + + const namespace = resource.metadata.namespace + + const res = namespace + ? await api.core.listNamespacedEvent(namespace, undefined, undefined, undefined, fieldSelector) + : await api.core.listEventForAllNamespaces(undefined, fieldSelector) + + const events = res.items + // Filter out old events (relating to prior versions of the resource) + .filter(e => + !minVersion + || !e.involvedObject!.resourceVersion + || parseInt(e.involvedObject!.resourceVersion, 10) > minVersion, + ) + + return events +} diff --git a/garden-service/src/plugins/kubernetes/status/pod.ts b/garden-service/src/plugins/kubernetes/status/pod.ts new file mode 100644 index 0000000000..a9e75365a9 --- /dev/null +++ b/garden-service/src/plugins/kubernetes/status/pod.ts @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { KubeApi, KubernetesError } from "../api" +import * as Bluebird from "bluebird" +import { KubernetesServerResource } from "../types" +import { V1Pod } from "@kubernetes/client-node" +import { some } from "lodash" +import { ResourceStatus } from "./status" +import chalk from "chalk" + +export const podLogLines = 20 + +export async function checkPodStatus( + resource: KubernetesServerResource, pods: KubernetesServerResource[], +): Promise { + for (const pod of pods) { + // TODO: detect unhealthy state (currently we just time out) + const ready = some(pod.status!.conditions!.map(c => c.type === "ready")) + if (!ready) { + return { state: "deploying", resource } + } + } + + return { state: "ready", resource } +} + +/** + * Get a formatted list of log tails for each of the specified pods. Used for debugging and error logs. + */ +export async function getPodLogs(api: KubeApi, namespace: string, podNames: string[]): Promise { + const allLogs = await Bluebird.map(podNames, async (name) => { + let containerName: string | undefined + + try { + const podRes = await api.core.readNamespacedPod(name, namespace) + const containerNames = podRes.spec.containers.map(c => c.name) + if (containerNames.length > 1) { + containerName = containerNames.filter(n => !n.match(/garden-/))[0] || containerNames[0] + } else { + containerName = containerNames[0] + } + } catch (err) { + if (err.code === 404) { + return "" + } else { + throw err + } + } + + // Putting 5000 bytes as a length limit in addition to the line limit, just as a precaution in case someone + // accidentally logs a binary file or something. + try { + const log = await api.core.readNamespacedPodLog( + name, namespace, containerName, false, 5000, undefined, false, undefined, podLogLines, + ) + return log ? chalk.blueBright(`\n****** ${name} ******\n`) + log : "" + } catch (err) { + if (err instanceof KubernetesError && err.message.includes("waiting to start")) { + return "" + } else { + throw err + } + } + }) + return allLogs.filter(l => l !== "").join("\n\n") +} diff --git a/garden-service/src/plugins/kubernetes/status/status.ts b/garden-service/src/plugins/kubernetes/status/status.ts new file mode 100644 index 0000000000..3500c10f80 --- /dev/null +++ b/garden-service/src/plugins/kubernetes/status/status.ts @@ -0,0 +1,406 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { diffString } from "json-diff" +import { DeploymentError } from "../../../exceptions" +import { PluginContext } from "../../../plugin-context" +import { ServiceState, combineStates } from "../../../types/service" +import { sleep, encodeYamlMulti, deepMap } from "../../../util/util" +import { KubeApi } from "../api" +import { KUBECTL_DEFAULT_TIMEOUT, kubectl } from "../kubectl" +import { getAppNamespace } from "../namespace" +import * as Bluebird from "bluebird" +import { KubernetesResource, KubernetesServerResource } from "../types" +import { zip, isArray, isPlainObject, pickBy, mapValues, flatten } from "lodash" +import { KubernetesProvider, KubernetesPluginContext } from "../config" +import { isSubset } from "../../../util/is-subset" +import { LogEntry } from "../../../logger/log-entry" +import { + V1ReplicationController, + V1ReplicaSet, + V1Pod, + V1PersistentVolumeClaim, + V1Service, +} from "@kubernetes/client-node" +import dedent = require("dedent") +import { getPods } from "../util" +import { checkWorkloadStatus } from "./workload" +import { checkPodStatus } from "./pod" + +export interface ResourceStatus { + state: ServiceState + resource: KubernetesServerResource + lastMessage?: string + warning?: true + logs?: string +} + +export interface StatusHandlerParams { + api: KubeApi + namespace: string + resource: KubernetesServerResource + log: LogEntry + resourceVersion?: number, +} + +interface ObjHandler { + (params: StatusHandlerParams): Promise +} + +// Handlers to check the rollout status for K8s objects where that applies. +// Using https://github.com/kubernetes/helm/blob/master/pkg/kube/wait.go as a reference here. +const objHandlers: { [kind: string]: ObjHandler } = { + DaemonSet: checkWorkloadStatus, + Deployment: checkWorkloadStatus, + StatefulSet: checkWorkloadStatus, + + PersistentVolumeClaim: async ({ resource }) => { + const pvc = >resource + const state: ServiceState = pvc.status.phase === "Bound" ? "ready" : "deploying" + return { state, resource } + }, + + Pod: async ({ resource }) => { + return checkPodStatus(resource, [>resource]) + }, + + ReplicaSet: async ({ api, namespace, resource }) => { + return checkPodStatus(resource, await getPods( + api, namespace, (>resource).spec.selector!.matchLabels!), + ) + }, + + ReplicationController: async ({ api, namespace, resource }) => { + return checkPodStatus(resource, await getPods( + api, namespace, (>resource).spec.selector), + ) + }, + + Service: async ({ resource }) => { + if (resource.spec.type === "ExternalName") { + return { state: "ready", resource } + } + + const service = >resource + + if (resource.spec.clusterIP !== "None" && service.spec.clusterIP === "") { + return { state: "deploying", resource } + } + + if (resource.spec.type === "LoadBalancer" && !service.status.loadBalancer!.ingress) { + return { state: "deploying", resource } + } + + return { state: "ready", resource } + }, +} + +/** + * Check if the specified Kubernetes objects are deployed and fully rolled out + */ +export async function checkResourceStatuses( + api: KubeApi, namespace: string, manifests: KubernetesResource[], log: LogEntry, +): Promise { + return Bluebird.map(manifests, async (manifest) => { + return checkResourceStatus(api, namespace, manifest, log) + }) +} + +export async function checkResourceStatus( + api: KubeApi, namespace: string, manifest: KubernetesResource, log: LogEntry, +) { + const handler = objHandlers[manifest.kind] + + if (manifest.metadata.namespace) { + namespace = manifest.metadata.namespace + } + + let resource: KubernetesServerResource + let resourceVersion: number | undefined + + try { + resource = await api.readBySpec(namespace, manifest, log) + resourceVersion = parseInt(resource.metadata.resourceVersion!, 10) + } catch (err) { + if (err.code === 404) { + return { state: "missing", resource: manifest } + } else { + throw err + } + } + + let status: ResourceStatus + if (handler) { + status = await handler({ api, namespace, resource, log, resourceVersion }) + } else { + // if there is no explicit handler to check the status, we assume there's no rollout phase to wait for + status = { state: "ready", resource: manifest } + } + + return status +} + +interface WaitParams { + ctx: PluginContext, + provider: KubernetesProvider, + serviceName: string, + resources: KubernetesResource[], + log: LogEntry, +} + +/** + * Wait until the rollout is complete for each of the given Kubernetes objects + */ +export async function waitForResources({ ctx, provider, serviceName, resources: objects, log }: WaitParams) { + let loops = 0 + let lastMessage: string | undefined + const startTime = new Date().getTime() + + const statusLine = log.info({ + symbol: "info", + section: serviceName, + msg: `Waiting for service to be ready...`, + }) + + const api = await KubeApi.factory(log, provider.config.context) + const namespace = await getAppNamespace(ctx, log, provider) + + while (true) { + await sleep(2000 + 1000 * loops) + loops += 1 + + const statuses = await checkResourceStatuses(api, namespace, objects, log) + + for (const status of statuses) { + if (status.state === "unhealthy") { + let msg = `Error deploying ${serviceName}: ${status.lastMessage}` + + if (status.logs) { + msg += "\n\n" + status.logs + } + + throw new DeploymentError(msg, { + serviceName, + status, + }) + } + + if (status.lastMessage && (!lastMessage || status.lastMessage !== lastMessage)) { + lastMessage = status.lastMessage + const symbol = status.warning === true ? "warning" : "info" + statusLine.setState({ + symbol, + msg: `${status.resource.kind}/${status.resource.metadata.name}: ${status.lastMessage}`, + }) + } + } + + if (combineStates(statuses.map(s => s.state)) === "ready") { + break + } + + const now = new Date().getTime() + + if (now - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) { + throw new DeploymentError(`Timed out waiting for ${serviceName} to deploy`, { statuses }) + } + } + + statusLine.setState({ symbol: "info", section: serviceName, msg: `Service deployed` }) +} + +interface ComparisonResult { + state: ServiceState + remoteObjects: KubernetesResource[] +} + +/** + * Check if each of the given Kubernetes objects matches what's installed in the cluster + */ +export async function compareDeployedObjects( + ctx: KubernetesPluginContext, api: KubeApi, namespace: string, resources: KubernetesResource[], log: LogEntry, + skipDiff: boolean, +): Promise { + // Unroll any `List` resource types + resources = flatten(resources.map((r: any) => r.apiVersion === "v1" && r.kind === "List" ? r.items : [r])) + + // Check if any resources are missing from the cluster. + const maybeDeployedObjects = await Bluebird.map( + resources, resource => getDeployedResource(ctx, ctx.provider, resource, log), + ) + const deployedObjects = maybeDeployedObjects.filter(o => o !== null) + + const result: ComparisonResult = { + state: "unknown", + remoteObjects: deployedObjects.filter(o => o !== null), + } + + const logDescription = (resource: KubernetesResource) => `${resource.kind}/${resource.metadata.name}` + + const missingObjectNames = zip(resources, maybeDeployedObjects) + .filter(([_, deployed]) => !deployed) + .map(([resource, _]) => logDescription(resource!)) + + if (missingObjectNames.length === resources.length) { + // All resources missing. + log.verbose(`All resources missing from cluster`) + result.state = "missing" + return result + } else if (missingObjectNames.length > 0) { + // One or more objects missing. + log.verbose(`Resource(s) ${missingObjectNames.join(", ")} missing from cluster`) + result.state = "outdated" + return result + } + + // From here, the state can only be "ready" or "outdated", so we proceed to compare the old & new specs. + + // TODO: The skipDiff parameter is a temporary workaround until we finish implementing diffing in a more reliable way. + if (!skipDiff) { + // First we try using `kubectl diff`, to avoid potential normalization issues (i.e. false negatives). This errors + // with exit code 1 if there is a mismatch, but may also fail with the same exit code for a number of other reasons, + // including the cluster not supporting dry-runs, certain CRDs not supporting dry-runs etc. + const yamlResources = await encodeYamlMulti(resources) + const context = ctx.provider.config.context + + try { + await kubectl.exec({ log, context, namespace, args: ["diff", "-f", "-"], input: Buffer.from(yamlResources) }) + + // If the commands exits succesfully, the check was successful and the diff is empty. + log.verbose(`kubectl diff indicates all resources match the deployed resources.`) + result.state = "ready" + return result + } catch (err) { + // Exited with non-zero code. Check for error messages on stderr. If one is there, the command was unable to + // complete the check, so we fall back to our own mechanism. Otherwise the command worked, but one or more + // resources are missing or outdated. + if (err.stderr && err.stderr.trim() !== "exit status 1") { + log.verbose(`kubectl diff failed: ${err.message}\n${err.stderr}`) + } else { + log.verbose(`kubectl diff indicates one or more resources are outdated.`) + log.silly(err.stdout) + result.state = "outdated" + return result + } + } + } + + // Using kubectl diff didn't work, so we fall back to our own comparison check, which works in _most_ cases, + // but doesn't exhaustively handle normalization issues. + log.verbose(`Getting currently deployed resources...`) + + const deployedObjectStatuses: ResourceStatus[] = await Bluebird.map( + deployedObjects, + async (resource) => checkResourceStatus(api, namespace, resource, log)) + + const deployedStates = deployedObjectStatuses.map(s => s.state) + if (deployedStates.find(s => s !== "ready")) { + + const descriptions = zip(deployedObjects, deployedStates) + .filter(([_, s]) => s !== "ready") + .map(([o, s]) => `${logDescription(o!)}: "${s}"`).join("\n") + + log.silly(dedent` + Resource(s) with non-ready status found in the cluster: + + ${descriptions}` + "\n") + + result.state = combineStates(deployedStates) + return result + } + + log.verbose(`Comparing expected and deployed resources...`) + + for (let [newSpec, existingSpec] of zip(resources, deployedObjects) as KubernetesResource[][]) { + // to avoid normalization issues, we convert all numeric values to strings and then compare + newSpec = deepMap(newSpec, v => typeof v === "number" ? v.toString() : v) + existingSpec = deepMap(existingSpec, v => typeof v === "number" ? v.toString() : v) + + // the API version may implicitly change when deploying + existingSpec.apiVersion = newSpec.apiVersion + + // the namespace property is silently dropped when added to non-namespaced + if (newSpec.metadata.namespace && existingSpec.metadata.namespace === undefined) { + delete newSpec.metadata.namespace + } + + if (!existingSpec.metadata.annotations) { + existingSpec.metadata.annotations = {} + } + + // handle auto-filled properties (this is a bit of a design issue in the K8s API) + if (newSpec.kind === "Service" && newSpec.spec.clusterIP === "") { + delete newSpec.spec.clusterIP + } + + // NOTE: this approach won't fly in the long run, but hopefully we can climb out of this mess when + // `kubectl diff` is ready, or server-side apply/diff is ready + if (newSpec.kind === "DaemonSet" || newSpec.kind === "Deployment" || newSpec.kind == "StatefulSet") { + // handle properties that are omitted in the response because they have the default value + // (another design issue in the K8s API) + if (newSpec.spec.minReadySeconds === 0) { + delete newSpec.spec.minReadySeconds + } + if (newSpec.spec.template && newSpec.spec.template.spec && newSpec.spec.template.spec.hostNetwork === false) { + delete newSpec.spec.template.spec.hostNetwork + } + } + + // clean null values + newSpec = removeNull(newSpec) + + if (!isSubset(existingSpec, newSpec)) { + if (newSpec) { + log.verbose(`Resource ${newSpec.metadata.name} is not a superset of deployed resource:`) + log.verbose(diffString(existingSpec, newSpec)) + } + // console.log(JSON.stringify(resource, null, 4)) + // console.log(JSON.stringify(existingSpec, null, 4)) + // console.log("----------------------------------------------------") + // throw new Error("bla") + result.state = "outdated" + return result + } + } + + log.verbose(`All resources match. Environment is ready.`) + + result.state = "ready" + return result +} + +async function getDeployedResource( + ctx: PluginContext, provider: KubernetesProvider, resource: KubernetesResource, log: LogEntry, +): Promise { + const api = await KubeApi.factory(log, provider.config.context) + const namespace = resource.metadata.namespace || await getAppNamespace(ctx, log, provider) + + try { + const res = await api.readBySpec(namespace, resource, log) + return res + } catch (err) { + if (err.code === 404) { + return null + } else { + throw err + } + } +} + +/** + * Recursively removes all null value properties from objects + */ +function removeNull(value: T | Iterable): T | Iterable | { [K in keyof T]: T[K] } { + if (isArray(value)) { + return value.map(removeNull) + } else if (isPlainObject(value)) { + return <{ [K in keyof T]: T[K] }>mapValues(pickBy(value, v => v !== null), removeNull) + } else { + return value + } +} diff --git a/garden-service/src/plugins/kubernetes/status/workload.ts b/garden-service/src/plugins/kubernetes/status/workload.ts new file mode 100644 index 0000000000..5cbbf9248c --- /dev/null +++ b/garden-service/src/plugins/kubernetes/status/workload.ts @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import chalk from "chalk" +import { flatten, sortBy } from "lodash" +import { KubernetesPod, KubernetesServerResource } from "../types" +import { + V1Deployment, + V1DaemonSet, + V1DaemonSetStatus, + V1StatefulSetStatus, + V1StatefulSet, + V1StatefulSetSpec, + V1DeploymentStatus, + V1Event, +} from "@kubernetes/client-node" +import dedent = require("dedent") +import { getWorkloadPods } from "../util" +import { getPodLogs, podLogLines } from "./pod" +import { ResourceStatus, StatusHandlerParams } from "./status" +import { getResourceEvents } from "./events" + +type Workload = KubernetesServerResource + +interface Condition { + message?: string + reason?: string +} + +/** + * Check the rollout status for the given Deployment, DaemonSet or StatefulSet. + * + * NOTE: This mostly replicates the logic in `kubectl rollout status`. Using that directly here + * didn't pan out, since it doesn't look for events and just times out when errors occur during rollout. + */ +export async function checkWorkloadStatus( + { api, namespace, resource }: StatusHandlerParams, +): Promise { + const workload = resource + + let _pods: KubernetesPod[] + let _events: V1Event[] + + const getPods = async () => { + if (!_pods) { + _pods = await getWorkloadPods(api, namespace, workload) + } + return _pods + } + + const getEvents = async () => { + if (!_events) { + // Get all relevant events for the workload + const workloadEvents = await getResourceEvents(api, workload) + const pods = await getPods() + const podEvents = flatten(await Promise.all(pods.map(pod => getResourceEvents(api, pod)))) + _events = sortBy([...workloadEvents, ...podEvents], e => e.metadata.creationTimestamp) + } + return _events + } + + const fail = async (lastMessage: string) => { + let logs = "" + + // List events + const events = await getEvents() + if (events.length > 0) { + logs += chalk.white("━━━ Events ━━━") + for (const event of events) { + const obj = event.involvedObject + const name = chalk.blueBright(`${obj.kind} ${obj.name}:`) + const msg = `${event.reason} - ${event.message}` + const colored = event.type === "Error" ? chalk.red(msg) : + event.type === "Warning" ? chalk.yellow(msg) : chalk.white(msg) + logs += `\n${name} ${colored}` + } + } + + // Attach pod logs for debug output + const podNames = (await getPods()).map(pod => pod.metadata.name) + const podLogs = await getPodLogs(api, namespace, podNames) || undefined + + if (podLogs) { + logs += chalk.white("\n\n━━━ Pod logs ━━━\n") + logs += chalk.gray(dedent` + + $ kubectl -n ${namespace} --context=${api.context} logs ${workload.kind.toLowerCase()}/${workload.metadata.name} + `) + "\n" + podLogs + } + + return { state: "unhealthy", lastMessage, logs, resource: workload } + } + + const failWithCondition = (condition: Condition) => { + return fail(`${condition.reason} - ${condition.message}`) + } + + // Check the reported rollout status on the workload resource itself. + const out = await getRolloutStatus(workload) + + // All set, nothing more to check! + if (out.state === "ready") { + return out + } + + // Catch timeout conditions + for (const condition of workload.status!.conditions || []) { + if (condition.status === "False" && condition.reason === "ProgressDeadlineExceeded") { + return failWithCondition(condition) + } + } + + // Look for warnings and fatal errors in pod statuses + for (const pod of await getPods()) { + const status = pod.status! + const containerStatuses = status.containerStatuses || [] + + for (const containerStatus of containerStatuses) { + const condition = containerStatus.state && containerStatus.state.waiting && containerStatus.state.waiting + if ( + condition && ( + condition.reason === "CrashLoopBackOff" || + condition.reason === "ImagePullBackOff" + ) + ) { + return failWithCondition(condition) + } + } + } + + // Look for warnings or failures in the events, + // so that we can display them or fail fast instead of timing out + for (let event of await getEvents()) { + if ( + event.type === "Error" || + event.type === "Failed" || + (event.type === "Warning" && ( + event.message!.includes("CrashLoopBackOff") || + event.message!.includes("ImagePullBackOff") || + event.message!.includes("DeadlineExceeded") || + event.reason === "BackOff" + )) + ) { + return failWithCondition(event) + } + + if (event.type === "Warning") { + out.warning = true + } + + let message = event.message + + if (event.reason === event.reason!.toUpperCase()) { + // some events like ingress events are formatted this way + message = `${event.reason} ${message}` + } + + if (message) { + out.lastMessage = message + } + } + + return out +} + +async function getRolloutStatus(workload: Workload) { + const out: ResourceStatus = { + state: "unhealthy", + resource: workload, + } + + out.state = "ready" + + // See `https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/rollout_status.go` for a reference + // for this logic. + if (workload.metadata.generation! > workload.status!.observedGeneration!) { + out.lastMessage = `Waiting for spec update to be observed...` + out.state = "deploying" + } else if (workload.kind === "DaemonSet") { + const status = workload.status + + const desired = status.desiredNumberScheduled || 0 + const updated = status.updatedNumberScheduled || 0 + const available = status.numberAvailable || 0 + + if (updated < desired) { + out.lastMessage = `Waiting for rollout: ${updated} out of ${desired} new pods updated...` + out.state = "deploying" + } else if (available < desired) { + out.lastMessage = `Waiting for rollout: ${available} out of ${desired} updated pods available...` + out.state = "deploying" + } + } else if (workload.kind === "StatefulSet") { + const status = workload.status + const workloadSpec = >workload.spec + + const replicas = status.replicas || 0 + const updated = status.updatedReplicas || 0 + const ready = status.readyReplicas || 0 + + if (replicas && ready < replicas) { + out.lastMessage = `Waiting for rollout: ${ready} out of ${replicas} new pods updated...` + out.state = "deploying" + } else if (workloadSpec.updateStrategy.type === "RollingUpdate" && workloadSpec.updateStrategy.rollingUpdate) { + if (replicas && workloadSpec.updateStrategy.rollingUpdate.partition) { + const desired = replicas - workloadSpec.updateStrategy.rollingUpdate.partition + if (updated < desired) { + out.lastMessage = + `Waiting for partitioned roll out to finish: ${updated} out of ${desired} new pods have been updated...` + out.state = "deploying" + } + } + } else if (status.updateRevision !== status.currentRevision) { + out.lastMessage = `Waiting for rolling update to complete...` + out.state = "deploying" + } + } else { + const status = workload.status + + const desired = status.replicas || 0 + const updated = status.updatedReplicas || 0 + const replicas = status.replicas || 0 + const available = status.availableReplicas || 0 + + if (updated < desired) { + out.lastMessage = `Waiting for rollout: ${updated} out of ${desired} new replicas updated...` + out.state = "deploying" + } else if (replicas > updated) { + out.lastMessage = `Waiting for rollout: ${replicas - updated} old replicas pending termination...` + out.state = "deploying" + } else if (available < updated) { + out.lastMessage = `Waiting for rollout: ${available} out of ${updated} updated replicas available...` + out.state = "deploying" + } + } + + return out +} diff --git a/garden-service/src/plugins/kubernetes/util.ts b/garden-service/src/plugins/kubernetes/util.ts index b0e7d0d855..44a809a49f 100644 --- a/garden-service/src/plugins/kubernetes/util.ts +++ b/garden-service/src/plugins/kubernetes/util.ts @@ -11,8 +11,9 @@ import { get, flatten, uniqBy } from "lodash" import { ChildProcess } from "child_process" import getPort = require("get-port") const AsyncLock = require("async-lock") +import { V1Pod } from "@kubernetes/client-node" -import { KubernetesResource, KubernetesWorkload, KubernetesPod } from "./types" +import { KubernetesResource, KubernetesWorkload, KubernetesPod, KubernetesServerResource } from "./types" import { splitLast } from "../../util/util" import { KubeApi } from "./api" import { PluginContext } from "../../plugin-context" @@ -68,12 +69,17 @@ export async function getWorkloadPods(api: KubeApi, namespace: string, resource: */ export async function getPods( api: KubeApi, namespace: string, selector: { [key: string]: string }, -): Promise { +): Promise[]> { const selectorString = Object.entries(selector).map(([k, v]) => `${k}=${v}`).join(",") const res = await api.core.listNamespacedPod( namespace, true, undefined, undefined, undefined, selectorString, ) - return res.items + return []>res.items.map(pod => { + // inexplicably, the API sometimes returns apiVersion and kind as undefined... + pod.apiVersion = "v1" + pod.kind = "Pod" + return pod + }) } /** diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index deef938f62..b9bab78f2b 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -27,7 +27,8 @@ import { KubernetesProvider } from "../kubernetes/config" import { getNamespace, getAppNamespace } from "../kubernetes/namespace" import { dumpYaml, findByName } from "../../util/util" import { KubeApi } from "../kubernetes/api" -import { waitForResources, checkWorkloadStatus } from "../kubernetes/status" +import { waitForResources } from "../kubernetes/status/status" +import { checkWorkloadStatus } from "../kubernetes/status/workload" import { CommonServiceSpec } from "../../config/service" import { GardenPlugin } from "../../types/plugin/plugin" import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider" @@ -427,7 +428,8 @@ async function getServiceStatus({ ctx, module, service, log }: GetServiceStatusP const container: any = findByName(deployment.spec.template.spec.containers, service.name) const envVersion = findByName(container.env, "GARDEN_VERSION") const version = envVersion ? envVersion.value : undefined - const status = await checkWorkloadStatus(api, namespace, deployment, log) + const resourceVersion = parseInt(deployment.metadata.resourceVersion!, 10) + const status = await checkWorkloadStatus({ api, namespace, resource: deployment, log, resourceVersion }) return { state: status.state, diff --git a/garden-service/src/task-graph.ts b/garden-service/src/task-graph.ts index 8499282dd7..eb083fc1e4 100644 --- a/garden-service/src/task-graph.ts +++ b/garden-service/src/task-graph.ts @@ -314,14 +314,14 @@ export class TaskGraph { } private logTaskError(node: TaskNode, err) { - const divider = padEnd("", 80, "—") + const divider = padEnd("", 80, "━") const error = toGardenError(err) const errorMessage = error.message.trim() const msg = - chalk.red(`\nFailed ${node.getDescription()}. Here is the output:\n${divider}\n`) + + chalk.red.bold(`\nFailed ${node.getDescription()}. Here is the output:\n${divider}\n`) + (hasAnsi(errorMessage) ? errorMessage : chalk.red(errorMessage)) + - chalk.red(`\n${divider}\n`) + chalk.red.bold(`\n${divider}\n`) this.log.error({ msg, error }) }