Skip to content

Commit

Permalink
fix(k8s): exec-ing and hot-reloading only worked for Deployments
Browse files Browse the repository at this point in the history 
Meaning, DaemonSet and StatefulSet workloads weren't handled right.

Recent changes for blue-green deployments also caused issues, which
are solved by these same changes.
  • Loading branch information
@edvald
edvald committed Sep 20, 2019
1 parent 37ecd0a commit 6d00df4
Show file tree
Hide file tree
Showing 10 changed files with 141 additions and 155 deletions.
6 changes: 3 additions & 3 deletions garden-service/src/plugins/kubernetes/commands/cleanup-cluster-registry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import { systemNamespace } from "../system"
import { PluginError } from "../../../exceptions"
import { apply, kubectl } from "../kubectl"
import { waitForResources } from "../status/status"
import { execInDeployment } from "../container/run"
import { execInWorkload } from "../container/run"
import { dedent, deline } from "../../../util/string"
import { execInBuilder, getBuilderPodName, BuilderExecParams, buildSyncDeploymentName } from "../container/build"
import { getPods } from "../util"
Expand Down Expand Up @@ -226,11 +226,11 @@ async function runRegistryGarbageCollection(ctx: KubernetesPluginContext, api: K

// Run garbage collection
log.info("Running garbage collection...")
await execInDeployment({
await execInWorkload({
provider,
log,
namespace: systemNamespace,
deploymentName: CLUSTER_REGISTRY_DEPLOYMENT_NAME,
workload: modifiedDeployment,
command: ["/bin/registry", "garbage-collect", "/etc/docker/registry/config.yml"],
interactive: false,
})
Expand Down
136 changes: 36 additions & 100 deletions garden-service/src/plugins/kubernetes/container/deployment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,27 @@
*/

import chalk from "chalk"
import Bluebird from "bluebird"
import { V1Container } from "@kubernetes/client-node"
import { Service, ServiceStatus } from "../../../types/service"
import { Service } from "../../../types/service"
import { extend, find, keyBy, merge, set } from "lodash"
import { ContainerModule, ContainerService } from "../../container/config"
import { createIngressResources } from "./ingress"
import { createServiceResources } from "./service"
import { waitForResources, compareDeployedObjects } from "../status/status"
import { apply, deleteObjectsByLabel } from "../kubectl"
import { apply, deleteObjectsBySelector } from "../kubectl"
import { getAppNamespace } from "../namespace"
import { PluginContext } from "../../../plugin-context"
import { KubeApi } from "../api"
import { KubernetesProvider, KubernetesPluginContext } from "../config"
import { configureHotReload } from "../hot-reload"
import { KubernetesResource, KubernetesServerResource } from "../types"
import { KubernetesWorkload } from "../types"
import { ConfigurationError } from "../../../exceptions"
import { getContainerServiceStatus, ContainerServiceStatus } from "./status"
import { containerHelpers } from "../../container/helpers"
import { LogEntry } from "../../../logger/log-entry"
import { DeployServiceParams } from "../../../types/plugin/service/deployService"
import { DeleteServiceParams } from "../../../types/plugin/service/deleteService"
import { millicpuToString, kilobytesToString, prepareEnvVars } from "../util"
import { millicpuToString, kilobytesToString, prepareEnvVars, workloadTypes } from "../util"
import { gardenAnnotationKey } from "../../../util/string"
import { RuntimeContext } from "../../../runtime-context"

Expand All @@ -54,11 +53,10 @@ export async function deployContainerServiceRolling(

const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
const { manifests } = await createContainerManifests(k8sCtx, log, service, runtimeContext, hotReload)

// TODO: use Helm instead of kubectl apply
const provider = k8sCtx.provider
const pruneSelector = "service=" + service.name
const pruneSelector = gardenAnnotationKey("service") + "=" + service.name

await apply({ log, provider, manifests, force, namespace, pruneSelector })

Expand All @@ -73,25 +71,15 @@ export async function deployContainerServiceRolling(
return getContainerServiceStatus(params)
}

// Given an array of k8s resources and a Garden service returns matching k8s resource
function getResourcesForService(items: KubernetesServerResource[], service): KubernetesServerResource[] {
return items.filter((resource) => {
return resource.metadata
&& resource.metadata.labels
&& resource.metadata.labels["module"] === service.module.name
&& resource.metadata.labels["service"] === service.name
})
}

export async function deployContainerServiceBlueGreen(
params: DeployServiceParams<ContainerModule>): Promise<ServiceStatus> {
params: DeployServiceParams<ContainerModule>): Promise<ContainerServiceStatus> {

const { ctx, service, runtimeContext, force, log, hotReload } = params
const k8sCtx = <KubernetesPluginContext>ctx
const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

// Create all the resource manifests for the Garden service which will be deployed
const manifests = await createContainerObjects(k8sCtx, log, service, runtimeContext, hotReload)
const { manifests } = await createContainerManifests(k8sCtx, log, service, runtimeContext, hotReload)

const provider = k8sCtx.provider
const api = await KubeApi.factory(log, provider)
Expand All @@ -118,14 +106,11 @@ export async function deployContainerServiceBlueGreen(
} else {
// A k8s service matching the current Garden service exist in the cluster.
// Proceeding with blue-green deployment
const newVersion = service.module.version.versionString
const versionKey = gardenAnnotationKey("version")

// Remove Service manifest from generated resources
const filteredManifests = manifests.filter(manifest => manifest.kind !== "Service")
// Retrieve new (yet-to-be-deployed) Deployment manifest
const deploymentManifest = find(manifests, (manifest) => {
return manifest.kind === "Deployment"
&& manifest.metadata.labels[gardenAnnotationKey("version")] === service.module.version.versionString
})

// Apply new Deployment manifest (deploy the Green version)
await apply({ log, provider, manifests: filteredManifests, force, namespace })
Expand All @@ -141,12 +126,12 @@ export async function deployContainerServiceBlueGreen(
const servicePatchBody = {
metadata: {
annotations: {
[gardenAnnotationKey("version")]: deploymentManifest.metadata.labels.version,
[versionKey]: newVersion,
},
},
spec: {
selector: {
[gardenAnnotationKey("version")]: deploymentManifest.metadata.labels.version,
[versionKey]: newVersion,
},
},
}
Expand Down Expand Up @@ -177,27 +162,17 @@ export async function deployContainerServiceBlueGreen(

// Clenup unused deployments:
// as a feature we delete all the deployments which don't match any deployed Service.

const deployments = await api.apps.listNamespacedDeployment(namespace)
// Retrieve all unused deployments for current service
const unusedDeployments = getResourcesForService(deployments.items, service)
.filter(deployment => deployment.metadata.labels
&& deployment.metadata.labels[gardenAnnotationKey("version")]
!== deploymentManifest.metadata.labels[gardenAnnotationKey("version")])

if (unusedDeployments) {
// Delete old Deployments (Blue)
await Bluebird.map(
unusedDeployments, oldDeployment => api.apps.deleteNamespacedDeployment(oldDeployment.metadata.name, namespace),
)
await waitForResources({
ctx: k8sCtx,
provider: k8sCtx.provider,
serviceName: `Cleanup deployments`,
resources: manifests,
log,
})
}
log.verbose(`Cleaning up old workloads`)
await deleteObjectsBySelector({
log,
provider,
namespace,
objectTypes: workloadTypes,
// Find workloads that match this service, but have a different version
selector:
`${gardenAnnotationKey("service")}=${service.name},` +
`${versionKey}!=${newVersion}`,
})
}
return getContainerServiceStatus(params)
}
Expand All @@ -223,8 +198,6 @@ export async function createContainerManifests(
for (const obj of manifests) {
set(obj, ["metadata", "labels", gardenAnnotationKey("module")], service.module.name)
set(obj, ["metadata", "labels", gardenAnnotationKey("service")], service.name)
set(obj, ["metadata", "labels", gardenAnnotationKey("generated")], "true")
set(obj, ["metadata", "labels", gardenAnnotationKey("version")], version.versionString)
set(obj, ["metadata", "annotations", gardenAnnotationKey("generated")], "true")
set(obj, ["metadata", "annotations", gardenAnnotationKey("version")], version.versionString)
}
Expand Down Expand Up @@ -310,20 +283,10 @@ export async function createWorkloadResource(
container.args = service.spec.args
}

// if (config.entrypoint) {
// container.command = [config.entrypoint]
// }

if (spec.healthCheck) {
configureHealthCheck(container, spec)
}

// if (service.privileged) {
// container.securityContext = {
// privileged: true,
// }
// }

if (spec.volumes && spec.volumes.length) {
configureVolumes(deployment, container, spec)
}
Expand Down Expand Up @@ -405,28 +368,30 @@ export async function createWorkloadResource(
return deployment
}

function deploymentConfig(service: Service, configuredReplicas: number, namespace: string): object {
function getDeploymentName(service: Service) {
return `${service.name}-${service.module.version.versionString}`
}

function deploymentConfig(service: Service, configuredReplicas: number, namespace: string): object {
const labels = {
module: service.module.name,
service: service.name,
[gardenAnnotationKey("module")]: service.module.name,
[gardenAnnotationKey("service")]: service.name,
[gardenAnnotationKey("version")]: service.module.version.versionString,
}

let selector: any = {
let selector = {
matchLabels: {
service: service.name,
[gardenAnnotationKey("service")]: service.name,
[gardenAnnotationKey("version")]: service.module.version.versionString,
},
}

selector.matchLabels[gardenAnnotationKey("version")] = service.module.version.versionString

// TODO: moar type-safety
return {
kind: "Deployment",
apiVersion: "apps/v1",
metadata: {
name: `${service.name}-${service.module.version.versionString}`,
name: getDeploymentName(service),
annotations: {
// we can use this to avoid overriding the replica count if it has been manually scaled
"garden.io/configured.replicas": configuredReplicas.toString(),
Expand All @@ -448,10 +413,6 @@ function deploymentConfig(service: Service, configuredReplicas: number, namespac
restartPolicy: "Always",
terminationGracePeriodSeconds: 5,
dnsPolicy: "ClusterFirst",
// TODO: support private registries
// imagePullSecrets: [
// { name: DOCKER_AUTH_SECRET_NAME },
// ],
volumes: [],
},
},
Expand Down Expand Up @@ -557,45 +518,20 @@ export function rsyncTargetPath(path: string) {
.replace(/\/*$/, "/")
}

export async function deleteService(params: DeleteServiceParams): Promise<ServiceStatus> {
export async function deleteService(params: DeleteServiceParams): Promise<ContainerServiceStatus> {
const { ctx, log, service } = params
const k8sCtx = <KubernetesPluginContext>ctx
const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)
const provider = k8sCtx.provider

await deleteContainerDeployment({ namespace, provider, serviceName: service.name, log })
await deleteObjectsByLabel({
await deleteObjectsBySelector({
log,
provider,
namespace,
labelKey: "service",
labelValue: service.name,
selector: `${gardenAnnotationKey("service")}=${service.name}`,
objectTypes: ["deployment", "replicaset", "pod", "service", "ingress", "daemonset"],
includeUninitialized: false,
})

}

export async function deleteContainerDeployment(
{ namespace, provider, serviceName, log }:
{ namespace: string, provider: KubernetesProvider, serviceName: string, log: LogEntry },
) {

let found = true
const api = await KubeApi.factory(log, provider)

try {
await api.extensions.deleteNamespacedDeployment(serviceName, namespace, <any>{})
} catch (err) {
if (err.code === 404) {
found = false
} else {
throw err
}
}

if (log) {
found ? log.setSuccess("Service deleted") : log.setWarn("Service not deployed")
}
return { state: "missing", detail: { remoteResources: [], workload: null } }
}
4 changes: 2 additions & 2 deletions garden-service/src/plugins/kubernetes/container/logs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { ContainerModule } from "../../container/config"
import { getAppNamespace } from "../namespace"
import { getAllLogs } from "../logs"
import { KubernetesPluginContext } from "../config"
import { createDeployment } from "./deployment"
import { createWorkloadResource } from "./deployment"
import { emptyRuntimeContext } from "../../../runtime-context"

export async function getServiceLogs(params: GetServiceLogsParams<ContainerModule>) {
Expand All @@ -20,7 +20,7 @@ export async function getServiceLogs(params: GetServiceLogsParams<ContainerModul
const provider = k8sCtx.provider
const namespace = await getAppNamespace(k8sCtx, log, provider)

const resources = [await createDeployment({
const resources = [await createWorkloadResource({
provider,
service,
// No need for the proper context here
Expand Down
18 changes: 9 additions & 9 deletions garden-service/src/plugins/kubernetes/container/run.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import { LogEntry } from "../../../logger/log-entry"
import { getWorkloadPods, prepareEnvVars } from "../util"
import { uniqByName } from "../../../util/util"
import { V1PodSpec } from "@kubernetes/client-node"
import { KubernetesWorkload } from "../types"

export async function execInService(params: ExecInServiceParams<ContainerModule>) {
const { ctx, log, service, command, interactive } = params
Expand All @@ -43,37 +44,36 @@ export async function execInService(params: ExecInServiceParams<ContainerModule>
const namespace = await getAppNamespace(k8sCtx, log, k8sCtx.provider)

// TODO: this check should probably live outside of the plugin
if (!includes(["ready", "outdated"], status.state)) {
if (!status.detail.workload || !includes(["ready", "outdated"], status.state)) {
throw new DeploymentError(`Service ${service.name} is not running`, {
name: service.name,
state: status.state,
})
}

return execInDeployment({ provider, log, namespace, deploymentName: service.name, command, interactive })
return execInWorkload({ provider, log, namespace, workload: status.detail.workload, command, interactive })
}

export async function execInDeployment(
{ provider, log, namespace, deploymentName, command, interactive }:
export async function execInWorkload(
{ provider, log, namespace, workload, command, interactive }:
{
provider: KubernetesProvider,
log: LogEntry,
namespace: string,
deploymentName: string,
workload: KubernetesWorkload,
command: string[],
interactive: boolean,
},
) {
const api = await KubeApi.factory(log, provider)
const deployment = await api.apps.readNamespacedDeployment(deploymentName, namespace)
const pods = await getWorkloadPods(api, namespace, deployment)
const pods = await getWorkloadPods(api, namespace, workload)

const pod = pods[0]

if (!pod) {
// This should not happen because of the prior status check, but checking to be sure
throw new DeploymentError(`Could not find running pod for ${deploymentName}`, {
deploymentName,
throw new DeploymentError(`Could not find running pod for ${workload.kind}/${workload.metadata.name}`, {
workload,
})
}

Expand Down
2 changes: 1 addition & 1 deletion garden-service/src/plugins/kubernetes/container/service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export async function createServiceResources(service: ContainerService, namespac
spec: {
ports: servicePorts,
selector: {
service: service.name,
[gardenAnnotationKey("service")]: service.name,
[gardenAnnotationKey("version")]: service.module.version.versionString,
},
type,
Expand Down
Loading

0 comments on commit 6d00df4

Please sign in to comment.