diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index 026484aeb2..7c84177f6a 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -536,9 +536,9 @@ Storage parameters to set for the in-cluster builder, container registry and cod These are all shared cluster-wide across all users and builds, so they should be resourced accordingly, factoring in how many concurrent builds you expect and how large your images and build contexts tend to be. -| Type | Required | Default | -| -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `object` | No | `{"builder":{"size":20480,"storageClass":null},"registry":{"size":20480,"storageClass":null},"sync":{"size":10240,"storageClass":null}}` | +| Type | Required | Default | +| -------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `object` | No | `{"builder":{"size":20480,"storageClass":null},"nfs":{"storageClass":null},"registry":{"size":20480,"storageClass":null},"sync":{"size":10240,"storageClass":null}}` | ### `providers[].storage.builder` @@ -572,6 +572,29 @@ Storage class to use for the volume. | -------- | -------- | ------- | | `string` | No | `null` | +### `providers[].storage.nfs` + +[providers](#providers) > [storage](#providersstorage) > nfs + +Storage parameters for the NFS provisioner, which we automatically create for the sync volume, _unless_ +you specify a `storageClass` for the sync volume. See the below `sync` parameter for more. + +Only applies when `buildMode` is set to `cluster-docker` or `kaniko`, ignored otherwise. + +| Type | Required | Default | +| -------- | -------- | ----------------------- | +| `object` | No | `{"storageClass":null}` | + +### `providers[].storage.nfs.storageClass` + +[providers](#providers) > [storage](#providersstorage) > [nfs](#providersstoragenfs) > storageClass + +Storage class to use as backing storage for NFS . + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `null` | + ### `providers[].storage.registry` [providers](#providers) > [storage](#providersstorage) > registry @@ -613,7 +636,7 @@ Storage parameters for the code sync volume, which build contexts are synced to in-cluster builds. Important: The storage class configured here has to support _ReadWriteMany_ access. -If you don't specify a storage class, Garden creates an NFS provisioner and provisions an ephemeral +If you don't specify a storage class, Garden creates an NFS provisioner and provisions an NFS volume for the sync data volume. Only applies when `buildMode` is set to `cluster-docker` or `kaniko`, ignored otherwise. @@ -936,6 +959,8 @@ providers: builder: size: 20480 storageClass: null + nfs: + storageClass: null registry: size: 20480 storageClass: null diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 036bda8b23..6fe19a883b 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -536,9 +536,9 @@ Storage parameters to set for the in-cluster builder, container registry and cod These are all shared cluster-wide across all users and builds, so they should be resourced accordingly, factoring in how many concurrent builds you expect and how large your images and build contexts tend to be. -| Type | Required | Default | -| -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `object` | No | `{"builder":{"size":20480,"storageClass":null},"registry":{"size":20480,"storageClass":null},"sync":{"size":10240,"storageClass":null}}` | +| Type | Required | Default | +| -------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `object` | No | `{"builder":{"size":20480,"storageClass":null},"nfs":{"storageClass":null},"registry":{"size":20480,"storageClass":null},"sync":{"size":10240,"storageClass":null}}` | ### `providers[].storage.builder` @@ -572,6 +572,29 @@ Storage class to use for the volume. | -------- | -------- | ------- | | `string` | No | `null` | +### `providers[].storage.nfs` + +[providers](#providers) > [storage](#providersstorage) > nfs + +Storage parameters for the NFS provisioner, which we automatically create for the sync volume, _unless_ +you specify a `storageClass` for the sync volume. See the below `sync` parameter for more. + +Only applies when `buildMode` is set to `cluster-docker` or `kaniko`, ignored otherwise. + +| Type | Required | Default | +| -------- | -------- | ----------------------- | +| `object` | No | `{"storageClass":null}` | + +### `providers[].storage.nfs.storageClass` + +[providers](#providers) > [storage](#providersstorage) > [nfs](#providersstoragenfs) > storageClass + +Storage class to use as backing storage for NFS . + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `null` | + ### `providers[].storage.registry` [providers](#providers) > [storage](#providersstorage) > registry @@ -613,7 +636,7 @@ Storage parameters for the code sync volume, which build contexts are synced to in-cluster builds. Important: The storage class configured here has to support _ReadWriteMany_ access. -If you don't specify a storage class, Garden creates an NFS provisioner and provisions an ephemeral +If you don't specify a storage class, Garden creates an NFS provisioner and provisions an NFS volume for the sync data volume. Only applies when `buildMode` is set to `cluster-docker` or `kaniko`, ignored otherwise. @@ -837,6 +860,8 @@ providers: builder: size: 20480 storageClass: null + nfs: + storageClass: null registry: size: 20480 storageClass: null diff --git a/garden-service/src/plugins/kubernetes/commands/cluster-init.ts b/garden-service/src/plugins/kubernetes/commands/cluster-init.ts index 1d239afa4b..4e68ec3f37 100644 --- a/garden-service/src/plugins/kubernetes/commands/cluster-init.ts +++ b/garden-service/src/plugins/kubernetes/commands/cluster-init.ts @@ -9,6 +9,8 @@ import { PluginCommand } from "../../../types/plugin/command" import { prepareSystem, getEnvironmentStatus } from "../init" import chalk from "chalk" +import { helm } from "../helm/helm-cli" +import { KubernetesPluginContext } from "../config" export const clusterInit: PluginCommand = { name: "cluster-init", @@ -34,6 +36,19 @@ export const clusterInit: PluginCommand = { }) } + const k8sCtx = ctx as KubernetesPluginContext + + log.info("Cleaning up old resources...") + + try { + await helm({ + ctx: k8sCtx, + log, + namespace: "garden-system", + args: ["delete", "--purge", "garden-nfs-provisioner"], + }) + } catch (_) { } + log.info(chalk.green("\nDone!")) return { result } diff --git a/garden-service/src/plugins/kubernetes/config.ts b/garden-service/src/plugins/kubernetes/config.ts index 9bf7722e97..05d77977d8 100644 --- a/garden-service/src/plugins/kubernetes/config.ts +++ b/garden-service/src/plugins/kubernetes/config.ts @@ -45,12 +45,13 @@ interface KubernetesResources { } interface KubernetesStorageSpec { - size: number + size?: number storageClass: string | null } interface KubernetesStorage { builder: KubernetesStorageSpec + nfs: KubernetesStorageSpec registry: KubernetesStorageSpec sync: KubernetesStorageSpec } @@ -125,6 +126,9 @@ export const defaultStorage: KubernetesStorage = { size: 20 * 1024, storageClass: null, }, + nfs: { + storageClass: null, + }, registry: { size: 20 * 1024, storageClass: null, @@ -176,7 +180,7 @@ const storageSchema = (defaults: KubernetesStorageSpec) => joi.object() .description("Volume size in megabytes."), storageClass: joi.string() .allow(null) - .default(null) + .default(defaults.storageClass) .description("Storage class to use for the volume."), }) .default(defaults) @@ -312,6 +316,20 @@ export const kubernetesConfigBase = providerConfigBaseSchema Only applies when \`buildMode\` is set to \`cluster-docker\`, ignored otherwise. `), + nfs: joi.object() + .keys({ + storageClass: joi.string() + .allow(null) + .default(null) + .description("Storage class to use as backing storage for NFS ."), + }) + .default({ storageClass: null }) + .description(dedent` + Storage parameters for the NFS provisioner, which we automatically create for the sync volume, _unless_ + you specify a \`storageClass\` for the sync volume. See the below \`sync\` parameter for more. + + Only applies when \`buildMode\` is set to \`cluster-docker\` or \`kaniko\`, ignored otherwise. + `), registry: storageSchema(defaultStorage.registry) .description(dedent` Storage parameters for the in-cluster Docker registry volume. Built images are stored here, so that they @@ -325,7 +343,7 @@ export const kubernetesConfigBase = providerConfigBaseSchema in-cluster builds. Important: The storage class configured here has to support _ReadWriteMany_ access. - If you don't specify a storage class, Garden creates an NFS provisioner and provisions an ephemeral + If you don't specify a storage class, Garden creates an NFS provisioner and provisions an NFS volume for the sync data volume. Only applies when \`buildMode\` is set to \`cluster-docker\` or \`kaniko\`, ignored otherwise. diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index a143384a9f..1fba40b838 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -25,7 +25,8 @@ import chalk from "chalk" import { deline } from "../../util/string" import { combineStates, ServiceStatusMap } from "../../types/service" -const nfsStorageClass = "garden-system-nfs" +// Note: We need to increment a version number here if we ever make breaking changes to the NFS provisioner StatefulSet +const nfsStorageClass = "garden-system-nfs-v2" /** * Performs the following actions to check environment status: @@ -162,7 +163,11 @@ export async function prepareSystem( // in the prepareEnvironment handler, instead of flagging as not ready here. This avoids blocking users where // there's variance in configuration between users of the same cluster, that often doesn't affect usage. if (!clusterInit && remoteCluster) { - if (combinedState === "outdated" && !serviceStates.includes("missing")) { + if ( + combinedState === "outdated" && + !serviceStates.includes("missing") && + !(ctx.command && ctx.command.name === "plugins" && ctx.command.args.command === "cluster-init") + ) { log.warn({ symbol: "warning", msg: chalk.yellow(deline` @@ -178,12 +183,6 @@ export async function prepareSystem( // We require manual init if we're installing any system services to remote clusters, to avoid conflicts // between users or unnecessary work. if (!clusterInit && remoteCluster && !systemReady) { - // Special-case so that this doesn't error when attempting to run the cluster init - const initCommandName = `plugins ${ctx.provider.name} cluster-init` - if (ctx.command && ctx.command.name === initCommandName) { - return {} - } - throw new KubernetesError(deline` One or more cluster-wide system services are missing or not ready. You need to run \`garden --env=${ctx.environmentName} plugins kubernetes cluster-init\` @@ -198,6 +197,8 @@ export async function prepareSystem( const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) const sysCtx = await sysGarden.getPluginContext(sysProvider) + await sysGarden.clearBuilds() + await installTiller({ ctx: sysCtx, provider: sysCtx.provider, log, force }) // Install system services @@ -231,6 +232,8 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams) } export function getKubernetesSystemVariables(config: KubernetesConfig) { + const syncStorageClass = config.storage.sync.storageClass || nfsStorageClass + return { "namespace": systemNamespace, @@ -241,22 +244,27 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) { "builder-limits-memory": megabytesToString(config.resources.builder.limits.memory), "builder-requests-cpu": millicpuToString(config.resources.builder.requests.cpu), "builder-requests-memory": megabytesToString(config.resources.builder.requests.memory), - "builder-storage-size": megabytesToString(config.storage.builder.size), + "builder-storage-size": megabytesToString(config.storage.builder.size!), "builder-storage-class": config.storage.builder.storageClass, + // We only use NFS for the build-sync volume, so we allocate the space we need for that plus 1GB for margin. + "nfs-storage-size": megabytesToString(config.storage.sync.size! + 1024), + "nfs-storage-class": config.storage.nfs.storageClass, + "registry-limits-cpu": millicpuToString(config.resources.registry.limits.cpu), "registry-limits-memory": megabytesToString(config.resources.registry.limits.memory), "registry-requests-cpu": millicpuToString(config.resources.registry.requests.cpu), "registry-requests-memory": megabytesToString(config.resources.registry.requests.memory), - "registry-storage-size": megabytesToString(config.storage.registry.size), + "registry-storage-size": megabytesToString(config.storage.registry.size!), "registry-storage-class": config.storage.registry.storageClass, "sync-limits-cpu": millicpuToString(config.resources.sync.limits.cpu), "sync-limits-memory": megabytesToString(config.resources.sync.limits.memory), "sync-requests-cpu": millicpuToString(config.resources.sync.requests.cpu), "sync-requests-memory": megabytesToString(config.resources.sync.requests.memory), - "sync-storage-size": megabytesToString(config.storage.sync.size), - "sync-storage-class": config.storage.sync.storageClass || nfsStorageClass, + "sync-storage-size": megabytesToString(config.storage.sync.size!), + "sync-storage-class": syncStorageClass, + "sync-volume-name": `garden-sync-${syncStorageClass}`, } } diff --git a/garden-service/static/kubernetes/system/build-sync/garden.yml b/garden-service/static/kubernetes/system/build-sync/garden.yml index 11b7bd9b7a..adddef091a 100644 --- a/garden-service/static/kubernetes/system/build-sync/garden.yml +++ b/garden-service/static/kubernetes/system/build-sync/garden.yml @@ -16,3 +16,5 @@ values: storage: request: ${var.sync-storage-size} storageClass: ${var.sync-storage-class} + pvc: + name: ${var.sync-volume-name} diff --git a/garden-service/static/kubernetes/system/build-sync/templates/deployment.yaml b/garden-service/static/kubernetes/system/build-sync/templates/deployment.yaml index 66f51b116b..a6eed9fb04 100644 --- a/garden-service/static/kubernetes/system/build-sync/templates/deployment.yaml +++ b/garden-service/static/kubernetes/system/build-sync/templates/deployment.yaml @@ -26,7 +26,7 @@ spec: volumes: - name: garden-build-sync persistentVolumeClaim: - claimName: garden-build-sync + claimName: {{ .Values.pvc.name }} containers: - name: sync image: "gardendev/rsync:0.1" diff --git a/garden-service/static/kubernetes/system/build-sync/templates/volume.yaml b/garden-service/static/kubernetes/system/build-sync/templates/volume.yaml index cbc9b1c472..7b29e88db5 100644 --- a/garden-service/static/kubernetes/system/build-sync/templates/volume.yaml +++ b/garden-service/static/kubernetes/system/build-sync/templates/volume.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: garden-build-sync + name: {{ .Values.pvc.name }} spec: accessModes: - ReadWriteMany diff --git a/garden-service/static/kubernetes/system/build-sync/values.yaml b/garden-service/static/kubernetes/system/build-sync/values.yaml index 74e86209ef..f733307178 100644 --- a/garden-service/static/kubernetes/system/build-sync/values.yaml +++ b/garden-service/static/kubernetes/system/build-sync/values.yaml @@ -22,6 +22,9 @@ resources: storage: request: 2Gi +pvc: + name: garden-build-sync + nodeSelector: {} tolerations: [] diff --git a/garden-service/static/kubernetes/system/docker-daemon/garden.yml b/garden-service/static/kubernetes/system/docker-daemon/garden.yml index a0a4df3245..5578edc67a 100644 --- a/garden-service/static/kubernetes/system/docker-daemon/garden.yml +++ b/garden-service/static/kubernetes/system/docker-daemon/garden.yml @@ -20,3 +20,6 @@ values: storage: size: ${var.builder-storage-size} storageClass: ${var.builder-storage-class} + buildSync: + volume: + name: ${var.sync-volume-name} diff --git a/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml index 2519987684..6bb455d65d 100644 --- a/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml +++ b/garden-service/static/kubernetes/system/docker-daemon/templates/deployment.yaml @@ -28,7 +28,7 @@ spec: claimName: garden-docker-data - name: garden-build-sync persistentVolumeClaim: - claimName: garden-build-sync + claimName: {{ .Values.buildSync.volume.name }} # - name: garden-registry-tls # secret: # secretName: foo diff --git a/garden-service/static/kubernetes/system/docker-daemon/values.yaml b/garden-service/static/kubernetes/system/docker-daemon/values.yaml index 641fa7af4a..65ca0d3636 100644 --- a/garden-service/static/kubernetes/system/docker-daemon/values.yaml +++ b/garden-service/static/kubernetes/system/docker-daemon/values.yaml @@ -27,6 +27,10 @@ storage: registry: hostname: garden-docker-registry +buildSync: + pvc: + name: garden-build-sync + nodeSelector: {} tolerations: [] diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/Chart.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/Chart.yaml new file mode 100644 index 0000000000..ec6ee0a869 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +appVersion: 2.2.3-0 +description: nfs-server-provisioner is an out-of-tree dynamic provisioner for Kubernetes. + You can use it to quickly & easily deploy shared storage that works almost anywhere. +home: https://github.com/kubernetes/charts/tree/master/stable/nfs-server-provisioner +keywords: +- nfs +- storage +maintainers: +- email: kiall@macinnes.ie + name: kiall +- email: joaocc-dev@live.com + name: joaocc +name: nfs-server-provisioner +sources: +- https://github.com/kubernetes-incubator/external-storage/tree/master/nfs +version: 0.4.0-0 diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/README.md b/garden-service/static/kubernetes/system/nfs-provisioner/README.md new file mode 100644 index 0000000000..d292fab475 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/README.md @@ -0,0 +1,204 @@ +# NFS Server Provisioner + +[NFS Server Provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs) +is an out-of-tree dynamic provisioner for Kubernetes. You can use it to quickly +& easily deploy shared storage that works almost anywhere. + +This chart will deploy the Kubernetes [external-storage projects](https://github.com/kubernetes-incubator/external-storage) +`nfs` provisioner. This provisioner includes a built in NFS server, and is not intended for connecting to a pre-existing +NFS server. If you have a pre-existing NFS Server, please consider using the [NFS Client Provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs-client) +instead. + +## TL;DR; + +```console +$ helm install stable/nfs-server-provisioner +``` + +> **Warning**: While installing in the default configuration will work, any data stored on +the dynamic volumes provisioned by this chart will not be persistent! + +## Introduction + +This chart bootstraps a [nfs-server-provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs) +deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) +package manager. + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install stable/nfs-server-provisioner --name my-release +``` + +The command deploys nfs-server-provisioner on the Kubernetes cluster in the default +configuration. The [configuration](#configuration) section lists the parameters +that can be configured during installation. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. + +## Configuration + +The following table lists the configurable parameters of the kibana chart and +their default values. + +| Parameter | Description | Default | +|:-------------------------------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------| +| `imagePullSecrets` | Specify image pull secrets | `nil` (does not add image pull secrets to deployed pods) | +| `image.repository` | The image repository to pull from | `quay.io/kubernetes_incubator/nfs-provisioner` | +| `image.tag` | The image tag to pull from | `v1.0.8` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `service.type` | service type | `ClusterIP` | +| `service.nfsPort` | TCP port on which the nfs-server-provisioner NFS service is exposed | `2049` | +| `service.mountdPort` | TCP port on which the nfs-server-provisioner mountd service is exposed | `20048` | +| `service.rpcbindPort` | TCP port on which the nfs-server-provisioner RPC service is exposed | `51413` | +| `service.nfsNodePort` | if `service.type` is `NodePort` and this is non-empty, sets the nfs-server-provisioner node port of the NFS service | `nil` | +| `service.mountdNodePort` | if `service.type` is `NodePort` and this is non-empty, sets the nfs-server-provisioner node port of the mountd service | `nil` | +| `service.rpcbindNodePort` | if `service.type` is `NodePort` and this is non-empty, sets the nfs-server-provisioner node port of the RPC service | `nil` | +| `persistence.enabled` | Enable config persistence using PVC | `false` | +| `persistence.storageClass` | PVC Storage Class for config volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for config volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for config volume | `1Gi` | +| `storageClass.create` | Enable creation of a StorageClass to consume this nfs-server-provisioner instance | `true` | +| `storageClass.provisionerName` | The provisioner name for the storageclass | `cluster.local/{release-name}-{chart-name}` | +| `storageClass.defaultClass` | Whether to set the created StorageClass as the clusters default StorageClass | `false` | +| `storageClass.name` | The name to assign the created StorageClass | `nfs` | +| `storageClass.allowVolumeExpansion` | Allow base storage PCV to be dynamically resizeable (set to null to disable ) | `true | +| `storageClass.parameters` | Parameters for StorageClass | `{}` | +| `storageClass.mountOptions` | Mount options for StorageClass | `[ "vers=4.1", "noatime" ]` | +| `storageClass.reclaimPolicy` | ReclaimPolicy field of the class, which can be either Delete or Retain | `Delete` | +| `resources` | Resource limits for nfs-server-provisioner pod | `{}` | +| `nodeSelector` | Map of node labels for pod assignment | `{}` | +| `tolerations` | List of node taints to tolerate | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | + +```console +$ helm install stable/nfs-server-provisioner --name my-release \ + --set=image.tag=v1.0.8,resources.limits.cpu=200m +``` + +Alternatively, a YAML file that specifies the values for the above parameters +can be provided while installing the chart. For example, + +```console +$ helm install stable/nfs-server-provisioner --name my-release -f values.yaml +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) as an example + +## Persistence + +The nfs-server-provisioner image stores it's configuration data, and importantly, **the dynamic volumes it +manages** `/export` path of the container. + +The chart mounts a [Persistent Volume](http://kubernetes.io/docs/user-guide/persistent-volumes/) +volume at this location. The volume can be created using dynamic volume +provisioning. However, **it is highly recommended** to explicitly specify +a storageclass to use rather than accept the clusters default, or pre-create +a volume for each replica. + +If this chart is deployed with more than 1 replica, `storageClass.defaultClass=true` +and `persistence.storageClass`, then the 2nd+ replica will end up using the 1st +replica to provision storage - which is likely never a desired outcome. + +## Recommended Persistence Configuration Examples + +The following is a recommended configuration example when another storage class +exists to provide persistence: + + persistence: + enabled: true + storageClass: "standard" + size: 200Gi + + storageClass: + defaultClass: true + +On many clusters, the cloud provider integration will create a "standard" storage +class which will create a volume (e.g. a Google Compute Engine Persistent Disk or +Amazon EBS volume) to provide persistence. + +--- + +The following is a recommended configuration example when another storage class +does not exist to provide persistence: + + persistence: + enabled: true + storageClass: "-" + size: 200Gi + + storageClass: + defaultClass: true + +In this configuration, a `PersistentVolume` must be created for each replica +to use. Installing the Helm chart, and then inspecting the `PersistentVolumeClaim`'s +created will provide the necessary names for your `PersistentVolume`'s to bind to. + +An example of the necessary `PersistentVolume`: + + apiVersion: v1 + kind: PersistentVolume + metadata: + name: data-nfs-server-provisioner-0 + spec: + capacity: + storage: 200Gi + accessModes: + - ReadWriteOnce + gcePersistentDisk: + fsType: "ext4" + pdName: "data-nfs-server-provisioner-0" + claimRef: + namespace: kube-system + name: data-nfs-server-provisioner-0 + +--- + +The following is a recommended configration example for running on bare metal with a hostPath volume: + + persistence: + enabled: true + storageClass: "-" + size: 200Gi + + storageClass: + defaultClass: true + + nodeSelector: + kubernetes.io/hostname: {node-name} + +In this configuration, a `PersistentVolume` must be created for each replica +to use. Installing the Helm chart, and then inspecting the `PersistentVolumeClaim`'s +created will provide the necessary names for your `PersistentVolume`'s to bind to. + +An example of the necessary `PersistentVolume`: + + apiVersion: v1 + kind: PersistentVolume + metadata: + name: data-nfs-server-provisioner-0 + spec: + capacity: + storage: 200Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /srv/volumes/data-nfs-server-provisioner-0 + claimRef: + namespace: kube-system + name: data-nfs-server-provisioner-0 + +> **Warning**: `hostPath` volumes cannot be migrated between machines by Kubernetes, as such, +in this example, we have restricted the `nfs-server-provisioner` pod to run on a single node. This +is unsuitable for production deployments. diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml b/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml index 8d62203fc1..430e1320cb 100644 --- a/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml +++ b/garden-service/static/kubernetes/system/nfs-provisioner/garden.yml @@ -2,13 +2,13 @@ kind: Module name: nfs-provisioner description: Provisioner for NFS volumes used by Garden system services type: helm -chart: stable/nfs-server-provisioner -version: "0.3.0" -releaseName: garden-nfs-provisioner +releaseName: garden-nfs-provisioner-v2 values: - nameOverride: garden-nfs-provisioner - fullnameOverride: garden-nfs-provisioner + nameOverride: garden-nfs-provisioner-v2 + fullnameOverride: garden-nfs-provisioner-v2 persistence: - enabled: false + enabled: true + size: ${var.nfs-storage-size} + storageClass: ${var.nfs-storage-class} storageClass: - name: garden-system-nfs + name: ${var.sync-storage-class} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/NOTES.txt b/garden-service/static/kubernetes/system/nfs-provisioner/templates/NOTES.txt new file mode 100644 index 0000000000..09376d2962 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/NOTES.txt @@ -0,0 +1,26 @@ +The NFS Provisioner service has now been installed. + +{{ if .Values.storageClass.create -}} +A storage class named '{{ .Values.storageClass.name }}' has now been created +and is available to provision dynamic volumes. + +You can use this storageclass by creating a `PersistentVolumeClaim` with the +correct storageClassName attribute. For example: + + --- + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-dynamic-volume-claim + spec: + storageClassName: "{{ .Values.storageClass.name }}" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi + +{{ else -}} +A storage class has NOT been created. You may create a custom `StorageClass` +resource with a `provisioner` attribute of `{{ template "nfs-provisioner.provisionerName" . }}`. +{{ end -}} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/_helpers.tpl b/garden-service/static/kubernetes/system/nfs-provisioner/templates/_helpers.tpl new file mode 100644 index 0000000000..cdc13bfa41 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "nfs-provisioner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nfs-provisioner.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nfs-provisioner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nfs-provisioner.provisionerName" -}} +{{- if .Values.storageClass.provisionerName -}} +{{- printf .Values.storageClass.provisionerName -}} +{{- else -}} +cluster.local/{{ template "nfs-provisioner.fullname" . -}} +{{- end -}} +{{- end -}} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/clusterrole.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/clusterrole.yaml new file mode 100644 index 0000000000..5affc4bab0 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/clusterrole.yaml @@ -0,0 +1,34 @@ +{{ if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "nfs-provisioner.fullname" . }} + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + resourceNames: ["nfs-provisioner"] + verbs: ["use"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +{{- end -}} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/deployment.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/deployment.yaml new file mode 100644 index 0000000000..9a7707cb48 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "nfs-provisioner.fullname" . }} + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + # TODO: Investigate how/if nfs-provisioner can be scaled out beyond 1 replica + replicas: 1 + strategy: + # We only want one instance at a time, because we're using the same volume for the data + type: Recreate + selector: + matchLabels: + app: {{ template "nfs-provisioner.name" . }} + release: {{ .Release.Name }} + serviceName: {{ template "nfs-provisioner.fullname" . }} + template: + metadata: + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + # NOTE: This is 10 seconds longer than the default nfs-provisioner --grace-period value of 90sec + terminationGracePeriodSeconds: 100 + serviceAccountName: {{ if .Values.rbac.create }}{{ template "nfs-provisioner.fullname" . }}{{ else }}{{ .Values.rbac.serviceAccountName | quote }}{{ end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: nfs + containerPort: 2049 + protocol: TCP + - name: mountd + containerPort: 20048 + protocol: TCP + - name: rpcbind-tcp + containerPort: 111 + protocol: TCP + - name: rpcbind-udp + containerPort: 111 + protocol: UDP + securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - SYS_RESOURCE + args: + - "-provisioner={{ template "nfs-provisioner.provisionerName" . }}" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_NAME + value: {{ template "nfs-provisioner.fullname" . }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: data + mountPath: /export + resources: + {{- with .Values.resources }} + resources: +{{ toYaml . | indent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: data +{{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "nfs-provisioner.fullname" . }}-data +{{- else }} + emptyDir: {} +{{- end }} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/rolebinding.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/rolebinding.yaml new file mode 100644 index 0000000000..fc88425f93 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "nfs-provisioner.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "nfs-provisioner.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "nfs-provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/service.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/service.yaml new file mode 100644 index 0000000000..b2d299e284 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/service.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "nfs-provisioner.fullname" . }} + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.nfsPort }} + targetPort: nfs + protocol: TCP + name: nfs +{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nfsNodePort))) }} + nodePort: {{ .Values.service.nfsNodePort }} +{{- end }} + - port: {{ .Values.service.mountdPort }} + targetPort: mountd + protocol: TCP + name: mountd +{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.mountdNodePort))) }} + nodePort: {{ .Values.service.mountdNodePort }} +{{- end }} + - port: {{ .Values.service.rpcbindPort }} + targetPort: rpcbind-tcp + protocol: TCP + name: rpcbind-tcp +{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.rpcbindNodePort))) }} + nodePort: {{ .Values.service.rpcbindNodePort }} +{{- end }} + - port: {{ .Values.service.rpcbindPort }} + targetPort: rpcbind-udp + protocol: UDP + name: rpcbind-udp +{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.rpcbindNodePort))) }} + nodePort: {{ .Values.service.rpcbindNodePort }} +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} + selector: + app: {{ template "nfs-provisioner.name" . }} + release: {{ .Release.Name }} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/serviceaccount.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/serviceaccount.yaml new file mode 100644 index 0000000000..79fd0ae0e1 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.rbac.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "nfs-provisioner.fullname" . }} +{{- end -}} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/storageclass.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/storageclass.yaml new file mode 100644 index 0000000000..093e650e5a --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/storageclass.yaml @@ -0,0 +1,28 @@ +{{ if .Values.storageClass.create -}} +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .Values.storageClass.name }} + labels: + app: {{ template "nfs-provisioner.name" . }} + chart: {{ template "nfs-provisioner.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.storageClass.defaultClass }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +{{- end }} +provisioner: {{ template "nfs-provisioner.provisionerName" . }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +{{ if .Values.storageClass.allowVolumeExpansion }} +allowVolumeExpansion: {{ .Values.storageClass.allowVolumeExpansion }} +{{ end }} +{{ end -}} +{{- with .Values.storageClass.parameters }} +parameters: +{{ toYaml . | indent 2 }} +{{- end }} +{{- with .Values.storageClass.mountOptions }} +mountOptions: +{{ toYaml . | indent 2 }} +{{- end }} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/templates/volume.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/templates/volume.yaml new file mode 100644 index 0000000000..d54216f522 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/templates/volume.yaml @@ -0,0 +1,20 @@ +{{- if .Values.persistence.enabled }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "nfs-provisioner.fullname" . }}-data +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} diff --git a/garden-service/static/kubernetes/system/nfs-provisioner/values.yaml b/garden-service/static/kubernetes/system/nfs-provisioner/values.yaml new file mode 100644 index 0000000000..1983857c64 --- /dev/null +++ b/garden-service/static/kubernetes/system/nfs-provisioner/values.yaml @@ -0,0 +1,88 @@ +# Default values for nfs-provisioner. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# imagePullSecrets: + +image: + repository: quay.io/kubernetes_incubator/nfs-provisioner + tag: v2.2.2 + pullPolicy: IfNotPresent + +service: + type: ClusterIP + + nfsPort: 2049 + mountdPort: 20048 + rpcbindPort: 51413 + # nfsNodePort: + # mountdNodePort: + # rpcbindNodePort: + + externalIPs: [] + +persistence: + enabled: true + + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + + accessMode: ReadWriteOnce + size: 1Gi + +## For creating the StorageClass automatically: +storageClass: + create: true + + ## Set a provisioner name. If unset, a name will be generated. + # provisionerName: + + ## Set StorageClass as the default StorageClass + ## Ignored if storageClass.create is false + defaultClass: false + + ## Set a StorageClass name + ## Ignored if storageClass.create is false + name: nfs + + # set to null to prevent expansion + allowVolumeExpansion: true + ## StorageClass parameters + parameters: {} + + mountOptions: + - vers=4.1 + - noatime + + ## ReclaimPolicy field of the class, which can be either Delete or Retain + reclaimPolicy: Delete + +## For RBAC support: +rbac: + create: true + + ## Ignored if rbac.create is true + ## + serviceAccountName: default + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {}