improvement(k8s): remove old system garden and improve local-k8s star…

…tup time (#5136)

* refactor: move constant `defaultSystemNamespace` to `constants.ts`

* chore: helpers for nginx integration

* refactor: re-arrange the list of the supported contexts

* chore: helper to apply manifests from yaml files

---
Co-authored-by: Vladimir Vagaytsev <[email protected]>

* chore(wip): getting rid of system services

---
Co-authored-by: Vladimir Vagaytsev <[email protected]>

* chore: rename some nginx deploy configs

* chore: rename some nginx deploy configs

Remove `garden` from the names.
These configs will be converted to the standard k8s configs.

* chore: move nginx configs to `static/kubernetes`

These configs must be converted to the standard k8s configs.

* fix(k8s): (wip) convert garden nginx deploy configs to k8s manifests

* chore: change file extension `yml` -> `yaml`

* refactor(k8s): (wip) split provider and environment config handlers

* fix(k3s): (wip) add missing nginx configuration

* fix(k8s): argument type for env config handler

* refactor(k8s): named type for k8s cluster type

* chore: remove unused constant

* chore: defensive type-check for k8s cluster type

* docs: update RBAC config guide

* chore: fix compile errors

* fix(k8s): use standard `getEnvironmentStatus` to ensure namespaces

* chore: add missing sha256 hash

* refactor: extract getter to build values for nginx helm chart

* refactor: extract some named constants

* refactor(k3s): gardenless k3s nginx setup

Convert `static/kubernetes/nginx-k3s.yaml` to function `getK3sNginxHelmValues`.

* refactor: gardenless ephemeral nginx setup

Convert `static/kubernetes/nginx-ephemeral.yaml` to function `getEphemeralNginxHelmValues`.

* chore: remove duplicate `--namespace` arg

It will be set by `helm` function.

* chore: remove garden config for system services

* fix: pass helm values in the proper format

* chore: remove unnecessary field from `LocalKubernetesConfig`

* chore: use explicit types for local k8s configs

* chore: do not install default nginx for local k8s clusters

A cluster-type specific installation will be done separately.

* chore: install and uninstall defaultBackend with nginx ingress controller

* chore: drop kind nginx support for old k8s versions (< 1.21)

* chore: fix ephemeral kubernetes

* chore: rename function

* chore: refactor installing ingress controllers

* refactor: rename module `integrations` => `nginx`

* refactor: extract new file `default-backend.ts`

To store default backend specific functions.

* refactor: rename file `nginx.ts` => `ingress-controller.ts`

Let the name be more specific.

* chore: use defensive type-checks

* fix: use specific helm values for k3s and generic cluster types

* chore: remove unnecessary calls of `defaultBackendInstall`

It's already called from inside `helmNginxInstall`.

* chore: remove unnecessary calls of `defaultBackendUninstall`

It's already called from inside `helmNginxUninstall`.

* chore: narrowed local var scope

* chore: re-arranged code

* refactor: extract named constant for Garden default-backend image

* chore: use named constant instead of hard-coded string

* chore: use `systemNamespace` var instead of hard-coded string

* refactor: move helm nginx helpers to the dedicated file

For better modularity and readability.

* refactor: move generic cluster nginx code to a dedicated file

For better modularity and readability.

* refactor: move k3s cluster nginx code to a dedicated file

For better modularity and readability.

* refactor: move microk8s cluster nginx code to a dedicated file

For better modularity and readability.

* refactor: move minikube cluster nginx code to a dedicated file

For better modularity and readability.

* chore: rename file `nginx-kind.ts` => `nginx-kind-manifests.ts`

* chore: move `nginx-kind-manifests.ts` to `nginx` package

* refactor: move kind cluster nginx code to a dedicated file

For better modularity and readability.

* refactor: move ephemeral cluster nginx code to a dedicated file

For better modularity and readability.

* refactor: extract function `helmIngressControllerReady`

To reduce code repetition.
It checks the readiness of ingress controller and default backend.

* chore: narrow scope of some local vars

* chore(k8s): remove unused static resources

* fix(k8s): uninstall only the ingress controller installed by Garden

We should only remove it if it was installed by Garden or this specific Garden project in the first place.

Co-authored-by: Anna Mager <[email protected]>

* chore: remove unused function

* refactor: make `getClusterType` side-effect free function

* fix(ephemeral): restore ingress controller installation

* refactor: more granular cluster types

Another one wiil be added for ephemeral cluster.

* chore: spacing + inline redundant local var

* refactor: configure ingress controllers in a single place

Always use base kubernetes provider's `prepareEnvironment` handler
to configure cluster-type-specific ingress controllers.

* refactor: ensure system namespace in the base kubernetes handler

To avoid repetitive code in the child-plugins.

* chore: use correct section name in logger

* chore: replace if/else with ternary operator

* chore: add some integ tests around ingress controllers

* chore: check status of main ingress controller deployed resources

* chore: wait for all nginx resources to be ready

* chore: explicit type for nginx helm values

* refactor: extract helper function `getNginxHelmMainResource`

* refactor: move constant to a dedicated file

Also applied a specific type.

* chore: use ESM imports (post-rebase corrections)

* chore: use narrow-typed named constants as Docker image names

* refactor: (1) introduce interface and classes for ingress setup

For DRY, better readability snf maintainability.

* refactor: (2) use interface and classes for ingress setup

* refactor: (3) remove unnecessary exports

* refactor: (4) inline functions into class methods

* refactor: (5) inline functions into class methods

* refactor: (6) convert interface to abstract class

* refactor: (7) introduce abstract method

* refactor: (8) inline functions into class methods

* chore: move `GardenIngressController` to standalone file

To avoid circular imports.

* refactor: (9) convert default backend functions to class

* refactor: (10) inline functions into class methods

* chore: more generic name for the ABC

* chore: re-arranged code

* refactor: (11) inline functions into class methods

* refactor: (12) change signature of `helmValuesGetter`

* refactor: (13) replace conditional logic with polymorphism

* refactor: rename method

* chore: add deprecation warning for `plugin kubernetes init-cluster` command

---------

Co-authored-by: Jon Edvald <[email protected]>
Co-authored-by: Anna Mager <[email protected]>
Co-authored-by: Anna Mager <[email protected]>

core/src/plugins/kubernetes/commands/cluster-init.ts

@@ -7,30 +7,33 @@
  */
 
 import type { PluginCommand } from "../../../plugin/command.js"
-import { prepareSystem, getEnvironmentStatus } from "../init.js"
+import { prepareEnvironment, getEnvironmentStatus } from "../init.js"
 import chalk from "chalk"
+import { emitNonRepeatableWarning } from "../../../warnings.js"
 
+// TODO: remove in 0.14
 export const clusterInit: PluginCommand = {
   name: "cluster-init",
-  description: "Initialize or update cluster-wide Garden services.",
+  description: "[DEPRECATED] Initialize or update cluster-wide Garden services.",
 
   title: ({ environmentName }) => {
     return `Initializing/updating cluster-wide services for ${chalk.white(environmentName)} environment`
   },
 
   handler: async ({ ctx, log }) => {
+    emitNonRepeatableWarning(log, "This command is now deprecated and will be removed in Garden 0.14.")
+
     const status = await getEnvironmentStatus({ ctx, log })
     let result = {}
 
     if (status.ready) {
       log.info("All services already initialized!")
     } else {
-      result = await prepareSystem({
+      result = await prepareEnvironment({
         ctx,
         log,
         force: true,
         status,
-        clusterInit: true,
       })
     }
 

core/src/plugins/kubernetes/commands/uninstall-garden-services.ts

@@ -8,9 +8,8 @@
 
 import chalk from "chalk"
 import type { PluginCommand } from "../../../plugin/command.js"
-import { getKubernetesSystemVariables } from "../init.js"
 import type { KubernetesPluginContext } from "../config.js"
-import { getSystemGarden } from "../system.js"
+import { ingressControllerUninstall } from "../nginx/ingress-controller.js"
 
 export const uninstallGardenServices: PluginCommand = {
   name: "uninstall-garden-services",
@@ -22,25 +21,13 @@ export const uninstallGardenServices: PluginCommand = {
 
   handler: async ({ ctx, log }) => {
     const k8sCtx = <KubernetesPluginContext>ctx
-    const variables = getKubernetesSystemVariables(k8sCtx.provider.config)
 
-    const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log)
-    const actions = await sysGarden.getActionRouter()
-
-    const graph = await sysGarden.getConfigGraph({ log, emit: false })
-    const deploys = graph.getDeploys()
-
-    log.info("")
-
-    const deployNames = deploys.map((s) => s.name)
-    const statuses = await actions.deleteDeploys({ graph, log, names: deployNames })
-
-    log.info("")
-
-    const environmentStatuses = await actions.provider.cleanupAll(log)
+    if (k8sCtx.provider.config.setupIngressController === "nginx") {
+      await ingressControllerUninstall(k8sCtx, log)
+    }
 
     log.info(chalk.green("\nDone!"))
 
-    return { result: { serviceStatuses: statuses, environmentStatuses } }
+    return { result: {} }
   },
 }

core/src/plugins/kubernetes/config.ts

@@ -9,7 +9,6 @@
 import type { StringMap } from "../../config/common.js"
 import {
   joi,
-  joiArray,
   joiIdentifier,
   joiIdentifierDescription,
   joiProviderName,
@@ -28,7 +27,6 @@ import {
 } from "../container/moduleConfig.js"
 import type { PluginContext } from "../../plugin-context.js"
 import { dedent, deline } from "../../util/string.js"
-import { defaultSystemNamespace } from "./system.js"
 import type { SyncableKind } from "./types.js"
 import { syncableKinds } from "./types.js"
 import type { BaseTaskSpec } from "../../config/task.js"
@@ -42,7 +40,9 @@ import type { SyncDefaults } from "./sync.js"
 import { syncDefaultsSchema } from "./sync.js"
 import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl.js"
 import { DOCS_BASE_URL } from "../../constants.js"
-import { defaultKanikoImageName } from "./constants.js"
+import { defaultKanikoImageName, defaultSystemNamespace } from "./constants.js"
+import type { LocalKubernetesClusterType } from "./local/config.js"
+import type { EphemeralKubernetesClusterType } from "./ephemeral/config.js"
 
 export interface ProviderSecretRef {
   name: string
@@ -122,6 +122,8 @@ export interface ClusterBuildkitCacheConfig {
   registry?: ContainerRegistryConfig
 }
 
+export type KubernetesClusterType = LocalKubernetesClusterType | EphemeralKubernetesClusterType
+
 export interface KubernetesConfig extends BaseProviderConfig {
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
@@ -173,8 +175,7 @@ export interface KubernetesConfig extends BaseProviderConfig {
   gardenSystemNamespace: string
   tlsCertificates: IngressTlsCertificate[]
   certManager?: CertManagerConfig
-  clusterType?: "kind" | "minikube" | "microk8s" | "k3s"
-  _systemServices: string[]
+  clusterType?: KubernetesClusterType
 }
 
 export type KubernetesProvider = Provider<KubernetesConfig>
@@ -633,7 +634,6 @@ export const kubernetesConfigBase = () =>
       tlsCertificates: joiSparseArray(tlsCertificateSchema())
         .unique("name")
         .description("One or more certificates to use for ingress."),
-      _systemServices: joiArray(joiIdentifier()).meta({ internal: true }),
       systemNodeSelector: joiStringMap(joi.string())
         .description(
           dedent`

core/src/plugins/kubernetes/constants.ts

@@ -42,6 +42,13 @@ export const buildkitRootlessImageName: DockerImageWithDigest =
   "gardendev/buildkit:v0.12.2-rootless@sha256:e30b7830078d51e66f1a861024dcc91f2ae5cb1108789c74d0e43ffe0d065b20"
 export const defaultKanikoImageName: DockerImageWithDigest =
   "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517"
+export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
+  "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
+export const defaultGardenIngressControllerImage: DockerImageWithDigest =
+  "k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"
+export const defaultGardenIngressControllerKubeWebhookCertGenImage: DockerImageWithDigest =
+  "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"
 
 export const buildkitDeploymentName = "garden-buildkit"
 export const buildkitContainerName = "buildkitd"
+export const defaultSystemNamespace = "garden-system"

core/src/plugins/kubernetes/container/build/local.ts

@@ -93,7 +93,7 @@ export const localBuild: BuildHandler = async (params) => {
 }
 
 /**
- * Loads a built local image to a local Kubernetes instance
+ * Loads a built local image to a local Kubernetes instance.
  */
 export async function loadToLocalK8s(params: BuildActionParams<"build", ContainerBuildAction>) {
   const { ctx, log, action } = params

core/src/plugins/kubernetes/ephemeral/config.ts

@@ -10,7 +10,6 @@ import chalk from "chalk"
 import fsExtra from "fs-extra"
 const { mkdirp, writeFile } = fsExtra
 import { load } from "js-yaml"
-import { remove } from "lodash-es"
 import moment from "moment"
 import { join } from "path"
 import { joi, joiProviderName } from "../../../config/common.js"
@@ -19,9 +18,13 @@ import { ConfigurationError } from "../../../exceptions.js"
 import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js"
 import { dedent } from "../../../util/string.js"
 import type { KubernetesConfig } from "../config.js"
+import { defaultResources } from "../config.js"
 import { namespaceSchema } from "../config.js"
 import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral.js"
 import { DEFAULT_GARDEN_CLOUD_DOMAIN } from "../../../constants.js"
+import { defaultSystemNamespace } from "../constants.js"
+
+export type EphemeralKubernetesClusterType = "ephemeral"
 
 export const configSchema = () =>
   providerConfigBaseSchema()
@@ -43,12 +46,7 @@ export const configSchema = () =>
 
 export async function configureProvider(params: ConfigureProviderParams<KubernetesConfig>) {
   const { base, log, projectName, ctx, config: baseConfig } = params
-  if (projectName === "garden-system") {
-    // avoid configuring ephemeral-kubernetes provider and creating ephemeral-cluster for garden-system project
-    return {
-      config: baseConfig,
-    }
-  }
+
   log.info(`Configuring ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider for project ${projectName}`)
   if (!ctx.cloudApi) {
     throw new ConfigurationError({
@@ -60,13 +58,16 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       message: `${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider is currently not supported for ${ctx.cloudApi.distroName}.`,
     })
   }
+
   // creating tmp dir .garden/ephemeral-kubernetes for storing kubeconfig
   const ephemeralClusterDirPath = join(ctx.gardenDirPath, "ephemeral-kubernetes")
   await mkdirp(ephemeralClusterDirPath)
+
   log.info("Retrieving ephemeral Kubernetes cluster")
   const createEphemeralClusterResponse = await ctx.cloudApi.createEphemeralCluster()
   const clusterId = createEphemeralClusterResponse.instanceMetadata.instanceId
   log.info(`Ephemeral Kubernetes cluster retrieved successfully`)
+
   const deadlineDateTime = moment(createEphemeralClusterResponse.instanceMetadata.deadline)
   const diffInNowAndDeadline = moment.duration(deadlineDateTime.diff(moment())).asMinutes().toFixed(1)
   log.info(
@@ -76,6 +77,7 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       )}`
     )
   )
+
   log.info("Fetching kubeconfig for the ephemeral cluster")
   const kubeConfig = await ctx.cloudApi.getKubeConfigForCluster(clusterId)
   const kubeconfigFileName = `${clusterId}-kubeconfig.yaml`
@@ -84,8 +86,7 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
   log.info(`Kubeconfig for ephemeral cluster saved at path: ${chalk.underline(kubeConfigPath)}`)
 
   const parsedKubeConfig: any = load(kubeConfig)
-  const currentContext = parsedKubeConfig["current-context"]
-  baseConfig.context = currentContext
+  baseConfig.context = parsedKubeConfig["current-context"]
   baseConfig.kubeconfig = kubeConfigPath
 
   // set deployment registry
@@ -94,15 +95,21 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
     namespace: createEphemeralClusterResponse.registry.repository,
     insecure: false,
   }
+
   // set imagePullSecrets
   baseConfig.imagePullSecrets = [
     {
       name: createEphemeralClusterResponse.registry.imagePullSecret.name,
       namespace: createEphemeralClusterResponse.registry.imagePullSecret.namespace,
     },
   ]
+
   // set build mode to kaniko
   baseConfig.buildMode = "kaniko"
+
+  // set resource requests and limits defaults for builder, sync and util
+  baseConfig.resources = defaultResources
+
   // set additional kaniko flags
   baseConfig.kaniko = {
     extraFlags: [
@@ -112,28 +119,24 @@ export async function configureProvider(params: ConfigureProviderParams<Kubernet
       "--force",
     ],
   }
-  // set setupIngressController to null while initializing kubernetes plugin
-  // as we use it later and configure it separately for ephemeral-kubernetes
+
+  // set default hostname
+  baseConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname
+
+  // use garden-system as system namespace for ephemeral-kubernetes
+  baseConfig.gardenSystemNamespace = defaultSystemNamespace
+
+  // set the proper cluster type explicitly
+  baseConfig.clusterType = "ephemeral"
+
   const kubernetesPluginConfig = {
     ...params,
     config: {
       ...baseConfig,
-      setupIngressController: null,
     },
   }
   const { config: updatedConfig } = await base!(kubernetesPluginConfig)
 
-  // setup ingress controller unless setupIngressController is set to false/null in provider config
-  if (baseConfig.setupIngressController) {
-    const _systemServices = updatedConfig._systemServices
-    const nginxServices = ["ingress-controller", "default-backend"]
-    remove(_systemServices, (s) => nginxServices.includes(s))
-    _systemServices.push("nginx-ephemeral")
-    updatedConfig.setupIngressController = "nginx"
-    // set default hostname
-    updatedConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname
-  }
-
   return {
     config: updatedConfig,
   }

core/src/plugins/kubernetes/helm/status.ts

@@ -28,7 +28,7 @@ import { gardenAnnotationKey } from "../../../util/string.js"
 
 export const gardenCloudAECPauseAnnotation = gardenAnnotationKey("aec-status")
 
-const helmStatusMap: { [status: string]: DeployState } = {
+export const helmStatusMap: { [status: string]: DeployState } = {
   unknown: "unknown",
   deployed: "ready",
   deleted: "missing",