fix(k8s): correctly handle Ingress API versions for container modules

We now auto-detect the most up-to-date supported Ingress API version
and create resources accordingly.

.circleci/config.yml

@@ -494,7 +494,7 @@ jobs:
       - run:
           name: Install kind
           command: |
-            curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-linux-amd64
+            curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-linux-amd64
             chmod +x kind-linux-amd64
             sudo mv kind-linux-amd64 /usr/local/bin/kind
       - run:
@@ -503,7 +503,7 @@ jobs:
             # Create the kind cluster with a custom config to enable the default ingress controller
             cat \<<EOF | kind create cluster \
                 --image <<parameters.kindNodeImage>> \
-                --wait=120s \
+                --wait=600s \
                 --config=-
 
             kind: Cluster
@@ -539,7 +539,7 @@ jobs:
             # Notes:
             # - We skip tests that only work for remote environments
             # - We currently don't support in-cluster building on kind.
-            yarn integ-kind -b
+            yarn integ-kind
       - run:
           name: Plugin tests
           command: yarn run test:plugins
@@ -604,7 +604,7 @@ jobs:
             # - Need to run with sudo to work with microk8s, because CircleCI doesn't allow us to log out
             #   and back in to add the circleci user to the microk8s group.
             # - We currently don't support in-cluster building on microk8s.
-            GARDEN_SKIP_TESTS="cluster-docker kaniko remote-only" sudo -E npm run integ -b
+            GARDEN_SKIP_TESTS="cluster-docker kaniko remote-only" sudo -E npm run integ
       - run:
           name: Plugin tests
           command: sudo -E npm run test:plugins
@@ -781,6 +781,10 @@ workflows:
     jobs:
       - test-kind:
           kindNodeImage: kindest/node:v1.20.2
+  test-kind-1.21:
+    jobs:
+      - test-kind:
+          kindNodeImage: kindest/node:v1.21.2
   commit:
     jobs:
       ### ALL BRANCHES ###

core/src/plugins/kubernetes/api.ts

@@ -33,6 +33,7 @@ import {
   V1Deployment,
   V1Service,
   Log,
+  NetworkingV1Api,
 } from "@kubernetes/client-node"
 import AsyncLock = require("async-lock")
 import request = require("request-promise")
@@ -86,12 +87,14 @@ const requestAgent = new Agent({ lookup })
 // NOTE: be warned, the API of the client library is very likely to change
 
 type K8sApi =
+  | ApiextensionsV1beta1Api
+  | AppsV1Api
+  | CoreApi
   | CoreV1Api
   | ExtensionsV1beta1Api
-  | RbacAuthorizationV1Api
-  | AppsV1Api
-  | ApiextensionsV1beta1Api
+  | NetworkingV1Api
   | PolicyV1beta1Api
+  | RbacAuthorizationV1Api
 type K8sApiConstructor<T extends K8sApi> = new (basePath?: string) => T
 
 const apiTypes: { [key: string]: K8sApiConstructor<any> } = {
@@ -101,6 +104,7 @@ const apiTypes: { [key: string]: K8sApiConstructor<any> } = {
   core: CoreV1Api,
   coreApi: CoreApi,
   extensions: ExtensionsV1beta1Api,
+  networking: NetworkingV1Api,
   policy: PolicyV1beta1Api,
   rbac: RbacAuthorizationV1Api,
 }
@@ -177,6 +181,7 @@ export class KubeApi {
   public core: WrappedApi<CoreV1Api>
   public coreApi: WrappedApi<CoreApi>
   public extensions: WrappedApi<ExtensionsV1beta1Api>
+  public networking: WrappedApi<NetworkingV1Api>
   public policy: WrappedApi<PolicyV1beta1Api>
   public rbac: WrappedApi<RbacAuthorizationV1Api>
 
@@ -304,18 +309,7 @@ export class KubeApi {
         }
       }))
 
-    const resource = resourceMap[kind]
-
-    if (!resource) {
-      const err = new KubernetesError(`Unrecognized resource type ${apiVersion}/${kind}`, {
-        apiVersion,
-        kind,
-      })
-      err.statusCode = 404
-      throw err
-    }
-
-    return resource
+    return resourceMap[kind]
   }
 
   async request({
@@ -503,6 +497,16 @@ export class KubeApi {
     namespace: string
   }) {
     const resourceInfo = await this.getApiResourceInfo(log, apiVersion, kind)
+
+    if (!resourceInfo) {
+      const err = new KubernetesError(`Unrecognized resource type ${apiVersion}/${kind}`, {
+        apiVersion,
+        kind,
+      })
+      err.statusCode = 404
+      throw err
+    }
+
     const basePath = getGroupBasePath(apiVersion)
 
     return resourceInfo.namespaced

core/src/plugins/kubernetes/config.ts

@@ -18,7 +18,7 @@ import {
   joiIdentifierDescription,
   joiSparseArray,
 } from "../../config/common"
-import { Provider, providerConfigBaseSchema, GenericProviderConfig } from "../../config/provider"
+import { Provider, providerConfigBaseSchema, BaseProviderConfig } from "../../config/provider"
 import {
   containerRegistryConfigSchema,
   ContainerRegistryConfig,
@@ -106,7 +106,7 @@ export interface NamespaceConfig {
   labels?: StringMap
 }
 
-export interface KubernetesConfig extends GenericProviderConfig {
+export interface KubernetesConfig extends BaseProviderConfig {
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
     rootless?: boolean
@@ -133,6 +133,7 @@ export interface KubernetesConfig extends GenericProviderConfig {
   kubeconfig?: string
   namespace?: NamespaceConfig
   registryProxyTolerations: V1Toleration[]
+  setupIngressController: string | null
   systemNodeSelector: { [key: string]: string }
   resources: KubernetesResources
   storage: KubernetesStorage

core/src/plugins/kubernetes/container/ingress.ts

@@ -17,8 +17,14 @@ import { ConfigurationError, PluginError } from "../../../exceptions"
 import { ensureSecret } from "../secrets"
 import { getHostnamesFromPem } from "../../../util/tls"
 import { KubernetesResource } from "../types"
-import { V1Secret } from "@kubernetes/client-node"
+import { ExtensionsV1beta1Ingress, V1Ingress, V1Secret } from "@kubernetes/client-node"
 import { LogEntry } from "../../../logger/log-entry"
+import chalk from "chalk"
+
+// Ingress API versions in descending order of preference
+// NOTE: We currently prefer the v1beta1 API version if available, since users may not have an IngressClass set up
+// correctly
+export const supportedIngressApiVersions = ["networking.k8s.io/v1beta1", "networking.k8s.io/v1", "extensions/v1beta1"]
 
 interface ServiceIngressWithCert extends ServiceIngress {
   spec: ContainerIngressSpec
@@ -38,60 +44,112 @@ export async function createIngressResources(
     return []
   }
 
-  const allIngresses = await getIngressesWithCert(service, api, provider)
-
-  return Bluebird.map(allIngresses, async (ingress, index) => {
-    const rules = [
-      {
-        host: ingress.hostname,
-        http: {
-          paths: [
-            {
-              path: ingress.path,
-              backend: {
-                serviceName: service.name,
-                servicePort: findByName(service.spec.ports, ingress.spec.port)!.servicePort,
-              },
-            },
-          ],
-        },
-      },
-    ]
+  // Detect the supported ingress version for the context
+  let apiVersion: string | undefined = undefined
 
-    const cert = ingress.certificate
-
-    const annotations = {
-      "ingress.kubernetes.io/force-ssl-redirect": !!cert + "",
+  for (const version of supportedIngressApiVersions) {
+    const resourceInfo = await api.getApiResourceInfo(log, version, "Ingress")
+    if (resourceInfo) {
+      apiVersion = version
     }
+  }
 
-    if (provider.config.ingressClass) {
-      annotations["kubernetes.io/ingress.class"] = provider.config.ingressClass
-    }
+  if (!apiVersion) {
+    log.warn(chalk.yellow(`Could not find a supported Ingress API version in the target cluster`))
+    return []
+  }
 
-    extend(annotations, ingress.spec.annotations)
+  const allIngresses = await getIngressesWithCert(service, api, provider)
 
-    const spec: any = { rules }
+  return Bluebird.map(allIngresses, async (ingress, index) => {
+    const cert = ingress.certificate
 
     if (!!cert) {
       // make sure the TLS secrets exist in this namespace
       await ensureSecret(api, cert.secretRef, namespace, log)
+    }
 
-      spec.tls = [
-        {
-          secretName: cert.secretRef.name,
+    if (apiVersion === "networking.k8s.io/v1") {
+      // The V1 API has a different shape than the beta API
+      // Note: We do not create the IngressClass resource automatically here!
+      const ingressResource: KubernetesResource<V1Ingress> = {
+        apiVersion,
+        kind: "Ingress",
+        metadata: {
+          name: `${service.name}-${index}`,
+          annotations: {
+            "ingress.kubernetes.io/force-ssl-redirect": !!cert + "",
+            ...ingress.spec.annotations,
+          },
+          namespace,
         },
-      ]
-    }
+        spec: {
+          ingressClassName: provider.config.ingressClass,
+          rules: [
+            {
+              host: ingress.hostname,
+              http: {
+                paths: [
+                  {
+                    path: ingress.path,
+                    pathType: "prefix",
+                    backend: {
+                      service: {
+                        name: service.name,
+                        port: {
+                          number: findByName(service.spec.ports, ingress.spec.port)!.servicePort,
+                        },
+                      },
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+          tls: cert ? [{ hosts: [ingress.hostname], secretName: cert.secretRef.name }] : undefined,
+        },
+      }
+      return ingressResource
+    } else {
+      const annotations = {
+        "ingress.kubernetes.io/force-ssl-redirect": !!cert + "",
+      }
 
-    return {
-      apiVersion: "extensions/v1beta1",
-      kind: "Ingress",
-      metadata: {
-        name: `${service.name}-${index}`,
-        annotations,
-        namespace,
-      },
-      spec,
+      if (provider.config.ingressClass) {
+        annotations["kubernetes.io/ingress.class"] = provider.config.ingressClass
+      }
+
+      extend(annotations, ingress.spec.annotations)
+
+      const ingressResource: KubernetesResource<ExtensionsV1beta1Ingress> = {
+        apiVersion: apiVersion!,
+        kind: "Ingress",
+        metadata: {
+          name: `${service.name}-${index}`,
+          annotations,
+          namespace,
+        },
+        spec: {
+          rules: [
+            {
+              host: ingress.hostname,
+              http: {
+                paths: [
+                  {
+                    path: ingress.path,
+                    backend: {
+                      serviceName: service.name,
+                      servicePort: <any>findByName(service.spec.ports, ingress.spec.port)!.servicePort,
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+          tls: cert ? [{ secretName: cert.secretRef.name }] : undefined,
+        },
+      }
+      return ingressResource
     }
   })
 }

core/src/plugins/kubernetes/kubernetes-module/common.ts

@@ -47,17 +47,10 @@ export async function getManifests({
         manifest.metadata = {}
       }
 
-      try {
-        const info = await api.getApiResourceInfo(log, manifest.apiVersion, manifest.kind)
+      const info = await api.getApiResourceInfo(log, manifest.apiVersion, manifest.kind)
 
-        if (info.namespaced) {
-          manifest.metadata.namespace = defaultNamespace
-        }
-      } catch (err) {
-        // We can get a 404 if a resource type can't be found, e.g. a missing CRD
-        if (err.statusCode !== 404) {
-          throw err
-        }
+      if (info?.namespaced) {
+        manifest.metadata.namespace = defaultNamespace
       }
     }
 

core/src/util/util.ts

@@ -36,6 +36,7 @@ const exitHookNames: string[] = [] // For debugging/testing/inspection purposes
 export type PickFromUnion<T, U extends T> = U
 export type ValueOf<T> = T[keyof T]
 export type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>
+export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>
 export type Diff<T, U> = T extends U ? never : T
 export type Mutable<T> = { -readonly [K in keyof T]: T[K] }
 export type Nullable<T> = { [P in keyof T]: T[P] | null }