feat(k8s): allow using registry mirror for utility images (#6552)

Garden uses various utility container images, e.g. for the K8s sync utility pod, and hosts them on Docker Hub. Some users have their own mirror of Docker Hub that they'd rather use instead of Docker Hub itself. This avoids issues with rate limiting among other things. This commit adds a config field on the Kubernetes provider spec that allows users to configure what registry is used. Note that it only allows users to specify the registry domain, not the repository, image name, or tag. So the assumption is that mirror has the exact same "layout" as Docker Hub itself which is usually (always?) the case. If needed, we can later expose more granular control, e.g. setting the image tag, but that's not required for now. Co-authored-by: Eyþór Magnússon <[email protected]>
garden-io · Oct 21, 2024 · 122371d · 122371d
1 parent c9e0e8d
commit 122371d
Show file tree

Hide file tree

Showing 27 changed files with 505 additions and 77 deletions.
diff --git a/core/src/plugins/kubernetes/commands/pull-image.ts b/core/src/plugins/kubernetes/commands/pull-image.ts
@@ -15,7 +15,7 @@ import { KubeApi } from "../api.js"
 import type { Log } from "../../../logger/log-entry.js"
 import { containerHelpers } from "../../container/helpers.js"
 import { PodRunner } from "../run.js"
-import { dockerAuthSecretKey, getK8sUtilImageName, systemDockerAuthSecretName } from "../constants.js"
+import { dockerAuthSecretKey, getK8sUtilImagePath, systemDockerAuthSecretName } from "../constants.js"
 import { getAppNamespace, getSystemNamespace } from "../namespace.js"
 import { randomString } from "../../../util/string.js"
 import type { PluginContext } from "../../../plugin-context.js"
@@ -155,7 +155,7 @@ export async function pullBuild(params: PullParams) {
         containers: [
           {
             name: "main",
-            image: getK8sUtilImageName(),
+            image: getK8sUtilImagePath(ctx.provider.config.utilImageRegistryDomain),
             command: ["sleep", "" + (imagePullTimeoutSeconds + 10)],
             volumeMounts: [
               {

diff --git a/core/src/plugins/kubernetes/config.ts b/core/src/plugins/kubernetes/config.ts
@@ -40,7 +40,7 @@ import type { SyncDefaults } from "./sync.js"
 import { syncDefaultsSchema } from "./sync.js"
 import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl.js"
 import { DOCS_BASE_URL } from "../../constants.js"
-import { defaultKanikoImageName, defaultSystemNamespace } from "./constants.js"
+import { defaultKanikoImageName, defaultUtilImageRegistryDomain, defaultSystemNamespace } from "./constants.js"
 import type { LocalKubernetesClusterType } from "./local/config.js"
 import type { EphemeralKubernetesClusterType } from "./ephemeral/config.js"
 
@@ -125,6 +125,7 @@ export interface ClusterBuildkitCacheConfig {
 export type KubernetesClusterType = LocalKubernetesClusterType | EphemeralKubernetesClusterType
 
 export interface KubernetesConfig extends BaseProviderConfig {
+  utilImageRegistryDomain: string
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
     cache: ClusterBuildkitCacheConfig[]
@@ -407,9 +408,20 @@ const buildkitCacheConfigurationSchema = () =>
       ),
   })
 
+export const utilImageRegistryDomainSpec = joi.string().default(defaultUtilImageRegistryDomain).description(dedent`
+    The container registry domain that should be used for pulling Garden utility images (such as the
+    image used in the Kubernetes sync utility Pod).
+
+    If you have your own Docker Hub registry mirror, you can set the domain here and the utility images
+    will be pulled from there. This can be useful to e.g. avoid Docker Hub rate limiting.
+
+    Otherwise the utility images are pulled directly from Docker Hub by default.
+  `)
+
 export const kubernetesConfigBase = () =>
   providerConfigBaseSchema()
     .keys({
+      utilImageRegistryDomain: utilImageRegistryDomainSpec,
       buildMode: joi
         .string()
         .valid("local-docker", "kaniko", "cluster-buildkit")

diff --git a/core/src/plugins/kubernetes/constants.ts b/core/src/plugins/kubernetes/constants.ts
@@ -10,9 +10,6 @@ import type { DockerImageWithDigest } from "../../util/string.js"
 import { gardenEnv } from "../../constants.js"
 import { makeDocsLinkPlain } from "../../docs/common.js"
 
-export const rsyncPortName = "garden-rsync"
-export const buildSyncVolumeName = `garden-sync`
-
 export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is 1MB
 // max ConfigMap data size is 1MB but we need to factor in overhead, plus in some cases the log is duplicated in
 // the outputs field, so we cap at 250kB.
@@ -24,50 +21,85 @@ export const PROXY_CONTAINER_SSH_TUNNEL_PORT_NAME = "garden-prx-ssh"
 
 export const systemDockerAuthSecretName = "builder-docker-config"
 export const dockerAuthSecretKey = ".dockerconfigjson"
-
 export const skopeoDaemonContainerName = "util"
-
 export const defaultIngressClass = "nginx"
+export const rsyncPortName = "garden-rsync"
+export const buildSyncVolumeName = `garden-sync`
+export const k8sSyncUtilContainerName = "garden-sync-init"
+export const buildkitDeploymentName = "garden-buildkit"
+export const buildkitContainerName = "buildkitd"
+export const defaultSystemNamespace = "garden-system"
+
+export const syncGuideRelPath = "guides/code-synchronization"
+export const syncGuideLink = makeDocsLinkPlain(syncGuideRelPath)
 
-// Docker images that Garden ships with
-export const k8sUtilImageNameLegacy: DockerImageWithDigest =
-  "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b"
-export const k8sUtilImageName: DockerImageWithDigest =
-  "gardendev/k8s-util:0.6.2@sha256:f51e7ce040e2e23bc0eaa7216e4d976f13786d96773ef7b8c8f349e7a63d74e9"
+export const defaultUtilImageRegistryDomain = "docker.io"
 
-export function getK8sUtilImageName(): DockerImageWithDigest {
-  return gardenEnv.GARDEN_ENABLE_NEW_SYNC ? k8sUtilImageName : k8sUtilImageNameLegacy
+function makeImagePath({
+  imageName,
+  registryDomain,
+}: {
+  imageName: DockerImageWithDigest
+  registryDomain: string
+}): DockerImageWithDigest {
+  const domainWithoutTrailingSlash = registryDomain.replace(/\/$/, "")
+
+  return `${domainWithoutTrailingSlash}/${imageName}`
 }
 
-export const k8sSyncUtilImageNameLegacy: DockerImageWithDigest =
-  "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578"
-export const k8sSyncUtilImageName: DockerImageWithDigest =
-  "gardendev/k8s-sync:0.2.2@sha256:9ebcd84df4a3a55ae0ba95051cab521d249a4d2d7a15d04da7301c888c02347b"
+export function getK8sUtilImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sUtilImageNameLegacy: DockerImageWithDigest =
+    "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b"
+  const k8sUtilImageName: DockerImageWithDigest =
+    "gardendev/k8s-util:0.6.2@sha256:f51e7ce040e2e23bc0eaa7216e4d976f13786d96773ef7b8c8f349e7a63d74e9"
 
-export const k8sSyncUtilContainerName = "garden-sync-init"
+  return gardenEnv.GARDEN_ENABLE_NEW_SYNC
+    ? makeImagePath({ imageName: k8sUtilImageName, registryDomain })
+    : makeImagePath({ imageName: k8sUtilImageNameLegacy, registryDomain })
+}
+
+export function getK8sSyncUtilImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sSyncUtilImageName: DockerImageWithDigest =
+    "gardendev/k8s-sync:0.2.2@sha256:9ebcd84df4a3a55ae0ba95051cab521d249a4d2d7a15d04da7301c888c02347b"
+  const k8sSyncUtilImageNameLegacy: DockerImageWithDigest =
+    "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578"
 
-export function getK8sSyncUtilImageName(): DockerImageWithDigest {
-  return gardenEnv.GARDEN_ENABLE_NEW_SYNC ? k8sSyncUtilImageName : k8sSyncUtilImageNameLegacy
+  return gardenEnv.GARDEN_ENABLE_NEW_SYNC
+    ? makeImagePath({ imageName: k8sSyncUtilImageName, registryDomain })
+    : makeImagePath({ imageName: k8sSyncUtilImageNameLegacy, registryDomain })
+}
+
+export function getK8sReverseProxyImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sReverseProxyImageName: DockerImageWithDigest =
+    "gardendev/k8s-reverse-proxy:0.1.1@sha256:2dff2275fc8c32cc0eba50eebd7ace6fdb007d9b3f4bd48d94355057324b2394"
+
+  return makeImagePath({ imageName: k8sReverseProxyImageName, registryDomain })
+}
+export function getBuildkitImagePath(registryDomain: string): DockerImageWithDigest {
+  const buildkitImageName: DockerImageWithDigest =
+    "gardendev/buildkit:v-0.16.0@sha256:ee7aa12e6fdba79ee9838631995fa7c5a12aba9091a0753dedfe891d430c8182"
+
+  return makeImagePath({ imageName: buildkitImageName, registryDomain })
+}
+
+export function getBuildkitRootlessImagePath(registryDomain: string): DockerImageWithDigest {
+  const buildkitRootlessImageName: DockerImageWithDigest =
+    "gardendev/buildkit:v-0.16.0-rootless@sha256:634506c016691b079e44614c5de65e0b0d4a98070304f6089e15f0279bfca411"
+
+  return makeImagePath({ imageName: buildkitRootlessImageName, registryDomain })
+}
+export function getDefaultGardenIngressControllerDefaultBackendImagePath(
+  registryDomain: string
+): DockerImageWithDigest {
+  const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
+    "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
+
+  return makeImagePath({ imageName: defaultGardenIngressControllerDefaultBackendImage, registryDomain })
 }
 
-export const k8sReverseProxyImageName: DockerImageWithDigest =
-  "gardendev/k8s-reverse-proxy:0.1.1@sha256:2dff2275fc8c32cc0eba50eebd7ace6fdb007d9b3f4bd48d94355057324b2394"
-export const buildkitImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v-0.16.0@sha256:ee7aa12e6fdba79ee9838631995fa7c5a12aba9091a0753dedfe891d430c8182"
-export const buildkitRootlessImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v-0.16.0-rootless@sha256:634506c016691b079e44614c5de65e0b0d4a98070304f6089e15f0279bfca411"
 export const defaultKanikoImageName: DockerImageWithDigest =
   "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517"
-export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
-  "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
 export const defaultGardenIngressControllerImage: DockerImageWithDigest =
   "k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"
 export const defaultGardenIngressControllerKubeWebhookCertGenImage: DockerImageWithDigest =
   "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"
-
-export const buildkitDeploymentName = "garden-buildkit"
-export const buildkitContainerName = "buildkitd"
-export const defaultSystemNamespace = "garden-system"
-
-export const syncGuideRelPath = "guides/code-synchronization"
-export const syncGuideLink = makeDocsLinkPlain(syncGuideRelPath)
diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -13,9 +13,9 @@ import {
   buildSyncVolumeName,
   buildkitContainerName,
   buildkitDeploymentName,
-  buildkitImageName,
-  buildkitRootlessImageName,
   dockerAuthSecretKey,
+  getBuildkitImagePath,
+  getBuildkitRootlessImagePath,
 } from "../../constants.js"
 import { KubeApi } from "../../api.js"
 import type { KubernetesDeployment } from "../../types.js"
@@ -472,7 +472,7 @@ export function getBuildkitDeployment(
           containers: [
             {
               name: buildkitContainerName,
-              image: buildkitImageName,
+              image: getBuildkitImagePath(provider.config.utilImageRegistryDomain),
               args: ["--addr", "unix:///run/buildkit/buildkitd.sock"],
               readinessProbe: {
                 exec: {
@@ -538,7 +538,7 @@ export function getBuildkitDeployment(
       "container.apparmor.security.beta.kubernetes.io/buildkitd": "unconfined",
       "container.seccomp.security.alpha.kubernetes.io/buildkitd": "unconfined",
     }
-    buildkitContainer.image = buildkitRootlessImageName
+    buildkitContainer.image = getBuildkitRootlessImagePath(provider.config.utilImageRegistryDomain)
     buildkitContainer.args = [
       "--addr",
       "unix:///run/user/1000/buildkit/buildkitd.sock",

diff --git a/core/src/plugins/kubernetes/container/build/common.ts b/core/src/plugins/kubernetes/container/build/common.ts
@@ -29,7 +29,7 @@ import type { Resolved } from "../../../../actions/types.js"
 import { stringifyResources } from "../util.js"
 import { getKubectlExecDestination } from "../../sync.js"
 import { getRunningDeploymentPod } from "../../util.js"
-import { buildSyncVolumeName, dockerAuthSecretKey, getK8sUtilImageName, rsyncPortName } from "../../constants.js"
+import { buildSyncVolumeName, dockerAuthSecretKey, getK8sUtilImagePath, rsyncPortName } from "../../constants.js"
 import { styles } from "../../../../logger/styles.js"
 import type { StringMap } from "../../../../config/common.js"
 import { LogLevel } from "../../../../logger/logger.js"
@@ -578,7 +578,7 @@ export function getBuilderServiceAccountSpec(namespace: string, annotations?: St
 export function getUtilContainer(authSecretName: string, provider: KubernetesProvider): V1Container {
   return {
     name: utilContainerName,
-    image: getK8sUtilImageName(),
+    image: getK8sUtilImagePath(provider.config.utilImageRegistryDomain),
     imagePullPolicy: "IfNotPresent",
     command: ["/rsync-server.sh"],
     env: [

diff --git a/core/src/plugins/kubernetes/container/build/kaniko.ts b/core/src/plugins/kubernetes/container/build/kaniko.ts
@@ -11,7 +11,7 @@ import {
   skopeoDaemonContainerName,
   dockerAuthSecretKey,
   defaultKanikoImageName,
-  getK8sUtilImageName,
+  getK8sUtilImagePath,
 } from "../../constants.js"
 import { KubeApi } from "../../api.js"
 import type { Log } from "../../../../logger/log-entry.js"
@@ -320,7 +320,7 @@ export function getKanikoBuilderPodManifest({
     initContainers: [
       {
         name: "init",
-        image: getK8sUtilImageName(),
+        image: getK8sUtilImagePath(provider.config.utilImageRegistryDomain),
         command: [
           "/bin/sh",
           "-c",

diff --git a/core/src/plugins/kubernetes/ephemeral/config.ts b/core/src/plugins/kubernetes/ephemeral/config.ts
@@ -17,7 +17,7 @@ import { ConfigurationError } from "../../../exceptions.js"
 import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js"
 import { dedent } from "../../../util/string.js"
 import type { KubernetesConfig } from "../config.js"
-import { defaultResources } from "../config.js"
+import { defaultResources, utilImageRegistryDomainSpec } from "../config.js"
 import { namespaceSchema } from "../config.js"
 import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral.js"
 import { DEFAULT_GARDEN_CLOUD_DOMAIN } from "../../../constants.js"
@@ -30,6 +30,7 @@ export const configSchema = () =>
   providerConfigBaseSchema()
     .keys({
       name: joiProviderName(EPHEMERAL_KUBERNETES_PROVIDER_NAME),
+      utilImageRegistryDomain: utilImageRegistryDomainSpec,
       namespace: namespaceSchema().description(
         "Specify which namespace to deploy services to (defaults to the project name). " +
           "Note that the framework generates other namespaces as well with this name as a prefix."

diff --git a/core/src/plugins/kubernetes/local-mode.ts b/core/src/plugins/kubernetes/local-mode.ts
@@ -14,7 +14,7 @@ import { remove, set } from "lodash-es"
 import type { BaseResource, KubernetesResource, SyncableResource, SyncableRuntimeAction } from "./types.js"
 import type { PrimitiveMap } from "../../config/common.js"
 import {
-  k8sReverseProxyImageName,
+  getK8sReverseProxyImagePath,
   PROXY_CONTAINER_SSH_TUNNEL_PORT,
   PROXY_CONTAINER_SSH_TUNNEL_PORT_NAME,
   PROXY_CONTAINER_USER_NAME,
@@ -382,16 +382,23 @@ function prepareLocalModePorts(): V1ContainerPort[] {
  * @param localModeEnvVars the list of localMode-specific environment variables
  * @param localModePorts the list of localMode-specific ports (e.g. ssh port for tunnel setup)
  */
-function patchSyncableManifest(
-  targetManifest: SyncableResource,
-  containerName: string,
-  localModeEnvVars: PrimitiveMap,
+function patchSyncableManifest({
+  targetManifest,
+  containerName,
+  localModeEnvVars,
+  localModePorts,
+  utilImageRegistryDomain,
+}: {
+  targetManifest: SyncableResource
+  containerName: string
+  localModeEnvVars: PrimitiveMap
   localModePorts: V1ContainerPort[]
-): void {
+  utilImageRegistryDomain: string
+}): void {
   const targetContainer = getResourceContainer(targetManifest, containerName)
 
   // use reverse proxy container image
-  targetContainer.image = k8sReverseProxyImageName
+  targetContainer.image = getK8sReverseProxyImagePath(utilImageRegistryDomain)
   // erase the original container command, the proxy container won't recognize it
   targetContainer.command = []
   // erase the original container arguments, the proxy container won't recognize them
@@ -462,7 +469,7 @@ export async function configureLocalMode(configParams: ConfigureLocalModeParams)
 
   // Logging this on the debug level because it can be displayed multiple times due to getServiceStatus checks
   log.debug(
-    `Configuring in local mode, proxy container ${styles.underline(k8sReverseProxyImageName)} will be deployed.`
+    `Configuring in local mode, proxy container ${styles.underline(getK8sReverseProxyImagePath(provider.config.utilImageRegistryDomain))} will be deployed.`
   )
 
   set(resolvedTarget, ["metadata", "annotations", gardenAnnotationKey("mode")], "local")
@@ -478,7 +485,13 @@ export async function configureLocalMode(configParams: ConfigureLocalModeParams)
   const localModeEnvVars = await prepareLocalModeEnvVars(portSpecs, keyPair)
   const localModePorts = prepareLocalModePorts()
 
-  patchSyncableManifest(resolvedTarget, targetContainer.name, localModeEnvVars, localModePorts)
+  patchSyncableManifest({
+    targetManifest: resolvedTarget,
+    containerName: targetContainer.name,
+    localModeEnvVars,
+    localModePorts,
+    utilImageRegistryDomain: provider.config.utilImageRegistryDomain,
+  })
 
   // Replace the original resource with the modified spec
   const preparedManifests = manifests

diff --git a/core/src/plugins/kubernetes/nginx/default-backend.ts b/core/src/plugins/kubernetes/nginx/default-backend.ts
@@ -12,7 +12,7 @@ import type { DeployState } from "../../../types/service.js"
 import { KubeApi } from "../api.js"
 import { checkResourceStatus, waitForResources } from "../status/status.js"
 import type { KubernetesDeployment, KubernetesService } from "../types.js"
-import { defaultGardenIngressControllerDefaultBackendImage } from "../constants.js"
+import { getDefaultGardenIngressControllerDefaultBackendImagePath } from "../constants.js"
 import { GardenIngressComponent } from "./ingress-controller-base.js"
 
 export class GardenDefaultBackend extends GardenIngressComponent {
@@ -111,7 +111,7 @@ function defaultBackendGetManifests(ctx: KubernetesPluginContext): {
         spec: {
           containers: [
             {
-              image: defaultGardenIngressControllerDefaultBackendImage,
+              image: getDefaultGardenIngressControllerDefaultBackendImagePath(provider.config.utilImageRegistryDomain),
               imagePullPolicy: "IfNotPresent",
               name: "default-backend",
               ports: [

diff --git a/core/src/plugins/kubernetes/sync.ts b/core/src/plugins/kubernetes/sync.ts
@@ -58,7 +58,7 @@ import { isConfiguredForSyncMode } from "./status/status.js"
 import type { PluginContext } from "../../plugin-context.js"
 import type { SyncConfig, SyncSession } from "../../mutagen.js"
 import { haltedStatuses, Mutagen, mutagenAgentPath, mutagenStatusDescriptions } from "../../mutagen.js"
-import { getK8sSyncUtilImageName, k8sSyncUtilContainerName, syncGuideLink } from "./constants.js"
+import { getK8sSyncUtilImagePath, k8sSyncUtilContainerName, syncGuideLink } from "./constants.js"
 import { isAbsolute, relative, resolve } from "path"
 import type { Resolved } from "../../actions/types.js"
 import { joinWithPosix } from "../../util/fs.js"
@@ -465,8 +465,8 @@ export async function configureSyncMode({
     if (!podSpec.imagePullSecrets) {
       podSpec.imagePullSecrets = []
     }
-    const k8sSyncUtilImageName = getK8sSyncUtilImageName()
 
+    const k8sSyncUtilImageName = getK8sSyncUtilImagePath(provider.config.utilImageRegistryDomain)
     if (!podSpec.initContainers.find((c) => c.image === k8sSyncUtilImageName)) {
       const initContainer: V1Container = {
         name: k8sSyncUtilContainerName,