feat(k8s): allow using registry mirror for utility images

core/src/plugins/kubernetes/commands/pull-image.ts

@@ -15,7 +15,7 @@ import { KubeApi } from "../api.js"
 import type { Log } from "../../../logger/log-entry.js"
 import { containerHelpers } from "../../container/helpers.js"
 import { PodRunner } from "../run.js"
-import { dockerAuthSecretKey, getK8sUtilImageName, systemDockerAuthSecretName } from "../constants.js"
+import { dockerAuthSecretKey, getK8sUtilImagePath, systemDockerAuthSecretName } from "../constants.js"
 import { getAppNamespace, getSystemNamespace } from "../namespace.js"
 import { randomString } from "../../../util/string.js"
 import type { PluginContext } from "../../../plugin-context.js"
@@ -155,7 +155,7 @@ export async function pullBuild(params: PullParams) {
         containers: [
           {
             name: "main",
-            image: getK8sUtilImageName(),
+            image: getK8sUtilImagePath(ctx.provider.config.utilImageRegistryDomain),
             command: ["sleep", "" + (imagePullTimeoutSeconds + 10)],
             volumeMounts: [
               {

core/src/plugins/kubernetes/config.ts

@@ -40,7 +40,7 @@ import type { SyncDefaults } from "./sync.js"
 import { syncDefaultsSchema } from "./sync.js"
 import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl.js"
 import { DOCS_BASE_URL } from "../../constants.js"
-import { defaultKanikoImageName, defaultSystemNamespace } from "./constants.js"
+import { defaultKanikoImageName, defaultUtilImageRegistryDomain, defaultSystemNamespace } from "./constants.js"
 import type { LocalKubernetesClusterType } from "./local/config.js"
 import type { EphemeralKubernetesClusterType } from "./ephemeral/config.js"
 
@@ -125,6 +125,7 @@ export interface ClusterBuildkitCacheConfig {
 export type KubernetesClusterType = LocalKubernetesClusterType | EphemeralKubernetesClusterType
 
 export interface KubernetesConfig extends BaseProviderConfig {
+  utilImageRegistryDomain: string
   buildMode: ContainerBuildMode
   clusterBuildkit?: {
     cache: ClusterBuildkitCacheConfig[]
@@ -407,9 +408,20 @@ const buildkitCacheConfigurationSchema = () =>
       ),
   })
 
+export const utilImageRegistryDomainSpec = joi.string().default(defaultUtilImageRegistryDomain).description(dedent`
+    The container registry domain that should be used for pulling Garden utility images (such as the
+    image used in the Kubernetes sync utility Pod).
+
+    If you have your own Docker Hub registry mirror, you can set the domain here and the utility images
+    will be pulled from there. This can be useful to e.g. avoid Docker Hub rate limiting.
+
+    Otherwise the utility images are pulled directly from Docker Hub by default.
+  `)
+
 export const kubernetesConfigBase = () =>
   providerConfigBaseSchema()
     .keys({
+      utilImageRegistryDomain: utilImageRegistryDomainSpec,
       buildMode: joi
         .string()
         .valid("local-docker", "kaniko", "cluster-buildkit")

core/src/plugins/kubernetes/constants.ts

@@ -10,9 +10,6 @@ import type { DockerImageWithDigest } from "../../util/string.js"
 import { gardenEnv } from "../../constants.js"
 import { makeDocsLinkPlain } from "../../docs/common.js"
 
-export const rsyncPortName = "garden-rsync"
-export const buildSyncVolumeName = `garden-sync`
-
 export const MAX_CONFIGMAP_DATA_SIZE = 1024 * 1024 // max ConfigMap data size is 1MB
 // max ConfigMap data size is 1MB but we need to factor in overhead, plus in some cases the log is duplicated in
 // the outputs field, so we cap at 250kB.
@@ -24,50 +21,85 @@ export const PROXY_CONTAINER_SSH_TUNNEL_PORT_NAME = "garden-prx-ssh"
 
 export const systemDockerAuthSecretName = "builder-docker-config"
 export const dockerAuthSecretKey = ".dockerconfigjson"
-
 export const skopeoDaemonContainerName = "util"
-
 export const defaultIngressClass = "nginx"
+export const rsyncPortName = "garden-rsync"
+export const buildSyncVolumeName = `garden-sync`
+export const k8sSyncUtilContainerName = "garden-sync-init"
+export const buildkitDeploymentName = "garden-buildkit"
+export const buildkitContainerName = "buildkitd"
+export const defaultSystemNamespace = "garden-system"
+
+export const syncGuideRelPath = "guides/code-synchronization"
+export const syncGuideLink = makeDocsLinkPlain(syncGuideRelPath)
 
-// Docker images that Garden ships with
-export const k8sUtilImageNameLegacy: DockerImageWithDigest =
-  "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b"
-export const k8sUtilImageName: DockerImageWithDigest =
-  "gardendev/k8s-util:0.6.2@sha256:f51e7ce040e2e23bc0eaa7216e4d976f13786d96773ef7b8c8f349e7a63d74e9"
+export const defaultUtilImageRegistryDomain = "docker.io"
 
-export function getK8sUtilImageName(): DockerImageWithDigest {
-  return gardenEnv.GARDEN_ENABLE_NEW_SYNC ? k8sUtilImageName : k8sUtilImageNameLegacy
+function makeImagePath({
+  imageName,
+  registryDomain,
+}: {
+  imageName: DockerImageWithDigest
+  registryDomain: string
+}): DockerImageWithDigest {
+  const domainWithoutTrailingSlash = registryDomain.replace(/\/$/, "")
+
+  return `${domainWithoutTrailingSlash}/${imageName}`
 }
 
-export const k8sSyncUtilImageNameLegacy: DockerImageWithDigest =
-  "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578"
-export const k8sSyncUtilImageName: DockerImageWithDigest =
-  "gardendev/k8s-sync:0.2.2@sha256:9ebcd84df4a3a55ae0ba95051cab521d249a4d2d7a15d04da7301c888c02347b"
+export function getK8sUtilImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sUtilImageNameLegacy: DockerImageWithDigest =
+    "gardendev/k8s-util:0.5.7@sha256:522da245a5e6ae7c711aa94f84fc83f82a8fdffbf6d8bc48f4d80fee0e0e631b"
+  const k8sUtilImageName: DockerImageWithDigest =
+    "gardendev/k8s-util:0.6.2@sha256:f51e7ce040e2e23bc0eaa7216e4d976f13786d96773ef7b8c8f349e7a63d74e9"
 
-export const k8sSyncUtilContainerName = "garden-sync-init"
+  return gardenEnv.GARDEN_ENABLE_NEW_SYNC
+    ? makeImagePath({ imageName: k8sUtilImageName, registryDomain })
+    : makeImagePath({ imageName: k8sUtilImageNameLegacy, registryDomain })
+}
+
+export function getK8sSyncUtilImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sSyncUtilImageName: DockerImageWithDigest =
+    "gardendev/k8s-sync:0.2.2@sha256:9ebcd84df4a3a55ae0ba95051cab521d249a4d2d7a15d04da7301c888c02347b"
+  const k8sSyncUtilImageNameLegacy: DockerImageWithDigest =
+    "gardendev/k8s-sync:0.1.5@sha256:28263cee5ac41acebb8c08f852c4496b15e18c0c94797d7a949a4453b5f91578"
 
-export function getK8sSyncUtilImageName(): DockerImageWithDigest {
-  return gardenEnv.GARDEN_ENABLE_NEW_SYNC ? k8sSyncUtilImageName : k8sSyncUtilImageNameLegacy
+  return gardenEnv.GARDEN_ENABLE_NEW_SYNC
+    ? makeImagePath({ imageName: k8sSyncUtilImageName, registryDomain })
+    : makeImagePath({ imageName: k8sSyncUtilImageNameLegacy, registryDomain })
+}
+
+export function getK8sReverseProxyImagePath(registryDomain: string): DockerImageWithDigest {
+  const k8sReverseProxyImageName: DockerImageWithDigest =
+    "gardendev/k8s-reverse-proxy:0.1.1@sha256:2dff2275fc8c32cc0eba50eebd7ace6fdb007d9b3f4bd48d94355057324b2394"
+
+  return makeImagePath({ imageName: k8sReverseProxyImageName, registryDomain })
+}
+export function getBuildkitImagePath(registryDomain: string): DockerImageWithDigest {
+  const buildkitImageName: DockerImageWithDigest =
+    "gardendev/buildkit:v-0.16.0@sha256:ee7aa12e6fdba79ee9838631995fa7c5a12aba9091a0753dedfe891d430c8182"
+
+  return makeImagePath({ imageName: buildkitImageName, registryDomain })
+}
+
+export function getBuildkitRootlessImagePath(registryDomain: string): DockerImageWithDigest {
+  const buildkitRootlessImageName: DockerImageWithDigest =
+    "gardendev/buildkit:v-0.16.0-rootless@sha256:634506c016691b079e44614c5de65e0b0d4a98070304f6089e15f0279bfca411"
+
+  return makeImagePath({ imageName: buildkitRootlessImageName, registryDomain })
+}
+export function getDefaultGardenIngressControllerDefaultBackendImagePath(
+  registryDomain: string
+): DockerImageWithDigest {
+  const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
+    "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
+
+  return makeImagePath({ imageName: defaultGardenIngressControllerDefaultBackendImage, registryDomain })
 }
 
-export const k8sReverseProxyImageName: DockerImageWithDigest =
-  "gardendev/k8s-reverse-proxy:0.1.1@sha256:2dff2275fc8c32cc0eba50eebd7ace6fdb007d9b3f4bd48d94355057324b2394"
-export const buildkitImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v-0.16.0@sha256:ee7aa12e6fdba79ee9838631995fa7c5a12aba9091a0753dedfe891d430c8182"
-export const buildkitRootlessImageName: DockerImageWithDigest =
-  "gardendev/buildkit:v-0.16.0-rootless@sha256:634506c016691b079e44614c5de65e0b0d4a98070304f6089e15f0279bfca411"
 export const defaultKanikoImageName: DockerImageWithDigest =
   "gcr.io/kaniko-project/executor:v1.11.0-debug@sha256:32ba2214921892c2fa7b5f9c4ae6f8f026538ce6b2105a93a36a8b5ee50fe517"
-export const defaultGardenIngressControllerDefaultBackendImage: DockerImageWithDigest =
-  "gardendev/default-backend:v0.1@sha256:1b02920425eea569c6be53bb2e3d2c1182243212de229be375da7a93594498cf"
 export const defaultGardenIngressControllerImage: DockerImageWithDigest =
   "k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"
 export const defaultGardenIngressControllerKubeWebhookCertGenImage: DockerImageWithDigest =
   "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660"
-
-export const buildkitDeploymentName = "garden-buildkit"
-export const buildkitContainerName = "buildkitd"
-export const defaultSystemNamespace = "garden-system"
-
-export const syncGuideRelPath = "guides/code-synchronization"
-export const syncGuideLink = makeDocsLinkPlain(syncGuideRelPath)

core/src/plugins/kubernetes/container/build/buildkit.ts

@@ -13,9 +13,9 @@ import {
   buildSyncVolumeName,
   buildkitContainerName,
   buildkitDeploymentName,
-  buildkitImageName,
-  buildkitRootlessImageName,
   dockerAuthSecretKey,
+  getBuildkitImagePath,
+  getBuildkitRootlessImagePath,
 } from "../../constants.js"
 import { KubeApi } from "../../api.js"
 import type { KubernetesDeployment } from "../../types.js"
@@ -472,7 +472,7 @@ export function getBuildkitDeployment(
           containers: [
             {
               name: buildkitContainerName,
-              image: buildkitImageName,
+              image: getBuildkitImagePath(provider.config.utilImageRegistryDomain),
               args: ["--addr", "unix:///run/buildkit/buildkitd.sock"],
               readinessProbe: {
                 exec: {
@@ -538,7 +538,7 @@ export function getBuildkitDeployment(
       "container.apparmor.security.beta.kubernetes.io/buildkitd": "unconfined",
       "container.seccomp.security.alpha.kubernetes.io/buildkitd": "unconfined",
     }
-    buildkitContainer.image = buildkitRootlessImageName
+    buildkitContainer.image = getBuildkitRootlessImagePath(provider.config.utilImageRegistryDomain)
     buildkitContainer.args = [
       "--addr",
       "unix:///run/user/1000/buildkit/buildkitd.sock",

core/src/plugins/kubernetes/container/build/common.ts

@@ -29,7 +29,7 @@ import type { Resolved } from "../../../../actions/types.js"
 import { stringifyResources } from "../util.js"
 import { getKubectlExecDestination } from "../../sync.js"
 import { getRunningDeploymentPod } from "../../util.js"
-import { buildSyncVolumeName, dockerAuthSecretKey, getK8sUtilImageName, rsyncPortName } from "../../constants.js"
+import { buildSyncVolumeName, dockerAuthSecretKey, getK8sUtilImagePath, rsyncPortName } from "../../constants.js"
 import { styles } from "../../../../logger/styles.js"
 import type { StringMap } from "../../../../config/common.js"
 import { LogLevel } from "../../../../logger/logger.js"
@@ -578,7 +578,7 @@ export function getBuilderServiceAccountSpec(namespace: string, annotations?: St
 export function getUtilContainer(authSecretName: string, provider: KubernetesProvider): V1Container {
   return {
     name: utilContainerName,
-    image: getK8sUtilImageName(),
+    image: getK8sUtilImagePath(provider.config.utilImageRegistryDomain),
     imagePullPolicy: "IfNotPresent",
     command: ["/rsync-server.sh"],
     env: [

core/src/plugins/kubernetes/container/build/kaniko.ts

@@ -11,7 +11,7 @@ import {
   skopeoDaemonContainerName,
   dockerAuthSecretKey,
   defaultKanikoImageName,
-  getK8sUtilImageName,
+  getK8sUtilImagePath,
 } from "../../constants.js"
 import { KubeApi } from "../../api.js"
 import type { Log } from "../../../../logger/log-entry.js"
@@ -320,7 +320,7 @@ export function getKanikoBuilderPodManifest({
     initContainers: [
       {
         name: "init",
-        image: getK8sUtilImageName(),
+        image: getK8sUtilImagePath(provider.config.utilImageRegistryDomain),
         command: [
           "/bin/sh",
           "-c",

core/src/plugins/kubernetes/ephemeral/config.ts

@@ -17,7 +17,7 @@ import { ConfigurationError } from "../../../exceptions.js"
 import type { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider.js"
 import { dedent } from "../../../util/string.js"
 import type { KubernetesConfig } from "../config.js"
-import { defaultResources } from "../config.js"
+import { defaultResources, utilImageRegistryDomainSpec } from "../config.js"
 import { namespaceSchema } from "../config.js"
 import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral.js"
 import { DEFAULT_GARDEN_CLOUD_DOMAIN } from "../../../constants.js"
@@ -30,6 +30,7 @@ export const configSchema = () =>
   providerConfigBaseSchema()
     .keys({
       name: joiProviderName(EPHEMERAL_KUBERNETES_PROVIDER_NAME),
+      utilImageRegistryDomain: utilImageRegistryDomainSpec,
       namespace: namespaceSchema().description(
         "Specify which namespace to deploy services to (defaults to the project name). " +
           "Note that the framework generates other namespaces as well with this name as a prefix."

core/src/plugins/kubernetes/local-mode.ts

@@ -14,7 +14,7 @@ import { remove, set } from "lodash-es"
 import type { BaseResource, KubernetesResource, SyncableResource, SyncableRuntimeAction } from "./types.js"
 import type { PrimitiveMap } from "../../config/common.js"
 import {
-  k8sReverseProxyImageName,
+  getK8sReverseProxyImagePath,
   PROXY_CONTAINER_SSH_TUNNEL_PORT,
   PROXY_CONTAINER_SSH_TUNNEL_PORT_NAME,
   PROXY_CONTAINER_USER_NAME,
@@ -382,16 +382,23 @@ function prepareLocalModePorts(): V1ContainerPort[] {
  * @param localModeEnvVars the list of localMode-specific environment variables
  * @param localModePorts the list of localMode-specific ports (e.g. ssh port for tunnel setup)
  */
-function patchSyncableManifest(
-  targetManifest: SyncableResource,
-  containerName: string,
-  localModeEnvVars: PrimitiveMap,
+function patchSyncableManifest({
+  targetManifest,
+  containerName,
+  localModeEnvVars,
+  localModePorts,
+  utilImageRegistryDomain,
+}: {
+  targetManifest: SyncableResource
+  containerName: string
+  localModeEnvVars: PrimitiveMap
   localModePorts: V1ContainerPort[]
-): void {
+  utilImageRegistryDomain: string
+}): void {
   const targetContainer = getResourceContainer(targetManifest, containerName)
 
   // use reverse proxy container image
-  targetContainer.image = k8sReverseProxyImageName
+  targetContainer.image = getK8sReverseProxyImagePath(utilImageRegistryDomain)
   // erase the original container command, the proxy container won't recognize it
   targetContainer.command = []
   // erase the original container arguments, the proxy container won't recognize them
@@ -462,7 +469,7 @@ export async function configureLocalMode(configParams: ConfigureLocalModeParams)
 
   // Logging this on the debug level because it can be displayed multiple times due to getServiceStatus checks
   log.debug(
-    `Configuring in local mode, proxy container ${styles.underline(k8sReverseProxyImageName)} will be deployed.`
+    `Configuring in local mode, proxy container ${styles.underline(getK8sReverseProxyImagePath(provider.config.utilImageRegistryDomain))} will be deployed.`
   )
 
   set(resolvedTarget, ["metadata", "annotations", gardenAnnotationKey("mode")], "local")
@@ -478,7 +485,13 @@ export async function configureLocalMode(configParams: ConfigureLocalModeParams)
   const localModeEnvVars = await prepareLocalModeEnvVars(portSpecs, keyPair)
   const localModePorts = prepareLocalModePorts()
 
-  patchSyncableManifest(resolvedTarget, targetContainer.name, localModeEnvVars, localModePorts)
+  patchSyncableManifest({
+    targetManifest: resolvedTarget,
+    containerName: targetContainer.name,
+    localModeEnvVars,
+    localModePorts,
+    utilImageRegistryDomain: provider.config.utilImageRegistryDomain,
+  })
 
   // Replace the original resource with the modified spec
   const preparedManifests = manifests

core/src/plugins/kubernetes/nginx/default-backend.ts

@@ -12,7 +12,7 @@ import type { DeployState } from "../../../types/service.js"
 import { KubeApi } from "../api.js"
 import { checkResourceStatus, waitForResources } from "../status/status.js"
 import type { KubernetesDeployment, KubernetesService } from "../types.js"
-import { defaultGardenIngressControllerDefaultBackendImage } from "../constants.js"
+import { getDefaultGardenIngressControllerDefaultBackendImagePath } from "../constants.js"
 import { GardenIngressComponent } from "./ingress-controller-base.js"
 
 export class GardenDefaultBackend extends GardenIngressComponent {
@@ -111,7 +111,7 @@ function defaultBackendGetManifests(ctx: KubernetesPluginContext): {
         spec: {
           containers: [
             {
-              image: defaultGardenIngressControllerDefaultBackendImage,
+              image: getDefaultGardenIngressControllerDefaultBackendImagePath(provider.config.utilImageRegistryDomain),
               imagePullPolicy: "IfNotPresent",
               name: "default-backend",
               ports: [

core/src/plugins/kubernetes/sync.ts

@@ -58,7 +58,7 @@ import { isConfiguredForSyncMode } from "./status/status.js"
 import type { PluginContext } from "../../plugin-context.js"
 import type { SyncConfig, SyncSession } from "../../mutagen.js"
 import { haltedStatuses, Mutagen, mutagenAgentPath, mutagenStatusDescriptions } from "../../mutagen.js"
-import { getK8sSyncUtilImageName, k8sSyncUtilContainerName, syncGuideLink } from "./constants.js"
+import { getK8sSyncUtilImagePath, k8sSyncUtilContainerName, syncGuideLink } from "./constants.js"
 import { isAbsolute, relative, resolve } from "path"
 import type { Resolved } from "../../actions/types.js"
 import { joinWithPosix } from "../../util/fs.js"
@@ -465,8 +465,8 @@ export async function configureSyncMode({
     if (!podSpec.imagePullSecrets) {
       podSpec.imagePullSecrets = []
     }
-    const k8sSyncUtilImageName = getK8sSyncUtilImageName()
 
+    const k8sSyncUtilImageName = getK8sSyncUtilImagePath(provider.config.utilImageRegistryDomain)
     if (!podSpec.initContainers.find((c) => c.image === k8sSyncUtilImageName)) {
       const initContainer: V1Container = {
         name: k8sSyncUtilContainerName,