fix(k8s): unnecessary socat sidecar being deployed with BuildKit

Fixes #2339
garden-io · May 10, 2021 · e4f22de · e4f22de
1 parent e17d936
commit e4f22de
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 10 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -848,6 +848,12 @@ workflows:
           project: gke
           environment: gke-kaniko-gcr
           requires: [build]
+      - e2e-project:
+          <<: *only-internal-prs
+          name: e2e-gke-buildkit-gcr
+          project: gke
+          environment: gke-buildkit-gcr
+          requires: [build]
       - e2e-project:
           <<: *only-internal-prs
           name: e2e-hot-reload

diff --git a/core/src/plugins/kubernetes/container/build/buildkit.ts b/core/src/plugins/kubernetes/container/build/buildkit.ts
@@ -23,7 +23,7 @@ import { LogEntry } from "../../../../logger/log-entry"
 import { waitForResources, compareDeployedResources } from "../../status/status"
 import { KubernetesProvider, KubernetesPluginContext } from "../../config"
 import { PluginContext } from "../../../../plugin-context"
-import { prepareDockerAuth, getRegistryHostname } from "../../init"
+import { prepareDockerAuth } from "../../init"
 import { BuildStatusHandler, skopeoBuildStatus, BuildHandler, syncToBuildSync, getSocatContainer } from "./common"
 import { getNamespaceStatus } from "../../namespace"
 import { containerHelpers } from "../../../container/helpers"
@@ -308,9 +308,10 @@ export function getBuildkitDeployment(provider: KubernetesProvider) {
     },
   }
 
-  // We need a proxy sidecar to be able to reach the in-cluster registry from the Pod
-  const registryHostname = getRegistryHostname(provider.config)
-  deployment.spec!.template.spec!.containers.push(getSocatContainer(registryHostname))
+  if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) {
+    // We need a proxy sidecar to be able to reach the in-cluster registry from the Pod
+    deployment.spec!.template.spec!.containers.push(getSocatContainer(provider))
+  }
 
   // Set the configured nodeSelector, if any
   if (!isEmpty(provider.config.clusterBuildkit?.nodeSelector)) {

diff --git a/core/src/plugins/kubernetes/container/build/common.ts b/core/src/plugins/kubernetes/container/build/common.ts
@@ -23,6 +23,7 @@ import { normalizeLocalRsyncPath } from "../../../../util/fs"
 import { exec } from "../../../../util/util"
 import { InternalError, RuntimeError } from "../../../../exceptions"
 import { LogEntry } from "../../../../logger/log-entry"
+import { getInClusterRegistryHostname } from "../../init"
 
 const inClusterRegistryPort = 5000
 
@@ -198,7 +199,9 @@ export async function getUtilDaemonPodRunner({
   })
 }
 
-export function getSocatContainer(registryHostname: string) {
+export function getSocatContainer(provider: KubernetesProvider) {
+  const registryHostname = getInClusterRegistryHostname(provider.config)
+
   return {
     name: "proxy",
     image: "gardendev/socat:0.1.0",

diff --git a/core/src/plugins/kubernetes/container/build/kaniko.ts b/core/src/plugins/kubernetes/container/build/kaniko.ts
@@ -21,7 +21,7 @@ import { getDockerAuthVolume } from "../../util"
 import { KubernetesProvider, KubernetesPluginContext, DEFAULT_KANIKO_IMAGE } from "../../config"
 import { ConfigurationError } from "../../../../exceptions"
 import { PodRunner } from "../../run"
-import { getRegistryHostname, getKubernetesSystemVariables } from "../../init"
+import { getKubernetesSystemVariables } from "../../init"
 import { Writable } from "stream"
 import { getSystemNamespace } from "../../namespace"
 import { dedent } from "../../../../util/string"
@@ -101,7 +101,6 @@ export async function runKaniko({
   const api = await KubeApi.factory(log, ctx, provider)
 
   const podName = makePodName("kaniko", namespace, module.name)
-  const registryHostname = getRegistryHostname(provider.config)
   const k8sSystemVars = getKubernetesSystemVariables(provider.config)
   const syncDataVolumeName = k8sSystemVars["sync-volume-name"]
   const commsVolumeName = "comms"
@@ -185,7 +184,7 @@ export async function runKaniko({
 
   if (provider.config.deploymentRegistry?.hostname === inClusterRegistryHostname) {
     spec.containers = spec.containers.concat([
-      getSocatContainer(registryHostname),
+      getSocatContainer(provider),
       // This is a workaround so that the kaniko executor can wait until socat starts, and so that the socat proxy
       // doesn't just keep running after the build finishes. Doing this in the kaniko Pod is currently not possible
       // because of https://github.com/GoogleContainerTools/distroless/issues/225

diff --git a/core/src/plugins/kubernetes/init.ts b/core/src/plugins/kubernetes/init.ts
@@ -377,7 +377,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
   return {
     "namespace": systemNamespace,
 
-    "registry-hostname": getRegistryHostname(config),
+    "registry-hostname": getInClusterRegistryHostname(config),
     "builder-mode": config.buildMode,
 
     "builder-limits-cpu": millicpuToString(config.resources.builder.limits.cpu),
@@ -415,7 +415,7 @@ export function getKubernetesSystemVariables(config: KubernetesConfig) {
   }
 }
 
-export function getRegistryHostname(config: KubernetesConfig) {
+export function getInClusterRegistryHostname(config: KubernetesConfig) {
   const systemNamespace = config.gardenSystemNamespace
   return `garden-docker-registry.${systemNamespace}.svc.cluster.local`
 }