Merge pull request #10510 from hakman/containerd-registry-mirrors-1.19

Manual cherry-pick of #10507: Add containerd option for registry mirrors

docs/cluster_spec.md

@@ -968,6 +968,23 @@ tar tf cri-containerd-cni-1.4.3-linux-amd64.tar.gz
     usr/local/sbin/runc
 ```
 
+### Registry Mirrors
+{{ kops_feature_table(kops_added_default='1.19') }}
+
+If you have many instances running, each time one of them pulls an image that is not present on the host, it will fetch it from the internet. By caching these images, you can keep the traffic within your local network and avoid egress bandwidth usage.
+
+See [Image Registry](https://github.com/containerd/containerd/blob/master/docs/cri/registry.md#configure-registry-endpoint) docs for more info.
+
+```yaml
+spec:
+  containerd:
+    registryMirrors:
+      docker.io:
+      - https://registry-1.docker.io
+      "*":
+      - http://HostIP2:Port2
+```
+
 ## Docker
 
 It is possible to override Docker daemon options for all masters and nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#DockerConfig) for the full list of options.

go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/miekg/coredns v0.0.0-20161111164017-20e25559d5ea
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/pelletier/go-toml v1.8.1
 	github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6
 	github.com/prometheus/client_golang v1.7.1
 	github.com/sergi/go-diff v1.0.0

go.sum

@@ -683,6 +683,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
 github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
+github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=

k8s/crds/kops.k8s.io_clusters.yaml

@@ -409,6 +409,13 @@ spec:
                         description: UrlArm64 overrides the URL for the ARM64 package.
                         type: string
                     type: object
+                  registryMirrors:
+                    additionalProperties:
+                      items:
+                        type: string
+                      type: array
+                    description: RegistryMirrors is list of image registries
+                    type: object
                   root:
                     description: Root directory for persistent data (default "/var/lib/containerd").
                     type: string

pkg/apis/kops/containerdconfig.go

@@ -26,6 +26,8 @@ type ContainerdConfig struct {
 	LogLevel *string `json:"logLevel,omitempty" flag:"log-level"`
 	// Packages overrides the URL and hash for the packages.
 	Packages *PackagesConfig `json:"packages,omitempty"`
+	// RegistryMirrors is list of image registries
+	RegistryMirrors map[string][]string `json:"registryMirrors,omitempty"`
 	// Root directory for persistent data (default "/var/lib/containerd").
 	Root *string `json:"root,omitempty" flag:"root"`
 	// SkipInstall prevents kOps from installing and modifying containerd in any way (default "false").

pkg/apis/kops/v1alpha2/containerdconfig.go

@@ -26,6 +26,8 @@ type ContainerdConfig struct {
 	LogLevel *string `json:"logLevel,omitempty" flag:"log-level"`
 	// Packages overrides the URL and hash for the packages.
 	Packages *PackagesConfig `json:"packages,omitempty"`
+	// RegistryMirrors is list of image registries
+	RegistryMirrors map[string][]string `json:"registryMirrors,omitempty"`
 	// Root directory for persistent data (default "/var/lib/containerd").
 	Root *string `json:"root,omitempty" flag:"root"`
 	// SkipInstall prevents kOps from installing and modifying containerd in any way (default "false").

pkg/model/components/BUILD.bazel

@@ -37,6 +37,7 @@ go_library(
         "//upup/pkg/fi/loader:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/blang/semver/v4:go_default_library",
+        "//vendor/github.com/pelletier/go-toml:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/model/components/containerd.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"github.com/blang/semver/v4"
-	"k8s.io/klog/v2"
+	"github.com/pelletier/go-toml"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/loader"
@@ -44,22 +44,25 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 	containerd := clusterSpec.Containerd
 
 	if clusterSpec.ContainerRuntime == "containerd" {
-		if b.IsKubernetesLT("1.18") {
-			klog.Warningf("kubernetes %s is untested with containerd", clusterSpec.KubernetesVersion)
-		}
-
-		// Set containerd based on Kubernetes version
+		// Set version based on Kubernetes version
 		if fi.StringValue(containerd.Version) == "" {
 			if b.IsKubernetesGTE("1.19") {
 				containerd.Version = fi.String("1.4.3")
 			} else {
 				containerd.Version = fi.String("1.3.9")
 			}
 		}
-
-		// Apply defaults for containerd running in container runtime mode
+		// Set default log level to INFO
 		containerd.LogLevel = fi.String("info")
-		containerd.ConfigOverride = fi.String("version = 2")
+		// Build config file for containerd running in CRI mode
+		if fi.StringValue(containerd.ConfigOverride) == "" {
+			config, _ := toml.Load("")
+			config.SetPath([]string{"version"}, int64(2))
+			for name, endpoints := range containerd.RegistryMirrors {
+				config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints)
+			}
+			containerd.ConfigOverride = fi.String(config.String())
+		}
 
 	} else if clusterSpec.ContainerRuntime == "docker" {
 		// Docker version should always be available
@@ -77,10 +80,12 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 				return nil
 			}
 		}
-
-		// Apply defaults for containerd running in Docker mode
+		// Set default log level to INFO
 		containerd.LogLevel = fi.String("info")
-		containerd.ConfigOverride = fi.String("disabled_plugins = [\"cri\"]\n")
+		// Build config file for containerd running in Docker mode
+		config, _ := toml.Load("")
+		config.SetPath([]string{"disabled_plugins"}, []string{"cri"})
+		containerd.ConfigOverride = fi.String(config.String())
 
 	} else {
 		// Unknown container runtime, should not install containerd

tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml

@@ -137,11 +137,31 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
   cloudConfig: null
   containerRuntime: containerd
   containerd:
-    configOverride: version = 2
+    configOverride: |
+      version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".registry]
+
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+              [plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"]
+                endpoint = ["http://HostIP2:Port2"]
+
+              [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+                endpoint = ["https://registry-1.docker.io"]
     logLevel: info
     packages:
       hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"
       urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    registryMirrors:
+      '*':
+      - http://HostIP2:Port2
+      docker.io:
+      - https://registry-1.docker.io
     version: 1.4.3
   docker:
     skipInstall: true
@@ -451,11 +471,31 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
   cloudConfig: null
   containerRuntime: containerd
   containerd:
-    configOverride: version = 2
+    configOverride: |
+      version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".registry]
+
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+              [plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"]
+                endpoint = ["http://HostIP2:Port2"]
+
+              [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+                endpoint = ["https://registry-1.docker.io"]
     logLevel: info
     packages:
       hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"
       urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    registryMirrors:
+      '*':
+      - http://HostIP2:Port2
+      docker.io:
+      - https://registry-1.docker.io
     version: 1.4.3
   docker:
     skipInstall: true

tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml

@@ -11,6 +11,11 @@ spec:
   configBase: memfs://clusters.example.com/containerd.example.com
   containerRuntime: containerd
   containerd:
+    registryMirrors:
+      docker.io:
+      - https://registry-1.docker.io
+      "*":
+      - http://HostIP2:Port2
     packages:
       urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
       hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"

tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml

@@ -137,7 +137,8 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properti
   cloudConfig: null
   containerRuntime: containerd
   containerd:
-    configOverride: version = 2
+    configOverride: |
+      version = 2
     logLevel: info
     version: 1.4.3
   docker:
@@ -448,7 +449,8 @@ Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplat
   cloudConfig: null
   containerRuntime: containerd
   containerd:
-    configOverride: version = 2
+    configOverride: |
+      version = 2
     logLevel: info
     version: 1.4.3
   docker: