Add --image-service-endpoint flag (#8279)

* Add --image-service-endpoint flag Problem: External container runtime can be set but image service endpoint is unchanged and also is not exposed as a flag. This is useful for using containerd snapshotters outside of the ones that have built-in support like stargz-snapshotter. Solution: Add a flag --image-service-endpoint and also default image service endpoint to container runtime endpoint if set. Signed-off-by: Edgar Lee <[email protected]> (cherry picked from commit fe18b1f) Signed-off-by: Brad Davidson <[email protected]>
k3s-io · Oct 17, 2023 · 4e77f40 · 4e77f40
1 parent 727071c
commit 4e77f40
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 17 deletions.
diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go
@@ -495,6 +495,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		Docker:                   envInfo.Docker,
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
+		ImageServiceEndpoint:     envInfo.ImageServiceEndpoint,
 		MultiClusterCIDR:         controlConfig.MultiClusterCIDR,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
@@ -526,24 +527,30 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.CRIDockerd.Root = filepath.Join(envInfo.DataDir, "agent", "cri-dockerd")
-	if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" {
-		switch nodeConfig.AgentConfig.Snapshotter {
-		case "overlayfs":
-			if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
-				return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
-					nodeConfig.Containerd.Root)
-			}
-		case "fuse-overlayfs":
-			if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
-				return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
-					nodeConfig.Containerd.Root)
-			}
-		case "stargz":
-			if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
-				return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
-					nodeConfig.Containerd.Root)
+	if !nodeConfig.Docker {
+		if nodeConfig.ImageServiceEndpoint != "" {
+			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ImageServiceEndpoint
+		} else if nodeConfig.ContainerRuntimeEndpoint == "" {
+			switch nodeConfig.AgentConfig.Snapshotter {
+			case "overlayfs":
+				if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+			case "fuse-overlayfs":
+				if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+			case "stargz":
+				if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+				nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
 			}
-			nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+		} else {
+			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint
 		}
 	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")

diff --git a/pkg/agent/run_linux.go b/pkg/agent/run_linux.go
@@ -37,5 +37,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
 	}
 
 	crp := "runtime-endpoint: " + cre + "\n"
+	ise := nodeConfig.ImageServiceEndpoint
+	if ise != "" && ise != cre {
+		crp += "image-endpoint: " + cre + "\n"
+	}
 	return os.WriteFile(agentConfDir+"/crictl.yaml", []byte(crp), 0600)
 }
diff --git a/pkg/agent/run_windows.go b/pkg/agent/run_windows.go
@@ -39,5 +39,9 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error {
 	}
 
 	crp := "runtime-endpoint: " + cre + "\n"
+	ise := nodeConfig.ImageServiceEndpoint
+	if ise != "" && ise != cre {
+		crp += "image-endpoint: " + cre + "\n"
+	}
 	return os.WriteFile(filepath.Join(agentConfDir, "crictl.yaml"), []byte(crp), 0600)
 }
diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go
@@ -27,6 +27,7 @@ type Agent struct {
 	Snapshotter              string
 	Docker                   bool
 	ContainerRuntimeEndpoint string
+	ImageServiceEndpoint     string
 	FlannelIface             string
 	FlannelConf              string
 	FlannelCniConfFile       string
@@ -114,6 +115,11 @@ var (
 		Usage:       "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path",
 		Destination: &AgentConfig.ContainerRuntimeEndpoint,
 	}
+	ImageServiceEndpointFlag = &cli.StringFlag{
+		Name:        "image-service-endpoint",
+		Usage:       "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.",
+		Destination: &AgentConfig.ImageServiceEndpoint,
+	}
 	PrivateRegistryFlag = &cli.StringFlag{
 		Name:        "private-registry",
 		Usage:       "(agent/runtime) Private registry configuration file",
@@ -247,6 +253,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			LBServerPortFlag,
 			ProtectKernelDefaultsFlag,
 			CRIEndpointFlag,
+			ImageServiceEndpointFlag,
 			PauseImageFlag,
 			SnapshotterFlag,
 			PrivateRegistryFlag,

diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go
@@ -497,6 +497,7 @@ var ServerFlags = []cli.Flag{
 	ImageCredProvConfigFlag,
 	DockerFlag,
 	CRIEndpointFlag,
+	ImageServiceEndpointFlag,
 	PauseImageFlag,
 	SnapshotterFlag,
 	PrivateRegistryFlag,

diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go
@@ -39,6 +39,7 @@ const (
 type Node struct {
 	Docker                   bool
 	ContainerRuntimeEndpoint string
+	ImageServiceEndpoint     string
 	NoFlannel                bool
 	SELinux                  bool
 	MultiClusterCIDR         bool