node-installer: run nydus snapshotter on bare metal platforms

internal/kuberesource/parts.go

@@ -50,12 +50,91 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 		return nil, fmt.Errorf("getting default runtime handler: %w", err)
 	}
 
+	tardevSnapshotter := Container().
+		WithName("tardev-snapshotter").
+		WithImage("ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest").
+		WithResources(ResourceRequirements().
+			WithMemoryLimitAndRequest(800),
+		).
+		WithVolumeMounts(
+			VolumeMount().
+				WithName("host-mount").
+				WithMountPath("/host"),
+			VolumeMount().
+				WithName("var-lib-containerd").
+				WithMountPath("/var/lib/containerd"),
+		).
+		WithArgs(
+			"tardev-snapshotter",
+			fmt.Sprintf("/var/lib/containerd/io.containerd.snapshotter.v1.tardev-%s", runtimeHandler),
+			fmt.Sprintf("/host/run/containerd/tardev-snapshotter-%s.sock", runtimeHandler),
+			"/host/var/run/containerd/containerd.sock",
+		).
+		WithEnv(
+			NewEnvVar("RUST_LOG", "tardev_snapshotter=trace"),
+		)
+	tardevSnapshotterVolumes := []*applycorev1.VolumeApplyConfiguration{
+		Volume().
+			WithName("var-lib-containerd").
+			WithHostPath(HostPathVolumeSource().
+				WithPath("/var/lib/containerd").
+				WithType(corev1.HostPathDirectory),
+			),
+	}
+
+	nydusSnapshotter := Container().
+		WithName("nydus-snapshotter").
+		WithImage("ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest").
+		WithResources(ResourceRequirements().
+			WithMemoryLimitAndRequest(800),
+		).
+		WithVolumeMounts(
+			VolumeMount().
+				WithName("host-mount").
+				WithMountPath("/host"),
+			VolumeMount().
+				WithName("var-lib-containerd").
+				WithMountPath("/var/lib/containerd"),
+			VolumeMount().
+				WithName("var-lib-nydus-snapshotter").
+				WithMountPath(fmt.Sprintf("/var/lib/nydus-snapshotter/%s", runtimeHandler)),
+		).
+		WithArgs(
+			"containerd-nydus-grpc",
+			// Snapshotter will write to this path and tell containerd to read from it, so
+			// path must be shared and the same on the host. See 'var-lib-nydus-snapshotter' volume.
+			fmt.Sprintf("--root=/var/lib/nydus-snapshotter/%s", runtimeHandler),
+			"--config=/share/nydus-snapshotter/config-coco-guest-pulling.toml",
+			fmt.Sprintf("--address=/host/run/containerd/containerd-nydus-grpc-%s.sock", runtimeHandler),
+			"--log-to-stdout",
+		)
+	nydusSnapshotterVolumes := []*applycorev1.VolumeApplyConfiguration{
+		Volume().
+			WithName("var-lib-containerd").
+			WithHostPath(HostPathVolumeSource().
+				WithPath("/var/lib/rancher/k3s/agent/containerd").
+				WithType(corev1.HostPathDirectory),
+			),
+		Volume().
+			WithName("var-lib-nydus-snapshotter").
+			WithHostPath(HostPathVolumeSource().
+				WithPath(fmt.Sprintf("/var/lib/nydus-snapshotter/%s", runtimeHandler)).
+				WithType(corev1.HostPathDirectoryOrCreate),
+			),
+	}
+
 	var nodeInstallerImageURL string
+	var snapshotter *applycorev1.ContainerApplyConfiguration
+	var snapshotterVolumes []*applycorev1.VolumeApplyConfiguration
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
+		snapshotter = tardevSnapshotter
+		snapshotterVolumes = tardevSnapshotterVolumes
 	case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
+		snapshotter = nydusSnapshotter
+		snapshotterVolumes = nydusSnapshotterVolumes
 	default:
 		return nil, fmt.Errorf("unsupported platform %q", platform)
 	}
@@ -87,43 +166,17 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 						WithCommand("/bin/node-installer", platform.String()),
 					).
 					WithContainers(
-						Container().
-							WithName("tardev-snapshotter").
-							WithImage("ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest").
-							WithResources(ResourceRequirements().
-								WithMemoryLimitAndRequest(800),
-							).
-							WithVolumeMounts(
-								VolumeMount().
-									WithName("host-mount").
-									WithMountPath("/host"),
-								VolumeMount().
-									WithName("var-lib-containerd").
-									WithMountPath("/var/lib/containerd"),
-							).
-							WithArgs(
-								"tardev-snapshotter",
-								fmt.Sprintf("/var/lib/containerd/io.containerd.snapshotter.v1.tardev-%s", runtimeHandler),
-								fmt.Sprintf("/host/run/containerd/tardev-snapshotter-%s.sock", runtimeHandler),
-								"/host/var/run/containerd/containerd.sock",
-							).
-							WithEnv(
-								NewEnvVar("RUST_LOG", "tardev_snapshotter=trace"),
-							),
+						snapshotter,
 					).
-					WithVolumes(
+					WithVolumes(append(
+						snapshotterVolumes,
 						Volume().
 							WithName("host-mount").
 							WithHostPath(HostPathVolumeSource().
 								WithPath("/").
 								WithType(corev1.HostPathDirectory),
 							),
-						Volume().
-							WithName("var-lib-containerd").
-							WithHostPath(HostPathVolumeSource().
-								WithPath("/var/lib/containerd").
-								WithType(corev1.HostPathDirectory),
-							),
+					)...,
 					),
 				),
 			),

justfile

@@ -27,13 +27,16 @@ initializer: (push "initializer")
 # Build the tardev-snapshotter, containerize and push it.
 tardev-snapshotter: (push "tardev-snapshotter")
 
+# Build the nydus-snapshotter, containerize and push it.
+nydus-snapshotter: (push "nydus-snapshotter")
+
 default_cli := "contrast.cli"
 default_deploy_target := "openssl"
 default_platform := "AKS-CLH-SNP"
 workspace_dir := "workspace"
 
 # Build the node-installer, containerize and push it.
-node-installer platform=default_platform: tardev-snapshotter
+node-installer platform=default_platform: tardev-snapshotter nydus-snapshotter
     #!/usr/bin/env bash
     case {{ platform }} in
         "AKS-CLH-SNP")

nodeinstaller/internal/config/containerd.go

@@ -10,41 +10,41 @@ type ContainerdConfig struct {
 	// Version of the config file
 	Version int `toml:"version"`
 	// Root is the path to a directory where containerd will store persistent data
-	Root string `toml:"root"`
+	Root string `toml:"root,omitempty"`
 	// State is the path to a directory where containerd will store transient data
-	State string `toml:"state"`
+	State string `toml:"state,omitempty"`
 	// TempDir is the path to a directory where to place containerd temporary files
-	TempDir string `toml:"temp"`
+	TempDir string `toml:"temp,omitempty"`
 	// PluginDir is the directory for dynamic plugins to be stored
-	PluginDir string `toml:"plugin_dir"`
+	PluginDir string `toml:"plugin_dir,omitempty"`
 	// GRPC configuration settings
-	GRPC any `toml:"grpc"`
+	GRPC any `toml:"grpc,omitempty"`
 	// TTRPC configuration settings
-	TTRPC any `toml:"ttrpc"`
+	TTRPC any `toml:"ttrpc,omitempty"`
 	// Debug and profiling settings
-	Debug any `toml:"debug"`
+	Debug Debug `toml:"debug,omitempty"`
 	// Metrics and monitoring settings
-	Metrics any `toml:"metrics"`
+	Metrics any `toml:"metrics,omitempty"`
 	// DisabledPlugins are IDs of plugins to disable. Disabled plugins won't be
 	// initialized and started.
-	DisabledPlugins []string `toml:"disabled_plugins"`
+	DisabledPlugins []string `toml:"disabled_plugins,omitempty"`
 	// RequiredPlugins are IDs of required plugins. Containerd exits if any
 	// required plugin doesn't exist or fails to be initialized or started.
-	RequiredPlugins []string `toml:"required_plugins"`
+	RequiredPlugins []string `toml:"required_plugins,omitempty"`
 	// Plugins provides plugin specific configuration for the initialization of a plugin
-	Plugins map[string]any `toml:"plugins"`
+	Plugins map[string]any `toml:"plugins,omitempty"`
 	// OOMScore adjust the containerd's oom score
-	OOMScore int `toml:"oom_score"`
+	OOMScore int `toml:"oom_score,omitempty"`
 	// Cgroup specifies cgroup information for the containerd daemon process
-	Cgroup any `toml:"cgroup"`
+	Cgroup any `toml:"cgroup,omitempty"`
 	// ProxyPlugins configures plugins which are communicated to over GRPC
-	ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"`
+	ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins,omitempty"`
 	// Timeouts specified as a duration
-	Timeouts map[string]string `toml:"timeouts"`
+	Timeouts map[string]string `toml:"timeouts,omitempty"`
 	// Imports are additional file path list to config files that can overwrite main config file fields
-	Imports []string `toml:"imports"`
+	Imports []string `toml:"imports,omitempty"`
 	// StreamProcessors configuration
-	StreamProcessors map[string]any `toml:"stream_processors"`
+	StreamProcessors map[string]any `toml:"stream_processors,omitempty"`
 }
 
 // ProxyPlugin provides a proxy plugin configuration.
@@ -96,3 +96,13 @@ type Runtime struct {
 	// podsandbox - means use Controller implementation from sbserver podsandbox package.
 	Sandboxer string `toml:"sandboxer,omitempty" json:"sandboxer,omitempty"`
 }
+
+// Debug provides debug configuration.
+type Debug struct {
+	Address string `toml:"address,omitempty"`
+	UID     int    `toml:"uid,omitempty"`
+	GID     int    `toml:"gid,omitempty"`
+	Level   string `toml:"level,omitempty"`
+	// Format represents the logging format. Supported values are 'text' and 'json'.
+	Format string `toml:"format,omitempty"`
+}

nodeinstaller/internal/constants/constants.go

@@ -67,6 +67,7 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool)
 		config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
 		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")}
+		config.Hypervisor["qemu"]["shared_fs"] = "none"
 		if debug {
 			config.Hypervisor["qemu"]["enable_debug"] = true
 			config.Hypervisor["qemu"]["kernel_params"] = " agent.log=debug initcall_debug"
@@ -85,7 +86,7 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, debug bool)
 		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
 		delete(config.Hypervisor["qemu"], "initrd")
 		config.Hypervisor["qemu"]["block_device_aio"] = "threads"
-		config.Hypervisor["qemu"]["shared_fs"] = "virtio-9p"
+		config.Hypervisor["qemu"]["shared_fs"] = "none"
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")}
 		config.Hypervisor["qemu"]["rootfs_type"] = "erofs"
 		if debug {
@@ -117,11 +118,11 @@ func ContainerdRuntimeConfigFragment(baseDir, snapshotter string, platform platf
 		Path:                         filepath.Join(baseDir, "bin", "containerd-shim-contrast-cc-v2"),
 		PodAnnotations:               []string{"io.katacontainers.*"},
 		PrivilegedWithoutHostDevices: true,
+		Snapshotter:                  snapshotter,
 	}
 
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
-		cfg.Snapshotter = snapshotter
 		cfg.Options = map[string]any{
 			"ConfigPath": filepath.Join(baseDir, "etc", "configuration-clh-snp.toml"),
 		}