Merge pull request containerd#11131 from Divya063/rw-cgroups

Enable Writable cgroups for unprivileged containers

integration/container_cgroup_writable_linux_test.go

@@ -0,0 +1,141 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/containerd/cgroups/v3"
+	"github.com/containerd/containerd/v2/integration/images"
+	"github.com/containerd/containerd/v2/integration/remote"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func newContainerdProcess(t *testing.T, cgroupWritable bool) *ctrdProc {
+	configDir := t.TempDir()
+	configPath := filepath.Join(configDir, "config.toml")
+	config := fmt.Sprintf(`
+version = 3
+
+[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
+  cgroup_writable = %t
+`,
+		cgroupWritable)
+
+	err := os.WriteFile(configPath, []byte(config), 0600)
+	require.NoError(t, err)
+
+	currentProc := newCtrdProc(t, "containerd", configDir, nil)
+	require.NoError(t, currentProc.isReady())
+
+	return currentProc
+}
+
+func TestContainerCgroupWritable(t *testing.T) {
+	t.Parallel()
+
+	if cgroups.Mode() != cgroups.Unified {
+		t.Skip("requires cgroup v2")
+	}
+
+	testCases := []struct {
+		name           string
+		cgroupWritable bool
+	}{
+		{
+			name:           "writable cgroup",
+			cgroupWritable: true,
+		},
+		{
+			name:           "readonly cgroup",
+			cgroupWritable: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			currentProc := newContainerdProcess(t, testCase.cgroupWritable)
+
+			// Get the runtime service
+			runtimeService, err := remote.NewRuntimeService(currentProc.grpcAddress(), 1*time.Minute)
+			require.NoError(t, err)
+
+			t.Cleanup(func() {
+				cleanupPods(t, runtimeService)
+				t.Log("Stopping containerd process")
+				require.NoError(t, currentProc.kill(syscall.SIGTERM))
+				require.NoError(t, currentProc.wait(5*time.Minute))
+			})
+
+			imageName := images.Get(images.BusyBox)
+			pullImagesByCRI(t, currentProc.criImageService(t), imageName)
+
+			// Create a test sandbox
+			sbConfig := &runtime.PodSandboxConfig{
+				Metadata: &runtime.PodSandboxMetadata{
+					Name:      "sandbox",
+					Namespace: "cgroup-writable",
+				},
+			}
+			sb, err := runtimeService.RunPodSandbox(sbConfig, "")
+			require.NoError(t, err)
+
+			containerName := "cgroup-writable-test"
+			cnConfig := &runtime.ContainerConfig{
+				Metadata: &runtime.ContainerMetadata{
+					Name: containerName,
+				},
+				Image: &runtime.ImageSpec{
+					Image: imageName,
+				},
+				Command: []string{"sh", "-c", "sleep 1d"},
+			}
+
+			cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
+			require.NoError(t, err)
+			defer func() {
+				assert.NoError(t, runtimeService.RemoveContainer(cn))
+			}()
+
+			require.NoError(t, runtimeService.StartContainer(cn))
+			defer func() {
+				assert.NoError(t, runtimeService.StopContainer(cn, 30))
+			}()
+
+			status, err := runtimeService.ContainerStatus(cn)
+			require.NoError(t, err)
+			assert.Equal(t, status.GetState(), runtime.ContainerState_CONTAINER_RUNNING)
+
+			// Execute a command to verify if cgroup is writable
+			_, stderr, err := runtimeService.ExecSync(cn, []string{"mkdir", "sys/fs/cgroup/dummy-group"}, 2)
+			if testCase.cgroupWritable {
+				require.NoError(t, err)
+				require.Empty(t, stderr)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, string(stderr), "mkdir: can't create directory 'sys/fs/cgroup/dummy-group': Read-only file system")
+			}
+		})
+	}
+}

internal/cri/config/config.go

@@ -32,6 +32,7 @@ import (
 	runcoptions "github.com/containerd/containerd/api/types/runc/options"
 	runtimeoptions "github.com/containerd/containerd/api/types/runtimeoptions/v1"
 	"github.com/containerd/containerd/v2/internal/cri/annotations"
+	"github.com/containerd/containerd/v2/internal/cri/opts"
 	"github.com/containerd/containerd/v2/pkg/deprecation"
 	"github.com/containerd/containerd/v2/plugins"
 )
@@ -102,6 +103,8 @@ type Runtime struct {
 	// to the runtime spec when the container when PrivilegedWithoutHostDevices is already enabled. Requires
 	// PrivilegedWithoutHostDevices to be enabled. Defaults to false.
 	PrivilegedWithoutHostDevicesAllDevicesAllowed bool `toml:"privileged_without_host_devices_all_devices_allowed" json:"privileged_without_host_devices_all_devices_allowed"`
+	// CgroupWritable enables writable cgroups in non-privileged containers
+	CgroupWritable bool `toml:"cgroup_writable" json:"cgroupWritable"`
 	// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
 	BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
 	// NetworkPluginConfDir is a directory containing the CNI network information for the runtime class.
@@ -529,6 +532,10 @@ func ValidateRuntimeConfig(ctx context.Context, c *RuntimeConfig) ([]deprecation
 	}
 
 	for k, r := range c.ContainerdConfig.Runtimes {
+		if r.CgroupWritable && !opts.IsCgroup2UnifiedMode() {
+			return warnings, fmt.Errorf("runtime %s: `cgroup_writable` is only supported on cgroup v2", k)
+		}
+
 		if !r.PrivilegedWithoutHostDevices && r.PrivilegedWithoutHostDevicesAllDevicesAllowed {
 			return warnings, errors.New("`privileged_without_host_devices_all_devices_allowed` requires `privileged_without_host_devices` to be enabled")
 		}

internal/cri/config/config_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
 
+	"github.com/containerd/containerd/v2/internal/cri/opts"
 	"github.com/containerd/containerd/v2/pkg/deprecation"
 )
 
@@ -189,6 +190,40 @@ func TestValidateConfig(t *testing.T) {
 			},
 			warnings: []deprecation.Warning{deprecation.CRIRegistryConfigs},
 		},
+		"cgroup_writable enabled": {
+			runtimeConfig: &RuntimeConfig{
+				ContainerdConfig: ContainerdConfig{
+					DefaultRuntimeName: RuntimeDefault,
+					Runtimes: map[string]Runtime{
+						RuntimeDefault: {
+							CgroupWritable: true,
+						},
+					},
+				},
+			},
+			runtimeExpectedErr: func() string {
+				if !opts.IsCgroup2UnifiedMode() {
+					return "`cgroup_writable` is only supported on cgroup v2"
+				}
+				return ""
+			}(),
+			runtimeExpected: func() *RuntimeConfig {
+				if !opts.IsCgroup2UnifiedMode() {
+					return nil
+				}
+				return &RuntimeConfig{
+					ContainerdConfig: ContainerdConfig{
+						DefaultRuntimeName: RuntimeDefault,
+						Runtimes: map[string]Runtime{
+							RuntimeDefault: {
+								CgroupWritable: true,
+								Sandboxer:      string(ModePodSandbox),
+							},
+						},
+					},
+				}
+			}(),
+		},
 		"privileged_without_host_devices_all_devices_allowed without privileged_without_host_devices": {
 			runtimeConfig: &RuntimeConfig{
 				ContainerdConfig: ContainerdConfig{

internal/cri/opts/spec_linux_opts.go

@@ -39,8 +39,7 @@ import (
 	"github.com/containerd/log"
 )
 
-// WithMounts sorts and adds runtime and CRI mounts to the spec
-func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string, handler *runtime.RuntimeHandler) oci.SpecOpts {
+func withMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string, handler *runtime.RuntimeHandler, cgroupWritable bool) oci.SpecOpts {
 	return func(ctx context.Context, client oci.Client, _ *containers.Container, s *runtimespec.Spec) (err error) {
 		// mergeMounts merge CRI mounts with extra mounts. If a mount destination
 		// is mounted by both a CRI mount and an extra mount, the CRI mount will
@@ -67,12 +66,15 @@ func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*ru
 		// shadow other mounts.
 		sort.Stable(orderedMounts(mounts))
 
-		// Mount cgroup into the container as readonly, which inherits docker's behavior.
+		mode := "ro"
+		if cgroupWritable {
+			mode = "rw"
+		}
 		s.Mounts = append(s.Mounts, runtimespec.Mount{
 			Source:      "cgroup",
 			Destination: "/sys/fs/cgroup",
 			Type:        "cgroup",
-			Options:     []string{"nosuid", "noexec", "nodev", "relatime", "ro"},
+			Options:     []string{"nosuid", "noexec", "nodev", "relatime", mode},
 		})
 
 		// Copy all mounts from default mounts, except for
@@ -214,6 +216,17 @@ func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*ru
 	}
 }
 
+// WithMounts sorts and adds runtime and CRI mounts to the spec
+func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string, handler *runtime.RuntimeHandler) oci.SpecOpts {
+	return withMounts(osi, config, extra, mountLabel, handler, false)
+
+}
+
+// WithMountsCgroupWritable sorts and adds runtime and CRI mounts to the spec if cgroup_writable is enabled.
+func WithMountsCgroupWritable(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string, handler *runtime.RuntimeHandler) oci.SpecOpts {
+	return withMounts(osi, config, extra, mountLabel, handler, true)
+}
+
 // Ensure mount point on which path is mounted, is shared.
 func ensureShared(path string, lookupMount func(string) (mount.Info, error)) error {
 	mountInfo, err := lookupMount(path)

internal/cri/opts/spec_nonlinux.go

@@ -40,3 +40,9 @@ func WithCDI(_ map[string]string, _ []*runtime.CDIDevice) oci.SpecOpts {
 		return nil
 	}
 }
+
+// IsCgroup2UnifiedMode returns whether we are running in cgroup v2 unified mode.
+// On non-Linux platforms, this always returns false.
+func IsCgroup2UnifiedMode() bool {
+	return false
+}

internal/cri/server/container_create.go

@@ -709,7 +709,14 @@ func (c *criService) buildLinuxSpec(
 		}
 	}()
 
-	specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel, runtimeHandler))
+	var ociSpecOpts oci.SpecOpts
+	if ociRuntime.CgroupWritable {
+		ociSpecOpts = customopts.WithMountsCgroupWritable(c.os, config, extraMounts, mountLabel, runtimeHandler)
+	} else {
+		ociSpecOpts = customopts.WithMounts(c.os, config, extraMounts, mountLabel, runtimeHandler)
+	}
+
+	specOpts = append(specOpts, ociSpecOpts)
 
 	if !c.config.DisableProcMount {
 		// Change the default masked/readonly paths to empty slices