Initial integration tests for cgroupv2

.travis.yml

@@ -34,6 +34,8 @@ matrix:
         - sudo ssh default sudo podman build -t test /vagrant
         # Mounting /lib/modules into the container is necessary as CRIU wants to load (via iptables) additional modules
         - sudo ssh default sudo podman run --privileged --cgroupns=private -v /lib/modules:/lib/modules:ro test make localunittest
+        # cgroupv2+systemd: test on vagrant host itself as we need systemd
+        - sudo ssh default -t 'cd /vagrant && sudo make localintegration RUNC_USE_SYSTEMD=yes'
   allow_failures:
     - go: tip
 

Vagrantfile

@@ -13,6 +13,12 @@ Vagrant.configure("2") do |config|
     v.cpus = 2
   end
   config.vm.provision "shell", inline: <<-SHELL
-    dnf install -y podman
+    cat << EOF | dnf -y shell
+config install_weak_deps: False
+update
+install podman make golang-go libseccomp-devel bats jq
+ts run
+EOF
+    dnf clean all
   SHELL
 end

tests/integration/cgroups.bats

@@ -14,18 +14,7 @@ function setup() {
     setup_busybox
 }
 
-function check_cgroup_value() {
-    cgroup=$1
-    source=$2
-    expected=$3
-
-    current=$(cat $cgroup/$source)
-    echo  $cgroup/$source
-    echo "current" $current "!?" "$expected"
-    [ "$current" -eq "$expected" ]
-}
-
-@test "runc update --kernel-memory (initialized)" {
+@test "runc update --kernel-memory{,-tcp} (initialized)" {
     [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup
     requires cgroups_kmem
 
@@ -34,7 +23,8 @@ function check_cgroup_value() {
     # Set some initial known values
     DATA=$(cat <<-EOF
     "memory": {
-        "kernel": 16777216
+        "kernel": 16777216,
+        "kernelTCP": 11534336
     },
 EOF
     )
@@ -45,12 +35,18 @@ EOF
     runc run -d --console-socket $CONSOLE_SOCKET test_cgroups_kmem
     [ "$status" -eq 0 ]
 
+    check_cgroup_value "memory.kmem.limit_in_bytes" 16777216
+    check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 11534336
+
     # update kernel memory limit
     runc update test_cgroups_kmem --kernel-memory 50331648
     [ "$status" -eq 0 ]
+    check_cgroup_value "memory.kmem.limit_in_bytes" 50331648
 
-	# check the value
-    check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
+    # update kernel memory tcp limit
+    runc update test_cgroups_kmem --kernel-memory-tcp 41943040
+    [ "$status" -eq 0 ]
+    check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 41943040
 }
 
 @test "runc update --kernel-memory (uninitialized)" {
@@ -71,7 +67,7 @@ EOF
         [ ! "$status" -eq 0 ]
     else
         [ "$status" -eq 0 ]
-        check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648
+        check_cgroup_value "memory.kmem.limit_in_bytes" 50331648
     fi
 }
 

tests/integration/events.bats

@@ -13,7 +13,8 @@ function teardown() {
 
 @test "events --stats" {
   # XXX: currently cgroups require root containers.
-  requires root
+  # TODO: support cgroup v2 memory.events
+  requires root cgroups_v1
 
   # run busybox detached
   runc run -d --console-socket $CONSOLE_SOCKET test_busybox
@@ -28,7 +29,8 @@ function teardown() {
 
 @test "events --interval default " {
   # XXX: currently cgroups require root containers.
-  requires root
+  # TODO: support cgroup v2 memory.events
+  requires root cgroups_v1
 
   # run busybox detached
   runc run -d --console-socket $CONSOLE_SOCKET test_busybox
@@ -55,7 +57,8 @@ function teardown() {
 
 @test "events --interval 1s " {
   # XXX: currently cgroups require root containers.
-  requires root
+  # TODO: support cgroup v2 memory.events
+  requires root cgroups_v1
 
   # run busybox detached
   runc run -d --console-socket $CONSOLE_SOCKET test_busybox
@@ -81,7 +84,8 @@ function teardown() {
 
 @test "events --interval 100ms " {
   # XXX: currently cgroups require root containers.
-  requires root
+  # TODO: support cgroup v2 memory.events
+  requires root cgroups_v1
 
   # run busybox detached
   runc run -d --console-socket $CONSOLE_SOCKET test_busybox

tests/integration/helpers.bash

@@ -36,20 +36,6 @@ ROOT=$(mktemp -d "$BATS_TMPDIR/runc.XXXXXX")
 # Path to console socket.
 CONSOLE_SOCKET="$BATS_TMPDIR/console.sock"
 
-# Cgroup paths
-CGROUP_MEMORY_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<MEMORY\>/ { print $5; exit }')
-CGROUP_CPU_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<CPU\>/ { print $5; exit }')
-if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then
-	CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope"
-else
-	CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup"
-fi
-CGROUP_MEMORY="${CGROUP_MEMORY_BASE_PATH}${CGROUPS_PATH}"
-
-# CONFIG_MEMCG_KMEM support
-KMEM="${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes"
-RT_PERIOD="${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us"
-
 # Check if we're in rootless mode.
 ROOTLESS=$(id -u)
 
@@ -119,14 +105,62 @@ function runc_rootless_cgroup() {
 	mv "$bundle/config.json"{.tmp,}
 }
 
-# Helper function to set cgroupsPath to the value of $CGROUPS_PATH
+function init_cgroup_paths() {
+	# init once
+	test -n "$CGROUP_UNIFIED" && return
+
+	if [ -n "${RUNC_USE_SYSTEMD}" ] ; then
+		REL_CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope"
+		OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test"
+	else
+		REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup"
+		OCI_CGROUPS_PATH=$REL_CGROUPS_PATH
+	fi
+
+	if stat -f -c %t /sys/fs/cgroup | grep -qFw 63677270; then
+		CGROUP_UNIFIED=yes
+		# "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers.
+		# - devices (since kernel 4.15)
+		# - freezer (since kernel 5.2)
+		# Assume these are always available, as it is hard to detect
+		CGROUP_SUBSYSTEMS=$(cat /sys/fs/cgroup/cgroup.controllers; echo devices freezer)
+		CGROUP_BASE_PATH=/sys/fs/cgroup
+		CGROUP_PATH=${CGROUP_BASE_PATH}${REL_CGROUPS_PATH}
+	else
+		CGROUP_UNIFIED=no
+		CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups)
+		for g in ${CGROUP_SUBSYSTEMS}; do
+			base_path=$(gawk '$(NF-2) == "cgroup" && $NF ~ /\<'${g}'\>/ { print $5; exit }' /proc/self/mountinfo)
+			test -z "$base_path" && continue
+			eval CGROUP_${g^^}_BASE_PATH="${base_path}"
+			eval CGROUP_${g^^}="${base_path}${REL_CGROUPS_PATH}"
+		done
+	fi
+}
+
+# Helper function to set cgroupsPath to the value of $OCI_CGROUPS_PATH
 function set_cgroups_path() {
   bundle="${1:-.}"
-  cgroups_path="/runc-cgroups-integration-test/test-cgroup"
-  if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then
-    cgroups_path="machine.slice:runc-cgroups:integration-test"
-  fi
-  sed -i 's#\("linux": {\)#\1\n    "cgroupsPath": "'"${cgroups_path}"'",#' "$bundle/config.json"
+  init_cgroup_paths
+  sed -i 's#\("linux": {\)#\1\n    "cgroupsPath": "'"${OCI_CGROUPS_PATH}"'",#' "$bundle/config.json"
+}
+
+# Helper to check a value in cgroups.
+function check_cgroup_value() {
+	source=$1
+	expected=$2
+
+	if [ "x$CGROUP_UNIFIED" = "xyes" ] ; then
+		cgroup=$CGROUP_PATH
+	else
+		ctrl=${source%%.*}
+		eval cgroup=\$CGROUP_${ctrl^^}
+	fi
+
+	current=$(cat $cgroup/$source)
+	echo $cgroup/$source
+	echo "current" $current "!?" "$expected"
+	[ "$current" = "$expected" ]
 }
 
 # Helper function to set a resources limit
@@ -177,15 +211,29 @@ function requires() {
 			fi
 			;;
 		cgroups_kmem)
-			if [ ! -e "$KMEM" ]; then
+			init_cgroup_paths
+			if [ ! -e "${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" ]; then
 				skip "Test requires ${var}"
 			fi
 			;;
 		cgroups_rt)
-			if [ ! -e "$RT_PERIOD" ]; then
+			init_cgroup_paths
+			if [ ! -e "${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" ]; then
 				skip "Test requires ${var}"
 			fi
 			;;
+		cgroups_v1)
+			init_cgroup_paths
+			if [ "$CGROUP_UNIFIED" != "no" ]; then
+				skip "Test requires cgroups v1"
+			fi
+			;;
+		cgroups_v2)
+			init_cgroup_paths
+			if [ "$CGROUP_UNIFIED" != "yes" ]; then
+				skip "Test requires cgroups v2 (unified)"
+			fi
+			;;
 		*)
 			fail "BUG: Invalid requires ${var}."
 			;;