diff --git a/.travis.yml b/.travis.yml index 3e12b820bfd..84213eadb28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,8 @@ matrix: - sudo ssh default sudo podman build -t test /vagrant # Mounting /lib/modules into the container is necessary as CRIU wants to load (via iptables) additional modules - sudo ssh default sudo podman run --privileged --cgroupns=private -v /lib/modules:/lib/modules:ro test make localunittest + # cgroupv2+systemd: test on vagrant host itself as we need systemd + - sudo ssh default -t 'cd /vagrant && sudo make localintegration RUNC_USE_SYSTEMD=yes' allow_failures: - go: tip diff --git a/Vagrantfile b/Vagrantfile index 165b078018d..aba0c0cd717 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,12 @@ Vagrant.configure("2") do |config| v.cpus = 2 end config.vm.provision "shell", inline: <<-SHELL - dnf install -y podman + cat << EOF | dnf -y shell +config install_weak_deps: False +update +install podman make golang-go libseccomp-devel bats jq +ts run +EOF + dnf clean all SHELL end diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index 17812abfdcc..6abb2ed6b02 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -14,18 +14,7 @@ function setup() { setup_busybox } -function check_cgroup_value() { - cgroup=$1 - source=$2 - expected=$3 - - current=$(cat $cgroup/$source) - echo $cgroup/$source - echo "current" $current "!?" "$expected" - [ "$current" -eq "$expected" ] -} - -@test "runc update --kernel-memory (initialized)" { +@test "runc update --kernel-memory{,-tcp} (initialized)" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup requires cgroups_kmem @@ -34,7 +23,8 @@ function check_cgroup_value() { # Set some initial known values DATA=$(cat <<-EOF "memory": { - "kernel": 16777216 + "kernel": 16777216, + "kernelTCP": 11534336 }, EOF ) @@ -45,12 +35,18 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_cgroups_kmem [ "$status" -eq 0 ] + check_cgroup_value "memory.kmem.limit_in_bytes" 16777216 + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 11534336 + # update kernel memory limit runc update test_cgroups_kmem --kernel-memory 50331648 [ "$status" -eq 0 ] + check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 - # check the value - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 + # update kernel memory tcp limit + runc update test_cgroups_kmem --kernel-memory-tcp 41943040 + [ "$status" -eq 0 ] + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 41943040 } @test "runc update --kernel-memory (uninitialized)" { @@ -71,7 +67,7 @@ EOF [ ! "$status" -eq 0 ] else [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 + check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 fi } diff --git a/tests/integration/events.bats b/tests/integration/events.bats index b3e6315b555..8f2f81a23d0 100644 --- a/tests/integration/events.bats +++ b/tests/integration/events.bats @@ -13,7 +13,8 @@ function teardown() { @test "events --stats" { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -28,7 +29,8 @@ function teardown() { @test "events --interval default " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -55,7 +57,8 @@ function teardown() { @test "events --interval 1s " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -81,7 +84,8 @@ function teardown() { @test "events --interval 100ms " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 8862dcb8b88..de8be4ba53a 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -36,20 +36,6 @@ ROOT=$(mktemp -d "$BATS_TMPDIR/runc.XXXXXX") # Path to console socket. CONSOLE_SOCKET="$BATS_TMPDIR/console.sock" -# Cgroup paths -CGROUP_MEMORY_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -CGROUP_CPU_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" -else - CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" -fi -CGROUP_MEMORY="${CGROUP_MEMORY_BASE_PATH}${CGROUPS_PATH}" - -# CONFIG_MEMCG_KMEM support -KMEM="${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" -RT_PERIOD="${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" - # Check if we're in rootless mode. ROOTLESS=$(id -u) @@ -119,14 +105,62 @@ function runc_rootless_cgroup() { mv "$bundle/config.json"{.tmp,} } -# Helper function to set cgroupsPath to the value of $CGROUPS_PATH +function init_cgroup_paths() { + # init once + test -n "$CGROUP_UNIFIED" && return + + if [ -n "${RUNC_USE_SYSTEMD}" ] ; then + REL_CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" + OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test" + else + REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" + OCI_CGROUPS_PATH=$REL_CGROUPS_PATH + fi + + if stat -f -c %t /sys/fs/cgroup | grep -qFw 63677270; then + CGROUP_UNIFIED=yes + # "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers. + # - devices (since kernel 4.15) + # - freezer (since kernel 5.2) + # Assume these are always available, as it is hard to detect + CGROUP_SUBSYSTEMS=$(cat /sys/fs/cgroup/cgroup.controllers; echo devices freezer) + CGROUP_BASE_PATH=/sys/fs/cgroup + CGROUP_PATH=${CGROUP_BASE_PATH}${REL_CGROUPS_PATH} + else + CGROUP_UNIFIED=no + CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups) + for g in ${CGROUP_SUBSYSTEMS}; do + base_path=$(gawk '$(NF-2) == "cgroup" && $NF ~ /\<'${g}'\>/ { print $5; exit }' /proc/self/mountinfo) + test -z "$base_path" && continue + eval CGROUP_${g^^}_BASE_PATH="${base_path}" + eval CGROUP_${g^^}="${base_path}${REL_CGROUPS_PATH}" + done + fi +} + +# Helper function to set cgroupsPath to the value of $OCI_CGROUPS_PATH function set_cgroups_path() { bundle="${1:-.}" - cgroups_path="/runc-cgroups-integration-test/test-cgroup" - if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - cgroups_path="machine.slice:runc-cgroups:integration-test" - fi - sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${cgroups_path}"'",#' "$bundle/config.json" + init_cgroup_paths + sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${OCI_CGROUPS_PATH}"'",#' "$bundle/config.json" +} + +# Helper to check a value in cgroups. +function check_cgroup_value() { + source=$1 + expected=$2 + + if [ "x$CGROUP_UNIFIED" = "xyes" ] ; then + cgroup=$CGROUP_PATH + else + ctrl=${source%%.*} + eval cgroup=\$CGROUP_${ctrl^^} + fi + + current=$(cat $cgroup/$source) + echo $cgroup/$source + echo "current" $current "!?" "$expected" + [ "$current" = "$expected" ] } # Helper function to set a resources limit @@ -177,15 +211,29 @@ function requires() { fi ;; cgroups_kmem) - if [ ! -e "$KMEM" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" ]; then skip "Test requires ${var}" fi ;; cgroups_rt) - if [ ! -e "$RT_PERIOD" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" ]; then skip "Test requires ${var}" fi ;; + cgroups_v1) + init_cgroup_paths + if [ "$CGROUP_UNIFIED" != "no" ]; then + skip "Test requires cgroups v1" + fi + ;; + cgroups_v2) + init_cgroup_paths + if [ "$CGROUP_UNIFIED" != "yes" ]; then + skip "Test requires cgroups v2 (unified)" + fi + ;; *) fail "BUG: Invalid requires ${var}." ;; diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 549b2ece831..61ce69376c8 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -19,9 +19,7 @@ function setup() { DATA=$(cat </ { print $5; exit }') - eval CGROUP_${g}="${base_path}${CGROUPS_PATH}" - done - - CGROUP_SYSTEM_MEMORY=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'MEMORY'\>/ { print $5; exit }') + # Set a few variables to make the code below work for both v1 and v2 + case $CGROUP_UNIFIED in + no) + MEM_LIMIT="memory.limit_in_bytes" + MEM_RESERVE="memory.soft_limit_in_bytes" + MEM_SWAP="memory.memsw.limit_in_bytes" + SYSTEM_MEM=$(cat "${CGROUP_MEMORY_BASE_PATH}/${MEM_LIMIT}") + SYSTEM_MEM_SWAP=$(cat "${CGROUP_MEMORY_BASE_PATH}/$MEM_SWAP") + ;; + yes) + MEM_LIMIT="memory.max" + MEM_RESERVE="memory.low" + MEM_SWAP="memory.swap.max" + SYSTEM_MEM="max" + SYSTEM_MEM_SWAP="max" + # checking swap is currently disabled for v2 + #CGROUP_MEMORY=$CGROUP_PATH + ;; + esac # check that initial values were properly set - check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000 - check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000 - check_cgroup_value $CGROUP_CPU "cpu.shares" 100 - check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.tcp.limit_in_bytes" 11534336 - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432 - check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824 - check_cgroup_value $CGROUP_PIDS "pids.max" 20 - - # update cpu-period - runc update test_update --cpu-period 900000 - [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 900000 - - # update cpu-quota - runc update test_update --cpu-quota 600000 - [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 600000 - - # update cpu-shares - runc update test_update --cpu-share 200 - [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.shares" 200 + check_cgroup_value "cpuset.cpus" 0 + check_cgroup_value $MEM_LIMIT 33554432 + check_cgroup_value $MEM_RESERVE 25165824 + check_cgroup_value "pids.max" 20 # update cpuset if supported (i.e. we're running on a multicore cpu) - cpu_count=$(grep '^processor' /proc/cpuinfo | wc -l) + cpu_count=$(grep -c '^processor' /proc/cpuinfo) if [ $cpu_count -gt 1 ]; then runc update test_update --cpuset-cpus "1" [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 1 + check_cgroup_value "cpuset.cpus" 1 fi # update memory limit runc update test_update --memory 67108864 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 67108864 + check_cgroup_value $MEM_LIMIT 67108864 runc update test_update --memory 50M [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 52428800 + check_cgroup_value $MEM_LIMIT 52428800 # update memory soft limit runc update test_update --memory-reservation 33554432 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 33554432 + check_cgroup_value "$MEM_RESERVE" 33554432 # Run swap memory tests if swap is available - if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then + if [ -f "$CGROUP_MEMORY/$MEM_SWAP" ]; then # try to remove memory swap limit runc update test_update --memory-swap -1 [ "$status" -eq 0 ] - # Get System memory swap limit - SYSTEM_MEMORY_SW=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.memsw.limit_in_bytes") - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} + check_cgroup_value "$MEM_SWAP" $SYSTEM_MEM_SWAP # update memory swap runc update test_update --memory-swap 96468992 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" 96468992 - fi; + check_cgroup_value "$MEM_SWAP" 96468992 + fi # try to remove memory limit runc update test_update --memory -1 [ "$status" -eq 0 ] - # Get System memory limit - SYSTEM_MEMORY=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.limit_in_bytes") - # check memory limited is gone - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" ${SYSTEM_MEMORY} + # check memory limit is gone + check_cgroup_value $MEM_LIMIT $SYSTEM_MEM # check swap memory limited is gone - if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY} + if [ -f "$CGROUP_MEMORY/$MEM_SWAP" ]; then + check_cgroup_value $MEM_SWAP $SYSTEM_MEM fi - # update kernel memory limit - runc update test_update --kernel-memory 50331648 - [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 - - # update kernel memory tcp limit - runc update test_update --kernel-memory-tcp 41943040 - [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.tcp.limit_in_bytes" 41943040 - # update pids limit runc update test_update --pids-limit 10 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_PIDS "pids.max" 10 + check_cgroup_value "pids.max" 10 - # Revert to the test initial value via json on stding + # Revert to the test initial value via json on stdin runc update -r - test_update < $BATS_TMPDIR/runc-cgroups-integration-test.json { "memory": { "limit": 33554432, - "reservation": 25165824, - "kernel": 16777216, - "kernelTCP": 11534336 + "reservation": 25165824 }, "cpu": { "shares": 100, @@ -224,33 +176,92 @@ EOF } } EOF -) - echo $DATA > $BATS_TMPDIR/runc-cgroups-integration-test.json runc update -r $BATS_TMPDIR/runc-cgroups-integration-test.json test_update [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000 - check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000 - check_cgroup_value $CGROUP_CPU "cpu.shares" 100 - check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.tcp.limit_in_bytes" 11534336 - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432 - check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824 - check_cgroup_value $CGROUP_PIDS "pids.max" 20 + check_cgroup_value "cpuset.cpus" 0 + check_cgroup_value $MEM_LIMIT 33554432 + check_cgroup_value $MEM_RESERVE 25165824 + check_cgroup_value "pids.max" 20 +} + +@test "update cgroup v1 cpu limits" { + [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup + requires cgroups_v1 + + # run a few busyboxes detached + runc run -d --console-socket $CONSOLE_SOCKET test_update + [ "$status" -eq 0 ] + + # check that initial values were properly set + check_cgroup_value "cpu.cfs_period_us" 1000000 + check_cgroup_value "cpu.cfs_quota_us" 500000 + check_cgroup_value "cpu.shares" 100 + + # update cpu-period + runc update test_update --cpu-period 900000 + [ "$status" -eq 0 ] + check_cgroup_value "cpu.cfs_period_us" 900000 + + # update cpu-quota + runc update test_update --cpu-quota 600000 + [ "$status" -eq 0 ] + check_cgroup_value "cpu.cfs_quota_us" 600000 + + # update cpu-shares + runc update test_update --cpu-share 200 + [ "$status" -eq 0 ] + check_cgroup_value "cpu.shares" 200 + + # Revert to the test initial value via json on stding + runc update -r - test_update < $BATS_TMPDIR/runc-cgroups-integration-test.json +{ + "cpu": { + "shares": 100, + "quota": 500000, + "period": 1000000 + } +} +EOF + + runc update -r $BATS_TMPDIR/runc-cgroups-integration-test.json test_update + [ "$status" -eq 0 ] + check_cgroup_value "cpu.cfs_period_us" 1000000 + check_cgroup_value "cpu.cfs_quota_us" 500000 + check_cgroup_value "cpu.shares" 100 } @test "update rt period and runtime" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup - requires cgroups_kmem cgroups_rt + requires cgroups_rt # run a detached busybox runc run -d --console-socket $CONSOLE_SOCKET test_update_rt [ "$status" -eq 0 ] - # get the cgroup paths - eval CGROUP_CPU="${CGROUP_CPU_BASE_PATH}${CGROUPS_PATH}" - runc update -r - test_update_rt <