Skip to content

Commit

Permalink
fix runc events error in cgroup v2
Browse files Browse the repository at this point in the history
Signed-off-by: lifubang <[email protected]>
  • Loading branch information
lifubang committed May 5, 2020
1 parent a57358e commit 0789d76
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 8 deletions.
7 changes: 7 additions & 0 deletions libcontainer/container_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,13 @@ func (c *linuxContainer) NotifyOOM() (<-chan struct{}, error) {
if c.config.RootlessCgroups {
logrus.Warn("getting OOM notifications may fail if you don't have the full access to cgroups")
}
if cgroups.IsCgroup2UnifiedMode() {
path, err := c.cgroupManager.GetUnifiedPath()
if err != nil {
return nil, err
}
return notifyOnOOMV2(path)
}
return notifyOnOOM(c.cgroupManager.GetPaths())
}

Expand Down
102 changes: 102 additions & 0 deletions libcontainer/notify_linux_v2.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// +build linux

package libcontainer

import (
"io/ioutil"
"path/filepath"
"strconv"
"strings"
"unsafe"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)

func getValueFromCgroup(path, key string) (int, error) {
content, err := ioutil.ReadFile(path)
if err != nil {
return 0, err
}

lines := strings.Split(string(content), "\n")
for _, line := range lines {
arr := strings.Split(line, " ")
if len(arr) == 2 && arr[0] == key {
return strconv.Atoi(arr[1])
}
}
return 0, nil
}

func registerMemoryEventV2(cgDir, evName, cgEvName string) (<-chan struct{}, error) {
eventControlPath := filepath.Join(cgDir, evName)
cgEvPath := filepath.Join(cgDir, cgEvName)
fd, err := unix.InotifyInit()
if err != nil {
return nil, errors.Wrap(err, "unable to init inotify")
}
// watching oom kill
evFd, err := unix.InotifyAddWatch(fd, eventControlPath, unix.IN_MODIFY)
if err != nil {
unix.Close(fd)
return nil, errors.Wrap(err, "unable to add inotify watch")
}
// Because no `unix.IN_DELETE|unix.IN_DELETE_SELF` event for cgroup file system, so watching all process exited
cgFd, err := unix.InotifyAddWatch(fd, cgEvPath, unix.IN_MODIFY)
if err != nil {
unix.Close(fd)
return nil, errors.Wrap(err, "unable to add inotify watch")
}
ch := make(chan struct{})
go func() {
var (
buffer [unix.SizeofInotifyEvent + unix.PathMax + 1]byte
offset uint32
)
defer func() {
unix.Close(fd)
close(ch)
}()

for {
n, err := unix.Read(fd, buffer[:])
if err != nil {
logrus.Warnf("unable to read event data from inotify, got error: %v", err)
return
}
if n < unix.SizeofInotifyEvent {
logrus.Warnf("we should read at least %d bytes from inotify, but got %d bytes.", unix.SizeofInotifyEvent, n)
return
}
offset = 0
for offset <= uint32(n-unix.SizeofInotifyEvent) {
rawEvent := (*unix.InotifyEvent)(unsafe.Pointer(&buffer[offset]))
offset += unix.SizeofInotifyEvent + uint32(rawEvent.Len)
if rawEvent.Mask&unix.IN_MODIFY != unix.IN_MODIFY {
continue
}
switch int(rawEvent.Wd) {
case evFd:
oom, err := getValueFromCgroup(eventControlPath, "oom_kill")
if err != nil || oom > 0 {
ch <- struct{}{}
}
case cgFd:
pids, err := getValueFromCgroup(cgEvPath, "populated")
if err != nil || pids == 0 {
return
}
}
}
}
}()
return ch, nil
}

// notifyOnOOMV2 returns channel on which you can expect event about OOM,
// if process died without OOM this channel will be closed.
func notifyOnOOMV2(path string) (<-chan struct{}, error) {
return registerMemoryEventV2(path, "memory.events", "cgroup.events")
}
54 changes: 46 additions & 8 deletions tests/integration/events.bats
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ function teardown() {

@test "events --stats" {
# XXX: currently cgroups require root containers.
# TODO: support cgroup v2 memory.events
requires root cgroups_v1
requires root
init_cgroup_paths

# run busybox detached
runc run -d --console-socket $CONSOLE_SOCKET test_busybox
Expand All @@ -29,8 +29,8 @@ function teardown() {

@test "events --interval default " {
# XXX: currently cgroups require root containers.
# TODO: support cgroup v2 memory.events
requires root cgroups_v1
requires root
init_cgroup_paths

# run busybox detached
runc run -d --console-socket $CONSOLE_SOCKET test_busybox
Expand All @@ -57,8 +57,8 @@ function teardown() {

@test "events --interval 1s " {
# XXX: currently cgroups require root containers.
# TODO: support cgroup v2 memory.events
requires root cgroups_v1
requires root
init_cgroup_paths

# run busybox detached
runc run -d --console-socket $CONSOLE_SOCKET test_busybox
Expand All @@ -84,8 +84,8 @@ function teardown() {

@test "events --interval 100ms " {
# XXX: currently cgroups require root containers.
# TODO: support cgroup v2 memory.events
requires root cgroups_v1
requires root
init_cgroup_paths

# run busybox detached
runc run -d --console-socket $CONSOLE_SOCKET test_busybox
Expand All @@ -111,3 +111,41 @@ function teardown() {
run eval "grep -q 'test_busybox' events.log"
[ "$status" -eq 0 ]
}

@test "events oom " {
# XXX: currently cgroups require root containers.
requires root
init_cgroup_paths

# Set some initial known values
DATA=$(cat <<EOF
"memory": {
"limit": 33554433,
"swap": 33554433,
"swappiness": 0
},
EOF
)
DATA=$(echo ${DATA} | sed 's/\n/\\n/g')
sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json

# run busybox detached
runc run -d --console-socket $CONSOLE_SOCKET test_busybox
[ "$status" -eq 0 ]

# spawn two sub processes (shells)
# the first sub process is an event logger that sends stats events to events.log
# the second sub process exec a memory hog process to cause a oom condition
# and waits for an oom event that includes test_busybox
# then kills the test_busybox container which causes the event logger to exit
(__runc events test_busybox > events.log) &
(
retry 10 1 eval "grep -q 'test_busybox' events.log"
__runc exec -d test_busybox sh -c 'VAR=$(seq 1 100000000)'
retry 10 1 eval "grep -q 'oom' events.log"
__runc delete -f test_busybox
) &
wait # wait for the above sub shells to finish

grep -q '{"type":"oom","id":"test_busybox"}' events.log
}

0 comments on commit 0789d76

Please sign in to comment.