From 9a56096a601c80d216b02e07021dc7237dfdd3b8 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Wed, 29 Nov 2017 18:44:31 +0800 Subject: [PATCH 01/10] add cgroups package Signed-off-by: Ma Shimiao --- cgroups/cgroups.go | 72 +++++++ cgroups/cgroups_v1.go | 473 ++++++++++++++++++++++++++++++++++++++++++ cgroups/cgroups_v2.go | 47 +++++ 3 files changed, 592 insertions(+) create mode 100644 cgroups/cgroups.go create mode 100644 cgroups/cgroups_v1.go create mode 100644 cgroups/cgroups_v2.go diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go new file mode 100644 index 000000000..fb6843f76 --- /dev/null +++ b/cgroups/cgroups.go @@ -0,0 +1,72 @@ +package cgroups + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +// Cgroup represents interfaces for cgroup validation +type Cgroup interface { + GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) + GetCPUData(cgPath string) (*rspec.LinuxCPU, error) + GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) + GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) + GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) + GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) + GetPidsData(cgPath string) (*rspec.LinuxPids, error) +} + +// FindCgroup gets cgroup root mountpoint +func FindCgroup() (Cgroup, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer f.Close() + + cgroupv2 := false + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + fields := strings.Split(text, " ") + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + numPostFields := len(postSeparatorFields) + + // This is an error as we can't detect if the mount is for "cgroup" + if numPostFields == 0 { + return nil, fmt.Errorf("Found no fields post '-' in %q", text) + } + + if postSeparatorFields[0] == "cgroup" { + // Check that the mount is properly formated. + if numPostFields < 3 { + return nil, fmt.Errorf("Error found less than 3 fields post '-' in %q", text) + } + + cg := &CgroupV1{ + MountPath: filepath.Dir(fields[4]), + } + return cg, nil + } else if postSeparatorFields[0] == "cgroup2" { + cgroupv2 = true + continue + //TODO cgroupv2 unimplemented + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + if cgroupv2 { + return nil, fmt.Errorf("cgroupv2 is not supported yet") + } + return nil, fmt.Errorf("cgroup is not found") +} diff --git a/cgroups/cgroups_v1.go b/cgroups/cgroups_v1.go new file mode 100644 index 000000000..96977b3db --- /dev/null +++ b/cgroups/cgroups_v1.go @@ -0,0 +1,473 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "regexp" + "strconv" + "strings" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +// CgroupV1 used for cgroupv1 validation +type CgroupV1 struct { + MountPath string +} + +func getDeviceID(id string) (int64, int64, error) { + elem := strings.Split(id, ":") + major, err := strconv.ParseInt(elem[0], 10, 64) + if err != nil { + return 0, 0, err + } + minor, err := strconv.ParseInt(elem[1], 10, 64) + if err != nil { + return 0, 0, err + } + return major, minor, nil +} + +// GetBlockIOData gets cgroup blockio data +func (cg *CgroupV1) GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) { + lb := &rspec.LinuxBlockIO{} + names := []string{"weight", "leaf_weight", "weight_device", "leaf_weight_device", "throttle.read_bps_device", "throttle.write_bps_device", "throttle.read_iops_device", "throttle.write_iops_device"} + for i, name := range names { + fileName := strings.Join([]string{"blkio", name}, ".") + filePath := filepath.Join(cg.MountPath, "blkio", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 16) + if err != nil { + return nil, err + } + weight := uint16(res) + lb.Weight = &weight + case 1: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 16) + if err != nil { + return nil, err + } + leafWeight := uint16(res) + lb.LeafWeight = &leafWeight + case 2: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(elem[1], 10, 16) + if err != nil { + return nil, err + } + weight := uint16(res) + lwd := rspec.LinuxWeightDevice{} + lwd.Major = major + lwd.Minor = minor + lwd.Weight = &weight + lb.WeightDevice = append(lb.WeightDevice, lwd) + } + case 3: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(elem[1], 10, 16) + if err != nil { + return nil, err + } + leafWeight := uint16(res) + exist := false + for i, wd := range lb.WeightDevice { + if wd.Major == major && wd.Minor == minor { + exist = true + lb.WeightDevice[i].LeafWeight = &leafWeight + break + } + } + if !exist { + lwd := rspec.LinuxWeightDevice{} + lwd.Major = major + lwd.Minor = minor + lwd.LeafWeight = &leafWeight + lb.WeightDevice = append(lb.WeightDevice, lwd) + } + } + case 4: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleReadBpsDevice = append(lb.ThrottleReadBpsDevice, ltd) + } + case 5: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleWriteBpsDevice = append(lb.ThrottleWriteBpsDevice, ltd) + } + case 6: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleReadIOPSDevice = append(lb.ThrottleReadIOPSDevice, ltd) + } + case 7: + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + major, minor, err := getDeviceID(elem[0]) + if err != nil { + return nil, err + } + rate, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + ltd := rspec.LinuxThrottleDevice{} + ltd.Major = major + ltd.Minor = minor + ltd.Rate = rate + lb.ThrottleWriteIOPSDevice = append(lb.ThrottleWriteIOPSDevice, ltd) + } + } + } + + return lb, nil +} + +// GetCPUData gets cgroup cpus data +func (cg *CgroupV1) GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { + lc := &rspec.LinuxCPU{} + names := []string{"shares", "cfs_quota_us", "cfs_period_us"} + for i, name := range names { + fileName := strings.Join([]string{"cpu", name}, ".") + filePath := filepath.Join(cg.MountPath, "cpu", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + shares := res + lc.Shares = &shares + case 1: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + quota := res + lc.Quota = "a + case 2: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + period := res + lc.Period = &period + } + } + // CONFIG_RT_GROUP_SCHED may be not set + // Can always get rt data from /proc + contents, err := ioutil.ReadFile("/proc/sys/kernel/sched_rt_period_us") + if err != nil { + return nil, err + } + rtPeriod, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lc.RealtimePeriod = &rtPeriod + contents, err = ioutil.ReadFile("/proc/sys/kernel/sched_rt_runtime_us") + if err != nil { + return nil, err + } + rtQuota, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lc.RealtimeRuntime = &rtQuota + + names = []string{"cpus", "mems"} + for i, name := range names { + fileName := strings.Join([]string{"cpuset", name}, ".") + filePath := filepath.Join(cg.MountPath, "cpuset", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + lc.Cpus = strings.TrimSpace(string(contents)) + case 1: + lc.Mems = strings.TrimSpace(string(contents)) + } + } + + return lc, nil +} + +// GetDevicesData gets cgroup devices data +func (cg *CgroupV1) GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) { + ld := []rspec.LinuxDeviceCgroup{} + + return ld, nil +} + +func inBytes(size string) (int64, error) { + KiB := 1024 + MiB := 1024 * KiB + GiB := 1024 * MiB + TiB := 1024 * GiB + PiB := 1024 * TiB + binaryMap := map[string]int64{"k": int64(KiB), "m": int64(MiB), "g": int64(GiB), "t": int64(TiB), "p": int64(PiB)} + sizeRegex := regexp.MustCompile(`^(\d+(\.\d+)*) ?([kKmMgGtTpP])?[bB]?$`) + matches := sizeRegex.FindStringSubmatch(size) + if len(matches) != 4 { + return -1, fmt.Errorf("invalid size: '%s'", size) + } + + byteSize, err := strconv.ParseFloat(matches[1], 64) + if err != nil { + return -1, err + } + + unitPrefix := strings.ToLower(matches[3]) + if mul, ok := binaryMap[unitPrefix]; ok { + byteSize *= float64(mul) + } + + return int64(byteSize), nil +} + +func getHugePageSize() ([]string, error) { + var pageSizes []string + sizeList := []string{"B", "kB", "MB", "GB", "TB", "PB"} + files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages") + if err != nil { + return pageSizes, err + } + for _, st := range files { + nameArray := strings.Split(st.Name(), "-") + pageSize, err := inBytes(nameArray[1]) + if err != nil { + return []string{}, err + } + size := float64(pageSize) + base := float64(1024.0) + i := 0 + unitsLimit := len(sizeList) - 1 + for size >= base && i < unitsLimit { + size = size / base + i++ + } + sizeString := fmt.Sprintf("%g%s", size, sizeList[i]) + pageSizes = append(pageSizes, sizeString) + } + + return pageSizes, nil +} + +// GetHugepageLimitData gets cgroup hugetlb data +func (cg *CgroupV1) GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) { + lh := []rspec.LinuxHugepageLimit{} + pageSizes, err := getHugePageSize() + if err != nil { + return lh, err + } + for _, pageSize := range pageSizes { + maxUsage := strings.Join([]string{"hugetlb", pageSize, "limit_in_bytes"}, ".") + filePath := filepath.Join(cg.MountPath, "hugetlb", cgPath, maxUsage) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return lh, err + } + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + pageLimit := rspec.LinuxHugepageLimit{} + pageLimit.Pagesize = pageSize + pageLimit.Limit = res + lh = append(lh, pageLimit) + } + + return lh, nil +} + +// GetMemoryData gets cgroup memory data +func (cg *CgroupV1) GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) { + lm := &rspec.LinuxMemory{} + names := []string{"limit_in_bytes", "soft_limit_in_bytes", "memsw.limit_in_bytes", "kmem.limit_in_bytes", "kmem.tcp.limit_in_bytes", "swappiness", "oom_control"} + for i, name := range names { + fileName := strings.Join([]string{"memory", name}, ".") + filePath := filepath.Join(cg.MountPath, "memory", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + switch i { + case 0: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + limit := res + lm.Limit = &limit + case 1: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + sLimit := res + lm.Reservation = &sLimit + case 2: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + swLimit := res + lm.Swap = &swLimit + case 3: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + kernelLimit := res + lm.Kernel = &kernelLimit + case 4: + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + tcpLimit := res + lm.KernelTCP = &tcpLimit + case 5: + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + swappiness := res + lm.Swappiness = &swappiness + case 6: + parts := strings.Split(string(contents), "\n") + part := strings.Split(parts[0], " ") + res, err := strconv.ParseInt(part[1], 10, 64) + if err != nil { + return nil, err + } + oom := false + if res == 1 { + oom = true + } + lm.DisableOOMKiller = &oom + } + } + + return lm, nil +} + +// GetNetworkData gets cgroup network data +func (cg *CgroupV1) GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { + ln := &rspec.LinuxNetwork{} + fileName := strings.Join([]string{"net_cls", "classid"}, ".") + filePath := filepath.Join(cg.MountPath, "net_cls", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + res, err := strconv.ParseUint(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + classid := uint32(res) + ln.ClassID = &classid + + fileName = strings.Join([]string{"net_prio", "ifpriomap"}, ".") + filePath = filepath.Join(cg.MountPath, "net_prio", cgPath, fileName) + contents, err = ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.Split(part, " ") + res, err := strconv.ParseUint(elem[1], 10, 64) + if err != nil { + return nil, err + } + lip := rspec.LinuxInterfacePriority{} + lip.Name = elem[0] + lip.Priority = uint32(res) + ln.Priorities = append(ln.Priorities, lip) + } + + return ln, nil +} + +// GetPidsData gets cgroup pids data +func (cg *CgroupV1) GetPidsData(cgPath string) (*rspec.LinuxPids, error) { + lp := &rspec.LinuxPids{} + fileName := strings.Join([]string{"pids", "max"}, ".") + filePath := filepath.Join(cg.MountPath, "pids", cgPath, fileName) + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + res, err := strconv.ParseInt(strings.TrimSpace(string(contents)), 10, 64) + if err != nil { + return nil, err + } + lp.Limit = res + + return lp, nil +} diff --git a/cgroups/cgroups_v2.go b/cgroups/cgroups_v2.go new file mode 100644 index 000000000..9d57625dc --- /dev/null +++ b/cgroups/cgroups_v2.go @@ -0,0 +1,47 @@ +package cgroups + +import ( + "fmt" + + rspec "github.com/opencontainers/runtime-spec/specs-go" +) + +// CgroupV2 used for cgroupv2 validation +type CgroupV2 struct { + MountPath string +} + +// GetBlockIOData gets cgroup blockio data +func GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetCPUData gets cgroup cpus data +func GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetDevicesData gets cgroup devices data +func GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetHugepageLimitData gets cgroup hugetlb data +func GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetMemoryData gets cgroup memory data +func (cg *CgroupV2) GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetNetworkData gets cgroup network data +func GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { + return nil, fmt.Errorf("unimplemented yet") +} + +// GetPidsData gets cgroup pids data +func GetPidsData(cgPath string) (*rspec.LinuxPids, error) { + return nil, fmt.Errorf("unimplemented yet") +} From 07118a8c93bd879ed3649888ee6843f14b043c85 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 11:54:58 +0800 Subject: [PATCH 02/10] add runtimeOutsideValidate Signed-off-by: Ma Shimiao --- validation/util/test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/validation/util/test.go b/validation/util/test.go index 3b544197c..dd4d83252 100644 --- a/validation/util/test.go +++ b/validation/util/test.go @@ -138,3 +138,41 @@ func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) { os.Stdout.Write(stdout) return nil } + +// RuntimeOutsideValidate validate runtime outside a container. +func RuntimeOutsideValidate(g *generate.Generator, f PreFunc) error { + bundleDir, err := PrepareBundle() + if err != nil { + return err + } + + r, err := NewRuntime(RuntimeCommand, bundleDir) + if err != nil { + os.RemoveAll(bundleDir) + return err + } + defer r.Clean(true, true) + err = r.SetConfig(g) + if err != nil { + return err + } + err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest")) + if err != nil { + return err + } + + r.SetID(uuid.NewV4().String()) + stderr, err := r.Create() + if err != nil { + os.Stderr.WriteString("failed to start the container\n") + os.Stderr.Write(stderr) + return err + } + + if f != nil { + if err := f("test"); err != nil { + return err + } + } + return nil +} From 4a57b0ff3ae8b9cba005361f379bd2b89e046aeb Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 14:25:04 +0800 Subject: [PATCH 03/10] add cgroup memory test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_memory.go | 57 ++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 validation/linux_cgroups_memory.go diff --git a/validation/linux_cgroups_memory.go b/validation/linux_cgroups_memory.go new file mode 100644 index 000000000..083435a64 --- /dev/null +++ b/validation/linux_cgroups_memory.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 50593792 + var swappiness uint64 = 50 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.SetLinuxResourcesMemoryLimit(limit) + g.SetLinuxResourcesMemoryReservation(limit) + g.SetLinuxResourcesMemorySwap(limit) + g.SetLinuxResourcesMemoryKernel(limit) + g.SetLinuxResourcesMemoryKernelTCP(limit) + g.SetLinuxResourcesMemorySwappiness(swappiness) + g.SetLinuxResourcesMemoryDisableOOMKiller(true) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lm, err := cg.GetMemoryData(path) + if err != nil { + return err + } + if limit != *lm.Limit { + return fmt.Errorf("memory limit is not set correctly, expect: %d, actual: %d", limit, *lm.Limit) + } + if limit != *lm.Reservation { + return fmt.Errorf("memory reservation is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Swap { + return fmt.Errorf("memory swap is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Kernel { + return fmt.Errorf("memory kernel is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if limit != *lm.KernelTCP { + return fmt.Errorf("memory kernelTCP is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if swappiness != *lm.Swappiness { + return fmt.Errorf("memory swappiness is not set correctly, expect: %d, actual: %d", swappiness, *lm.Swappiness) + } + if true != *lm.DisableOOMKiller { + return fmt.Errorf("memory oom is not set correctly, expect: %t, actual: %t", true, *lm.DisableOOMKiller) + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} From 432615a089b5eb89595948fcba005bb2284ef9e1 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 14:42:03 +0800 Subject: [PATCH 04/10] add cgroup hugetlb test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_hugetlb.go | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 validation/linux_cgroups_hugetlb.go diff --git a/validation/linux_cgroups_hugetlb.go b/validation/linux_cgroups_hugetlb.go new file mode 100644 index 000000000..96f67e5fd --- /dev/null +++ b/validation/linux_cgroups_hugetlb.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + page := "1GB" + var limit uint64 = 56892210544640 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.AddLinuxResourcesHugepageLimit(page, limit) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lhd, err := cg.GetHugepageLimitData(path) + if err != nil { + return err + } + for _, lhl := range lhd { + if lhl.Pagesize == page && lhl.Limit != limit { + return fmt.Errorf("hugepage %s limit is not set correctly, expect: %d, actual: %d", page, limit, lhl.Limit) + } + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} From f1e02ff6b3bca42a9fa33f918f55cf97da3ab193 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 17:59:27 +0800 Subject: [PATCH 05/10] add cgroup blkio test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_blkio.go | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 validation/linux_cgroups_blkio.go diff --git a/validation/linux_cgroups_blkio.go b/validation/linux_cgroups_blkio.go new file mode 100644 index 000000000..d3ad815dd --- /dev/null +++ b/validation/linux_cgroups_blkio.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var weight uint16 = 500 + var leafWeight uint16 = 300 + var major, minor int64 = 8, 0 + var rate uint64 = 102400 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.SetLinuxResourcesBlockIOWeight(weight) + g.SetLinuxResourcesBlockIOLeafWeight(leafWeight) + g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight) + g.AddLinuxResourcesBlockIOLeafWeightDevice(major, minor, leafWeight) + g.AddLinuxResourcesBlockIOThrottleReadBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lbd, err := cg.GetBlockIOData(path) + if err != nil { + return err + } + if *lbd.Weight != weight { + return fmt.Errorf("blkio weight is not set correctly, expect: %d, actual: %d", weight, lbd.Weight) + } + if *lbd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight is not set correctly, expect: %d, actual: %d", weight, lbd.LeafWeight) + } + + found := false + for _, wd := range lbd.WeightDevice { + if wd.Major == major && wd.Minor == minor { + found = true + if *wd.Weight != weight { + return fmt.Errorf("blkio weight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, weight, wd.Weight) + } + if *wd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, leafWeight, wd.LeafWeight) + } + } + } + if !found { + return fmt.Errorf("blkio weightDevice for %d:%d is not set", major, minor) + } + + found = false + for _, trbd := range lbd.ThrottleReadBpsDevice { + if trbd.Major == major && trbd.Minor == minor { + found = true + if trbd.Rate != rate { + return fmt.Errorf("blkio read bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read bps for %d:%d is not set", major, minor) + } + + found = false + for _, twbd := range lbd.ThrottleWriteBpsDevice { + if twbd.Major == major && twbd.Minor == minor { + found = true + if twbd.Rate != rate { + return fmt.Errorf("blkio write bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write bps for %d:%d is not set", major, minor) + } + + found = false + for _, trid := range lbd.ThrottleReadIOPSDevice { + if trid.Major == major && trid.Minor == minor { + found = true + if trid.Rate != rate { + return fmt.Errorf("blkio read iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read iops for %d:%d is not set", major, minor) + } + + found = false + for _, twid := range lbd.ThrottleWriteIOPSDevice { + if twid.Major == major && twid.Minor == minor { + found = true + if twid.Rate != rate { + return fmt.Errorf("blkio write iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write iops for %d:%d is not set", major, minor) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} From b712995985254a38efd60472822110387b440b63 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 19:16:59 +0800 Subject: [PATCH 06/10] add cgroup cpus test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_cpus.go | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 validation/linux_cgroups_cpus.go diff --git a/validation/linux_cgroups_cpus.go b/validation/linux_cgroups_cpus.go new file mode 100644 index 000000000..6d442e8be --- /dev/null +++ b/validation/linux_cgroups_cpus.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var shares uint64 = 1024 + var period uint64 = 100000 + var quota int64 = 50000 + var cpus, mems string = "0-1", "0" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.SetLinuxResourcesCPUShares(shares) + g.SetLinuxResourcesCPUQuota(quota) + g.SetLinuxResourcesCPUPeriod(period) + g.SetLinuxResourcesCPUCpus(cpus) + g.SetLinuxResourcesCPUMems(mems) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lcd, err := cg.GetCPUData(path) + if err != nil { + return err + } + if *lcd.Shares != shares { + return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, lcd.Shares) + } + if *lcd.Quota != quota { + return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, lcd.Quota) + } + if *lcd.Period != period { + return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, lcd.Period) + } + if lcd.Cpus != cpus { + return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus) + } + if lcd.Mems != mems { + return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} From 72e67e5bfcf9ace7bd0937397e51afe83d3c1742 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 19:43:32 +0800 Subject: [PATCH 07/10] add cgroup pids test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_pids.go | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 validation/linux_cgroups_pids.go diff --git a/validation/linux_cgroups_pids.go b/validation/linux_cgroups_pids.go new file mode 100644 index 000000000..c00c6b37f --- /dev/null +++ b/validation/linux_cgroups_pids.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 1000 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.SetLinuxResourcesPidsLimit(limit) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lpd, err := cg.GetPidsData(path) + if err != nil { + return err + } + if lpd.Limit != limit { + return fmt.Errorf("pids limit is not set correctly, expect: %d, actual: %d", limit, lpd.Limit) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} From 63510447a39599351df404b5bf4c87d5f73a5257 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Tue, 5 Dec 2017 19:57:56 +0800 Subject: [PATCH 08/10] add cgroup network test for runtime Signed-off-by: Ma Shimiao --- validation/linux_cgroups_network.go | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 validation/linux_cgroups_network.go diff --git a/validation/linux_cgroups_network.go b/validation/linux_cgroups_network.go new file mode 100644 index 000000000..7ca3cd78d --- /dev/null +++ b/validation/linux_cgroups_network.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var id, prio uint32 = 255, 10 + ifName := "lo" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath("/test") + g.SetLinuxResourcesNetworkClassID(id) + err := util.RuntimeOutsideValidate(g, func(path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lnd, err := cg.GetNetworkData(path) + if err != nil { + return err + } + if *lnd.ClassID != id { + return fmt.Errorf("network ID is not set correctly, expect: %d, actual: %d", id, lnd.ClassID) + } + found := false + for _, lip := range lnd.Priorities { + if lip.Name == ifName { + found = true + if lip.Priority != prio { + return fmt.Errorf("network priority for %s is not set correctly, expect: %d, actual: %d", ifName, prio, lip.Priority) + } + } + } + if !found { + return fmt.Errorf("network priority for %s is not set correctly", ifName) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} From 1aebc097a85bef14eaecf4cdbd817b14d2ab3f60 Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Wed, 6 Dec 2017 10:38:12 +0800 Subject: [PATCH 09/10] update to support relative cgrouppath test Signed-off-by: Ma Shimiao --- cgroups/cgroups.go | 46 +++++++++++--- cgroups/cgroups_v1.go | 94 ++++++++++++++++++++++++++--- cgroups/cgroups_v2.go | 16 ++--- validation/linux_cgroups_blkio.go | 6 +- validation/linux_cgroups_cpus.go | 6 +- validation/linux_cgroups_hugetlb.go | 6 +- validation/linux_cgroups_memory.go | 6 +- validation/linux_cgroups_network.go | 6 +- validation/linux_cgroups_pids.go | 6 +- validation/util/test.go | 11 +++- 10 files changed, 161 insertions(+), 42 deletions(-) diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go index fb6843f76..e108f3106 100644 --- a/cgroups/cgroups.go +++ b/cgroups/cgroups.go @@ -3,6 +3,7 @@ package cgroups import ( "bufio" "fmt" + "io/ioutil" "os" "path/filepath" "strings" @@ -10,15 +11,22 @@ import ( rspec "github.com/opencontainers/runtime-spec/specs-go" ) +var ( + // AbsCgroupPath is absolute path for container's cgroup mount + AbsCgroupPath = "/cgrouptest" + // RelCgroupPath is relative path for container's cgroup mount + RelCgroupPath = "testdir/cgrouptest/container" +) + // Cgroup represents interfaces for cgroup validation type Cgroup interface { - GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) - GetCPUData(cgPath string) (*rspec.LinuxCPU, error) - GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) - GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) - GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) - GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) - GetPidsData(cgPath string) (*rspec.LinuxPids, error) + GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) + GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) + GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) + GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) + GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) + GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) + GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) } // FindCgroup gets cgroup root mountpoint @@ -70,3 +78,27 @@ func FindCgroup() (Cgroup, error) { } return nil, fmt.Errorf("cgroup is not found") } + +// GetSubsystemPath gets path of subsystem +func GetSubsystemPath(pid int, subsystem string) (string, error) { + contents, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) + if err != nil { + return "", err + } + + parts := strings.Split(strings.TrimSpace(string(contents)), "\n") + for _, part := range parts { + elem := strings.SplitN(part, ":", 3) + if len(elem) < 3 { + continue + } + subelems := strings.Split(elem[1], ",") + for _, subelem := range subelems { + if subelem == subsystem { + return elem[2], nil + } + } + } + + return "", fmt.Errorf("subsystem %s not found", subsystem) +} diff --git a/cgroups/cgroups_v1.go b/cgroups/cgroups_v1.go index 96977b3db..3f056821c 100644 --- a/cgroups/cgroups_v1.go +++ b/cgroups/cgroups_v1.go @@ -30,12 +30,22 @@ func getDeviceID(id string) (int64, int64, error) { } // GetBlockIOData gets cgroup blockio data -func (cg *CgroupV1) GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) { +func (cg *CgroupV1) GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) { lb := &rspec.LinuxBlockIO{} names := []string{"weight", "leaf_weight", "weight_device", "leaf_weight_device", "throttle.read_bps_device", "throttle.write_bps_device", "throttle.read_iops_device", "throttle.write_iops_device"} for i, name := range names { fileName := strings.Join([]string{"blkio", name}, ".") filePath := filepath.Join(cg.MountPath, "blkio", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "blkio") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "blkio") + } + filePath = filepath.Join(cg.MountPath, "blkio", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -182,12 +192,22 @@ func (cg *CgroupV1) GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) { } // GetCPUData gets cgroup cpus data -func (cg *CgroupV1) GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { +func (cg *CgroupV1) GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) { lc := &rspec.LinuxCPU{} names := []string{"shares", "cfs_quota_us", "cfs_period_us"} for i, name := range names { fileName := strings.Join([]string{"cpu", name}, ".") filePath := filepath.Join(cg.MountPath, "cpu", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "cpu") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "cpu") + } + filePath = filepath.Join(cg.MountPath, "cpu", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -241,6 +261,16 @@ func (cg *CgroupV1) GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { for i, name := range names { fileName := strings.Join([]string{"cpuset", name}, ".") filePath := filepath.Join(cg.MountPath, "cpuset", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "cpuset") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "cpuset") + } + filePath = filepath.Join(cg.MountPath, "cpuset", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -257,7 +287,7 @@ func (cg *CgroupV1) GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { } // GetDevicesData gets cgroup devices data -func (cg *CgroupV1) GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) { +func (cg *CgroupV1) GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) { ld := []rspec.LinuxDeviceCgroup{} return ld, nil @@ -318,7 +348,7 @@ func getHugePageSize() ([]string, error) { } // GetHugepageLimitData gets cgroup hugetlb data -func (cg *CgroupV1) GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) { +func (cg *CgroupV1) GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) { lh := []rspec.LinuxHugepageLimit{} pageSizes, err := getHugePageSize() if err != nil { @@ -327,6 +357,16 @@ func (cg *CgroupV1) GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLi for _, pageSize := range pageSizes { maxUsage := strings.Join([]string{"hugetlb", pageSize, "limit_in_bytes"}, ".") filePath := filepath.Join(cg.MountPath, "hugetlb", cgPath, maxUsage) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "hugetlb") + if err != nil { + return lh, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "hugetlb") + } + filePath = filepath.Join(cg.MountPath, "hugetlb", subPath, maxUsage) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return lh, err @@ -345,12 +385,22 @@ func (cg *CgroupV1) GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLi } // GetMemoryData gets cgroup memory data -func (cg *CgroupV1) GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) { +func (cg *CgroupV1) GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) { lm := &rspec.LinuxMemory{} names := []string{"limit_in_bytes", "soft_limit_in_bytes", "memsw.limit_in_bytes", "kmem.limit_in_bytes", "kmem.tcp.limit_in_bytes", "swappiness", "oom_control"} for i, name := range names { fileName := strings.Join([]string{"memory", name}, ".") filePath := filepath.Join(cg.MountPath, "memory", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "memory") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "memory") + } + filePath = filepath.Join(cg.MountPath, "memory", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -417,10 +467,20 @@ func (cg *CgroupV1) GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) { } // GetNetworkData gets cgroup network data -func (cg *CgroupV1) GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { +func (cg *CgroupV1) GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) { ln := &rspec.LinuxNetwork{} fileName := strings.Join([]string{"net_cls", "classid"}, ".") filePath := filepath.Join(cg.MountPath, "net_cls", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "net_cls") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "net_cls") + } + filePath = filepath.Join(cg.MountPath, "net_cls", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -434,6 +494,16 @@ func (cg *CgroupV1) GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { fileName = strings.Join([]string{"net_prio", "ifpriomap"}, ".") filePath = filepath.Join(cg.MountPath, "net_prio", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "net_prio") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "net_prio") + } + filePath = filepath.Join(cg.MountPath, "net_prio", subPath, fileName) + } contents, err = ioutil.ReadFile(filePath) if err != nil { return nil, err @@ -455,10 +525,20 @@ func (cg *CgroupV1) GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { } // GetPidsData gets cgroup pids data -func (cg *CgroupV1) GetPidsData(cgPath string) (*rspec.LinuxPids, error) { +func (cg *CgroupV1) GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) { lp := &rspec.LinuxPids{} fileName := strings.Join([]string{"pids", "max"}, ".") filePath := filepath.Join(cg.MountPath, "pids", cgPath, fileName) + if !filepath.IsAbs(cgPath) { + subPath, err := GetSubsystemPath(pid, "pids") + if err != nil { + return nil, err + } + if !strings.Contains(subPath, RelCgroupPath) { + return nil, fmt.Errorf("cgroup subsystem %s is not mounted as expected", "pids") + } + filePath = filepath.Join(cg.MountPath, "pids", subPath, fileName) + } contents, err := ioutil.ReadFile(filePath) if err != nil { return nil, err diff --git a/cgroups/cgroups_v2.go b/cgroups/cgroups_v2.go index 9d57625dc..d15e647d9 100644 --- a/cgroups/cgroups_v2.go +++ b/cgroups/cgroups_v2.go @@ -12,36 +12,36 @@ type CgroupV2 struct { } // GetBlockIOData gets cgroup blockio data -func GetBlockIOData(cgPath string) (*rspec.LinuxBlockIO, error) { +func GetBlockIOData(pid int, cgPath string) (*rspec.LinuxBlockIO, error) { return nil, fmt.Errorf("unimplemented yet") } // GetCPUData gets cgroup cpus data -func GetCPUData(cgPath string) (*rspec.LinuxCPU, error) { +func GetCPUData(pid int, cgPath string) (*rspec.LinuxCPU, error) { return nil, fmt.Errorf("unimplemented yet") } // GetDevicesData gets cgroup devices data -func GetDevicesData(cgPath string) ([]rspec.LinuxDeviceCgroup, error) { +func GetDevicesData(pid int, cgPath string) ([]rspec.LinuxDeviceCgroup, error) { return nil, fmt.Errorf("unimplemented yet") } // GetHugepageLimitData gets cgroup hugetlb data -func GetHugepageLimitData(cgPath string) ([]rspec.LinuxHugepageLimit, error) { +func GetHugepageLimitData(pid int, cgPath string) ([]rspec.LinuxHugepageLimit, error) { return nil, fmt.Errorf("unimplemented yet") } // GetMemoryData gets cgroup memory data -func (cg *CgroupV2) GetMemoryData(cgPath string) (*rspec.LinuxMemory, error) { +func (cg *CgroupV2) GetMemoryData(pid int, cgPath string) (*rspec.LinuxMemory, error) { return nil, fmt.Errorf("unimplemented yet") } // GetNetworkData gets cgroup network data -func GetNetworkData(cgPath string) (*rspec.LinuxNetwork, error) { +func GetNetworkData(pid int, cgPath string) (*rspec.LinuxNetwork, error) { return nil, fmt.Errorf("unimplemented yet") } -// GetPidsData gets cgroup pids data -func GetPidsData(cgPath string) (*rspec.LinuxPids, error) { +// GetPidsData gets cgroup pid ints data +func GetPidsData(pid int, cgPath string) (*rspec.LinuxPids, error) { return nil, fmt.Errorf("unimplemented yet") } diff --git a/validation/linux_cgroups_blkio.go b/validation/linux_cgroups_blkio.go index d3ad815dd..40ae1d016 100644 --- a/validation/linux_cgroups_blkio.go +++ b/validation/linux_cgroups_blkio.go @@ -13,7 +13,7 @@ func main() { var major, minor int64 = 8, 0 var rate uint64 = 102400 g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.SetLinuxResourcesBlockIOWeight(weight) g.SetLinuxResourcesBlockIOLeafWeight(leafWeight) g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight) @@ -22,12 +22,12 @@ func main() { g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate) g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate) g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lbd, err := cg.GetBlockIOData(path) + lbd, err := cg.GetBlockIOData(pid, path) if err != nil { return err } diff --git a/validation/linux_cgroups_cpus.go b/validation/linux_cgroups_cpus.go index 6d442e8be..32728a205 100644 --- a/validation/linux_cgroups_cpus.go +++ b/validation/linux_cgroups_cpus.go @@ -13,18 +13,18 @@ func main() { var quota int64 = 50000 var cpus, mems string = "0-1", "0" g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.SetLinuxResourcesCPUShares(shares) g.SetLinuxResourcesCPUQuota(quota) g.SetLinuxResourcesCPUPeriod(period) g.SetLinuxResourcesCPUCpus(cpus) g.SetLinuxResourcesCPUMems(mems) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lcd, err := cg.GetCPUData(path) + lcd, err := cg.GetCPUData(pid, path) if err != nil { return err } diff --git a/validation/linux_cgroups_hugetlb.go b/validation/linux_cgroups_hugetlb.go index 96f67e5fd..ae1b81d09 100644 --- a/validation/linux_cgroups_hugetlb.go +++ b/validation/linux_cgroups_hugetlb.go @@ -11,14 +11,14 @@ func main() { page := "1GB" var limit uint64 = 56892210544640 g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.AddLinuxResourcesHugepageLimit(page, limit) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lhd, err := cg.GetHugepageLimitData(path) + lhd, err := cg.GetHugepageLimitData(pid, path) if err != nil { return err } diff --git a/validation/linux_cgroups_memory.go b/validation/linux_cgroups_memory.go index 083435a64..7293d6388 100644 --- a/validation/linux_cgroups_memory.go +++ b/validation/linux_cgroups_memory.go @@ -11,7 +11,7 @@ func main() { var limit int64 = 50593792 var swappiness uint64 = 50 g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.SetLinuxResourcesMemoryLimit(limit) g.SetLinuxResourcesMemoryReservation(limit) g.SetLinuxResourcesMemorySwap(limit) @@ -19,12 +19,12 @@ func main() { g.SetLinuxResourcesMemoryKernelTCP(limit) g.SetLinuxResourcesMemorySwappiness(swappiness) g.SetLinuxResourcesMemoryDisableOOMKiller(true) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lm, err := cg.GetMemoryData(path) + lm, err := cg.GetMemoryData(pid, path) if err != nil { return err } diff --git a/validation/linux_cgroups_network.go b/validation/linux_cgroups_network.go index 7ca3cd78d..9cbd0d59b 100644 --- a/validation/linux_cgroups_network.go +++ b/validation/linux_cgroups_network.go @@ -11,14 +11,14 @@ func main() { var id, prio uint32 = 255, 10 ifName := "lo" g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.SetLinuxResourcesNetworkClassID(id) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lnd, err := cg.GetNetworkData(path) + lnd, err := cg.GetNetworkData(pid, path) if err != nil { return err } diff --git a/validation/linux_cgroups_pids.go b/validation/linux_cgroups_pids.go index c00c6b37f..3e1fb780c 100644 --- a/validation/linux_cgroups_pids.go +++ b/validation/linux_cgroups_pids.go @@ -10,14 +10,14 @@ import ( func main() { var limit int64 = 1000 g := util.GetDefaultGenerator() - g.SetLinuxCgroupsPath("/test") + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) g.SetLinuxResourcesPidsLimit(limit) - err := util.RuntimeOutsideValidate(g, func(path string) error { + err := util.RuntimeOutsideValidate(g, cgroups.AbsCgroupPath, func(pid int, path string) error { cg, err := cgroups.FindCgroup() if err != nil { return err } - lpd, err := cg.GetPidsData(path) + lpd, err := cg.GetPidsData(pid, path) if err != nil { return err } diff --git a/validation/util/test.go b/validation/util/test.go index dd4d83252..e18c3dbc5 100644 --- a/validation/util/test.go +++ b/validation/util/test.go @@ -24,6 +24,9 @@ var ( // but before creating the container. type PreFunc func(string) error +// AfterFunc validate container's outside environment after created +type AfterFunc func(int, string) error + func init() { runtimeInEnv := os.Getenv("RUNTIME") if runtimeInEnv != "" { @@ -140,7 +143,7 @@ func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) { } // RuntimeOutsideValidate validate runtime outside a container. -func RuntimeOutsideValidate(g *generate.Generator, f PreFunc) error { +func RuntimeOutsideValidate(g *generate.Generator, cgroupPath string, f AfterFunc) error { bundleDir, err := PrepareBundle() if err != nil { return err @@ -170,7 +173,11 @@ func RuntimeOutsideValidate(g *generate.Generator, f PreFunc) error { } if f != nil { - if err := f("test"); err != nil { + state, err := r.State() + if err != nil { + return err + } + if err := f(state.Pid, cgroupPath); err != nil { return err } } From 7c09b4ca1cd890ccf17cb56697caae409b39f9ce Mon Sep 17 00:00:00 2001 From: Ma Shimiao Date: Wed, 6 Dec 2017 11:14:17 +0800 Subject: [PATCH 10/10] add relative cgroupath test Signed-off-by: Ma Shimiao --- validation/linux_cgroups_relative_blkio.go | 115 +++++++++++++++++++ validation/linux_cgroups_relative_cpus.go | 52 +++++++++ validation/linux_cgroups_relative_hugetlb.go | 35 ++++++ validation/linux_cgroups_relative_memory.go | 57 +++++++++ validation/linux_cgroups_relative_network.go | 47 ++++++++ validation/linux_cgroups_relative_pids.go | 33 ++++++ 6 files changed, 339 insertions(+) create mode 100644 validation/linux_cgroups_relative_blkio.go create mode 100644 validation/linux_cgroups_relative_cpus.go create mode 100644 validation/linux_cgroups_relative_hugetlb.go create mode 100644 validation/linux_cgroups_relative_memory.go create mode 100644 validation/linux_cgroups_relative_network.go create mode 100644 validation/linux_cgroups_relative_pids.go diff --git a/validation/linux_cgroups_relative_blkio.go b/validation/linux_cgroups_relative_blkio.go new file mode 100644 index 000000000..4f2b56abd --- /dev/null +++ b/validation/linux_cgroups_relative_blkio.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var weight uint16 = 500 + var leafWeight uint16 = 300 + var major, minor int64 = 8, 0 + var rate uint64 = 102400 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesBlockIOWeight(weight) + g.SetLinuxResourcesBlockIOLeafWeight(leafWeight) + g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight) + g.AddLinuxResourcesBlockIOLeafWeightDevice(major, minor, leafWeight) + g.AddLinuxResourcesBlockIOThrottleReadBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate) + g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lbd, err := cg.GetBlockIOData(pid, path) + if err != nil { + return err + } + if *lbd.Weight != weight { + return fmt.Errorf("blkio weight is not set correctly, expect: %d, actual: %d", weight, lbd.Weight) + } + if *lbd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight is not set correctly, expect: %d, actual: %d", weight, lbd.LeafWeight) + } + + found := false + for _, wd := range lbd.WeightDevice { + if wd.Major == major && wd.Minor == minor { + found = true + if *wd.Weight != weight { + return fmt.Errorf("blkio weight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, weight, wd.Weight) + } + if *wd.LeafWeight != leafWeight { + return fmt.Errorf("blkio leafWeight for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, leafWeight, wd.LeafWeight) + } + } + } + if !found { + return fmt.Errorf("blkio weightDevice for %d:%d is not set", major, minor) + } + + found = false + for _, trbd := range lbd.ThrottleReadBpsDevice { + if trbd.Major == major && trbd.Minor == minor { + found = true + if trbd.Rate != rate { + return fmt.Errorf("blkio read bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read bps for %d:%d is not set", major, minor) + } + + found = false + for _, twbd := range lbd.ThrottleWriteBpsDevice { + if twbd.Major == major && twbd.Minor == minor { + found = true + if twbd.Rate != rate { + return fmt.Errorf("blkio write bps for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twbd.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write bps for %d:%d is not set", major, minor) + } + + found = false + for _, trid := range lbd.ThrottleReadIOPSDevice { + if trid.Major == major && trid.Minor == minor { + found = true + if trid.Rate != rate { + return fmt.Errorf("blkio read iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, trid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio read iops for %d:%d is not set", major, minor) + } + + found = false + for _, twid := range lbd.ThrottleWriteIOPSDevice { + if twid.Major == major && twid.Minor == minor { + found = true + if twid.Rate != rate { + return fmt.Errorf("blkio write iops for %d:%d is not set correctly, expect: %d, actual: %d", major, minor, rate, twid.Rate) + } + } + } + if !found { + return fmt.Errorf("blkio write iops for %d:%d is not set", major, minor) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_cpus.go b/validation/linux_cgroups_relative_cpus.go new file mode 100644 index 000000000..8102b77f9 --- /dev/null +++ b/validation/linux_cgroups_relative_cpus.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var shares uint64 = 1024 + var period uint64 = 100000 + var quota int64 = 50000 + var cpus, mems string = "0-1", "0" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesCPUShares(shares) + g.SetLinuxResourcesCPUQuota(quota) + g.SetLinuxResourcesCPUPeriod(period) + g.SetLinuxResourcesCPUCpus(cpus) + g.SetLinuxResourcesCPUMems(mems) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lcd, err := cg.GetCPUData(pid, path) + if err != nil { + return err + } + if *lcd.Shares != shares { + return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, lcd.Shares) + } + if *lcd.Quota != quota { + return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, lcd.Quota) + } + if *lcd.Period != period { + return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, lcd.Period) + } + if lcd.Cpus != cpus { + return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus) + } + if lcd.Mems != mems { + return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_hugetlb.go b/validation/linux_cgroups_relative_hugetlb.go new file mode 100644 index 000000000..5b888241f --- /dev/null +++ b/validation/linux_cgroups_relative_hugetlb.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + page := "1GB" + var limit uint64 = 56892210544640 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.AddLinuxResourcesHugepageLimit(page, limit) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lhd, err := cg.GetHugepageLimitData(pid, path) + if err != nil { + return err + } + for _, lhl := range lhd { + if lhl.Pagesize == page && lhl.Limit != limit { + return fmt.Errorf("hugepage %s limit is not set correctly, expect: %d, actual: %d", page, limit, lhl.Limit) + } + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_memory.go b/validation/linux_cgroups_relative_memory.go new file mode 100644 index 000000000..578aa96dc --- /dev/null +++ b/validation/linux_cgroups_relative_memory.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 50593792 + var swappiness uint64 = 50 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesMemoryLimit(limit) + g.SetLinuxResourcesMemoryReservation(limit) + g.SetLinuxResourcesMemorySwap(limit) + g.SetLinuxResourcesMemoryKernel(limit) + g.SetLinuxResourcesMemoryKernelTCP(limit) + g.SetLinuxResourcesMemorySwappiness(swappiness) + g.SetLinuxResourcesMemoryDisableOOMKiller(true) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lm, err := cg.GetMemoryData(pid, path) + if err != nil { + return err + } + if limit != *lm.Limit { + return fmt.Errorf("memory limit is not set correctly, expect: %d, actual: %d", limit, *lm.Limit) + } + if limit != *lm.Reservation { + return fmt.Errorf("memory reservation is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Swap { + return fmt.Errorf("memory swap is not set correctly, expect: %d, actual: %d", limit, *lm.Reservation) + } + if limit != *lm.Kernel { + return fmt.Errorf("memory kernel is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if limit != *lm.KernelTCP { + return fmt.Errorf("memory kernelTCP is not set correctly, expect: %d, actual: %d", limit, *lm.Kernel) + } + if swappiness != *lm.Swappiness { + return fmt.Errorf("memory swappiness is not set correctly, expect: %d, actual: %d", swappiness, *lm.Swappiness) + } + if true != *lm.DisableOOMKiller { + return fmt.Errorf("memory oom is not set correctly, expect: %t, actual: %t", true, *lm.DisableOOMKiller) + } + return nil + }) + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_network.go b/validation/linux_cgroups_relative_network.go new file mode 100644 index 000000000..8d1985b3a --- /dev/null +++ b/validation/linux_cgroups_relative_network.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var id, prio uint32 = 255, 10 + ifName := "lo" + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesNetworkClassID(id) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lnd, err := cg.GetNetworkData(pid, path) + if err != nil { + return err + } + if *lnd.ClassID != id { + return fmt.Errorf("network ID is not set correctly, expect: %d, actual: %d", id, lnd.ClassID) + } + found := false + for _, lip := range lnd.Priorities { + if lip.Name == ifName { + found = true + if lip.Priority != prio { + return fmt.Errorf("network priority for %s is not set correctly, expect: %d, actual: %d", ifName, prio, lip.Priority) + } + } + } + if !found { + return fmt.Errorf("network priority for %s is not set correctly", ifName) + } + + return nil + }) + + if err != nil { + util.Fatal(err) + } +} diff --git a/validation/linux_cgroups_relative_pids.go b/validation/linux_cgroups_relative_pids.go new file mode 100644 index 000000000..f8ec4b145 --- /dev/null +++ b/validation/linux_cgroups_relative_pids.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + + "github.com/opencontainers/runtime-tools/cgroups" + "github.com/opencontainers/runtime-tools/validation/util" +) + +func main() { + var limit int64 = 1000 + g := util.GetDefaultGenerator() + g.SetLinuxCgroupsPath(cgroups.RelCgroupPath) + g.SetLinuxResourcesPidsLimit(limit) + err := util.RuntimeOutsideValidate(g, cgroups.RelCgroupPath, func(pid int, path string) error { + cg, err := cgroups.FindCgroup() + if err != nil { + return err + } + lpd, err := cg.GetPidsData(pid, path) + if err != nil { + return err + } + if lpd.Limit != limit { + return fmt.Errorf("pids limit is not set correctly, expect: %d, actual: %d", limit, lpd.Limit) + } + return nil + }) + + if err != nil { + util.Fatal(err) + } +}