diff --git a/libcontainer/SPEC.md b/libcontainer/SPEC.md index 221545c01dc..9bf1e8d59e3 100644 --- a/libcontainer/SPEC.md +++ b/libcontainer/SPEC.md @@ -143,6 +143,7 @@ system resources like cpu, memory, and device access. | freezer | 1 | | hugetlb | 1 | | pids | 1 | +| intel_rdt | 1 | All cgroup subsystem are joined so that statistics can be collected from diff --git a/libcontainer/cgroups/fs/apply_raw.go b/libcontainer/cgroups/fs/apply_raw.go index 21646e5710f..a08848af424 100644 --- a/libcontainer/cgroups/fs/apply_raw.go +++ b/libcontainer/cgroups/fs/apply_raw.go @@ -30,6 +30,7 @@ var ( &NetPrioGroup{}, &PerfEventGroup{}, &FreezerGroup{}, + &IntelRdtGroup{}, } CgroupProcesses = "cgroup.procs" HugePageSizes, _ = cgroups.GetHugePageSize() diff --git a/libcontainer/cgroups/fs/intel_rdt.go b/libcontainer/cgroups/fs/intel_rdt.go new file mode 100644 index 00000000000..7c59fb5acfa --- /dev/null +++ b/libcontainer/cgroups/fs/intel_rdt.go @@ -0,0 +1,75 @@ +// +build linux + +package fs + +import ( + "fmt" + "strconv" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/configs" +) + +type IntelRdtGroup struct { +} + +func (s *IntelRdtGroup) Name() string { + return "intel_rdt" +} + +func (s *IntelRdtGroup) Apply(d *cgroupData) error { + dir, err := d.join("intel_rdt") + if err != nil { + if !cgroups.IsNotFound(err) { + return err + } + // We will not return err here when: + // 1. The h/w platform doesn't support Intel RDT/CAT feature, + // intel_rdt cgroup is not enabled in kernel. + // 2. intel_rdt cgroup is not mounted + return nil + } + + if err := s.Set(dir, d.config); err != nil { + return err + } + + return nil +} + +func (s *IntelRdtGroup) Set(path string, cgroup *configs.Cgroup) error { + // The valid CBM (capacity bitmask) is a *contiguous bits set* and + // number of bits that can be set is less than the max bit. The max + // bits in the CBM is varied among supported Intel platforms. + // + // By default the child cgroups inherit the CBM from parent. The CBM + // in a child cgroup should be a subset of the CBM in parent. Kernel + // will check if it is valid when writing. + // + // e.g., 0xfffff in root cgroup indicates the max bits of CBM is 20 + // bits, which mapping to entire L3 cache capacity. Some valid CBM + // values to Set in children cgroup: 0xf, 0xf0, 0x3ff, 0x1f00 and etc. + if cgroup.Resources.IntelRdtL3Cbm != 0 { + l3CbmStr := fmt.Sprintf("0x%s", strconv.FormatUint(cgroup.Resources.IntelRdtL3Cbm, 16)) + if err := writeFile(path, "intel_rdt.l3_cbm", l3CbmStr); err != nil { + return err + } + } + + return nil +} + +func (s *IntelRdtGroup) Remove(d *cgroupData) error { + return removePath(d.path("intel_rdt")) +} + +func (s *IntelRdtGroup) GetStats(path string, stats *cgroups.Stats) error { + value, err := getCgroupParamUintHex(path, "intel_rdt.l3_cbm") + if err != nil { + return fmt.Errorf("failed to parse intel_rdt.l3_cbm - %s", err) + } + + stats.IntelRdtStats.L3Cbm = value + + return nil +} diff --git a/libcontainer/cgroups/fs/intel_rdt_test.go b/libcontainer/cgroups/fs/intel_rdt_test.go new file mode 100644 index 00000000000..748750a7e91 --- /dev/null +++ b/libcontainer/cgroups/fs/intel_rdt_test.go @@ -0,0 +1,62 @@ +// +build linux + +package fs + +import ( + "strconv" + "testing" + + "github.com/opencontainers/runc/libcontainer/cgroups" +) + +func TestIntelRdtSetL3Cbm(t *testing.T) { + helper := NewCgroupTestUtil("intel_rdt", t) + defer helper.cleanup() + + const ( + l3CbmBefore = 0xf + l3CbmAfter = 0xf0 + ) + + helper.writeFileContents(map[string]string{ + "intel_rdt.l3_cbm": strconv.FormatUint(l3CbmBefore, 16), + }) + + helper.CgroupData.config.Resources.IntelRdtL3Cbm = l3CbmAfter + intelrdt := &IntelRdtGroup{} + if err := intelrdt.Set(helper.CgroupPath, helper.CgroupData.config); err != nil { + t.Fatal(err) + } + + value, err := getCgroupParamUintHex(helper.CgroupPath, "intel_rdt.l3_cbm") + if err != nil { + t.Fatalf("Failed to parse intel_rdt.l3_cbm - %s", err) + } + + if value != l3CbmAfter { + t.Fatal("Got the wrong value, set intel_rdt.l3_cbm failed.") + } +} + +func TestIntelRdtStats(t *testing.T) { + helper := NewCgroupTestUtil("intel_rdt", t) + defer helper.cleanup() + + const ( + l3CbmContents = 0x1f00 + ) + + helper.writeFileContents(map[string]string{ + "intel_rdt.l3_cbm": strconv.FormatUint(l3CbmContents, 16), + }) + + intelrdt := &IntelRdtGroup{} + stats := *cgroups.NewStats() + if err := intelrdt.GetStats(helper.CgroupPath, &stats); err != nil { + t.Fatal(err) + } + + if stats.IntelRdtStats.L3Cbm != l3CbmContents { + t.Fatalf("Expected '0x%x', got '0x%x' for intel_rdt.l3_cbm", l3CbmContents, stats.IntelRdtStats.L3Cbm) + } +} diff --git a/libcontainer/cgroups/fs/utils.go b/libcontainer/cgroups/fs/utils.go index 852b18391d0..04b8def0d49 100644 --- a/libcontainer/cgroups/fs/utils.go +++ b/libcontainer/cgroups/fs/utils.go @@ -68,6 +68,22 @@ func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { return res, nil } +// Gets a single hex uint64 value from the specified cgroup file. +func getCgroupParamUintHex(cgroupPath, cgroupFile string) (uint64, error) { + fileName := filepath.Join(cgroupPath, cgroupFile) + contents, err := ioutil.ReadFile(fileName) + if err != nil { + return 0, err + } + + hexStr := strings.TrimSpace(strings.TrimPrefix(string(contents), "0x")) + res, err := parseUint(hexStr, 16, 64) + if err != nil { + return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName) + } + return res, nil +} + // Gets a string value from the specified cgroup file func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) { contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile)) diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go index 54ace4185dd..e60b663d040 100644 --- a/libcontainer/cgroups/stats.go +++ b/libcontainer/cgroups/stats.go @@ -84,13 +84,18 @@ type HugetlbStats struct { Failcnt uint64 `json:"failcnt"` } +type IntelRdtStats struct { + L3Cbm uint64 `json:"l3_cbm,omitempty"` +} + type Stats struct { CpuStats CpuStats `json:"cpu_stats,omitempty"` MemoryStats MemoryStats `json:"memory_stats,omitempty"` PidsStats PidsStats `json:"pids_stats,omitempty"` BlkioStats BlkioStats `json:"blkio_stats,omitempty"` // the map is in the format "size of hugepage: stats of the hugepage" - HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` + HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` + IntelRdtStats IntelRdtStats `json:"intel_rdt_stats,omitempty"` } func NewStats() *Stats { diff --git a/libcontainer/cgroups/systemd/apply_systemd.go b/libcontainer/cgroups/systemd/apply_systemd.go index 3161639f218..8ac0543c9d0 100644 --- a/libcontainer/cgroups/systemd/apply_systemd.go +++ b/libcontainer/cgroups/systemd/apply_systemd.go @@ -63,6 +63,7 @@ var subsystems = subsystemSet{ &fs.NetPrioGroup{}, &fs.NetClsGroup{}, &fs.NameGroup{GroupName: "name=systemd"}, + &fs.IntelRdtGroup{}, } const ( @@ -251,7 +252,7 @@ func (m *Manager) Apply(pid int) error { return err } - // we need to manually join the freezer, net_cls, net_prio, pids and cpuset cgroup in systemd + // we need to manually join the freezer, net_cls, net_prio cpuset and intel_rdt cgroup in systemd // because it does not currently support it via the dbus api. if err := joinFreezer(c, pid); err != nil { return err @@ -279,6 +280,11 @@ func (m *Manager) Apply(pid int) error { if err := joinPerfEvent(c, pid); err != nil { return err } + + if err := joinIntelRdt(c, pid); err != nil { + return err + } + // FIXME: Systemd does have `BlockIODeviceWeight` property, but we got problem // using that (at least on systemd 208, see https://github.com/opencontainers/runc/libcontainer/pull/354), // so use fs work around for now. @@ -603,3 +609,24 @@ func joinPerfEvent(c *configs.Cgroup, pid int) error { } return nil } + +func joinIntelRdt(c *configs.Cgroup, pid int) error { + path, err := join(c, "intel_rdt", pid) + if err != nil { + if !cgroups.IsNotFound(err) { + return err + } + // We will not return err here when: + // 1. The h/w platform doesn't support Intel RDT/CAT feature, + // intel_rdt cgroup is not enabled in kernel. + // 2. intel_rdt cgroup is not mounted + return nil + } + + IntelRdt, err := subsystems.Get("intel_rdt") + if err != nil { + return err + } + + return IntelRdt.Set(path, c) +} diff --git a/libcontainer/configs/cgroup_unix.go b/libcontainer/configs/cgroup_unix.go index 40a033f35bc..acd21fa4d79 100644 --- a/libcontainer/configs/cgroup_unix.go +++ b/libcontainer/configs/cgroup_unix.go @@ -118,4 +118,7 @@ type Resources struct { // Set class identifier for container's network packets NetClsClassid string `json:"net_cls_classid"` + + // L3 cache capacity bitmask (CBM) for container + IntelRdtL3Cbm uint64 `json:"intel_rdt_l3_cbm"` } diff --git a/spec.go b/spec.go index 2836e7c8582..e937eb28d5f 100644 --- a/spec.go +++ b/spec.go @@ -480,6 +480,10 @@ func createCgroupConfig(name string, spec *specs.LinuxSpec) (*configs.Cgroup, er }) } } + if r.IntelRdt != nil { + c.Resources.IntelRdtL3Cbm = *r.IntelRdt.L3Cbm + } + return c, nil }