Skip to content

Commit

Permalink
Add mon groups for resctrl.
Browse files Browse the repository at this point in the history 
"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON
group that is their ancestor.

More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Paweł Szulik <[email protected]>
  • Loading branch information
Paweł Szulik committed Aug 24, 2020
1 parent e5f2eae commit 397e410
Show file tree
Hide file tree
Showing 3 changed files with 207 additions and 95 deletions.
262 changes: 167 additions & 95 deletions libcontainer/intelrdt/intelrdt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"strings"
"sync"

"github.com/sirupsen/logrus"

"github.com/opencontainers/runc/libcontainer/configs"
)

Expand Down Expand Up @@ -167,10 +169,17 @@ type IntelRdtManager struct {
Config *configs.Config
Id string
Path string
Type string
}

const (
IntelRdtTasks = "tasks"
IntelRdtTasks = "tasks"
MonitoringGroupDirectoryName = "mon_groups"

// See "Resource alloc and monitor groups" part in:
// https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
MonitoringGroupType = "MON"
ControlGroupType = "CTRL_MON"
)

var (
Expand Down Expand Up @@ -548,26 +557,47 @@ func GetIntelRdtPath(id string) (string, error) {
return path, nil
}

// Get the 'container_id" path in Intel RDT "monitoring group" filesystem.
func getMonGroupIntelRdtPath(id string) (string, error) {
rootPath, err := getIntelRdtRoot()
if err != nil {
return "", err
}

path := filepath.Join(rootPath, MonitoringGroupDirectoryName, id)
return path, nil
}

// Applies Intel RDT configuration to the process with the specified pid
func (m *IntelRdtManager) Apply(pid int) (err error) {
// If intelRdt is not specified in config, we do nothing
if m.Config.IntelRdt == nil {
switch m.Type {
case ControlGroupType:
// If intelRdt is not specified in config, we do nothing
if m.Config.IntelRdt == nil {
return nil
}
rdtData, err := getIntelRdtData(m.Config, pid)
if err != nil && !IsNotFound(err) {
return err
}

m.mu.Lock()
defer m.mu.Unlock()
path, err := rdtData.join(m.Id)
if err != nil {
return err
}

m.Path = path
return nil
}
d, err := getIntelRdtData(m.Config, pid)
if err != nil && !IsNotFound(err) {
return err
}

m.mu.Lock()
defer m.mu.Unlock()
path, err := d.join(m.Id)
if err != nil {
return err
case MonitoringGroupType:
m.mu.Lock()
defer m.mu.Unlock()
return WriteIntelRdtTasks(m.Path, pid)
}

m.Path = path
return nil
return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type)
}

// Destroys the Intel RDT 'container_id' group
Expand All @@ -584,94 +614,124 @@ func (m *IntelRdtManager) Destroy() error {
// Returns Intel RDT path to save in a state file and to be able to
// restore the object later
func (m *IntelRdtManager) GetPath() string {
var err error
if m.Path == "" {
m.Path, _ = GetIntelRdtPath(m.Id)
switch m.Type {
case ControlGroupType:
m.Path, err = GetIntelRdtPath(m.Id)
if err != nil {
logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err)
}
case MonitoringGroupType:
flattedContainerID := strings.Replace(m.Id, "/", "-", -1)
m.Path, err = getMonGroupIntelRdtPath(flattedContainerID)
if err != nil {
logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err)
}
}
}
return m.Path
}

// Returns statistics for Intel RDT
func (m *IntelRdtManager) GetStats() (*Stats, error) {
// If intelRdt is not specified in config
if m.Config.IntelRdt == nil {
return nil, nil
}

m.mu.Lock()
defer m.mu.Unlock()
stats := NewStats()

rootPath, err := getIntelRdtRoot()
if err != nil {
return nil, err
}
// The read-only L3 cache and memory bandwidth schemata in root
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
if err != nil {
return nil, err
}
schemaRootStrings := strings.Split(tmpRootStrings, "\n")

// The L3 cache and memory bandwidth schemata in 'container_id' group
containerPath := m.GetPath()
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
if err != nil {
return nil, err
}
schemaStrings := strings.Split(tmpStrings, "\n")
switch m.Type {
case ControlGroupType:
containerPath := m.GetPath()

if IsCatEnabled() {
// The read-only L3 cache information
l3CacheInfo, err := getL3CacheInfo()
err := getMonitoringStats(containerPath, stats)
if err != nil {
return nil, err
}
stats.L3CacheInfo = l3CacheInfo

// The read-only L3 cache schema in root
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "L3") {
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
// If intelRdt is not specified in config
if m.Config != nil {
if m.Config.IntelRdt == nil {
return nil, nil
}
}

// The L3 cache schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "L3") {
stats.L3CacheSchema = strings.TrimSpace(schema)
rootPath, err := getIntelRdtRoot()
if err != nil {
return nil, err
}
}
}
// The read-only L3 cache and memory bandwidth schemata in root
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
if err != nil {
return nil, err
}
schemaRootStrings := strings.Split(tmpRootStrings, "\n")

if IsMbaEnabled() {
// The read-only memory bandwidth information
memBwInfo, err := getMemBwInfo()
if err != nil {
return nil, err
}
stats.MemBwInfo = memBwInfo
// The L3 cache and memory bandwidth schemata in 'container_id' group

// The read-only memory bandwidth information
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "MB") {
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
if err != nil {
return nil, err
}
}
schemaStrings := strings.Split(tmpStrings, "\n")

if IsCatEnabled() {
// The read-only L3 cache information
l3CacheInfo, err := getL3CacheInfo()
if err != nil {
return nil, err
}
stats.L3CacheInfo = l3CacheInfo

// The memory bandwidth schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "MB") {
stats.MemBwSchema = strings.TrimSpace(schema)
// The read-only L3 cache schema in root
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "L3") {
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
}
}

// The L3 cache schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "L3") {
stats.L3CacheSchema = strings.TrimSpace(schema)
}
}
}

if IsMbaEnabled() {
// The read-only memory bandwidth information
memBwInfo, err := getMemBwInfo()
if err != nil {
return nil, err
}
stats.MemBwInfo = memBwInfo

// The read-only memory bandwidth information
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "MB") {
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
}
}

// The memory bandwidth schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "MB") {
stats.MemBwSchema = strings.TrimSpace(schema)
}
}
}
}
}

err = getMonitoringStats(containerPath, stats)
if err != nil {
return nil, err
}
return stats, nil

case MonitoringGroupType:
path := m.GetPath()

return stats, nil
err := getMonitoringStats(path, stats)
if err != nil {
return nil, err
}
return stats, nil
}
return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type)
}

// Set Intel RDT "resource control" filesystem as configured.
Expand Down Expand Up @@ -721,34 +781,46 @@ func (m *IntelRdtManager) Set(container *configs.Config) error {
// For example, on a two-socket machine, the schema line could be
// "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on
// socket 0 and 7000 MBps memory bandwidth limit on socket 1.
if container.IntelRdt != nil {
path := m.GetPath()
l3CacheSchema := container.IntelRdt.L3CacheSchema
memBwSchema := container.IntelRdt.MemBwSchema
switch m.Type {
case ControlGroupType:
if container.IntelRdt != nil {
path := m.GetPath()
l3CacheSchema := container.IntelRdt.L3CacheSchema
memBwSchema := container.IntelRdt.MemBwSchema

// Write a single joint schema string to schemata file
if l3CacheSchema != "" && memBwSchema != "" {
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
return NewLastCmdError(err)
}
}

// Write a single joint schema string to schemata file
if l3CacheSchema != "" && memBwSchema != "" {
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
return NewLastCmdError(err)
// Write only L3 cache schema string to schemata file
if l3CacheSchema != "" && memBwSchema == "" {
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
return NewLastCmdError(err)
}
}
}

// Write only L3 cache schema string to schemata file
if l3CacheSchema != "" && memBwSchema == "" {
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
return NewLastCmdError(err)
// Write only memory bandwidth schema string to schemata file
if l3CacheSchema == "" && memBwSchema != "" {
if err := writeFile(path, "schemata", memBwSchema); err != nil {
return NewLastCmdError(err)
}
}
}

// Write only memory bandwidth schema string to schemata file
if l3CacheSchema == "" && memBwSchema != "" {
if err := writeFile(path, "schemata", memBwSchema); err != nil {
return NewLastCmdError(err)
}
case MonitoringGroupType:
if _, err := os.Stat(m.GetPath()); err == nil {
return nil
}
if err := os.Mkdir(m.GetPath(), 0755); err != nil {
return err
}
return nil
}

return nil
return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type)
}

func (raw *intelRdtData) join(id string) (string, error) {
Expand Down
39 changes: 39 additions & 0 deletions libcontainer/intelrdt/intelrdt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package intelrdt

import (
"errors"
"strings"
"testing"
)
Expand Down Expand Up @@ -120,3 +121,41 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
t.Fatal("Got the wrong value, set 'schemata' failed.")
}
}

func TestIntelRdtManager_GetStats_NotSupported_Type(t *testing.T) {
manager := IntelRdtManager{
Type: "not_supported",
Id: "id",
}

_, err := manager.GetStats()

expectedError := errors.New("couldn't obtain stats from: \"id\" resctrl manager of type: \"not_supported\"")

if err == nil {
t.Fatalf("Expected error: %v, got nil.", expectedError)
}

if err.Error() != expectedError.Error() {
t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
}
}

func Test_IntelRdtManager_Set_NotSupported_Type(t *testing.T) {
manager := IntelRdtManager{
Type: "not_supported",
Id: "id",
}

err := manager.Set(nil)

expectedError := errors.New("couldn't set configuration for: \"id\" resctrl manager of type: \"not_supported\"")

if err == nil {
t.Fatalf("Expected error: %v, got nil.", expectedError)
}

if err.Error() != expectedError.Error() {
t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
}
}
1 change: 1 addition & 0 deletions update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,7 @@ other options are ignored.
Config: &config,
Id: container.ID(),
Path: state.IntelRdtPath,
Type: intelrdt.ControlGroupType,
}
if err := intelRdtManager.Apply(state.InitProcessPid); err != nil {
return err
Expand Down

0 comments on commit 397e410

Please sign in to comment.