Skip to content

Commit

Permalink
Add mon groups for resctrl.
Browse files Browse the repository at this point in the history 
Signed-off-by: Paweł Szulik <[email protected]>
  • Loading branch information
Paweł Szulik committed Jul 22, 2020
1 parent f8749ba commit 62ccdb1
Show file tree
Hide file tree
Showing 46 changed files with 18,738 additions and 95 deletions.
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ require (
github.com/pkg/errors v0.9.1
github.com/seccomp/libseccomp-golang v0.9.1
github.com/sirupsen/logrus v1.6.0
github.com/stretchr/testify v1.6.1
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2
// NOTE: urfave/cli must be <= v1.22.1 due to a regression: https://github.com/urfave/cli/issues/1092
github.com/urfave/cli v1.22.1
github.com/vishvananda/netlink v1.1.0
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
)
10 changes: 10 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
Expand Down Expand Up @@ -42,8 +43,12 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
Expand All @@ -59,5 +64,10 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE=
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
260 changes: 165 additions & 95 deletions libcontainer/intelrdt/intelrdt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"strings"
"sync"

"github.com/sirupsen/logrus"

"github.com/opencontainers/runc/libcontainer/configs"
)

Expand Down Expand Up @@ -167,10 +169,16 @@ type IntelRdtManager struct {
Config *configs.Config
Id string
Path string
Type string
}

const (
IntelRdtTasks = "tasks"
IntelRdtTasks = "tasks"
MonGroupDirectoryName = "mon_groups"
// Monitoring group.
MON = "MON"
// Control group.
CTRL_MON = "CTRL_MON"
)

var (
Expand Down Expand Up @@ -548,26 +556,47 @@ func GetIntelRdtPath(id string) (string, error) {
return path, nil
}

// Get the 'container_id" path in Intel RDT "monitoring group" filesystem.
func GetMonGroupIntelRdtPath(id string) (string, error) {
rootPath, err := getIntelRdtRoot()
if err != nil {
return "", err
}

path := filepath.Join(rootPath, MonGroupDirectoryName, id)
return path, nil
}

// Applies Intel RDT configuration to the process with the specified pid
func (m *IntelRdtManager) Apply(pid int) (err error) {
// If intelRdt is not specified in config, we do nothing
if m.Config.IntelRdt == nil {
switch m.Type {
case CTRL_MON:
// If intelRdt is not specified in config, we do nothing
if m.Config.IntelRdt == nil {
return nil
}
rdtData, err := getIntelRdtData(m.Config, pid)
if err != nil && !IsNotFound(err) {
return err
}

m.mu.Lock()
defer m.mu.Unlock()
path, err := rdtData.join(m.Id)
if err != nil {
return err
}

m.Path = path
return nil
}
d, err := getIntelRdtData(m.Config, pid)
if err != nil && !IsNotFound(err) {
return err
}

m.mu.Lock()
defer m.mu.Unlock()
path, err := d.join(m.Id)
if err != nil {
return err
case MON:
m.mu.Lock()
defer m.mu.Unlock()
return WriteIntelRdtTasks(m.Path, pid)
}

m.Path = path
return nil
return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type)
}

// Destroys the Intel RDT 'container_id' group
Expand All @@ -584,94 +613,124 @@ func (m *IntelRdtManager) Destroy() error {
// Returns Intel RDT path to save in a state file and to be able to
// restore the object later
func (m *IntelRdtManager) GetPath() string {
var err error
if m.Path == "" {
m.Path, _ = GetIntelRdtPath(m.Id)
switch m.Type {
case CTRL_MON:
m.Path, err = GetIntelRdtPath(m.Id)
if err != nil {
logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err)
}
case MON:
flattedContainerID := strings.Replace(m.Id, "/", "-", -1)
m.Path, err = GetMonGroupIntelRdtPath(flattedContainerID)
if err != nil {
logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err)
}
}
}
return m.Path
}

// Returns statistics for Intel RDT
func (m *IntelRdtManager) GetStats() (*Stats, error) {
// If intelRdt is not specified in config
if m.Config.IntelRdt == nil {
return nil, nil
}

m.mu.Lock()
defer m.mu.Unlock()
stats := NewStats()

rootPath, err := getIntelRdtRoot()
if err != nil {
return nil, err
}
// The read-only L3 cache and memory bandwidth schemata in root
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
if err != nil {
return nil, err
}
schemaRootStrings := strings.Split(tmpRootStrings, "\n")

// The L3 cache and memory bandwidth schemata in 'container_id' group
containerPath := m.GetPath()
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
if err != nil {
return nil, err
}
schemaStrings := strings.Split(tmpStrings, "\n")
switch m.Type {
case CTRL_MON:
containerPath := m.GetPath()

if IsCatEnabled() {
// The read-only L3 cache information
l3CacheInfo, err := getL3CacheInfo()
err := getMonitoringStats(containerPath, stats)
if err != nil {
return nil, err
}
stats.L3CacheInfo = l3CacheInfo

// The read-only L3 cache schema in root
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "L3") {
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
// If intelRdt is not specified in config
if m.Config != nil {
if m.Config.IntelRdt == nil {
return nil, nil
}
}

// The L3 cache schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "L3") {
stats.L3CacheSchema = strings.TrimSpace(schema)
rootPath, err := getIntelRdtRoot()
if err != nil {
return nil, err
}
}
}
// The read-only L3 cache and memory bandwidth schemata in root
tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
if err != nil {
return nil, err
}
schemaRootStrings := strings.Split(tmpRootStrings, "\n")

if IsMbaEnabled() {
// The read-only memory bandwidth information
memBwInfo, err := getMemBwInfo()
if err != nil {
return nil, err
}
stats.MemBwInfo = memBwInfo
// The L3 cache and memory bandwidth schemata in 'container_id' group

// The read-only memory bandwidth information
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "MB") {
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
if err != nil {
return nil, err
}
}
schemaStrings := strings.Split(tmpStrings, "\n")

if IsCatEnabled() {
// The read-only L3 cache information
l3CacheInfo, err := getL3CacheInfo()
if err != nil {
return nil, err
}
stats.L3CacheInfo = l3CacheInfo

// The memory bandwidth schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "MB") {
stats.MemBwSchema = strings.TrimSpace(schema)
// The read-only L3 cache schema in root
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "L3") {
stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
}
}

// The L3 cache schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "L3") {
stats.L3CacheSchema = strings.TrimSpace(schema)
}
}
}

if IsMbaEnabled() {
// The read-only memory bandwidth information
memBwInfo, err := getMemBwInfo()
if err != nil {
return nil, err
}
stats.MemBwInfo = memBwInfo

// The read-only memory bandwidth information
for _, schemaRoot := range schemaRootStrings {
if strings.Contains(schemaRoot, "MB") {
stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
}
}

// The memory bandwidth schema in 'container_id' group
for _, schema := range schemaStrings {
if strings.Contains(schema, "MB") {
stats.MemBwSchema = strings.TrimSpace(schema)
}
}
}
}
}

err = getMonitoringStats(containerPath, stats)
if err != nil {
return nil, err
}
return stats, nil

case MON:
path := m.GetPath()

return stats, nil
err := getMonitoringStats(path, stats)
if err != nil {
return nil, err
}
return stats, nil
}
return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type)
}

// Set Intel RDT "resource control" filesystem as configured.
Expand Down Expand Up @@ -721,34 +780,45 @@ func (m *IntelRdtManager) Set(container *configs.Config) error {
// For example, on a two-socket machine, the schema line could be
// "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on
// socket 0 and 7000 MBps memory bandwidth limit on socket 1.
if container.IntelRdt != nil {
path := m.GetPath()
l3CacheSchema := container.IntelRdt.L3CacheSchema
memBwSchema := container.IntelRdt.MemBwSchema
switch m.Type {
case CTRL_MON:
if container.IntelRdt != nil {
path := m.GetPath()
l3CacheSchema := container.IntelRdt.L3CacheSchema
memBwSchema := container.IntelRdt.MemBwSchema

// Write a single joint schema string to schemata file
if l3CacheSchema != "" && memBwSchema != "" {
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
return NewLastCmdError(err)
}
}

// Write a single joint schema string to schemata file
if l3CacheSchema != "" && memBwSchema != "" {
if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
return NewLastCmdError(err)
// Write only L3 cache schema string to schemata file
if l3CacheSchema != "" && memBwSchema == "" {
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
return NewLastCmdError(err)
}
}
}

// Write only L3 cache schema string to schemata file
if l3CacheSchema != "" && memBwSchema == "" {
if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
return NewLastCmdError(err)
// Write only memory bandwidth schema string to schemata file
if l3CacheSchema == "" && memBwSchema != "" {
if err := writeFile(path, "schemata", memBwSchema); err != nil {
return NewLastCmdError(err)
}
}
}

// Write only memory bandwidth schema string to schemata file
if l3CacheSchema == "" && memBwSchema != "" {
if err := writeFile(path, "schemata", memBwSchema); err != nil {
return NewLastCmdError(err)
}
case MON:
if _, err := os.Stat(m.GetPath()); err == nil {
return nil
}
if err := os.Mkdir(m.GetPath(), 0755); err != nil {
return err
}
}

return nil
return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type)
}

func (raw *intelRdtData) join(id string) (string, error) {
Expand Down
Loading

0 comments on commit 62ccdb1

Please sign in to comment.