intelrdt: add mbm stats

Signed-off-by: Paweł Szulik <[email protected]>

events.go

@@ -161,6 +161,9 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema
 		}
+		if intelrdt.IsMbmEnabled() {
+			s.IntelRdt.MBMStats = is.MBMStats
+		}
 	}
 
 	s.NetworkInterfaces = ls.Interfaces

libcontainer/intelrdt/intelrdt.go

@@ -55,6 +55,10 @@ import (
  * |   |   |-- cbm_mask
  * |   |   |-- min_cbm_bits
  * |   |   |-- num_closids
+ * |   |-- L3_MON
+ * |   |   |-- max_threshold_occupancy
+ * |   |   |-- mon_features
+ * |   |   |-- num_rmids
  * |   |-- MB
  * |       |-- bandwidth_gran
  * |       |-- delay_linear
@@ -221,6 +225,17 @@ func init() {
 			isMbaEnabled = true
 		}
 	}
+
+	if flagsSet.MBMTotal || flagsSet.MBMLocal {
+		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil {
+			isMbmEnabled = true
+		}
+
+		enabledMonFeatures, err = getMonFeatures(intelRdtRoot)
+		if err != nil {
+			return
+		}
+	}
 }
 
 // Return the mount point path of Intel RDT "resource control" filesysem
@@ -300,6 +315,10 @@ func isIntelRdtMounted() bool {
 type cpuInfoFlags struct {
 	CAT bool // Cache Allocation Technology
 	MBA bool // Memory Bandwidth Allocation
+
+	// Memory Bandwidth Monitoring related.
+	MBMTotal bool
+	MBMLocal bool
 }
 
 func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
@@ -325,6 +344,10 @@ func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
 					infoFlags.CAT = true
 				case "mba":
 					infoFlags.MBA = true
+				case "cqm_mbm_total":
+					infoFlags.MBMTotal = true
+				case "cqm_mbm_local":
+					infoFlags.MBMLocal = true
 				}
 			}
 			return infoFlags, nil
@@ -589,7 +612,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 	schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
 	// The L3 cache and memory bandwidth schemata in 'container_id' group
-	tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata")
+	containerPath := m.GetPath()
+	tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
 	if err != nil {
 		return nil, err
 	}
@@ -641,6 +665,14 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 		}
 	}
 
+	if IsMbmEnabled() {
+		mbmStats, err := getMBMStats(containerPath)
+		if err != nil {
+			return stats, err
+		}
+		stats.MBMStats = mbmStats
+	}
+
 	return stats, nil
 }
 

libcontainer/intelrdt/mbm.go

@@ -0,0 +1,113 @@
+// +build linux
+
+package intelrdt
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	// The flag to indicate if Intel RDT/MBM is enabled
+	isMbmEnabled bool
+
+	enabledMonFeatures monFeatures
+)
+
+type monFeatures struct {
+	mbmTotalBytes bool
+	mbmLocalBytes bool
+	llcOccupancy  bool
+}
+
+// Check if Intel RDT/MBM is enabled
+func IsMbmEnabled() bool {
+	return isMbmEnabled
+}
+
+func getMonFeatures(intelRdtRoot string) (monFeatures, error) {
+	file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features"))
+	defer file.Close()
+	if err != nil {
+		return monFeatures{}, err
+	}
+	return parseMonFeatures(file)
+}
+
+func parseMonFeatures(reader io.Reader) (monFeatures, error) {
+	scanner := bufio.NewScanner(reader)
+
+	monFeatures := monFeatures{}
+
+	for scanner.Scan() {
+
+		switch feature := scanner.Text(); feature {
+
+		case "mbm_total_bytes":
+			monFeatures.mbmTotalBytes = true
+		case "mbm_local_bytes":
+			monFeatures.mbmLocalBytes = true
+		case "llc_occupancy":
+			monFeatures.llcOccupancy = true
+		default:
+			logrus.Warnf(fmt.Sprintf("Unsupported RDT Memory Bandwidth Monitoring (MBM) feature: %s", feature))
+		}
+	}
+
+	return monFeatures, scanner.Err()
+}
+
+func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) {
+	mbmStats := []MBMNumaNodeStats{}
+
+	numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data"))
+	if err != nil {
+		return &mbmStats, err
+	}
+
+	for _, file := range numaFiles {
+		if file.IsDir() {
+			numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name()))
+			if err != nil {
+				return &mbmStats, nil
+			}
+			mbmStats = append(mbmStats, *numaStats)
+		}
+	}
+
+	return &mbmStats, nil
+}
+
+func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) {
+	stats := &MBMNumaNodeStats{}
+	if enabledMonFeatures.mbmTotalBytes {
+		mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMTotalBytes = mbmTotalBytes
+	}
+
+	if enabledMonFeatures.mbmLocalBytes {
+		mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMLocalBytes = mbmLocalBytes
+	}
+
+	if enabledMonFeatures.llcOccupancy {
+		llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy")
+		if err != nil {
+			return nil, err
+		}
+		stats.LLCOccupancy = llcOccupancy
+	}
+	return stats, nil
+}

libcontainer/intelrdt/mbm_test.go

@@ -0,0 +1,137 @@
+// +build linux
+
+package intelrdt
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestParseMonFeatures(t *testing.T) {
+	t.Run("All features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(
+			strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy"))
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{true, true, true}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Cannot gather all features!")
+		}
+	})
+
+	t.Run("No features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(strings.NewReader(""))
+
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{false, false, false}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Expected no features available but there is any!")
+		}
+	})
+}
+
+func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) {
+	testDir, err := ioutil.TempDir("", "rdt_mbm_test")
+	if err != nil {
+		return "", err
+	}
+	monDataPath := filepath.Join(testDir, "mon_data")
+
+	for _, numa := range NUMANodes {
+		numaPath := filepath.Join(monDataPath, numa)
+		err = os.MkdirAll(numaPath, os.ModePerm)
+		if err != nil {
+			return "", err
+		}
+
+		for fileName, value := range mocks {
+			err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777)
+			if err != nil {
+				return "", err
+			}
+		}
+
+	}
+
+	return testDir, nil
+}
+
+func TestGetMbmStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"mbm_total_bytes": 9123911,
+		"mbm_local_bytes": 2361361,
+		"llc_occupancy":   123013,
+	}
+
+	mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedMBM)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.mbmTotalBytes = true
+		enabledMonFeatures.mbmLocalBytes = true
+		enabledMonFeatures.llcOccupancy = true
+
+		stats, err := getMBMStats(mockedMBM)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(*stats) != len(mocksNUMANodesToCreate) {
+			t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v",
+				len(mocksNUMANodesToCreate), len(*stats))
+		}
+
+		checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
+			if got.MBMTotalBytes != expected.MBMTotalBytes {
+				t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
+					expected.MBMTotalBytes,
+					got.MBMTotalBytes)
+			}
+
+			if got.MBMLocalBytes != expected.MBMLocalBytes {
+				t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
+					expected.MBMLocalBytes,
+					got.MBMLocalBytes)
+			}
+
+			if got.LLCOccupancy != expected.LLCOccupancy {
+				t.Fatalf("Wrong value of llc_occupancy. Expected: %v but got: %v",
+					expected.LLCOccupancy,
+					got.LLCOccupancy)
+			}
+		}
+
+		expectedStats := MBMNumaNodeStats{
+			MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
+			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
+			LLCOccupancy:  mocksFilesToCreate["llc_occupancy"],
+		}
+
+		checkStatCorrection((*stats)[0], expectedStats, t)
+		checkStatCorrection((*stats)[1], expectedStats, t)
+	})
+
+}

libcontainer/intelrdt/stats.go

@@ -15,6 +15,17 @@ type MemBwInfo struct {
 	NumClosids    uint64 `json:"num_closids,omitempty"`
 }
 
+type MBMNumaNodeStats struct {
+	// The 'mbm_total_bytes' in 'container_id' group
+	MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"`
+
+	// The 'mbm_local_bytes' in 'container_id' group
+	MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"`
+
+	// The 'llc_occupancy' in 'container_id' group
+	LLCOccupancy uint64 `json:"llc_occupancy,omitempty"`
+}
+
 type Stats struct {
 	// The read-only L3 cache information
 	L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"`
@@ -33,6 +44,9 @@ type Stats struct {
 
 	// The memory bandwidth schema in 'container_id' group
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
+
+	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
+	MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
 }
 
 func NewStats() *Stats {

types/events.go

@@ -1,5 +1,7 @@
 package types
 
+import "github.com/opencontainers/runc/libcontainer/intelrdt"
+
 // Event struct for encoding the event data to json.
 type Event struct {
 	Type string      `json:"type"`
@@ -113,6 +115,9 @@ type IntelRdt struct {
 
 	// The memory bandwidth schema in 'container_id' group
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
+
+	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
+	MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"`
 }
 
 type NetworkInterface struct {