Merge pull request #2292 from Creatone/creatone/extend-intelrdt

Add RDT Memory Bandwidth Monitoring (MBM) and Cache Monitoring Technology (CMT) statistics.

events.go

@@ -163,6 +163,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema
 		}
+		if intelrdt.IsMBMEnabled() {
+			s.IntelRdt.MBMStats = is.MBMStats
+		}
+		if intelrdt.IsCMTEnabled() {
+			s.IntelRdt.CMTStats = is.CMTStats
+		}
 	}
 
 	s.NetworkInterfaces = ls.Interfaces

libcontainer/intelrdt/cmt.go

@@ -0,0 +1,22 @@
+package intelrdt
+
+var (
+	cmtEnabled bool
+)
+
+// Check if Intel RDT/CMT is enabled.
+func IsCMTEnabled() bool {
+	return cmtEnabled
+}
+
+func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) {
+	stats := &CMTNumaNodeStats{}
+
+	llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy")
+	if err != nil {
+		return nil, err
+	}
+	stats.LLCOccupancy = llcOccupancy
+
+	return stats, nil
+}

libcontainer/intelrdt/cmt_test.go

@@ -0,0 +1,56 @@
+package intelrdt
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetCMTNumaNodeStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"llc_occupancy": 9123911,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.llcOccupancy = true
+
+		stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
+			}
+			stats = append(stats, *other)
+		}
+
+		expectedStats := CMTNumaNodeStats{
+			LLCOccupancy: mocksFilesToCreate["llc_occupancy"],
+		}
+
+		checkCMTStatCorrection(stats[0], expectedStats, t)
+		checkCMTStatCorrection(stats[1], expectedStats, t)
+	})
+}
+
+func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) {
+	if got.LLCOccupancy != expected.LLCOccupancy {
+		t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v",
+			expected.LLCOccupancy,
+			got.LLCOccupancy)
+	}
+}

libcontainer/intelrdt/intelrdt.go

@@ -55,6 +55,10 @@ import (
  * |   |   |-- cbm_mask
  * |   |   |-- min_cbm_bits
  * |   |   |-- num_closids
+ * |   |-- L3_MON
+ * |   |   |-- max_threshold_occupancy
+ * |   |   |-- mon_features
+ * |   |   |-- num_rmids
  * |   |-- MB
  * |       |-- bandwidth_gran
  * |       |-- delay_linear
@@ -191,8 +195,7 @@ type intelRdtData struct {
 // Check if Intel RDT sub-features are enabled in init()
 func init() {
 	// 1. Check if hardware and kernel support Intel RDT sub-features
-	// "cat_l3" flag for CAT and "mba" flag for MBA
-	isCatFlagSet, isMbaFlagSet, err := parseCpuInfoFile("/proc/cpuinfo")
+	flagsSet, err := parseCpuInfoFile("/proc/cpuinfo")
 	if err != nil {
 		return
 	}
@@ -207,7 +210,7 @@ func init() {
 	// "resource control" filesystem. Intel RDT sub-features can be
 	// selectively disabled or enabled by kernel command line
 	// (e.g., rdt=!l3cat,mba) in 4.14 and newer kernel
-	if isCatFlagSet {
+	if flagsSet.CAT {
 		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3")); err == nil {
 			isCatEnabled = true
 		}
@@ -217,11 +220,23 @@ func init() {
 		// MBA should be enabled because MBA Software Controller
 		// depends on MBA
 		isMbaEnabled = true
-	} else if isMbaFlagSet {
+	} else if flagsSet.MBA {
 		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "MB")); err == nil {
 			isMbaEnabled = true
 		}
 	}
+
+	if flagsSet.MBMTotal || flagsSet.MBMLocal {
+		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil {
+			mbmEnabled = true
+			cmtEnabled = true
+		}
+
+		enabledMonFeatures, err = getMonFeatures(intelRdtRoot)
+		if err != nil {
+			return
+		}
+	}
 }
 
 // Return the mount point path of Intel RDT "resource control" filesysem
@@ -298,13 +313,21 @@ func isIntelRdtMounted() bool {
 	return true
 }
 
-func parseCpuInfoFile(path string) (bool, bool, error) {
-	isCatFlagSet := false
-	isMbaFlagSet := false
+type cpuInfoFlags struct {
+	CAT bool // Cache Allocation Technology
+	MBA bool // Memory Bandwidth Allocation
+
+	// Memory Bandwidth Monitoring related.
+	MBMTotal bool
+	MBMLocal bool
+}
+
+func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
+	infoFlags := cpuInfoFlags{}
 
 	f, err := os.Open(path)
 	if err != nil {
-		return false, false, err
+		return infoFlags, err
 	}
 	defer f.Close()
 
@@ -319,19 +342,23 @@ func parseCpuInfoFile(path string) (bool, bool, error) {
 			for _, flag := range flags {
 				switch flag {
 				case "cat_l3":
-					isCatFlagSet = true
+					infoFlags.CAT = true
 				case "mba":
-					isMbaFlagSet = true
+					infoFlags.MBA = true
+				case "cqm_mbm_total":
+					infoFlags.MBMTotal = true
+				case "cqm_mbm_local":
+					infoFlags.MBMLocal = true
 				}
 			}
-			return isCatFlagSet, isMbaFlagSet, nil
+			return infoFlags, nil
 		}
 	}
 	if err := s.Err(); err != nil {
-		return false, false, err
+		return infoFlags, err
 	}
 
-	return isCatFlagSet, isMbaFlagSet, nil
+	return infoFlags, nil
 }
 
 func parseUint(s string, base, bitSize int) (uint64, error) {
@@ -586,7 +613,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 	schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
 	// The L3 cache and memory bandwidth schemata in 'container_id' group
-	tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata")
+	containerPath := m.GetPath()
+	tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
 	if err != nil {
 		return nil, err
 	}
@@ -638,6 +666,11 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 		}
 	}
 
+	err = getMonitoringStats(containerPath, stats)
+	if err != nil {
+		return nil, err
+	}
+
 	return stats, nil
 }
 

libcontainer/intelrdt/mbm.go

@@ -0,0 +1,34 @@
+// +build linux
+
+package intelrdt
+
+var (
+	// The flag to indicate if Intel RDT/MBM is enabled
+	mbmEnabled bool
+)
+
+// Check if Intel RDT/MBM is enabled.
+func IsMBMEnabled() bool {
+	return mbmEnabled
+}
+
+func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) {
+	stats := &MBMNumaNodeStats{}
+	if enabledMonFeatures.mbmTotalBytes {
+		mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMTotalBytes = mbmTotalBytes
+	}
+
+	if enabledMonFeatures.mbmLocalBytes {
+		mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMLocalBytes = mbmLocalBytes
+	}
+
+	return stats, nil
+}

libcontainer/intelrdt/mbm_test.go

@@ -0,0 +1,68 @@
+// +build linux
+
+package intelrdt
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetMBMNumaNodeStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"mbm_total_bytes": 9123911,
+		"mbm_local_bytes": 2361361,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.mbmTotalBytes = true
+		enabledMonFeatures.mbmLocalBytes = true
+
+		stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
+			}
+			stats = append(stats, *other)
+		}
+
+		expectedStats := MBMNumaNodeStats{
+			MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
+			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
+		}
+
+		checkMBMStatCorrection(stats[0], expectedStats, t)
+		checkMBMStatCorrection(stats[1], expectedStats, t)
+	})
+}
+
+func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
+	if got.MBMTotalBytes != expected.MBMTotalBytes {
+		t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
+			expected.MBMTotalBytes,
+			got.MBMTotalBytes)
+	}
+
+	if got.MBMLocalBytes != expected.MBMLocalBytes {
+		t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
+			expected.MBMLocalBytes,
+			got.MBMLocalBytes)
+	}
+
+}