Add host memory and CPU usage metrics

probe/host/reporter.go

@@ -4,6 +4,7 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/weaveworks/scope/common/mtime"
 	"github.com/weaveworks/scope/report"
 )
 
@@ -18,17 +19,16 @@ const (
 	Load1         = "load1"
 	Load5         = "load5"
 	Load15        = "load15"
+	CPUUsage      = "cpu_usage_percent"
+	MemUsage      = "mem_usage_percent"
 )
 
 // Exposed for testing.
 const (
-	ProcUptime = "/proc/uptime"
-	ProcLoad   = "/proc/loadavg"
-)
-
-// Exposed for testing.
-var (
-	Now = func() string { return time.Now().UTC().Format(time.RFC3339Nano) }
+	ProcUptime  = "/proc/uptime"
+	ProcLoad    = "/proc/loadavg"
+	ProcStat    = "/proc/stat"
+	ProcMemInfo = "/proc/meminfo"
 )
 
 // Reporter generates Reports containing the host topology.
@@ -72,15 +72,22 @@ func (r *Reporter) Report() (report.Report, error) {
 		return rep, err
 	}
 
+	now := mtime.Now()
+	metrics := GetLoad(now)
+	cpuUsage, max := GetCPUUsagePercent()
+	metrics[CPUUsage] = report.MakeMetric().Add(now, cpuUsage).WithMax(max)
+	memUsage, max := GetMemoryUsagePercent()
+	metrics[MemUsage] = report.MakeMetric().Add(now, memUsage).WithMax(max)
+
 	rep.Host.AddNode(report.MakeHostNodeID(r.hostID), report.MakeNodeWith(map[string]string{
-		Timestamp:     Now(),
+		Timestamp:     mtime.Now().UTC().Format(time.RFC3339Nano),
 		HostName:      r.hostName,
 		OS:            runtime.GOOS,
 		KernelVersion: kernel,
 		Uptime:        uptime.String(),
 	}).WithSets(report.Sets{
 		LocalNetworks: report.MakeStringSet(localCIDRs...),
-	}).WithMetrics(GetLoad()))
+	}).WithMetrics(metrics))
 
 	return rep, nil
 }

probe/host/reporter_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/weaveworks/scope/common/mtime"
 	"github.com/weaveworks/scope/probe/host"
 	"github.com/weaveworks/scope/report"
 	"github.com/weaveworks/scope/test"
@@ -18,40 +19,47 @@ func TestReporter(t *testing.T) {
 		version   = "version"
 		network   = "192.168.0.0/16"
 		hostID    = "hostid"
-		now       = "now"
 		hostname  = "hostname"
 		timestamp = time.Now()
 		load      = report.Metrics{
-			host.Load1:  report.MakeMetric().Add(timestamp, 1.0),
-			host.Load5:  report.MakeMetric().Add(timestamp, 5.0),
-			host.Load15: report.MakeMetric().Add(timestamp, 15.0),
+			host.Load1:    report.MakeMetric().Add(timestamp, 1.0),
+			host.Load5:    report.MakeMetric().Add(timestamp, 5.0),
+			host.Load15:   report.MakeMetric().Add(timestamp, 15.0),
+			host.CPUUsage: report.MakeMetric().Add(timestamp, 30.0).WithMax(100.0),
+			host.MemUsage: report.MakeMetric().Add(timestamp, 60.0).WithMax(100.0),
 		}
 		uptime      = "278h55m43s"
 		kernel      = "release version"
 		_, ipnet, _ = net.ParseCIDR(network)
 		localNets   = report.Networks([]*net.IPNet{ipnet})
 	)
 
+	mtime.NowForce(timestamp)
+	defer mtime.NowReset()
+
 	var (
-		oldGetKernelVersion = host.GetKernelVersion
-		oldGetLoad          = host.GetLoad
-		oldGetUptime        = host.GetUptime
-		oldNow              = host.Now
+		oldGetKernelVersion      = host.GetKernelVersion
+		oldGetLoad               = host.GetLoad
+		oldGetUptime             = host.GetUptime
+		oldGetCPUUsagePercent    = host.GetCPUUsagePercent
+		oldGetMemoryUsagePercent = host.GetMemoryUsagePercent
 	)
 	defer func() {
 		host.GetKernelVersion = oldGetKernelVersion
 		host.GetLoad = oldGetLoad
 		host.GetUptime = oldGetUptime
-		host.Now = oldNow
+		host.GetCPUUsagePercent = oldGetCPUUsagePercent
+		host.GetMemoryUsagePercent = oldGetMemoryUsagePercent
 	}()
 	host.GetKernelVersion = func() (string, error) { return release + " " + version, nil }
-	host.GetLoad = func() report.Metrics { return load }
+	host.GetLoad = func(time.Time) report.Metrics { return load }
 	host.GetUptime = func() (time.Duration, error) { return time.ParseDuration(uptime) }
-	host.Now = func() string { return now }
+	host.GetCPUUsagePercent = func() (float64, float64) { return 30.0, 100.0 }
+	host.GetMemoryUsagePercent = func() (float64, float64) { return 60.0, 100.0 }
 
 	want := report.MakeReport()
 	want.Host.AddNode(report.MakeHostNodeID(hostID), report.MakeNodeWith(map[string]string{
-		host.Timestamp:     now,
+		host.Timestamp:     timestamp.UTC().Format(time.RFC3339Nano),
 		host.HostName:      hostname,
 		host.OS:            runtime.GOOS,
 		host.Uptime:        uptime,

probe/host/system_darwin.go

@@ -30,12 +30,11 @@ var GetKernelVersion = func() (string, error) {
 }
 
 // GetLoad returns the current load averages as metrics.
-var GetLoad = func() report.Metrics {
+var GetLoad = func(now time.Time) report.Metrics {
 	out, err := exec.Command("w").CombinedOutput()
 	if err != nil {
 		return nil
 	}
-	now := time.Now()
 	matches := loadRe.FindAllStringSubmatch(string(out), -1)
 	if matches == nil || len(matches) < 1 || len(matches[0]) < 4 {
 		return nil
@@ -84,3 +83,13 @@ var GetUptime = func() (time.Duration, error) {
 	}
 	return (time.Duration(d) * 24 * time.Hour) + (time.Duration(h) * time.Hour) + (time.Duration(m) * time.Minute), nil
 }
+
+// GetCPUUsagePercent returns the percent cpu usage and max (ie #cpus * 100)
+var GetCPUUsagePercent = func() (float64, float64) {
+	return 0.0, 0.0
+}
+
+// GetMemoryUsagePercent returns the percent memory usage and max (ie 100)
+var GetMemoryUsagePercent = func() (float64, float64) {
+	return 0.0, 0.0
+}

probe/host/system_linux.go

@@ -8,6 +8,8 @@ import (
 	"syscall"
 	"time"
 
+	linuxproc "github.com/c9s/goprocinfo/linux"
+
 	"github.com/weaveworks/scope/report"
 )
 
@@ -24,12 +26,11 @@ var GetKernelVersion = func() (string, error) {
 }
 
 // GetLoad returns the current load averages as metrics.
-var GetLoad = func() report.Metrics {
+var GetLoad = func(now time.Time) report.Metrics {
 	buf, err := ioutil.ReadFile("/proc/loadavg")
 	if err != nil {
 		return nil
 	}
-	now := time.Now()
 	toks := strings.Fields(string(buf))
 	if len(toks) < 3 {
 		return nil
@@ -72,3 +73,42 @@ var GetUptime = func() (time.Duration, error) {
 
 	return time.Duration(uptime) * time.Second, nil
 }
+
+var previousStat = linuxproc.CPUStat{}
+
+// GetCPUUsagePercent returns the percent cpu usage and max (ie #cpus * 100)
+var GetCPUUsagePercent = func() (float64, float64) {
+	stat, err := linuxproc.ReadStat(ProcStat)
+	if err != nil {
+		return 0.0, 0.0
+	}
+
+	// From http://stackoverflow.com/questions/23367857/accurate-calculation-of-cpu-usage-given-in-percentage-in-linux
+	var (
+		currentStat = stat.CPUStatAll
+		prevIdle    = previousStat.Idle + previousStat.IOWait
+		idle        = currentStat.Idle + currentStat.IOWait
+		prevNonIdle = (previousStat.User + previousStat.Nice + previousStat.System +
+			previousStat.IRQ + previousStat.SoftIRQ + previousStat.Steal)
+		nonIdle = (currentStat.User + currentStat.Nice + currentStat.System +
+			currentStat.IRQ + currentStat.SoftIRQ + currentStat.Steal)
+		prevTotal = prevIdle + prevNonIdle
+		total     = idle + nonIdle
+		// differentiate: actual value minus the previous one
+		totald = total - prevTotal
+		idled  = idle - prevIdle
+	)
+	previousStat = currentStat
+	return float64(totald-idled) * 100. / float64(totald), float64(len(stat.CPUStats)) * 100.
+}
+
+// GetMemoryUsagePercent returns the percent memory usage and max (ie 100)
+var GetMemoryUsagePercent = func() (float64, float64) {
+	meminfo, err := linuxproc.ReadMemInfo(ProcMemInfo)
+	if err != nil {
+		return 0.0, 0.0
+	}
+
+	used := meminfo.MemTotal - meminfo.MemFree - meminfo.Buffers - meminfo.Cached
+	return float64(used) * 100. / float64(meminfo.MemTotal), 100.
+}

probe/host/system_test.go

@@ -3,6 +3,7 @@ package host_test
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/weaveworks/scope/probe/host"
 )
@@ -19,7 +20,7 @@ func TestGetKernelVersion(t *testing.T) {
 }
 
 func TestGetLoad(t *testing.T) {
-	have := host.GetLoad()
+	have := host.GetLoad(time.Now())
 	if len(have) != 3 {
 		t.Fatalf("Expected 3 metrics, but got: %v", have)
 	}

render/detailed_node.go

@@ -346,7 +346,9 @@ func processOriginTable(nmd report.Node, addHostTag bool, addContainerTag bool)
 	}, len(rows) > 0 || commFound || pidFound
 }
 
-func sparklineRow(human string, metric report.Metric, format func(report.Metric) (report.Metric, string)) Row {
+type formatter func(report.Metric) (report.Metric, string)
+
+func sparklineRow(human string, metric report.Metric, format formatter) Row {
 	if format == nil {
 		format = formatDefault
 	}
@@ -510,14 +512,19 @@ func hostOriginTable(nmd report.Node) (Table, bool) {
 	}
 
 	rows := []Row{}
-	for _, tuple := range []struct{ key, human string }{
-		{host.Load1, "Load (1m)"},
-		{host.Load5, "Load (5m)"},
-		{host.Load15, "Load (15m)"},
+	for _, tuple := range []struct {
+		key, human string
+		fmt        formatter
+	}{
+		{host.Load1, "Load (1m)", nil},
+		{host.Load5, "Load (5m)", nil},
+		{host.Load15, "Load (15m)", nil},
+		{host.CPUUsage, "CPU Usage", formatPercent},
+		{host.MemUsage, "Memory Usage", formatPercent},
 	} {
 		if val, ok := nmd.Metrics[tuple.key]; ok {
 			val.Max = maxLoad
-			rows = append(rows, sparklineRow(tuple.human, val, nil))
+			rows = append(rows, sparklineRow(tuple.human, val, tuple.fmt))
 		}
 	}
 	for _, tuple := range []struct{ key, human string }{

report/metrics.go

@@ -70,6 +70,17 @@ func (m Metric) WithFirst(t time.Time) Metric {
 	}
 }
 
+// WithMax returns a fresh copy of m, with Max set to max
+func (m Metric) WithMax(max float64) Metric {
+	return Metric{
+		Samples: m.Samples,
+		Max:     max,
+		Min:     m.Min,
+		First:   m.First,
+		Last:    m.Last,
+	}
+}
+
 // Len returns the number of samples in the metric.
 func (m Metric) Len() int {
 	if m.Samples == nil {