diff --git a/README.md b/README.md index 11e3720..0942d03 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,15 @@ Number of bytes of memory used. The extra label `memtype` can have three values *swapped*: Field VmSwap from /proc/[pid]/status, translated from KB to bytes. +If gathering smaps file is enabled, two additional values for `memtype` are added: + +*proportionalResident*: Sum of "Pss" fields from /proc/[pid]/smaps, whose doc says: + +> The "proportional set size" (PSS) of a process is the count of pages it has +> in memory, where each page is divided by the number of processes sharing it. + +*proportionalSwapped*: Sum of "SwapPss" fields from /proc/[pid]/smaps + ### open_filedesc gauge Number of file descriptors, based on counting how many entries are in the directory diff --git a/cmd/process-exporter/main.go b/cmd/process-exporter/main.go index c2d9cf5..0a8f343 100644 --- a/cmd/process-exporter/main.go +++ b/cmd/process-exporter/main.go @@ -293,6 +293,8 @@ func main() { "if a proc is tracked, track with it any children that aren't part of their own group") threads = flag.Bool("threads", true, "report on per-threadname metrics as well") + smaps = flag.Bool("gather-smaps", false, + "gather metrics from smaps file, which contains proportional resident memory size") man = flag.Bool("man", false, "print manual") configPath = flag.String("config.path", "", @@ -355,7 +357,7 @@ func main() { matchnamer = namemapper } - pc, err := NewProcessCollector(*procfsPath, *children, *threads, matchnamer, *recheck, *debug) + pc, err := NewProcessCollector(*procfsPath, *children, *threads, *smaps, matchnamer, *recheck, *debug) if err != nil { log.Fatalf("Error initializing: %v", err) } @@ -399,6 +401,7 @@ type ( scrapeChan chan scrapeRequest *proc.Grouper threads bool + smaps bool source proc.Source scrapeErrors int scrapeProcReadErrors int @@ -411,6 +414,7 @@ func NewProcessCollector( procfsPath string, children bool, threads bool, + smaps bool, n common.MatchNamer, recheck bool, debug bool, @@ -419,11 +423,14 @@ func NewProcessCollector( if err != nil { return nil, err } + + fs.GatherSMaps = smaps p := &NamedProcessCollector{ scrapeChan: make(chan scrapeRequest), Grouper: proc.NewGrouper(n, children, threads, recheck, debug), source: fs, threads: threads, + smaps: smaps, debug: debug, } @@ -540,6 +547,13 @@ func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(count), gname, wchan) } + if p.smaps { + ch <- prometheus.MustNewConstMetric(membytesDesc, + prometheus.GaugeValue, float64(gcounts.Memory.ProportionalBytes), gname, "proportionalResident") + ch <- prometheus.MustNewConstMetric(membytesDesc, + prometheus.GaugeValue, float64(gcounts.Memory.ProportionalSwapBytes), gname, "proportionalSwapped") + } + if p.threads { for _, thr := range gcounts.Threads { ch <- prometheus.MustNewConstMetric(threadCountDesc, diff --git a/go.mod b/go.mod index 118a5cb..127a96f 100644 --- a/go.mod +++ b/go.mod @@ -18,4 +18,4 @@ require ( gopkg.in/yaml.v2 v2.2.1 ) -replace github.com/prometheus/procfs => github.com/PierreF/procfs v0.0.12-0.20200406134856-80381c9e9fc9 +replace github.com/prometheus/procfs => github.com/PierreF/procfs v0.0.12-0.20200408075604-52118802aeee diff --git a/go.sum b/go.sum index 901139f..c2d7d3d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/PierreF/procfs v0.0.12-0.20200406134856-80381c9e9fc9 h1:qPadqUpRwZsFrSPBI7FwfGqRc7mxBBxuH65pSQqUg00= -github.com/PierreF/procfs v0.0.12-0.20200406134856-80381c9e9fc9/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/PierreF/procfs v0.0.12-0.20200408075604-52118802aeee h1:DrMsac2QdXsP8iXw5EbMwd8wCBkLtWICA381SKE59g0= +github.com/PierreF/procfs v0.0.12-0.20200408075604-52118802aeee/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= diff --git a/proc/grouper.go b/proc/grouper.go index 970f682..7fe76fc 100644 --- a/proc/grouper.go +++ b/proc/grouper.go @@ -66,6 +66,8 @@ func groupadd(grp Group, ts Update) Group { grp.Memory.ResidentBytes += ts.Memory.ResidentBytes grp.Memory.VirtualBytes += ts.Memory.VirtualBytes grp.Memory.VmSwapBytes += ts.Memory.VmSwapBytes + grp.Memory.ProportionalBytes += ts.Memory.ProportionalBytes + grp.Memory.ProportionalSwapBytes += ts.Memory.ProportionalSwapBytes if ts.Filedesc.Open != -1 { grp.OpenFDs += uint64(ts.Filedesc.Open) } diff --git a/proc/grouper_test.go b/proc/grouper_test.go index 758dc27..5b3f10f 100644 --- a/proc/grouper_test.go +++ b/proc/grouper_test.go @@ -45,30 +45,30 @@ func TestGrouperBasic(t *testing.T) { }{ { []IDInfo{ - piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0}, + piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0}, Filedesc{4, 400}, 2, States{Other: 1}), - piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0}, + piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0, 0, 0}, Filedesc{40, 400}, 3, States{Waiting: 1}), }, GroupByName{ - "g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0}, starttime, + "g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0, 0, 0}, starttime, 4, 0.01, 2, nil}, - "g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0}, starttime, + "g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0, 0, 0}, starttime, 40, 0.1, 3, nil}, }, }, { []IDInfo{ piinfost(p1, n1, Counts{2, 3, 4, 5, 6, 7, 0, 0}, - Memory{6, 7, 0}, Filedesc{100, 400}, 4, States{Zombie: 1}), + Memory{6, 7, 0, 0, 0}, Filedesc{100, 400}, 4, States{Zombie: 1}), piinfost(p2, n2, Counts{4, 5, 6, 7, 8, 9, 0, 0}, - Memory{9, 8, 0}, Filedesc{400, 400}, 2, States{Running: 1}), + Memory{9, 8, 0, 0, 0}, Filedesc{400, 400}, 2, States{Running: 1}), }, GroupByName{ "g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{Zombie: 1}, msi{}, 1, - Memory{6, 7, 0}, starttime, 100, 0.25, 4, nil}, + Memory{6, 7, 0, 0, 0}, starttime, 100, 0.25, 4, nil}, "g2": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1}, msi{}, 1, - Memory{9, 8, 0}, starttime, 400, 1, 2, nil}, + Memory{9, 8, 0, 0, 0}, starttime, 400, 1, 2, nil}, }, }, } @@ -95,10 +95,10 @@ func TestGrouperProcJoin(t *testing.T) { }{ { []IDInfo{ - piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0}, Filedesc{4, 400}, 2), + piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2), }, GroupByName{ - "g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0}, starttime, 4, 0.01, 2, nil}, + "g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0}, starttime, 4, 0.01, 2, nil}, }, }, { // The counts for pid2 won't be factored into the total yet because we only add @@ -106,24 +106,24 @@ func TestGrouperProcJoin(t *testing.T) { // affected though. []IDInfo{ piinfost(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, - Memory{3, 4, 0}, Filedesc{4, 400}, 2, States{Running: 1}), + Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}), piinfost(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, - Memory{1, 2, 0}, Filedesc{40, 400}, 3, States{Sleeping: 1}), + Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3, States{Sleeping: 1}), }, GroupByName{ "g1": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1, Sleeping: 1}, msi{}, 2, - Memory{4, 6, 0}, starttime, 44, 0.1, 5, nil}, + Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil}, }, }, { []IDInfo{ piinfost(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, - Memory{1, 5, 0}, Filedesc{4, 400}, 2, States{Running: 1}), + Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}), piinfost(p2, n2, Counts{2, 2, 2, 2, 2, 2, 0, 0}, - Memory{2, 4, 0}, Filedesc{40, 400}, 3, States{Running: 1}), + Memory{2, 4, 0, 0, 0}, Filedesc{40, 400}, 3, States{Running: 1}), }, GroupByName{ "g1": Group{Counts{4, 4, 4, 4, 4, 4, 0, 0}, States{Running: 2}, msi{}, 2, - Memory{3, 9, 0}, starttime, 44, 0.1, 5, nil}, + Memory{3, 9, 0, 0, 0}, starttime, 44, 0.1, 5, nil}, }, }, } @@ -150,18 +150,18 @@ func TestGrouperNonDecreasing(t *testing.T) { }{ { []IDInfo{ - piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0}, Filedesc{4, 400}, 2), - piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0}, Filedesc{40, 400}, 3), + piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2), + piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3), }, GroupByName{ - "g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0}, starttime, 44, 0.1, 5, nil}, + "g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil}, }, }, { []IDInfo{ - piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0}, Filedesc{4, 400}, 2), + piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2), }, GroupByName{ - "g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0}, starttime, 4, 0.01, 2, nil}, + "g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0}, starttime, 4, 0.01, 2, nil}, }, }, { []IDInfo{}, diff --git a/proc/read.go b/proc/read.go index bae7301..80b107d 100644 --- a/proc/read.go +++ b/proc/read.go @@ -49,9 +49,11 @@ type ( // Memory describes a proc's memory usage. Memory struct { - ResidentBytes uint64 - VirtualBytes uint64 - VmSwapBytes uint64 + ResidentBytes uint64 + VirtualBytes uint64 + VmSwapBytes uint64 + ProportionalBytes uint64 + ProportionalSwapBytes uint64 } // Filedesc describes a proc's file descriptor usage and soft limit. @@ -187,9 +189,10 @@ type ( // FS implements Source. FS struct { procfs.FS - BootTime uint64 - MountPoint string - debug bool + BootTime uint64 + MountPoint string + GatherSMaps bool + debug bool } ) @@ -474,13 +477,25 @@ func (p proc) GetMetrics() (Metrics, int, error) { softerrors |= 1 } + memory := Memory{ + ResidentBytes: uint64(stat.ResidentMemory()), + VirtualBytes: uint64(stat.VirtualMemory()), + VmSwapBytes: uint64(status.VmSwap), + } + + if p.proccache.fs.GatherSMaps { + smaps, err := p.Proc.ProcSMaps() + if err != nil { + softerrors |= 1 + } else { + memory.ProportionalBytes = smaps.PssSum() + memory.ProportionalSwapBytes = smaps.SwapPssSum() + } + } + return Metrics{ Counts: counts, - Memory: Memory{ - ResidentBytes: uint64(stat.ResidentMemory()), - VirtualBytes: uint64(stat.VirtualMemory()), - VmSwapBytes: uint64(status.VmSwap), - }, + Memory: memory, Filedesc: Filedesc{ Open: int64(numfds), Limit: uint64(limits.OpenFiles), @@ -554,7 +569,7 @@ func NewFS(mountPoint string, debug bool) (*FS, error) { if err != nil { return nil, err } - return &FS{fs, stat.BootTime, mountPoint, debug}, nil + return &FS{fs, stat.BootTime, mountPoint, false, debug}, nil } func (fs *FS) threadFs(pid int) (*FS, error) { @@ -563,7 +578,7 @@ func (fs *FS) threadFs(pid int) (*FS, error) { if err != nil { return nil, err } - return &FS{tfs, fs.BootTime, mountPoint, false}, nil + return &FS{tfs, fs.BootTime, mountPoint, fs.GatherSMaps, false}, nil } // AllProcs implements Source. diff --git a/proc/tracker_test.go b/proc/tracker_test.go index e28a148..cd5993c 100644 --- a/proc/tracker_test.go +++ b/proc/tracker_test.go @@ -99,15 +99,15 @@ func TestTrackerMetrics(t *testing.T) { want Update }{ { - piinfost(p, n, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0}, + piinfost(p, n, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0}, Filedesc{1, 10}, 9, States{Sleeping: 1}), - Update{n, Delta{}, Memory{7, 8, 0}, Filedesc{1, 10}, tm, + Update{n, Delta{}, Memory{7, 8, 0, 0, 0}, Filedesc{1, 10}, tm, 9, States{Sleeping: 1}, msi{}, nil}, }, { - piinfost(p, n, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{1, 2, 0}, + piinfost(p, n, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{2, 20}, 1, States{Running: 1}), - Update{n, Delta{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0}, + Update{n, Delta{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{2, 20}, tm, 1, States{Running: 1}, msi{}, nil}, }, }