From dbef8ce898c40cacc59b64a1ac22f16184340c26 Mon Sep 17 00:00:00 2001 From: Phil Frost Date: Tue, 25 Jun 2019 10:04:03 -0700 Subject: [PATCH] Parse task queue stats from /proc/schedstat These are useful as an indication of CPU contention and scheduler latency. Signed-off-by: Phil Frost --- fixtures.ttar | 27 +++++++++ proc.go | 9 +++ schedstat.go | 128 ++++++++++++++++++++++++++++++++++++++++++ schedstat_test.go | 139 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 schedstat.go create mode 100644 schedstat_test.go diff --git a/fixtures.ttar b/fixtures.ttar index 8ebd5b977..b74374484 100644 --- a/fixtures.ttar +++ b/fixtures.ttar @@ -168,6 +168,11 @@ SymlinkTo: net:[4026531993] Path: fixtures/proc/26231/root SymlinkTo: / # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/26231/schedstat +Lines: 1 +411605849 93680043 79 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/26231/stat Lines: 1 26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0 @@ -300,6 +305,18 @@ Lines: 1 com.github.uiautomatorNULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTENULLBYTEEOF Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/26233/schedstat +Lines: 8 + ____________________________________ +< this is a malformed schedstat file > + ------------------------------------ + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: fixtures/proc/584 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -609,6 +626,16 @@ some avg10=0.10 avg60=2.00 avg300=3.85 total=15 full avg10=0.20 avg60=3.00 avg300=4.95 total=25 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/schedstat +Lines: 6 +version 15 +timestamp 15819019232 +cpu0 498494191 0 3533438552 2553969831 3853684107 2465731542 2045936778163039 343796328169361 4767485306 +domain0 00000000,00000003 212499247 210112015 1861015 1860405436 536440 369895 32599 210079416 25368550 24241256 384652 927363878 807233 6366 1647 24239609 2122447165 1886868564 121112060 2848625533 125678146 241025 1032026 1885836538 2545 12 2533 0 0 0 0 0 0 1387952561 21076581 0 +cpu1 518377256 0 4155211005 2778589869 10466382 2867629021 1904686152592476 364107263788241 5145567945 +domain0 00000000,00000003 217653037 215526982 1577949 1580427380 557469 393576 28538 215498444 28721913 27662819 371153 870843407 745912 5523 1639 27661180 2331056874 2107732788 111442342 652402556 123615235 196159 1045245 2106687543 2400 3 2397 0 0 0 0 0 0 1437804657 26220076 0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/self SymlinkTo: 26231 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/proc.go b/proc.go index a0aee5956..b7c79cf77 100644 --- a/proc.go +++ b/proc.go @@ -300,3 +300,12 @@ func (p Proc) FileDescriptorsInfo() (ProcFDInfos, error) { return fdinfos, nil } + +// Schedstat returns task scheduling information for the process. +func (p Proc) Schedstat() (ProcSchedstat, error) { + contents, err := ioutil.ReadFile(p.path("schedstat")) + if err != nil { + return ProcSchedstat{}, err + } + return parseProcSchedstat(string(contents)) +} diff --git a/schedstat.go b/schedstat.go new file mode 100644 index 000000000..86396c0d9 --- /dev/null +++ b/schedstat.go @@ -0,0 +1,128 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "errors" + "os" + "regexp" + "strconv" +) + +var ( + cpuLineRE = regexp.MustCompile(`cpu(\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+) (\d+)`) + procLineRE = regexp.MustCompile(`(\d+) (\d+) (\d+)`) +) + +// Schedstat contains scheduler statistics from /proc/schedstats +// +// See +// https://www.kernel.org/doc/Documentation/scheduler/sched-stats.txt +// for a detailed description of what these numbers mean. +type Schedstat struct { + CPUs []*SchedstatCPU +} + +// SchedstatCPU contains the values from one "cpu" line +type SchedstatCPU struct { + CPUNum string + + RunningJiffies uint64 + WaitingJiffies uint64 + RunTimeslices uint64 +} + +// ProcSchedstat contains the values from /proc//schedstat +type ProcSchedstat struct { + RunningJiffies uint64 + WaitingJiffies uint64 + RunTimeslices uint64 +} + +func (fs FS) Schedstat() (*Schedstat, error) { + file, err := os.Open(fs.proc.Path("schedstat")) + if err != nil { + return nil, err + } + defer file.Close() + + stats := &Schedstat{} + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + match := cpuLineRE.FindStringSubmatch(scanner.Text()) + if match != nil { + cpu := &SchedstatCPU{} + cpu.CPUNum = match[1] + + cpu.RunningJiffies, err = strconv.ParseUint(match[8], 10, 64) + if err != nil { + continue + } + + cpu.WaitingJiffies, err = strconv.ParseUint(match[9], 10, 64) + if err != nil { + continue + } + + cpu.RunTimeslices, err = strconv.ParseUint(match[10], 10, 64) + if err != nil { + continue + } + + stats.CPUs = append(stats.CPUs, cpu) + } + } + + return stats, nil +} + +func parseProcSchedstat(contents string) (stats ProcSchedstat, err error) { + match := procLineRE.FindStringSubmatch(contents) + + if match != nil { + stats.RunningJiffies, err = strconv.ParseUint(match[1], 10, 64) + if err != nil { + return + } + + stats.WaitingJiffies, err = strconv.ParseUint(match[2], 10, 64) + if err != nil { + return + } + + stats.RunTimeslices, err = strconv.ParseUint(match[3], 10, 64) + return + } + + err = errors.New("could not parse schedstat") + return +} + +func (stat *SchedstatCPU) RunningSeconds() float64 { + return float64(stat.RunningJiffies) / userHZ +} + +func (stat *SchedstatCPU) WaitingSeconds() float64 { + return float64(stat.WaitingJiffies) / userHZ +} + +func (stat *ProcSchedstat) RunningSeconds() float64 { + return float64(stat.RunningJiffies) / userHZ +} + +func (stat *ProcSchedstat) WaitingSeconds() float64 { + return float64(stat.WaitingJiffies) / userHZ +} diff --git a/schedstat_test.go b/schedstat_test.go new file mode 100644 index 000000000..42e43f1c2 --- /dev/null +++ b/schedstat_test.go @@ -0,0 +1,139 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import "testing" + +func TestSchedstat(t *testing.T) { + stats, err := getProcFixtures(t).Schedstat() + if err != nil { + t.Fatal(err) + } + + if len(stats.CPUs) != 2 { + t.Errorf("expected 2 CPUs, got %v", len(stats.CPUs)) + } + + var cpu *SchedstatCPU + for _, cpu = range stats.CPUs { + if cpu.CPUNum == "0" { + break + } + } + + if cpu == nil || cpu.CPUNum != "0" { + t.Error("could not find cpu0") + } + + if want, have := uint64(2045936778163039), cpu.RunningJiffies; want != have { + t.Errorf("want RunningJiffies %v, have %v", want, have) + } + if want, have := float64(2045936778163039)/userHZ, cpu.RunningSeconds(); want != have { + t.Errorf("want RunningSeconds() %v, have %v", want, have) + } + + if want, have := uint64(343796328169361), cpu.WaitingJiffies; want != have { + t.Errorf("want WaitingJiffies %v, have %v", want, have) + } + if want, have := float64(343796328169361)/userHZ, cpu.WaitingSeconds(); want != have { + t.Errorf("want WaitingSeconds() %v, have %v", want, have) + } + + if want, have := uint64(4767485306), cpu.RunTimeslices; want != have { + t.Errorf("want RunTimeslices %v, have %v", want, have) + } +} + +func TestProcSchedstat(t *testing.T) { + p1, err := getProcFixtures(t).Proc(26231) + if err != nil { + t.Fatal(err) + } + + schedstat, err := p1.Schedstat() + if err != nil { + t.Fatal(err) + } + + if want, have := uint64(411605849), schedstat.RunningJiffies; want != have { + t.Errorf("want RunningJiffies %v, have %v", want, have) + } + if want, have := float64(411605849)/userHZ, schedstat.RunningSeconds(); want != have { + t.Errorf("want RunningSeconds() %v, have %v", want, have) + } + + if want, have := uint64(93680043), schedstat.WaitingJiffies; want != have { + t.Errorf("want WaitingJiffies %v, have %v", want, have) + } + if want, have := float64(93680043)/userHZ, schedstat.WaitingSeconds(); want != have { + t.Errorf("want WaitingSeconds() %v, have %v", want, have) + } + + if want, have := uint64(79), schedstat.RunTimeslices; want != have { + t.Errorf("want RunTimeslices %v, have %v", want, have) + } +} + +func TestProcSchedstatErrors(t *testing.T) { + p1, err := getProcFixtures(t).Proc(26232) + if err != nil { + t.Fatal(err) + } + + _, err = p1.Schedstat() + if err == nil { + t.Error("proc 26232 doesn't have schedstat -- should have gotten an error") + } + + p2, err := getProcFixtures(t).Proc(26233) + if err != nil { + t.Fatal(err) + } + + _, err = p2.Schedstat() + if err == nil { + t.Error("proc 26233 has malformed schedstat -- should have gotten an error") + } +} + +// schedstat can have a 2nd line: it should be ignored +func TestProcSchedstatMultipleLines(t *testing.T) { + schedstat, err := parseProcSchedstat("123 456 789\n10 11\n") + if err != nil { + t.Fatal(err) + } + if want, have := uint64(123), schedstat.RunningJiffies; want != have { + t.Errorf("want RunningJiffies %v, have %v", want, have) + } + if want, have := uint64(456), schedstat.WaitingJiffies; want != have { + t.Errorf("want WaitingJiffies %v, have %v", want, have) + } + if want, have := uint64(789), schedstat.RunTimeslices; want != have { + t.Errorf("want RunTimeslices %v, have %v", want, have) + } +} + +func TestProcSchedstatUnparsableInt(t *testing.T) { + if _, err := parseProcSchedstat("abc 456 789\n"); err == nil { + t.Error("schedstat should have been unparsable\n") + } + + if _, err := parseProcSchedstat("123 abc 789\n"); err == nil { + t.Error("schedstat should have been unparsable\n") + } + + if _, err := parseProcSchedstat("123 456 abc\n"); err == nil { + t.Error("schedstat should have been unparsable\n") + } +}