From f3fd6a8d5019f39c8292870c36009fc9d55d6a33 Mon Sep 17 00:00:00 2001 From: Yuriy Vasiyarov Date: Mon, 25 Nov 2019 19:11:57 +0300 Subject: [PATCH] Issue #2616: ZFS Zpool properties on Linux --- plugins/inputs/zfs/README.md | 10 ++- plugins/inputs/zfs/zfs.go | 99 ++++++++++++++++++++++++++++ plugins/inputs/zfs/zfs_freebsd.go | 88 +++---------------------- plugins/inputs/zfs/zfs_linux.go | 94 ++++++++++++++++---------- plugins/inputs/zfs/zfs_linux_test.go | 99 ++++++++++++++++++++++++++-- 5 files changed, 268 insertions(+), 122 deletions(-) diff --git a/plugins/inputs/zfs/README.md b/plugins/inputs/zfs/README.md index b60711e305e69..26272a46897fe 100644 --- a/plugins/inputs/zfs/README.md +++ b/plugins/inputs/zfs/README.md @@ -1,7 +1,7 @@ # ZFS plugin This ZFS plugin provides metrics from your ZFS filesystems. It supports ZFS on -Linux and FreeBSD. It gets ZFS stat from `/proc/spl/kstat/zfs` on Linux and +Linux and FreeBSD. It gets ZFS stat from `/proc/spl/kstat/zfs` and zpool on Linux and from `sysctl` and `zpool` on FreeBSD. ### Configuration: @@ -180,7 +180,7 @@ each pool. #### Pool Metrics (optional) -On Linux (reference: kstat accumulated time and queue length statistics): +On Linux (reference: kstat accumulated time and queue length statistics) and zpool statistics: - zfs_pool - nread (integer, bytes) @@ -195,6 +195,12 @@ On Linux (reference: kstat accumulated time and queue length statistics): - rupdate (integer, timestamp) - wcnt (integer, count) - rcnt (integer, count) + - allocated (integer, bytes) + - capacity (integer, bytes) + - dedupratio (float, ratio) + - free (integer, bytes) + - size (integer, bytes) + - fragmentation (integer, percent) On FreeBSD: diff --git a/plugins/inputs/zfs/zfs.go b/plugins/inputs/zfs/zfs.go index 8e6bec4644932..03751a44a593f 100644 --- a/plugins/inputs/zfs/zfs.go +++ b/plugins/inputs/zfs/zfs.go @@ -1,5 +1,13 @@ package zfs +import ( + "bytes" + "fmt" + "os/exec" + "strconv" + "strings" +) + type Sysctl func(metric string) ([]string, error) type Zpool func() ([]string, error) @@ -30,6 +38,97 @@ func (z *Zfs) SampleConfig() string { return sampleConfig } +func (z *Zfs) getZpoolStats() (map[string]map[string]interface{}, error) { + + poolFields := map[string]map[string]interface{}{} + + lines, err := z.zpool() + if err != nil { + return poolFields, err + } + + for _, line := range lines { + col := strings.Split(line, "\t") + if len(col) != 8 { + continue + } + + health := col[1] + name := col[0] + + fields := map[string]interface{}{ + "name": name, + "health": health, + } + + if health == "UNAVAIL" { + + fields["size"] = int64(0) + + } else { + + size, err := strconv.ParseInt(col[2], 10, 64) + if err != nil { + return poolFields, fmt.Errorf("Error parsing size: %s", err) + } + fields["size"] = size + + alloc, err := strconv.ParseInt(col[3], 10, 64) + if err != nil { + return poolFields, fmt.Errorf("Error parsing allocation: %s", err) + } + fields["allocated"] = alloc + + free, err := strconv.ParseInt(col[4], 10, 64) + if err != nil { + return poolFields, fmt.Errorf("Error parsing free: %s", err) + } + fields["free"] = free + + frag, err := strconv.ParseInt(strings.TrimSuffix(col[5], "%"), 10, 0) + if err != nil { // This might be - for RO devs + frag = 0 + } + fields["fragmentation"] = frag + + capval, err := strconv.ParseInt(col[6], 10, 0) + if err != nil { + return poolFields, fmt.Errorf("Error parsing capacity: %s", err) + } + fields["capacity"] = capval + + dedup, err := strconv.ParseFloat(strings.TrimSuffix(col[7], "x"), 32) + if err != nil { + return poolFields, fmt.Errorf("Error parsing dedupratio: %s", err) + } + fields["dedupratio"] = dedup + } + poolFields[name] = fields + } + + return poolFields, nil +} + func (z *Zfs) Description() string { return "Read metrics of ZFS from arcstats, zfetchstats, vdev_cache_stats, and pools" } + +func run(command string, args ...string) ([]string, error) { + cmd := exec.Command(command, args...) + var outbuf, errbuf bytes.Buffer + cmd.Stdout = &outbuf + cmd.Stderr = &errbuf + err := cmd.Run() + + stdout := strings.TrimSpace(outbuf.String()) + stderr := strings.TrimSpace(errbuf.String()) + + if _, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("%s error: %s", command, stderr) + } + return strings.Split(stdout, "\n"), nil +} + +func zpool() ([]string, error) { + return run("zpool", []string{"list", "-Hp", "-o", "name,health,size,alloc,free,fragmentation,capacity,dedupratio"}...) +} diff --git a/plugins/inputs/zfs/zfs_freebsd.go b/plugins/inputs/zfs/zfs_freebsd.go index 51c20682e832b..7855faf6e2189 100644 --- a/plugins/inputs/zfs/zfs_freebsd.go +++ b/plugins/inputs/zfs/zfs_freebsd.go @@ -3,9 +3,7 @@ package zfs import ( - "bytes" "fmt" - "os/exec" "strconv" "strings" @@ -14,71 +12,25 @@ import ( ) func (z *Zfs) gatherPoolStats(acc telegraf.Accumulator) (string, error) { - - lines, err := z.zpool() + poolFields, err := z.getZpoolStats() if err != nil { return "", err } pools := []string{} - for _, line := range lines { - col := strings.Split(line, "\t") - - pools = append(pools, col[0]) + for name, _ := range poolFields { + pools = append(pools, name) } if z.PoolMetrics { - for _, line := range lines { - col := strings.Split(line, "\t") - if len(col) != 8 { - continue + for name, fields := range poolFields { + tags := map[string]string{ + "pool": name, + "health": fields["health"].(string), } - tags := map[string]string{"pool": col[0], "health": col[1]} - fields := map[string]interface{}{} - - if tags["health"] == "UNAVAIL" { - - fields["size"] = int64(0) - - } else { - - size, err := strconv.ParseInt(col[2], 10, 64) - if err != nil { - return "", fmt.Errorf("Error parsing size: %s", err) - } - fields["size"] = size - - alloc, err := strconv.ParseInt(col[3], 10, 64) - if err != nil { - return "", fmt.Errorf("Error parsing allocation: %s", err) - } - fields["allocated"] = alloc - - free, err := strconv.ParseInt(col[4], 10, 64) - if err != nil { - return "", fmt.Errorf("Error parsing free: %s", err) - } - fields["free"] = free - - frag, err := strconv.ParseInt(strings.TrimSuffix(col[5], "%"), 10, 0) - if err != nil { // This might be - for RO devs - frag = 0 - } - fields["fragmentation"] = frag - - capval, err := strconv.ParseInt(col[6], 10, 0) - if err != nil { - return "", fmt.Errorf("Error parsing capacity: %s", err) - } - fields["capacity"] = capval - - dedup, err := strconv.ParseFloat(strings.TrimSuffix(col[7], "x"), 32) - if err != nil { - return "", fmt.Errorf("Error parsing dedupratio: %s", err) - } - fields["dedupratio"] = dedup - } + delete(fields, "name") + delete(fields, "health") acc.AddFields("zfs_pool", fields, tags) } @@ -93,11 +45,11 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { kstatMetrics = []string{"arcstats", "zfetchstats", "vdev_cache_stats"} } - tags := map[string]string{} poolNames, err := z.gatherPoolStats(acc) if err != nil { return err } + tags := map[string]string{"pools": poolNames} tags["pools"] = poolNames fields := make(map[string]interface{}) @@ -117,26 +69,6 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { return nil } -func run(command string, args ...string) ([]string, error) { - cmd := exec.Command(command, args...) - var outbuf, errbuf bytes.Buffer - cmd.Stdout = &outbuf - cmd.Stderr = &errbuf - err := cmd.Run() - - stdout := strings.TrimSpace(outbuf.String()) - stderr := strings.TrimSpace(errbuf.String()) - - if _, ok := err.(*exec.ExitError); ok { - return nil, fmt.Errorf("%s error: %s", command, stderr) - } - return strings.Split(stdout, "\n"), nil -} - -func zpool() ([]string, error) { - return run("zpool", []string{"list", "-Hp", "-o", "name,health,size,alloc,free,fragmentation,capacity,dedupratio"}...) -} - func sysctl(metric string) ([]string, error) { return run("sysctl", []string{"-q", fmt.Sprintf("kstat.zfs.misc.%s", metric)}...) } diff --git a/plugins/inputs/zfs/zfs_linux.go b/plugins/inputs/zfs/zfs_linux.go index 276880d7dea97..96a0b997fa976 100644 --- a/plugins/inputs/zfs/zfs_linux.go +++ b/plugins/inputs/zfs/zfs_linux.go @@ -31,27 +31,16 @@ func getPools(kstatPath string) []poolInfo { return pools } -func getTags(pools []poolInfo) map[string]string { - var poolNames string - - for _, pool := range pools { - if len(poolNames) != 0 { - poolNames += "::" - } - poolNames += pool.name - } - - return map[string]string{"pools": poolNames} -} +func getSinglePoolKstat(pool poolInfo) (map[string]interface{}, error) { + fields := make(map[string]interface{}) -func gatherPoolStats(pool poolInfo, acc telegraf.Accumulator) error { lines, err := internal.ReadLines(pool.ioFilename) if err != nil { - return err + return fields, err } if len(lines) != 3 { - return err + return fields, err } keys := strings.Fields(lines[1]) @@ -60,24 +49,21 @@ func gatherPoolStats(pool poolInfo, acc telegraf.Accumulator) error { keyCount := len(keys) if keyCount != len(values) { - return fmt.Errorf("Key and value count don't match Keys:%v Values:%v", keys, values) + return fields, fmt.Errorf("Key and value count don't match Keys:%v Values:%v", keys, values) } - tag := map[string]string{"pool": pool.name} - fields := make(map[string]interface{}) for i := 0; i < keyCount; i++ { value, err := strconv.ParseInt(values[i], 10, 64) if err != nil { - return err + return fields, err } fields[keys[i]] = value } - acc.AddFields("zfs_pool", fields, tag) - return nil + return fields, nil } -func (z *Zfs) Gather(acc telegraf.Accumulator) error { +func (z *Zfs) getKstatMetrics() []string { kstatMetrics := z.KstatMetrics if len(kstatMetrics) == 0 { // vdev_cache_stats is deprecated @@ -86,26 +72,23 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { kstatMetrics = []string{"abdstats", "arcstats", "dnodestats", "dbufcachestats", "dmu_tx", "fm", "vdev_mirror_stats", "zfetchstats", "zil"} } + return kstatMetrics +} +func (z *Zfs) getKstatPath() string { kstatPath := z.KstatPath if len(kstatPath) == 0 { kstatPath = "/proc/spl/kstat/zfs" } + return kstatPath +} - pools := getPools(kstatPath) - tags := getTags(pools) - - if z.PoolMetrics { - for _, pool := range pools { - err := gatherPoolStats(pool, acc) - if err != nil { - return err - } - } - } - +func (z *Zfs) gatherZfsKstats(acc telegraf.Accumulator, poolNames string) error { + tags := map[string]string{"pools": poolNames} fields := make(map[string]interface{}) - for _, metric := range kstatMetrics { + kstatPath := z.getKstatPath() + + for _, metric := range z.getKstatMetrics() { lines, err := internal.ReadLines(kstatPath + "/" + metric) if err != nil { continue @@ -131,8 +114,47 @@ func (z *Zfs) Gather(acc telegraf.Accumulator) error { return nil } +func (z *Zfs) Gather(acc telegraf.Accumulator) error { + + //Gather pools metrics from kstats + poolFields, err := z.getZpoolStats() + if err != nil { + return err + } + + poolNames := []string{} + pools := getPools(z.getKstatPath()) + for _, pool := range pools { + poolNames = append(poolNames, pool.name) + + if z.PoolMetrics { + + //Merge zpool list with kstats + fields, err := getSinglePoolKstat(pool) + if err != nil { + return err + } else { + for k, v := range poolFields[pool.name] { + fields[k] = v + } + tags := map[string]string{ + "pool": pool.name, + "health": fields["health"].(string), + } + + delete(fields, "name") + delete(fields, "health") + + acc.AddFields("zfs_pool", fields, tags) + } + } + } + + return z.gatherZfsKstats(acc, strings.Join(poolNames, "::")) +} + func init() { inputs.Add("zfs", func() telegraf.Input { - return &Zfs{} + return &Zfs{zpool: zpool} }) } diff --git a/plugins/inputs/zfs/zfs_linux_test.go b/plugins/inputs/zfs/zfs_linux_test.go index 133d1cafa53c9..1b7cce29c5bf4 100644 --- a/plugins/inputs/zfs/zfs_linux_test.go +++ b/plugins/inputs/zfs/zfs_linux_test.go @@ -245,6 +245,29 @@ preferred_not_found 4 43 var testKstatPath = os.TempDir() + "/telegraf/proc/spl/kstat/zfs" +// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio +var zpool_output = []string{ + "HOME ONLINE 319975063552 202555169792 117419893760 22 63 1.00", + "STORAGE ONLINE 15942918602752 1172931735552 14769986867200 12 7 1.00", +} + +func mock_zpool_one_pool() ([]string, error) { + return zpool_output[0:1], nil +} + +func mock_zpool() ([]string, error) { + return zpool_output, nil +} + +// $ zpool list -Hp -o name,health,size,alloc,free,fragmentation,capacity,dedupratio +var zpool_output_unavail = []string{ + "HOME UNAVAIL - - - - - -", +} + +func mock_zpool_unavail() ([]string, error) { + return zpool_output_unavail, nil +} + func TestZfsPoolMetrics(t *testing.T) { err := os.MkdirAll(testKstatPath, 0755) require.NoError(t, err) @@ -262,20 +285,21 @@ func TestZfsPoolMetrics(t *testing.T) { var acc testutil.Accumulator - z := &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} + z := &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, zpool: mock_zpool_one_pool} err = z.Gather(&acc) require.NoError(t, err) require.False(t, acc.HasMeasurement("zfs_pool")) acc.Metrics = nil - z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, PoolMetrics: true} + z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, PoolMetrics: true, zpool: mock_zpool_one_pool} err = z.Gather(&acc) require.NoError(t, err) //one pool, all metrics tags := map[string]string{ - "pool": "HOME", + "pool": "HOME", + "health": "ONLINE", } acc.AssertContainsTaggedFields(t, "zfs_pool", poolMetrics, tags) @@ -321,7 +345,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { "pools": "HOME", } - z := &Zfs{KstatPath: testKstatPath} + z := &Zfs{KstatPath: testKstatPath, zpool: mock_zpool_one_pool} err = z.Gather(&acc) require.NoError(t, err) @@ -339,7 +363,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { "pools": "HOME::STORAGE", } - z = &Zfs{KstatPath: testKstatPath} + z = &Zfs{KstatPath: testKstatPath, zpool: mock_zpool} acc2 := testutil.Accumulator{} err = z.Gather(&acc2) require.NoError(t, err) @@ -350,7 +374,7 @@ func TestZfsGeneratesMetrics(t *testing.T) { intMetrics = getKstatMetricsArcOnly() //two pools, one metric - z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}} + z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, zpool: mock_zpool} acc3 := testutil.Accumulator{} err = z.Gather(&acc3) require.NoError(t, err) @@ -361,6 +385,45 @@ func TestZfsGeneratesMetrics(t *testing.T) { require.NoError(t, err) } +func TestZfsPoolUnavailMetrics(t *testing.T) { + err := os.MkdirAll(testKstatPath, 0755) + require.NoError(t, err) + + err = os.MkdirAll(testKstatPath+"/HOME", 0755) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/HOME/io", []byte(pool_ioContents), 0644) + require.NoError(t, err) + + err = ioutil.WriteFile(testKstatPath+"/arcstats", []byte(arcstatsContents), 0644) + require.NoError(t, err) + + poolMetrics := getPoolUnavailMetrics() + + var acc testutil.Accumulator + + z := &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, zpool: mock_zpool_unavail} + err = z.Gather(&acc) + require.NoError(t, err) + + require.False(t, acc.HasMeasurement("zfs_pool")) + acc.Metrics = nil + + z = &Zfs{KstatPath: testKstatPath, KstatMetrics: []string{"arcstats"}, PoolMetrics: true, zpool: mock_zpool_unavail} + err = z.Gather(&acc) + require.NoError(t, err) + + //one pool, all metrics + tags := map[string]string{ + "pool": "HOME", + "health": "UNAVAIL", + } + acc.AssertContainsTaggedFields(t, "zfs_pool", poolMetrics, tags) + + err = os.RemoveAll(os.TempDir() + "/telegraf") + require.NoError(t, err) +} + func getKstatMetricsArcOnly() map[string]interface{} { return map[string]interface{}{ "arcstats_hits": int64(5968846374), @@ -523,6 +586,29 @@ func getKstatMetricsAll() map[string]interface{} { } func getPoolMetrics() map[string]interface{} { + return map[string]interface{}{ + "nread": int64(1884160), + "nwritten": int64(6450688), + "reads": int64(22), + "writes": int64(978), + "wtime": int64(272187126), + "wlentime": int64(2850519036), + "wupdate": int64(2263669418655), + "rtime": int64(424226814), + "rlentime": int64(2850519036), + "rupdate": int64(2263669871823), + "wcnt": int64(0), + "rcnt": int64(0), + "allocated": int64(202555169792), + "capacity": int64(63), + "dedupratio": float64(1.0), + "fragmentation": int64(22), + "free": int64(117419893760), + "size": int64(319975063552), + } +} + +func getPoolUnavailMetrics() map[string]interface{} { return map[string]interface{}{ "nread": int64(1884160), "nwritten": int64(6450688), @@ -536,5 +622,6 @@ func getPoolMetrics() map[string]interface{} { "rupdate": int64(2263669871823), "wcnt": int64(0), "rcnt": int64(0), + "size": int64(0), } }