updates for zfsonlinux 0.7.5 (#779)

* updates for zfsonlinux 0.7.5

* add constants for KSTAT_DATA_* types

* added e2e test for negative values represented by uint64 that can result from ZFS bugs

collector/fixtures/proc/spl/kstat/zfs/abdstats

@@ -0,0 +1,23 @@
+7 1 0x01 21 5712 73163810083184 309946154984654
+name                            type data
+struct_size                     4    2520
+linear_cnt                      4    62
+linear_data_size                4    223232
+scatter_cnt                     4    1
+scatter_data_size               4    16384
+scatter_chunk_waste             4    0
+scatter_order_0                 4    0
+scatter_order_1                 4    0
+scatter_order_2                 4    1
+scatter_order_3                 4    0
+scatter_order_4                 4    0
+scatter_order_5                 4    0
+scatter_order_6                 4    0
+scatter_order_7                 4    0
+scatter_order_8                 4    0
+scatter_order_9                 4    0
+scatter_order_10                4    0
+scatter_page_multi_chunk        4    0
+scatter_page_multi_zone         4    0
+scatter_page_alloc_retry        4    0
+scatter_sg_table_retry          4    0

collector/fixtures/proc/spl/kstat/zfs/dbuf_stats

@@ -0,0 +1,65 @@
+15 1 0x01 63 17136 73163812943503 309964267073187
+name                            type data
+dbuf_cache_count                4    27
+dbuf_cache_size                 4    302080
+dbuf_cache_size_max             4    394240
+dbuf_cache_max_bytes            4    62834368
+dbuf_cache_lowater_bytes        4    56550932
+dbuf_cache_hiwater_bytes        4    69117804
+dbuf_cache_total_evicts         4    0
+dbuf_cache_level_0              4    27
+dbuf_cache_level_1              4    0
+dbuf_cache_level_2              4    0
+dbuf_cache_level_3              4    0
+dbuf_cache_level_4              4    0
+dbuf_cache_level_5              4    0
+dbuf_cache_level_6              4    0
+dbuf_cache_level_7              4    0
+dbuf_cache_level_8              4    0
+dbuf_cache_level_9              4    0
+dbuf_cache_level_10             4    0
+dbuf_cache_level_11             4    0
+dbuf_cache_level_0_bytes        4    302080
+dbuf_cache_level_1_bytes        4    0
+dbuf_cache_level_2_bytes        4    0
+dbuf_cache_level_3_bytes        4    0
+dbuf_cache_level_4_bytes        4    0
+dbuf_cache_level_5_bytes        4    0
+dbuf_cache_level_6_bytes        4    0
+dbuf_cache_level_7_bytes        4    0
+dbuf_cache_level_8_bytes        4    0
+dbuf_cache_level_9_bytes        4    0
+dbuf_cache_level_10_bytes       4    0
+dbuf_cache_level_11_bytes       4    0
+hash_hits                       4    108807
+hash_misses                     4    1851
+hash_collisions                 4    0
+hash_elements                   4    55
+hash_elements_max               4    55
+hash_chains                     4    0
+hash_chain_max                  4    0
+hash_insert_race                4    0
+hash_dbuf_level_0               4    37
+hash_dbuf_level_1               4    10
+hash_dbuf_level_2               4    2
+hash_dbuf_level_3               4    2
+hash_dbuf_level_4               4    2
+hash_dbuf_level_5               4    2
+hash_dbuf_level_6               4    0
+hash_dbuf_level_7               4    0
+hash_dbuf_level_8               4    0
+hash_dbuf_level_9               4    0
+hash_dbuf_level_10              4    0
+hash_dbuf_level_11              4    0
+hash_dbuf_level_0_bytes         4    465920
+hash_dbuf_level_1_bytes         4    1310720
+hash_dbuf_level_2_bytes         4    262144
+hash_dbuf_level_3_bytes         4    262144
+hash_dbuf_level_4_bytes         4    262144
+hash_dbuf_level_5_bytes         4    262144
+hash_dbuf_level_6_bytes         4    0
+hash_dbuf_level_7_bytes         4    0
+hash_dbuf_level_8_bytes         4    0
+hash_dbuf_level_9_bytes         4    0
+hash_dbuf_level_10_bytes        4    0
+hash_dbuf_level_11_bytes        4    0

collector/fixtures/proc/spl/kstat/zfs/dnodestats

@@ -0,0 +1,30 @@
+10 1 0x01 28 7616 73163810135894 309969103316276
+name                            type data
+dnode_hold_dbuf_hold            4    0
+dnode_hold_dbuf_read            4    0
+dnode_hold_alloc_hits           4    37617
+dnode_hold_alloc_misses         4    0
+dnode_hold_alloc_interior       4    0
+dnode_hold_alloc_lock_retry     4    0
+dnode_hold_alloc_lock_misses    4    0
+dnode_hold_alloc_type_none      4    0
+dnode_hold_free_hits            4    0
+dnode_hold_free_misses          4    0
+dnode_hold_free_lock_misses     4    0
+dnode_hold_free_lock_retry      4    0
+dnode_hold_free_overflow        4    0
+dnode_hold_free_refcount        4    0
+dnode_hold_free_txg             4    0
+dnode_allocate                  4    0
+dnode_reallocate                4    0
+dnode_buf_evict                 4    17
+dnode_alloc_next_chunk          4    0
+dnode_alloc_race                4    0
+dnode_alloc_next_block          4    0
+dnode_move_invalid              4    0
+dnode_move_recheck1             4    0
+dnode_move_recheck2             4    0
+dnode_move_special              4    0
+dnode_move_handle               4    0
+dnode_move_rwlock               4    0
+dnode_move_active               4    0

collector/fixtures/proc/spl/kstat/zfs/vdev_mirror_stats

@@ -0,0 +1,9 @@
+18 1 0x01 7 1904 73163813004224 309980651991187
+name                            type data
+rotating_linear                 4    0
+rotating_offset                 4    0
+rotating_seek                   4    0
+non_rotating_linear             4    0
+non_rotating_seek               4    0
+preferred_found                 4    0
+preferred_not_found             4    94

collector/fixtures/proc/spl/kstat/zfs/zil

@@ -8,7 +8,7 @@ zil_itx_indirect_bytes          4    0
 zil_itx_copied_count            4    0
 zil_itx_copied_bytes            4    0
 zil_itx_needcopy_count          4    0
-zil_itx_needcopy_bytes          4    0
+zil_itx_needcopy_bytes          4    18446744073709537686
 zil_itx_metaslab_normal_count   4    0
 zil_itx_metaslab_normal_bytes   4    0
 zil_itx_metaslab_slog_count     4    0

collector/zfs.go

@@ -44,13 +44,17 @@ func NewZFSCollector() (Collector, error) {
 		linuxProcpathBase: "spl/kstat/zfs",
 		linuxZpoolIoPath:  "/*/io",
 		linuxPathMap: map[string]string{
-			"zfs_arc":        "arcstats",
-			"zfs_dmu_tx":     "dmu_tx",
-			"zfs_fm":         "fm",
-			"zfs_zfetch":     "zfetchstats",
-			"zfs_vdev_cache": "vdev_cache_stats",
-			"zfs_xuio":       "xuio_stats",
-			"zfs_zil":        "zil",
+			"zfs_abd":         "abdstats",
+			"zfs_arc":         "arcstats",
+			"zfs_dbuf":        "dbuf_stats",
+			"zfs_dmu_tx":      "dmu_tx",
+			"zfs_dnode":       "dnodestats",
+			"zfs_fm":          "fm",
+			"zfs_vdev_cache":  "vdev_cache_stats", // vdev_cache is deprecated
+			"zfs_vdev_mirror": "vdev_mirror_stats",
+			"zfs_xuio":        "xuio_stats", // no known consumers of the XUIO interface on Linux exist
+			"zfs_zfetch":      "zfetchstats",
+			"zfs_zil":         "zil",
 		},
 	}, nil
 }
@@ -60,7 +64,8 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) error {
 		if err := c.updateZfsStats(subsystem, ch); err != nil {
 			if err == errZFSNotAvailable {
 				log.Debug(err)
-				return nil
+				// ZFS /proc files are added as new features to ZFS arrive, it is ok to continue
+				continue
 			}
 			return err
 		}

collector/zfs_linux.go

@@ -26,10 +26,26 @@ import (
 	"github.com/prometheus/common/log"
 )
 
+// constants from https://github.com/zfsonlinux/zfs/blob/master/lib/libspl/include/sys/kstat.h
+// kept as strings for comparison thus avoiding conversion to int
+const (
+	KSTAT_DATA_CHAR   = "0"
+	KSTAT_DATA_INT32  = "1"
+	KSTAT_DATA_UINT32 = "2"
+	KSTAT_DATA_INT64  = "3"
+	KSTAT_DATA_UINT64 = "4"
+	KSTAT_DATA_LONG   = "5"
+	KSTAT_DATA_ULONG  = "6"
+	KSTAT_DATA_STRING = "7"
+)
+
 func (c *zfsCollector) openProcFile(path string) (*os.File, error) {
 	file, err := os.Open(procFilePath(path))
 	if err != nil {
-		log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", procFilePath(path))
+		// file not found error can occur if:
+		// 1. zfs module is not loaded
+		// 2. zfs version does not have the feature with metrics -- ok to ignore
+		log.Debugf("Cannot open %q for reading", procFilePath(path))
 		return nil, errZFSNotAvailable
 	}
 	return file, nil
@@ -60,7 +76,8 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error {
 	for _, zpoolPath := range zpoolPaths {
 		file, err := os.Open(zpoolPath)
 		if err != nil {
-			log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", zpoolPath)
+			// this file should exist, but there is a race where an exporting pool can remove the files -- ok to ignore
+			log.Debugf("Cannot open %q for reading", zpoolPath)
 			return errZFSNotAvailable
 		}
 
@@ -93,14 +110,16 @@ func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler
 			continue
 		}
 
-		key := fmt.Sprintf("kstat.zfs.misc.%s.%s", fmtExt, parts[0])
-
-		value, err := strconv.ParseUint(parts[2], 10, 64)
-		if err != nil {
-			return fmt.Errorf("could not parse expected integer value for %q", key)
+		// kstat data type (column 2) should be KSTAT_DATA_UINT64, otherwise ignore
+		// TODO: when other KSTAT_DATA_* types arrive, much of this will need to be restructured
+		if parts[1] == KSTAT_DATA_UINT64 {
+			key := fmt.Sprintf("kstat.zfs.misc.%s.%s", fmtExt, parts[0])
+			value, err := strconv.ParseUint(parts[2], 10, 64)
+			if err != nil {
+				return fmt.Errorf("could not parse expected integer value for %q", key)
+			}
+			handler(zfsSysctl(key), value)
 		}
-		handler(zfsSysctl(key), value)
-
 	}
 	if !parseLine {
 		return fmt.Errorf("did not parse a single %q metric", fmtExt)

collector/zfs_linux_test.go

@@ -310,3 +310,147 @@ func TestZpoolParsing(t *testing.T) {
 		t.Fatal("Zpool parsing handler was not called for some expected sysctls")
 	}
 }
+
+func TestAbdstatsParsing(t *testing.T) {
+	abdstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/abdstats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer abdstatsFile.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseProcfsFile(abdstatsFile, "abdstats", func(s zfsSysctl, v uint64) {
+
+		if s != zfsSysctl("kstat.zfs.misc.abdstats.linear_data_size") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != uint64(223232) {
+			t.Fatalf("Incorrect value parsed from procfs abdstats data")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("ABDStats parsing handler was not called for some expected sysctls")
+	}
+}
+
+func TestDbufstatsParsing(t *testing.T) {
+	dbufstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/dbuf_stats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dbufstatsFile.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseProcfsFile(dbufstatsFile, "dbufstats", func(s zfsSysctl, v uint64) {
+
+		if s != zfsSysctl("kstat.zfs.misc.dbufstats.hash_hits") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != uint64(108807) {
+			t.Fatalf("Incorrect value parsed from procfs dbufstats data")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("DbufStats parsing handler was not called for some expected sysctls")
+	}
+}
+
+func TestDnodestatsParsing(t *testing.T) {
+	dnodestatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/dnodestats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dnodestatsFile.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseProcfsFile(dnodestatsFile, "dnodestats", func(s zfsSysctl, v uint64) {
+
+		if s != zfsSysctl("kstat.zfs.misc.dnodestats.dnode_hold_alloc_hits") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != uint64(37617) {
+			t.Fatalf("Incorrect value parsed from procfs dnodestats data")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("Dnodestats parsing handler was not called for some expected sysctls")
+	}
+}
+
+func TestVdevMirrorstatsParsing(t *testing.T) {
+	vdevMirrorStatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/vdev_mirror_stats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer vdevMirrorStatsFile.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseProcfsFile(vdevMirrorStatsFile, "vdev_mirror_stats", func(s zfsSysctl, v uint64) {
+
+		if s != zfsSysctl("kstat.zfs.misc.vdev_mirror_stats.preferred_not_found") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != uint64(94) {
+			t.Fatalf("Incorrect value parsed from procfs vdev_mirror_stats data")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("VdevMirrorStats parsing handler was not called for some expected sysctls")
+	}
+}