zfs-0.7.7 squashed patchset

This is a squashed patchset for zfs-0.7.7.  The individual commits are
in the tonyhutter:zfs-0.7.7-hutter branch.  I squashed the commits so
that buildbot wouldn't have to run against each one, and because
github/builbot seem to have a maximum limit of 30 commits they can
test from a PR.

- Fix MMP write frequency for large pools #7205 #7289
- Handle zio_resume and mmp => off #7286
- Fix zfs-kmod builds when using rpm >= 4.14 #7284
- zdb and inuse tests don't pass with real disks #6939 #7261
- Take user namespaces into account in policy checks #6800 #7270
- Detect long config lock acquisition in mmp #7212
- Linux 4.16 compat: get_disk_and_module() #7264
- Change checksum & IO delay ratelimit values #7252
- Increment zil_itx_needcopy_bytes properly #6988 #7176
- Fix some typos #7237
- Fix zpool(8) list example to match actual format #7244
- Add SMART self-test results to zpool status -c #7178
- Add scrub after resilver zed script #4662 #7086
- Fix free memory calculation on v3.14+ #7170
- Report duration and error in mmp_history entries #7190
- Do not initiate MMP writes while pool is suspended #7182
- Linux 4.16 compat: use correct *_dec_and_test()
- Allow modprobe to fail when called within systemd #7174
- Add SMART attributes for SSD and NVMe #7183 #7193
- Correct count_uberblocks in mmp.kshlib #7191
- Fix config issues: frame size and headers #7169
- Clarify zinject(8) explanation of -e #7172
- OpenZFS 8857 - zio_remove_child() panic due to already destroyed
  parent zio #7168
- 'zfs receive' fails with "dataset is busy" #7129 #7154
- contrib/initramfs: add missing conf.d/zfs #7158
- mmp should use a fixed tag for spa_config locks #6530 #7155
- Handle zap_add() failures in mixed case mode #7011 #7054
- Fix zdb -ed on objset for exported pool #7099 #6464
- Fix zdb -E segfault #7099
- Fix zdb -R decompression #7099 #4984
- Fix racy assignment of zcb.zcb_haderrors #7099
- Fix zle_decompress out of bound access #7099
- Fix zdb -c traverse stop on damaged objset root #7099
- Linux 4.11 compat: avoid refcount_t name conflict #7148
- Linux 4.16 compat: inode_set_iversion() #7148
- OpenZFS 8966 - Source file zfs_acl.c, function zfs_aclset_common
  contains a use after end of the lifetime of a local variable #7141
- Remove deprecated zfs_arc_p_aggressive_disable #7135
- Fix default libdir for Debian/Ubuntu #7083 #7101
- Bug fix in qat_compress.c for vmalloc addr check #7125
- Fix systemd_ RPM macros usage on Debian-based distributions #7074
  #7100
- Emit an error message before MMP suspends pool #7048
- ZTS: Fix create-o_ashift test case #6924 #6977
- Fix --with-systemd on Debian-based distributions (#6963) #6591 #6963
- Remove vn_rename and vn_remove dependency openzfs/spl#648 #6753
- Add support for "--enable-code-coverage" option #6670
- Make "-fno-inline" compile option more accessible #6605
- Add configure option to enable gcov analysis #6642
- Implement --enable-debuginfo to force debuginfo #2734
- Make --enable-debug fail when given bogus args #2734

Signed-off-by: Tony Hutter <[email protected]>
Requires-spl: refs/pull/690/head

.github/codecov.yml

@@ -0,0 +1,9 @@
+codecov:
+  strict_yaml_branch: master # only use the latest copy on master branch
+
+comment: off
+
+coverage:
+  status:
+    project: off
+    patch: off

.gitignore

@@ -19,6 +19,8 @@
 *.mod.c
 *~
 *.swp
+*.gcno
+*.gcda
 .deps
 .libs
 .dirstamp

Makefile.am

@@ -23,16 +23,18 @@ EXTRA_DIST  = autogen.sh copy-builtin
 EXTRA_DIST += config/config.awk config/rpm.am config/deb.am config/tgz.am
 EXTRA_DIST += META DISCLAIMER COPYRIGHT README.markdown OPENSOLARIS.LICENSE
 
+@CODE_COVERAGE_RULES@
+
 distclean-local::
 	-$(RM) -R autom4te*.cache
 	-find . \( -name SCCS -o -name BitKeeper -o -name .svn -o -name CVS \
 		-o -name .pc -o -name .hg -o -name .git \) -prune -o \
 		\( -name '*.orig' -o -name '*.rej' -o -name '*~' \
 		-o -name '*.bak' -o -name '#*#' -o -name '.*.orig' \
-		-o -name '.*.rej' -o -name '.script-config' -o -size 0 \
-		-o -name '*%' -o -name '.*.cmd' -o -name 'core' \
-		-o -name 'Makefile' -o -name 'Module.symvers' \
-		-o -name '*.order' -o -name '*.markers' \) \
+		-o -name '.*.rej' -o -size 0 -o -name '*%' -o -name '.*.cmd' \
+		-o -name 'core' -o -name 'Makefile' -o -name 'Module.symvers' \
+		-o -name '*.order' -o -name '*.markers' -o -name '*.gcda' \
+		-o -name '*.gcno' \) \
 		-type f -print | xargs $(RM)
 
 dist-hook:

cmd/arcstat/arcstat.py

@@ -280,7 +280,7 @@ def init():
                 "outfile",
                 "help",
                 "verbose",
-                "seperator",
+                "separator",
                 "columns"
             ]
         )
@@ -299,7 +299,7 @@ def init():
             hflag = True
         if opt in ('-v', '--verbose'):
             vflag = True
-        if opt in ('-s', '--seperator'):
+        if opt in ('-s', '--separator'):
             sep = arg
             i += 1
         if opt in ('-f', '--columns'):

cmd/dbufstat/dbufstat.py

@@ -474,7 +474,7 @@ def main():
                 "help",
                 "infile",
                 "outfile",
-                "seperator",
+                "separator",
                 "types",
                 "verbose",
                 "extended"
@@ -499,7 +499,7 @@ def main():
             ofile = arg
         if opt in ('-r', '--raw'):
             raw += 1
-        if opt in ('-s', '--seperator'):
+        if opt in ('-s', '--separator'):
             sep = arg
         if opt in ('-t', '--types'):
             tflag = True

cmd/zdb/zdb.c

@@ -2716,17 +2716,17 @@ dump_label(const char *dev)
 		exit(1);
 	}
 
-	if (ioctl(fd, BLKFLSBUF) != 0)
-		(void) printf("failed to invalidate cache '%s' : %s\n", path,
-		    strerror(errno));
-
 	if (fstat64_blk(fd, &statbuf) != 0) {
 		(void) printf("failed to stat '%s': %s\n", path,
 		    strerror(errno));
 		(void) close(fd);
 		exit(1);
 	}
 
+	if (S_ISBLK(statbuf.st_mode) && ioctl(fd, BLKFLSBUF) != 0)
+		(void) printf("failed to invalidate cache '%s' : %s\n", path,
+		    strerror(errno));
+
 	avl_create(&config_tree, cksum_record_compare,
 	    sizeof (cksum_record_t), offsetof(cksum_record_t, link));
 	avl_create(&uberblock_tree, cksum_record_compare,
@@ -3313,7 +3313,7 @@ dump_block_stats(spa_t *spa)
 	uint64_t norm_alloc, norm_space, total_alloc, total_found;
 	int flags = TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | TRAVERSE_HARD;
 	boolean_t leaks = B_FALSE;
-	int e, c;
+	int e, c, err;
 	bp_embedded_type_t i;
 
 	(void) printf("\nTraversing all blocks %s%s%s%s%s...\n\n",
@@ -3354,7 +3354,7 @@ dump_block_stats(spa_t *spa)
 
 	zcb.zcb_totalasize = metaslab_class_get_alloc(spa_normal_class(spa));
 	zcb.zcb_start = zcb.zcb_lastprint = gethrtime();
-	zcb.zcb_haderrors |= traverse_pool(spa, 0, flags, zdb_blkptr_cb, &zcb);
+	err = traverse_pool(spa, 0, flags, zdb_blkptr_cb, &zcb);
 
 	/*
 	 * If we've traversed the data blocks then we need to wait for those
@@ -3370,6 +3370,12 @@ dump_block_stats(spa_t *spa)
 		}
 	}
 
+	/*
+	 * Done after zio_wait() since zcb_haderrors is modified in
+	 * zdb_blkptr_done()
+	 */
+	zcb.zcb_haderrors |= err;
+
 	if (zcb.zcb_haderrors) {
 		(void) printf("\nError counts:\n\n");
 		(void) printf("\t%5s  %s\n", "errno", "count");
@@ -3889,13 +3895,6 @@ zdb_vdev_lookup(vdev_t *vdev, char *path)
 	return (NULL);
 }
 
-/* ARGSUSED */
-static int
-random_get_pseudo_bytes_cb(void *buf, size_t len, void *unused)
-{
-	return (random_get_pseudo_bytes(buf, len));
-}
-
 /*
  * Read a block from a pool and print it out.  The syntax of the
  * block descriptor is:
@@ -4058,17 +4057,8 @@ zdb_read_block(char *thing, spa_t *spa)
 		 * every decompress function at every inflated blocksize.
 		 */
 		enum zio_compress c;
-		void *pbuf2 = umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
 		void *lbuf2 = umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
 
-		abd_copy_to_buf(pbuf2, pabd, psize);
-
-		VERIFY0(abd_iterate_func(pabd, psize, SPA_MAXBLOCKSIZE - psize,
-		    random_get_pseudo_bytes_cb, NULL));
-
-		VERIFY0(random_get_pseudo_bytes((uint8_t *)pbuf2 + psize,
-		    SPA_MAXBLOCKSIZE - psize));
-
 		/*
 		 * XXX - On the one hand, with SPA_MAXBLOCKSIZE at 16MB,
 		 * this could take a while and we should let the user know
@@ -4078,25 +4068,39 @@ zdb_read_block(char *thing, spa_t *spa)
 		for (lsize = psize + SPA_MINBLOCKSIZE;
 		    lsize <= SPA_MAXBLOCKSIZE; lsize += SPA_MINBLOCKSIZE) {
 			for (c = 0; c < ZIO_COMPRESS_FUNCTIONS; c++) {
+				/*
+				 * ZLE can easily decompress non zle stream.
+				 * So have an option to disable it.
+				 */
+				if (c == ZIO_COMPRESS_ZLE &&
+				    getenv("ZDB_NO_ZLE"))
+					continue;
+
 				(void) fprintf(stderr,
 				    "Trying %05llx -> %05llx (%s)\n",
 				    (u_longlong_t)psize, (u_longlong_t)lsize,
 				    zio_compress_table[c].ci_name);
+
+				/*
+				 * We randomize lbuf2, and decompress to both
+				 * lbuf and lbuf2. This way, we will know if
+				 * decompression fill exactly to lsize.
+				 */
+				VERIFY0(random_get_pseudo_bytes(lbuf2, lsize));
+
 				if (zio_decompress_data(c, pabd,
 				    lbuf, psize, lsize) == 0 &&
-				    zio_decompress_data_buf(c, pbuf2,
+				    zio_decompress_data(c, pabd,
 				    lbuf2, psize, lsize) == 0 &&
 				    bcmp(lbuf, lbuf2, lsize) == 0)
 					break;
 			}
 			if (c != ZIO_COMPRESS_FUNCTIONS)
 				break;
 		}
-
-		umem_free(pbuf2, SPA_MAXBLOCKSIZE);
 		umem_free(lbuf2, SPA_MAXBLOCKSIZE);
 
-		if (lsize <= psize) {
+		if (lsize > SPA_MAXBLOCKSIZE) {
 			(void) printf("Decompress of %s failed\n", thing);
 			goto out;
 		}
@@ -4135,11 +4139,12 @@ zdb_embedded_block(char *thing)
 {
 	blkptr_t bp;
 	unsigned long long *words = (void *)&bp;
-	char buf[SPA_MAXBLOCKSIZE];
+	char *buf;
 	int err;
 
-	memset(&bp, 0, sizeof (blkptr_t));
+	buf = umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL);
 
+	bzero(&bp, sizeof (bp));
 	err = sscanf(thing, "%llx:%llx:%llx:%llx:%llx:%llx:%llx:%llx:"
 	    "%llx:%llx:%llx:%llx:%llx:%llx:%llx:%llx",
 	    words + 0, words + 1, words + 2, words + 3,
@@ -4157,6 +4162,7 @@ zdb_embedded_block(char *thing)
 		exit(1);
 	}
 	zdb_dump_block_raw(buf, BPE_GET_LSIZE(&bp), 0);
+	umem_free(buf, SPA_MAXBLOCKSIZE);
 }
 
 int
@@ -4171,7 +4177,7 @@ main(int argc, char **argv)
 	int error = 0;
 	char **searchdirs = NULL;
 	int nsearch = 0;
-	char *target;
+	char *target, *target_pool;
 	nvlist_t *policy = NULL;
 	uint64_t max_txg = UINT64_MAX;
 	int flags = ZFS_IMPORT_MISSING_LOG;
@@ -4374,6 +4380,20 @@ main(int argc, char **argv)
 	error = 0;
 	target = argv[0];
 
+	if (strpbrk(target, "/@") != NULL) {
+		size_t targetlen;
+
+		target_pool = strdup(target);
+		*strpbrk(target_pool, "/@") = '\0';
+
+		target_is_spa = B_FALSE;
+		targetlen = strlen(target);
+		if (targetlen && target[targetlen - 1] == '/')
+			target[targetlen - 1] = '\0';
+	} else {
+		target_pool = target;
+	}
+
 	if (dump_opt['e']) {
 		importargs_t args = { 0 };
 		nvlist_t *cfg = NULL;
@@ -4382,8 +4402,10 @@ main(int argc, char **argv)
 		args.path = searchdirs;
 		args.can_be_active = B_TRUE;
 
-		error = zpool_tryimport(g_zfs, target, &cfg, &args);
+		error = zpool_tryimport(g_zfs, target_pool, &cfg, &args);
+
 		if (error == 0) {
+
 			if (nvlist_add_nvlist(cfg,
 			    ZPOOL_REWIND_POLICY, policy) != 0) {
 				fatal("can't open '%s': %s",
@@ -4398,19 +4420,13 @@ main(int argc, char **argv)
 				(void) printf("\nConfiguration for import:\n");
 				dump_nvlist(cfg, 8);
 			}
-			error = spa_import(target, cfg, NULL,
+			error = spa_import(target_pool, cfg, NULL,
 			    flags | ZFS_IMPORT_SKIP_MMP);
 		}
 	}
 
-	if (strpbrk(target, "/@") != NULL) {
-		size_t targetlen;
-
-		target_is_spa = B_FALSE;
-		targetlen = strlen(target);
-		if (targetlen && target[targetlen - 1] == '/')
-			target[targetlen - 1] = '\0';
-	}
+	if (target_pool != target)
+		free(target_pool);
 
 	if (error == 0) {
 		if (target_is_spa || dump_opt['R']) {

cmd/zed/Makefile.am

@@ -69,7 +69,8 @@ dist_zedexec_SCRIPTS = \
 	zed.d/statechange-notify.sh \
 	zed.d/vdev_clear-led.sh \
 	zed.d/vdev_attach-led.sh \
-	zed.d/pool_import-led.sh
+	zed.d/pool_import-led.sh \
+	zed.d/resilver_finish-start-scrub.sh
 
 zedconfdefaults = \
 	all-syslog.sh \
@@ -80,7 +81,8 @@ zedconfdefaults = \
 	statechange-notify.sh \
 	vdev_clear-led.sh \
 	vdev_attach-led.sh \
-	pool_import-led.sh
+	pool_import-led.sh \
+	resilver_finish-start-scrub.sh
 
 install-data-hook:
 	$(MKDIR_P) "$(DESTDIR)$(zedconfdir)"

cmd/zed/zed.d/resilver_finish-start-scrub.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# resilver_finish-start-scrub.sh
+# Run a scrub after a resilver
+#
+# Exit codes:
+# 1: Internal error
+# 2: Script wasn't enabled in zed.rc
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ "${ZED_SCRUB_AFTER_RESILVER}" = "1" ] || exit 2
+[ -n "${ZEVENT_POOL}" ] || exit 1
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 1
+zed_check_cmd "${ZPOOL}" || exit 1
+
+zed_log_msg "Starting scrub after resilver on ${ZEVENT_POOL}"
+"${ZPOOL}" scrub "${ZEVENT_POOL}"

cmd/zed/zed.d/zed.rc

@@ -86,6 +86,9 @@
 #
 ZED_USE_ENCLOSURE_LEDS=1
 
+##
+# Run a scrub after every resilver
+#ZED_SCRUB_AFTER_RESILVER=1
 
 ##
 # The syslog priority (e.g., specified as a "facility.level" pair).

cmd/zed/zed_conf.c

@@ -155,6 +155,8 @@ _zed_conf_display_help(const char *prog, int got_err)
 	    "Run daemon in the foreground.");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-M",
 	    "Lock all pages in memory.");
+	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-P",
+	    "$PATH for ZED to use (only used by ZTS).");
 	fprintf(fp, "%*c%*s %s\n", w1, 0x20, -w2, "-Z",
 	    "Zero state file.");
 	fprintf(fp, "\n");
@@ -247,7 +249,7 @@ _zed_conf_parse_path(char **resultp, const char *path)
 void
 zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 {
-	const char * const opts = ":hLVc:d:p:s:vfFMZ";
+	const char * const opts = ":hLVc:d:p:P:s:vfFMZ";
 	int opt;
 
 	if (!zcp || !argv || !argv[0])
@@ -275,6 +277,9 @@ zed_conf_parse_opts(struct zed_conf *zcp, int argc, char **argv)
 		case 'p':
 			_zed_conf_parse_path(&zcp->pid_file, optarg);
 			break;
+		case 'P':
+			_zed_conf_parse_path(&zcp->path, optarg);
+			break;
 		case 's':
 			_zed_conf_parse_path(&zcp->state_file, optarg);
 			break;

cmd/zed/zed_conf.h

@@ -37,6 +37,7 @@ struct zed_conf {
 	int		state_fd;		/* fd to state file */
 	libzfs_handle_t	*zfs_hdl;		/* handle to libzfs */
 	int		zevent_fd;		/* fd for access to zevents */
+	char		*path;		/* custom $PATH for zedlets to use */
 };
 
 struct zed_conf *zed_conf_create(void);