Merge pull request openzfs#21 from KlaraSystems/ddtload-sync

Ddtload

cmd/zpool/zpool_main.c

@@ -82,7 +82,7 @@ static int zpool_do_remove(int, char **);
 static int zpool_do_labelclear(int, char **);
 
 static int zpool_do_checkpoint(int, char **);
-static int zpool_do_ddtload(int, char **);
+static int zpool_do_load(int, char **);
 
 static int zpool_do_list(int, char **);
 static int zpool_do_iostat(int, char **);
@@ -145,7 +145,6 @@ typedef enum {
 	HELP_CLEAR,
 	HELP_CREATE,
 	HELP_CHECKPOINT,
-	HELP_DDTLOAD,
 	HELP_DESTROY,
 	HELP_DETACH,
 	HELP_EXPORT,
@@ -154,6 +153,7 @@ typedef enum {
 	HELP_IOSTAT,
 	HELP_LABELCLEAR,
 	HELP_LIST,
+	HELP_LOAD,
 	HELP_OFFLINE,
 	HELP_ONLINE,
 	HELP_REPLACE,
@@ -281,7 +281,7 @@ static zpool_command_t command_table[] = {
 	{ "labelclear",	zpool_do_labelclear,	HELP_LABELCLEAR		},
 	{ NULL },
 	{ "checkpoint",	zpool_do_checkpoint,	HELP_CHECKPOINT		},
-	{ "ddtload",	zpool_do_ddtload,	HELP_DDTLOAD		},
+	{ "load",	zpool_do_load,		HELP_LOAD		},
 	{ NULL },
 	{ "list",	zpool_do_list,		HELP_LIST		},
 	{ "iostat",	zpool_do_iostat,	HELP_IOSTAT		},
@@ -342,8 +342,6 @@ get_usage(zpool_help_t idx)
 		    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 	case HELP_CHECKPOINT:
 		return (gettext("\tcheckpoint [--discard] <pool> ...\n"));
-	case HELP_DDTLOAD:
-		return (gettext("\tddtload <pool>\n"));
 	case HELP_DESTROY:
 		return (gettext("\tdestroy [-f] <pool>\n"));
 	case HELP_DETACH:
@@ -372,6 +370,9 @@ get_usage(zpool_help_t idx)
 		return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 		    "[-T d|u] [pool] ... \n"
 		    "\t    [interval [count]]\n"));
+	case HELP_LOAD:
+		return (gettext("\tload <type> [<type opts>] <pool>\n"
+		    "\tload ddt <pool>\n"));
 	case HELP_OFFLINE:
 		return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 	case HELP_ONLINE:
@@ -1831,7 +1832,7 @@ typedef struct status_cbdata {
 	boolean_t	cb_literal;
 	boolean_t	cb_explain;
 	boolean_t	cb_first;
-	boolean_t	cb_dedup_stats;
+	int		cb_dedup_stats;
 	boolean_t	cb_print_status;
 	boolean_t	cb_print_slow_ios;
 	boolean_t	cb_print_vdev_init;
@@ -2955,41 +2956,72 @@ zpool_do_checkpoint(int argc, char **argv)
 
 #define	CHECKPOINT_OPT	1024
 
+enum zpool_load_type {
+	ZPOOL_LOAD_TYPE_DDT,
+};
+
 /*
- * zpool ddtload <pool>
+ * zpool load <type> [<type opts> ...] <pool>
  *
  * Loads the DDT table of the specified pool.
  */
 int
-zpool_do_ddtload(int argc, char **argv)
+zpool_do_load(int argc, char **argv)
 {
 	char *pool;
+	char *typestr;
+	enum zpool_load_type type;
 	zpool_handle_t *zhp;
-	int err;
+	int err = 0;
 
 	argc--;
 	argv++;
 	if (argc < 1) {
+		(void) fprintf(stderr, gettext("missing type argument\n"));
+		usage(B_FALSE);
+	}
+	if (argc < 2) {
 		(void) fprintf(stderr, gettext("missing pool argument\n"));
 		usage(B_FALSE);
 	}
-	if (argc > 1) {
+	if (argc > 2) {
 		(void) fprintf(stderr, gettext("too many arguments\n"));
 		usage(B_FALSE);
 	}
 
-	pool = argv[0];
+	typestr = argv[0];
+	pool = argv[1];
+
+	if (strcmp(typestr, "ddt") == 0) {
+		type = ZPOOL_LOAD_TYPE_DDT;
+	} else {
+		(void) fprintf(stderr, gettext("unsupported load type\n"));
+		usage(B_FALSE);
+	}
 
 	if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
 		/* As a special case, check for use of '/' in the name */
-		if (strchr(pool, '/') != NULL) {
-			(void) fprintf(stderr, gettext("'zpool ddtload' "
-			    "doesn't work on datasets.\n"));
+		switch (type) {
+		case ZPOOL_LOAD_TYPE_DDT:
+			if (strchr(pool, '/') != NULL) {
+				(void) fprintf(stderr, gettext("This load "
+				    "type doesn't work on datasets.\n"));
+			}
+			break;
+		default:
+			break;
 		}
 		return (1);
 	}
 
-	err = zpool_ddtload(zhp);
+	switch (type) {
+	case ZPOOL_LOAD_TYPE_DDT:
+		err = zpool_ddtload(zhp);
+		break;
+	default:
+		break;
+	}
+
 	zpool_close(zhp);
 
 	return (err);
@@ -5619,6 +5651,7 @@ print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
 	case ZPOOL_PROP_EXPANDSZ:
 	case ZPOOL_PROP_CHECKPOINT:
 	case ZPOOL_PROP_DEDUPRATIO:
+	case ZPOOL_PROP_DEDUPCACHED:
 		if (value == 0)
 			(void) strlcpy(propval, "-", sizeof (propval));
 		else
@@ -7361,13 +7394,17 @@ print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
 }
 
 static void
-print_dedup_stats(nvlist_t *config)
+print_dedup_stats(zpool_handle_t *zhp, nvlist_t *config, boolean_t literal)
 {
 	ddt_histogram_t *ddh;
 	ddt_stat_t *dds;
 	ddt_object_t *ddo;
 	uint_t c;
-	char dspace[6], mspace[6];
+	/* Extra space provided for literal display */
+	char dspace[32], mspace[32], cspace[32];
+	uint64_t cspace_prop;
+	enum zfs_nicenum_format format;
+	zprop_source_t src;
 
 	/*
 	 * If the pool was faulted then we may not have been able to
@@ -7385,12 +7422,26 @@ print_dedup_stats(nvlist_t *config)
 		return;
 	}
 
-	zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
-	zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
-	(void) printf("DDT entries %llu, size %s on disk, %s in core\n",
+	/*
+	 * Squash cached size into in-core size to handle race.
+	 * Only include cached size if it is available.
+	 */
+	cspace_prop = MIN(zpool_get_prop_int(zhp, ZPOOL_PROP_DEDUPCACHED, &src),
+	    ddo->ddo_mspace);
+	format = literal ? ZFS_NICENUM_RAW : ZFS_NICENUM_1024;
+	zfs_nicenum_format(cspace_prop, cspace, sizeof (cspace), format);
+	zfs_nicenum_format(ddo->ddo_dspace, dspace, sizeof (dspace), format);
+	zfs_nicenum_format(ddo->ddo_mspace, mspace, sizeof (mspace), format);
+	(void) printf("DDT entries %llu, size %s on disk, %s in core",
 	    (u_longlong_t)ddo->ddo_count,
 	    dspace,
 	    mspace);
+	if (src != ZPROP_SRC_DEFAULT) {
+		(void) printf(", %s cached (%.02f%%)",
+		    cspace,
+		    (double)cspace_prop / (double)ddo->ddo_mspace * 100.0);
+	}
+	(void) printf("\n");
 
 	verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
 	    (uint64_t **)&dds, &c) == 0);
@@ -7426,6 +7477,10 @@ status_callback(zpool_handle_t *zhp, void *data)
 	uint_t c;
 	vdev_stat_t *vs;
 
+	/* If dedup stats were requested, also fetch dedupcached. */
+	if (cbp->cb_dedup_stats > 1)
+		zpool_add_propname(zhp, "dedupcached");
+
 	config = zpool_get_config(zhp, NULL);
 	reason = zpool_get_status(zhp, &msgid, &errata);
 
@@ -7813,7 +7868,7 @@ status_callback(zpool_handle_t *zhp, void *data)
 		}
 
 		if (cbp->cb_dedup_stats)
-			print_dedup_stats(config);
+			print_dedup_stats(zhp, config, cbp->cb_literal);
 	} else {
 		(void) printf(gettext("config: The configuration cannot be "
 		    "determined.\n"));
@@ -7903,7 +7958,7 @@ zpool_do_status(int argc, char **argv)
 			cb.cb_explain = B_TRUE;
 			break;
 		case 'D':
-			cb.cb_dedup_stats = B_TRUE;
+			cb.cb_dedup_stats++;
 			break;
 		case 't':
 			cb.cb_print_vdev_trim = B_TRUE;

configure.ac

@@ -254,6 +254,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/cli_root/zpool_import/blockfiles/Makefile
 	tests/zfs-tests/tests/functional/cli_root/zpool_initialize/Makefile
 	tests/zfs-tests/tests/functional/cli_root/zpool_labelclear/Makefile
+	tests/zfs-tests/tests/functional/cli_root/zpool_load/Makefile
 	tests/zfs-tests/tests/functional/cli_root/zpool/Makefile
 	tests/zfs-tests/tests/functional/cli_root/zpool_offline/Makefile
 	tests/zfs-tests/tests/functional/cli_root/zpool_online/Makefile

include/libzfs.h

@@ -396,6 +396,7 @@ extern nvlist_t *zpool_get_config(zpool_handle_t *, nvlist_t **);
 extern nvlist_t *zpool_get_features(zpool_handle_t *);
 extern int zpool_refresh_stats(zpool_handle_t *, boolean_t *);
 extern int zpool_get_errlog(zpool_handle_t *, nvlist_t **);
+extern void zpool_add_propname(zpool_handle_t *, char *);
 
 /*
  * Import and export functions

include/libzfs_impl.h

@@ -97,11 +97,14 @@ struct zfs_handle {
  */
 #define	ZFS_IS_VOLUME(zhp) ((zhp)->zfs_head_type == ZFS_TYPE_VOLUME)
 
+#define	ZHP_MAX_PROPNAMES	4
 struct zpool_handle {
 	libzfs_handle_t *zpool_hdl;
 	zpool_handle_t *zpool_next;
 	char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
 	int zpool_state;
+	unsigned int zpool_n_propnames;
+	char *zpool_propnames[ZHP_MAX_PROPNAMES];
 	size_t zpool_config_size;
 	nvlist_t *zpool_config;
 	nvlist_t *zpool_old_config;

include/sys/arc.h

@@ -286,6 +286,12 @@ zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 arc_prune_t *arc_add_prune_callback(arc_prune_func_t *func, void *private);
 void arc_remove_prune_callback(arc_prune_t *p);
 void arc_freed(spa_t *spa, const blkptr_t *bp);
+int arc_cached(spa_t *spa, const blkptr_t *bp);
+#define	ARC_CACHED_EMBEDDED	(1U << 0)
+#define	ARC_CACHED_IN_L1	(1U << 1)
+#define	ARC_CACHED_IN_MRU	(1U << 2)
+#define	ARC_CACHED_IN_MFU	(1U << 3)
+#define	ARC_CACHED_IN_L2	(1U << 4)
 
 void arc_flush(spa_t *spa, boolean_t retry);
 void arc_tempreserve_clear(uint64_t reserve);

include/sys/ddt.h

@@ -217,6 +217,7 @@ extern void ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg);
 
 extern uint64_t ddt_get_dedup_dspace(spa_t *spa);
 extern uint64_t ddt_get_pool_dedup_ratio(spa_t *spa);
+extern uint64_t ddt_get_pool_dedup_cached(spa_t *spa);
 
 extern int ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde,
     ddt_phys_t *ddp_willref);

include/sys/dmu.h

@@ -502,6 +502,11 @@ void dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
 
 
 int dmu_object_remap_indirects(objset_t *os, uint64_t object, uint64_t txg);
+/*
+ * Get an estimated cache size for an object.  Caller must expect races.
+ */
+void dmu_object_cached_size(objset_t *os, uint64_t object,
+    uint64_t *l1sz, uint64_t *l2sz);
 
 void dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
     void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
@@ -887,6 +892,8 @@ extern int zfs_max_recordsize;
  */
 void dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
 	uint64_t len, enum zio_priority pri);
+int dmu_prefetch_wait(objset_t *os, uint64_t object, uint64_t offset,
+    uint64_t size, uint32_t flags);
 
 typedef struct dmu_object_info {
 	/* All sizes are in bytes unless otherwise indicated. */

include/sys/dmu_traverse.h

@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  */
 
 #ifndef	_SYS_DMU_TRAVERSE_H
@@ -71,6 +71,20 @@ int traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
 int traverse_pool(spa_t *spa,
     uint64_t txg_start, int flags, blkptr_cb_t func, void *arg);
 
+/*
+ * Note that this calculation cannot overflow with the current maximum indirect
+ * block size (128k).  If that maximum is increased to 1M, however, this
+ * calculation can overflow, and handling would need to be added to ensure
+ * continued correctness.
+ */
+static inline uint64_t
+bp_span_in_blocks(uint8_t indblkshift, uint64_t level)
+{
+	unsigned int shift = level * (indblkshift - SPA_BLKPTRSHIFT);
+	ASSERT3U(shift, <, 64);
+	return (1ULL << shift);
+}
+
 #ifdef	__cplusplus
 }
 #endif

include/sys/fs/zfs.h

@@ -245,6 +245,7 @@ typedef enum {
 	ZPOOL_PROP_CHECKPOINT,
 	ZPOOL_PROP_LOAD_GUID,
 	ZPOOL_PROP_AUTOTRIM,
+	ZPOOL_PROP_DEDUPCACHED,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 
@@ -1374,6 +1375,12 @@ typedef enum {
  */
 #define	ZPOOL_HIDDEN_ARGS	"hidden_args"
 
+/*
+ * The following are names used when invoking ZFS_IOC_POOL_GET_PROPS.
+ */
+#define	ZPOOL_GET_PROPS_NAMES		"get_props_names"
+#define	ZPOOL_GET_PROPS_EXCLUSIVE	"get_props_exclusive"
+
 /*
  * The following are names used when invoking ZFS_IOC_POOL_INITIALIZE.
  */

include/sys/spa.h

@@ -1155,6 +1155,8 @@ extern void spa_boot_init(void);
 /* properties */
 extern int spa_prop_set(spa_t *spa, nvlist_t *nvp);
 extern int spa_prop_get(spa_t *spa, nvlist_t **nvp);
+extern int spa_prop_get_nvlist(spa_t *spa, char **props,
+    unsigned int n_props, nvlist_t **outnvl);
 extern void spa_prop_clear_bootfs(spa_t *spa, uint64_t obj, dmu_tx_t *tx);
 extern void spa_configfile_set(spa_t *, nvlist_t *, boolean_t);
 

lib/libzfs/libzfs_pool.c

@@ -76,6 +76,14 @@ zpool_get_all_props(zpool_handle_t *zhp)
 
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 
+	if (zhp->zpool_n_propnames > 0) {
+		nvlist_t *innvl = fnvlist_alloc();
+		fnvlist_add_string_array(innvl, ZPOOL_GET_PROPS_NAMES,
+		    zhp->zpool_propnames, zhp->zpool_n_propnames);
+		if (zcmd_write_src_nvlist(hdl, &zc, innvl) != 0)
+			return (-1);
+	}
+
 	if (zcmd_alloc_dst_nvlist(hdl, &zc, 0) != 0)
 		return (-1);
 
@@ -4123,6 +4131,15 @@ zbookmark_mem_compare(const void *a, const void *b)
 	return (memcmp(a, b, sizeof (zbookmark_phys_t)));
 }
 
+void
+zpool_add_propname(zpool_handle_t *zhp, char *propname)
+{
+
+	assert(zhp->zpool_n_propnames < ZHP_MAX_PROPNAMES);
+	zhp->zpool_propnames[zhp->zpool_n_propnames] = propname;
+	zhp->zpool_n_propnames++;
+}
+
 /*
  * Retrieve the persistent error log, uniquify the members, and return to the
  * caller.