Illumos 2605 want to resume interrupted zfs send

Reviewed by: George Wilson <[email protected]> Reviewed by: Paul Dagnelie <[email protected]> Reviewed by: Richard Elling <[email protected]> Reviewed by: Xin Li <[email protected]> Reviewed by: Arne Jansen <[email protected]> Approved by: Dan McDonald <[email protected]> References: https://www.illumos.org/issues/2605 illumos/illumos-gate@9c3fd12 Porting notes: The tests from Illumos in usr/src/test/zfs-tests were left out Further the following files which don't exist in ZoL: usr/src/cmd/truss/expound.c usr/src/lib/libzfs/common/mapfile-vers usr/src/lib/libzfs_core/common/mapfile-vers usr/src/pkg/manifests/system-test-zfstest.mf From a readability point of view the changes in [module/zfs/dmu_traverse.c] from ecfb0b5 Fix misuse of input argument in traverse_visitbp appear to be the optimum - also there's no functional change, when going with upstream the compiler would likely complain about ISO C90 errors - so only switching cdnp = buf->b_data; to child_dnp = buf->b_data; diverged code base from Illumos: [cmd/zstreamdump/zstreamdump.c] 37f8a88 Illumos 5746 - more checksumming in zfs send [include/sys/zfs_ioctl.h] 0cee240 Speed up 'zfs list -t snapshot -o name -s name' [module/zfs/zfs_ioctl.c] 13fe019 Illumos 3464 zfs synctask code needs restructuring [module/zfs/dmu_send.c] b58986e Use large stacks when available 044baf0 Use taskq for dump_bytes() ISO C90 - mixed declarations and code: struct send_block_record *to_data; struct receive_ign_obj_node *n; account for 'missing braces around initializer' in 'struct send_thread_arg to_arg =' ISO C90 - mixed declarations and code: void *payload = NULL; nvlist_t *nvl = fnvlist_alloc(); uint64_t one = 1; uint64_t zero = 1; char recvname[ZFS_MAXNAMELEN]; uint64_t val; uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen; size_t payload_len = 0; [module/zfs/dsl_dataset.c] Illumos 4185 add new cryptographic checksums to ZFS: SHA-512, Skein, Edon-R hasn't landed yet, thus ' #include <sys/zio_checksum.h> is still missing, ' #include <sys/dmu_send.h> however is needed further replaced fletcher_4_native(compressed, compressed_size, NULL, &cksum); with fletcher_4_native(compressed, compressed_size, &cksum); ctx_template argument was removed (purpose: the template will be reused to make initialization more efficient) C99 or C11 mode: declare 'int i' at the beginning of the function. ISO C90 - mixed declarations and code: char buf[256]; zio_cksum_t cksum; char *propval = kmem_asprintf("%u-%llx-%llx-%s", char recvname[ZFS_MAXNAMELEN]; dsl_dataset_t *recv_ds; [libzfs_sendrecv.c] ' #include <sha2.h> isn't listed in the header list since sha2, sha256 is implemented differently in ZoL. replace fletcher_4_native(compressed, len, NULL, &cksum); with fletcher_4_native(compressed, len, &cksum); declare 'int nread, i;' at the beginning of the function b8864a2 Fix gcc cast warnings [module/zcommon/zfs_prop.c] 11b9ec2 Add full SELinux support [module/zfs/dmu_traverse.c] ecfb0b5 Fix misuse of input argument in traverse_visitbp 79c76d5 Change KM_PUSHPAGE -> KM_SLEEP a168788 Reduce stack for traverse_visitbp() recursion b8d06fc Switch KM_SLEEP to KM_PUSHPAGE 47050a8 Fix stack traverse_impl() ISO C90: place 'uint64_t obj, error;' at the beginning, further replace 'err' with 'error' [man/man8/zfs.8] Part of the sections for 'zfs receive' and 'zfs send' was rewritten and reordered to approximate upstream. Ported-by: kernelOfTruth [email protected]
openzfs · Jan 14, 2016 · 8d0c7dd · 8d0c7dd
1 parent d21f279
commit 8d0c7dd
Show file tree

Hide file tree

Showing 22 changed files with 1,401 additions and 267 deletions.
diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c
@@ -248,10 +248,11 @@ get_usage(zfs_help_t idx)
 	case HELP_PROMOTE:
 		return (gettext("\tpromote <clone-filesystem>\n"));
 	case HELP_RECEIVE:
-		return (gettext("\treceive [-vnFu] <filesystem|volume|"
+		return (gettext("\treceive [-vnsFu] <filesystem|volume|"
 		    "snapshot>\n"
-		    "\treceive [-vnFu] [-o origin=<snapshot>] [-d | -e] "
-		    "<filesystem>\n"));
+		    "\treceive [-vnsFu] [-o origin=<snapshot>] [-d | -e] "
+		    "<filesystem>\n"
+		    "\treceive -A <filesystem|volume>\n"));
 	case HELP_RENAME:
 		return (gettext("\trename [-f] <filesystem|volume|snapshot> "
 		    "<filesystem|volume|snapshot>\n"
@@ -263,7 +264,8 @@ get_usage(zfs_help_t idx)
 		return (gettext("\tsend [-DnPpRvLe] [-[iI] snapshot] "
 		    "<snapshot>\n"
 		    "\tsend [-Le] [-i snapshot|bookmark] "
-		    "<filesystem|volume|snapshot>\n"));
+		    "<filesystem|volume|snapshot>\n"
+		    "\tsend [-nvPe] -t <receive_resume_token>\n"));
 	case HELP_SET:
 		return (gettext("\tset <property=value> ... "
 		    "<filesystem|volume|snapshot> ...\n"));
@@ -3687,6 +3689,7 @@ zfs_do_send(int argc, char **argv)
 {
 	char *fromname = NULL;
 	char *toname = NULL;
+	char *resume_token = NULL;
 	char *cp;
 	zfs_handle_t *zhp;
 	sendflags_t flags = { 0 };
@@ -3695,7 +3698,7 @@ zfs_do_send(int argc, char **argv)
 	boolean_t extraverbose = B_FALSE;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":i:I:RDpvnPLe")) != -1) {
+	while ((c = getopt(argc, argv, ":i:I:RDpvnPLet:")) != -1) {
 		switch (c) {
 		case 'i':
 			if (fromname)
@@ -3736,6 +3739,9 @@ zfs_do_send(int argc, char **argv)
 		case 'e':
 			flags.embed_data = B_TRUE;
 			break;
+		case 't':
+			resume_token = optarg;
+			break;
 		case ':':
 			(void) fprintf(stderr, gettext("missing argument for "
 			    "'%c' option\n"), optopt);
@@ -3751,14 +3757,28 @@ zfs_do_send(int argc, char **argv)
 	argc -= optind;
 	argv += optind;
 
-	/* check number of arguments */
-	if (argc < 1) {
-		(void) fprintf(stderr, gettext("missing snapshot argument\n"));
-		usage(B_FALSE);
-	}
-	if (argc > 1) {
-		(void) fprintf(stderr, gettext("too many arguments\n"));
-		usage(B_FALSE);
+	if (resume_token != NULL) {
+		if (fromname != NULL || flags.replicate || flags.props ||
+		    flags.dedup) {
+			(void) fprintf(stderr,
+			    gettext("invalid flags combined with -t\n"));
+			usage(B_FALSE);
+		}
+		if (argc != 0) {
+			(void) fprintf(stderr, gettext("no additional "
+			    "arguments are permitted with -t\n"));
+			usage(B_FALSE);
+		}
+	} else {
+		if (argc < 1) {
+			(void) fprintf(stderr,
+			    gettext("missing snapshot argument\n"));
+			usage(B_FALSE);
+		}
+		if (argc > 1) {
+			(void) fprintf(stderr, gettext("too many arguments\n"));
+			usage(B_FALSE);
+		}
 	}
 
 	if (!flags.dryrun && isatty(STDOUT_FILENO)) {
@@ -3768,6 +3788,11 @@ zfs_do_send(int argc, char **argv)
 		return (1);
 	}
 
+	if (resume_token != NULL) {
+		return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
+		    resume_token));
+	}
+
 	/*
 	 * Special case sending a filesystem, or from a bookmark.
 	 */
@@ -3873,23 +3898,23 @@ zfs_do_send(int argc, char **argv)
 }
 
 /*
- * zfs receive [-vnFu] [-d | -e] <fs@snap>
- *
  * Restore a backup stream from stdin.
  */
 static int
 zfs_do_receive(int argc, char **argv)
 {
 	int c, err;
 	recvflags_t flags = { 0 };
+	boolean_t abort_resumable = B_FALSE;
+
 	nvlist_t *props;
 	nvpair_t *nvp = NULL;
 
 	if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
 		nomem();
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":o:denuvF")) != -1) {
+	while ((c = getopt(argc, argv, ":o:denuvFsA")) != -1) {
 		switch (c) {
 		case 'o':
 			if (parseprop(props, optarg) != 0)
@@ -3911,9 +3936,15 @@ zfs_do_receive(int argc, char **argv)
 		case 'v':
 			flags.verbose = B_TRUE;
 			break;
+		case 's':
+			flags.resumable = B_TRUE;
+			break;
 		case 'F':
 			flags.force = B_TRUE;
 			break;
+		case 'A':
+			abort_resumable = B_TRUE;
+			break;
 		case ':':
 			(void) fprintf(stderr, gettext("missing argument for "
 			    "'%c' option\n"), optopt);
@@ -3946,14 +3977,51 @@ zfs_do_receive(int argc, char **argv)
 		}
 	}
 
+	if (abort_resumable) {
+		if (flags.isprefix || flags.istail || flags.dryrun ||
+		    flags.resumable || flags.nomount) {
+			(void) fprintf(stderr, gettext("invalid option"));
+			usage(B_FALSE);
+		}
+
+		char namebuf[ZFS_MAXNAMELEN];
+		(void) snprintf(namebuf, sizeof (namebuf),
+		    "%s/%%recv", argv[0]);
+
+		if (zfs_dataset_exists(g_zfs, namebuf,
+		    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) {
+			zfs_handle_t *zhp = zfs_open(g_zfs,
+			    namebuf, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+			if (zhp == NULL)
+				return (1);
+			err = zfs_destroy(zhp, B_FALSE);
+		} else {
+			zfs_handle_t *zhp = zfs_open(g_zfs,
+			    argv[0], ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+			if (zhp == NULL)
+				usage(B_FALSE);
+			if (!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) ||
+			    zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+			    NULL, 0, NULL, NULL, 0, B_TRUE) == -1) {
+				(void) fprintf(stderr,
+				    gettext("'%s' does not have any "
+				    "resumable receive state to abort\n"),
+				    argv[0]);
+				return (1);
+			}
+			err = zfs_destroy(zhp, B_FALSE);
+		}
+
+		return (err != 0);
+	}
+
 	if (isatty(STDIN_FILENO)) {
 		(void) fprintf(stderr,
 		    gettext("Error: Backup stream can not be read "
 		    "from a terminal.\n"
 		    "You must redirect standard input.\n"));
 		return (1);
 	}
-
 	err = zfs_receive(g_zfs, argv[0], props, &flags, STDIN_FILENO, NULL);
 
 	return (err != 0);
@@ -5784,6 +5852,24 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
 		return (0);
 	}
 
+	/*
+	 * If this filesystem is inconsistent and has a receive resume
+	 * token, we can not mount it.
+	 */
+	if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
+	    zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+	    NULL, 0, NULL, NULL, 0, B_TRUE) == 0) {
+		if (!explicit)
+			return (0);
+
+		(void) fprintf(stderr, gettext("cannot %s '%s': "
+		    "Contains partially-completed state from "
+		    "\"zfs receive -r\", which can be resumed with "
+		    "\"zfs send -t\"\n"),
+		    cmdname, zfs_get_name(zhp));
+		return (1);
+	}
+
 	/*
 	 * At this point, we have verified that the mountpoint and/or
 	 * shareopts are appropriate for auto management. If the

diff --git a/cmd/zstreamdump/zstreamdump.c b/cmd/zstreamdump/zstreamdump.c
@@ -127,7 +127,7 @@ read_hdr(dmu_replay_record_t *drr, zio_cksum_t *cksum)
 		    (longlong_t)saved_cksum.zc_word[1],
 		    (longlong_t)saved_cksum.zc_word[2],
 		    (longlong_t)saved_cksum.zc_word[3]);
-		exit(1);
+		return (0);
 	}
 	return (sizeof (*drr));
 }
@@ -347,8 +347,7 @@ main(int argc, char *argv[])
 			if (verbose)
 				(void) printf("\n");
 
-			if ((DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
-			    DMU_COMPOUNDSTREAM) && drr->drr_payloadlen != 0) {
+			if (drr->drr_payloadlen != 0) {
 				nvlist_t *nv;
 				int sz = drr->drr_payloadlen;
 

diff --git a/include/libzfs.h b/include/libzfs.h
@@ -631,6 +631,10 @@ typedef boolean_t (snapfilter_cb_t)(zfs_handle_t *, void *);
 extern int zfs_send(zfs_handle_t *, const char *, const char *,
     sendflags_t *, int, snapfilter_cb_t, void *, nvlist_t **);
 extern int zfs_send_one(zfs_handle_t *, const char *, int, enum lzc_send_flags);
+extern int zfs_send_resume(libzfs_handle_t *, sendflags_t *, int outfd,
+    const char *);
+extern nvlist_t *zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl,
+    const char *token);
 
 extern int zfs_promote(zfs_handle_t *);
 extern int zfs_hold(zfs_handle_t *, const char *, const char *,
@@ -671,6 +675,12 @@ typedef struct recvflags {
 	/* set "canmount=off" on all modified filesystems */
 	boolean_t canmountoff;
 
+	/*
+	 * Mark the file systems as "resumable" and do not destroy them if the
+	 * receive is interrupted
+	 */
+	boolean_t resumable;
+
 	/* byteswap flag is used internally; callers need not specify */
 	boolean_t byteswap;
 

diff --git a/include/libzfs_core.h b/include/libzfs_core.h
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef	_LIBZFS_CORE_H
@@ -58,7 +58,11 @@ enum lzc_send_flags {
 };
 
 int lzc_send(const char *, const char *, int, enum lzc_send_flags);
+int lzc_send_resume(const char *, const char *, int,
+    enum lzc_send_flags, uint64_t, uint64_t);
 int lzc_receive(const char *, nvlist_t *, const char *, boolean_t, int);
+int lzc_receive_resumable(const char *, nvlist_t *, const char *,
+    boolean_t, int);
 int lzc_send_space(const char *, const char *, uint64_t *);
 
 boolean_t lzc_exists(const char *);

diff --git a/include/sys/dmu_impl.h b/include/sys/dmu_impl.h
@@ -24,7 +24,7 @@
  */
 /*
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_DMU_IMPL_H
@@ -272,6 +272,8 @@ typedef struct dmu_sendarg {
 	uint64_t dsa_featureflags;
 	uint64_t dsa_last_data_object;
 	uint64_t dsa_last_data_offset;
+	uint64_t dsa_resume_object;
+	uint64_t dsa_resume_offset;
 } dmu_sendarg_t;
 
 void dmu_object_zapify(objset_t *, uint64_t, dmu_object_type_t, dmu_tx_t *);

diff --git a/include/sys/dmu_send.h b/include/sys/dmu_send.h
@@ -36,10 +36,13 @@ struct vnode;
 struct dsl_dataset;
 struct drr_begin;
 struct avl_tree;
+struct dmu_replay_record;
 
-int dmu_send(const char *tosnap, const char *fromsnap,
-    boolean_t embedok, boolean_t large_block_ok,
-    int outfd, struct vnode *vp, offset_t *off);
+extern const char *recv_clone_name;
+
+int dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
+    boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff,
+    struct vnode *vp, offset_t *off);
 int dmu_send_estimate(struct dsl_dataset *ds, struct dsl_dataset *fromds,
     uint64_t *sizep);
 int dmu_send_estimate_from_txg(struct dsl_dataset *ds, uint64_t fromtxg,
@@ -50,21 +53,24 @@ int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
 
 typedef struct dmu_recv_cookie {
 	struct dsl_dataset *drc_ds;
+	struct dmu_replay_record *drc_drr_begin;
 	struct drr_begin *drc_drrb;
 	const char *drc_tofs;
 	const char *drc_tosnap;
 	boolean_t drc_newfs;
 	boolean_t drc_byteswap;
 	boolean_t drc_force;
+	boolean_t drc_resumable;
 	struct avl_tree *drc_guid_to_ds_map;
 	zio_cksum_t drc_cksum;
 	uint64_t drc_newsnapobj;
 	void *drc_owner;
 	cred_t *drc_cred;
 } dmu_recv_cookie_t;
 
-int dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
-    boolean_t force, char *origin, dmu_recv_cookie_t *drc);
+int dmu_recv_begin(char *tofs, char *tosnap,
+    struct dmu_replay_record *drr_begin,
+    boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc);
 int dmu_recv_stream(dmu_recv_cookie_t *drc, struct vnode *vp, offset_t *voffp,
     int cleanup_fd, uint64_t *action_handlep);
 int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner);

diff --git a/include/sys/dmu_traverse.h b/include/sys/dmu_traverse.h
@@ -54,6 +54,8 @@ typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 
 int traverse_dataset(struct dsl_dataset *ds,
     uint64_t txg_start, int flags, blkptr_cb_t func, void *arg);
+int traverse_dataset_resume(struct dsl_dataset *ds, uint64_t txg_start,
+    zbookmark_phys_t *resume, int flags, blkptr_cb_t func, void *arg);
 int traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
     uint64_t txg_start, zbookmark_phys_t *resume, int flags,
     blkptr_cb_t func, void *arg);