From 298f8f4426a0b52be63cbff2e36cff794c6e529a Mon Sep 17 00:00:00 2001 From: Steven Hartland Date: Thu, 14 Nov 2013 09:37:37 -0800 Subject: [PATCH] 1867: behlendorf/issue-1350 - Adds support for property overrides (WIP) Adds support for property overrides (-o property=value), property excludes (-x property) and dataset limits (-l ) to zfs receive. Both -o and -x options mirror the functionality already available in Oracle's ZFS implementation which is also mentioned in the upstream feature request #2745: The -l option allows receive to be limited to specific datasets within the stream effectively allowing partial restores from a multi dataset stream. References: https://www.illumos.org/issues/2745 Ported-by: Milan-Benes Issue #1350 NOTES: Man page updates missing and must be added, see patch: http://blog.multiplay.co.uk/dropzone/freebsd/zfs-recv-properties.patch --- cmd/zfs/zfs_main.c | 67 ++++++++- include/libzfs.h | 2 +- include/libzfs_impl.h | 2 + lib/libzfs/libzfs_dataset.c | 2 +- lib/libzfs/libzfs_sendrecv.c | 271 ++++++++++++++++++++++++++++++----- 5 files changed, 303 insertions(+), 41 deletions(-) diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index bfc9e1c62a4a..4672f76cf2c0 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -247,9 +247,9 @@ get_usage(zfs_help_t idx) case HELP_PROMOTE: return (gettext("\tpromote \n")); case HELP_RECEIVE: - return (gettext("\treceive [-vnFus] \n" - "\treceive [-vnFus] [-d | -e] \n")); + return (gettext("\treceive [-vnFus] [-d | -e] [-o ] " + "... [-x ] ... [-l ] ... " + " \n")); case HELP_RENAME: return (gettext("\trename [-f] " "\n" @@ -478,6 +478,21 @@ usage(boolean_t requested) exit(requested ? 0 : 2); } +static int +parsepropname(nvlist_t *props) +{ + char *propname = optarg; + + if (nvlist_lookup_string(props, propname, NULL) == 0) { + (void) fprintf(stderr, gettext("property '%s' " + "specified multiple times\n"), propname); + return (-1); + } + if (nvlist_add_boolean(props, propname)) + nomem(); + return (0); +} + static int parseprop(nvlist_t *props) { @@ -3853,18 +3868,24 @@ zfs_do_send(int argc, char **argv) } /* - * zfs receive [-vnFus] [-d | -e] + * zfs receive [-vnFus] [-d | -e] [-l ] ... + * [-o property=value] ... [-x property] ... * * Restore a backup stream from stdin. */ static int zfs_do_receive(int argc, char **argv) { + nvlist_t *props, *limitds; int c, err; recvflags_t flags = { 0 }; + if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0) + nomem(); + if (nvlist_alloc(&limitds, NV_UNIQUE_NAME, 0) != 0) + nomem(); /* check options */ - while ((c = getopt(argc, argv, ":denuvFs")) != -1) { + while ((c = getopt(argc, argv, ":del:no:uvx:Fs")) != -1) { switch (c) { case 's': flags.skip = B_TRUE; @@ -3876,15 +3897,32 @@ zfs_do_receive(int argc, char **argv) flags.isprefix = B_TRUE; flags.istail = B_TRUE; break; + case 'l': + if (parsepropname(limitds)) { + err = 1; + goto recverror; + } + break; case 'n': flags.dryrun = B_TRUE; break; + case 'o': + if (parseprop(props)) { + err = 1; + goto recverror; + } + break; case 'u': flags.nomount = B_TRUE; break; case 'v': flags.verbose = B_TRUE; break; + case 'x': + if (parsepropname(props)) { + err = 1; + goto recverror; + } case 'F': flags.force = B_TRUE; break; @@ -3921,7 +3959,24 @@ zfs_do_receive(int argc, char **argv) return (1); } - err = zfs_receive(g_zfs, argv[0], &flags, STDIN_FILENO, NULL); + if (nvlist_empty(props)) { + nvlist_free(props); + props = NULL; + } + + if (nvlist_empty(limitds)) { + nvlist_free(limitds); + limitds = NULL; + } + + err = zfs_receive(g_zfs, argv[0], &flags, STDIN_FILENO, props, limitds, NULL); + +recverror: + if (props != NULL) + nvlist_free(props); + + if (limitds != NULL) + nvlist_free(limitds); return (err != 0); } diff --git a/include/libzfs.h b/include/libzfs.h index 126ccb3b73f9..aa4d1bdf10bd 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -679,7 +679,7 @@ typedef struct recvflags { } recvflags_t; extern int zfs_receive(libzfs_handle_t *, const char *, recvflags_t *, - int, avl_tree_t *); + int, nvlist_t *, nvlist_t *, avl_tree_t *); typedef enum diff_flags { ZFS_DIFF_PARSEABLE = 0x1, diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index be132f1b905f..db351670e63f 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -189,6 +189,8 @@ int changelist_haszonedchild(prop_changelist_t *); void remove_mountpoint(zfs_handle_t *); int create_parents(libzfs_handle_t *, char *, int); +int check_parents(libzfs_handle_t *hdl, const char *path, uint64_t *zoned, + boolean_t accept_ancestor, int *prefixlen); boolean_t isa_child_of(const char *dataset, const char *parent); zfs_handle_t *make_dataset_handle(libzfs_handle_t *, const char *); diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index 3712d9872725..652beba1d05d 100644 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -2869,7 +2869,7 @@ parent_name(const char *path, char *buf, size_t buflen) * 'zoned' property, which is used to validate property settings when creating * new datasets. */ -static int +int check_parents(libzfs_handle_t *hdl, const char *path, uint64_t *zoned, boolean_t accept_ancestor, int *prefixlen) { diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 76aff504715c..658ab96d3c23 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -63,7 +63,8 @@ extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *); static int zfs_receive_impl(libzfs_handle_t *, const char *, recvflags_t *, - int, const char *, nvlist_t *, avl_tree_t *, char **, int, uint64_t *); + int, nvlist_t *, nvlist_t *, const char *, nvlist_t *, avl_tree_t *, + char **, int, uint64_t *); static const zio_cksum_t zero_cksum = { { 0 } }; @@ -2039,7 +2040,7 @@ created_before(libzfs_handle_t *hdl, avl_tree_t *avl, static int recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, recvflags_t *flags, nvlist_t *stream_nv, avl_tree_t *stream_avl, - nvlist_t *renamed) + nvlist_t *renamed, nvlist_t *limitds) { nvlist_t *local_nv, *deleted = NULL; avl_tree_t *local_avl; @@ -2091,6 +2092,12 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, &parent_fromsnap_guid)); (void) nvlist_lookup_uint64(nvfs, "origin", &originguid); + if (limitds && !nvlist_exists(limitds, fsname)) { + if (flags->verbose) + (void) printf("skipping replication of %s\n", fsname); + continue; + } + /* * First find the stream's fs, so we can check for * a different origin (due to "zfs promote") @@ -2347,8 +2354,9 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, static int zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, - recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc, - char **top_zfs, int cleanup_fd, uint64_t *action_handlep) + recvflags_t *flags, nvlist_t *props, nvlist_t *limitds, + dmu_replay_record_t *drr, zio_cksum_t *zc, char **top_zfs, int cleanup_fd, + uint64_t *action_handlep) { nvlist_t *stream_nv = NULL; avl_tree_t *stream_avl = NULL; @@ -2468,7 +2476,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, } softerr = recv_incremental_replication(hdl, tofs, flags, - stream_nv, stream_avl, renamed); + stream_nv, stream_avl, renamed, limitds); /* Unmount renamed filesystems before receiving. */ while ((pair = nvlist_next_nvpair(renamed, @@ -2514,8 +2522,8 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, * recv_skip() and return 0). */ error = zfs_receive_impl(hdl, destname, flags, fd, - sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd, - action_handlep); + props, limitds, sendfs, stream_nv, stream_avl, top_zfs, + cleanup_fd, action_handlep); if (error == ENODATA) { error = 0; break; @@ -2529,7 +2537,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, * renames again. */ softerr = recv_incremental_replication(hdl, tofs, flags, - stream_nv, stream_avl, NULL); + stream_nv, stream_avl, NULL, limitds); } out: @@ -2641,21 +2649,182 @@ recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap) return (-1); } +/* + * Calculate a list of properties for the current dataset taking into account + * its current properties (props) and the external (exprops) + */ +static int +props_override(char *dsname, nvlist_t *props, nvlist_t *exprops, + nvlist_t **npropsp, recvflags_t *flags, libzfs_handle_t *hdl, + zfs_type_t type, uint64_t zoned, zfs_handle_t *zhp, + const char *errbuf) +{ + nvlist_t *doprops, *goprops, *dxprops, *gxprops, *nprops, *vprops; + nvpair_t *pair; + char *strval; + int ret = 0; + + if (nvlist_empty(props) || nvlist_empty(exprops)) + return (0); /* No properties */ + + if (nvlist_dup(props, &nprops, 0) != 0) + return (-1); + + VERIFY(nvlist_alloc(&doprops, NV_UNIQUE_NAME, 0) == 0); + VERIFY(nvlist_alloc(&goprops, NV_UNIQUE_NAME, 0) == 0); + VERIFY(nvlist_alloc(&dxprops, NV_UNIQUE_NAME, 0) == 0); + VERIFY(nvlist_alloc(&gxprops, NV_UNIQUE_NAME, 0) == 0); + + /* build lists to process in order */ + pair = nvlist_next_nvpair(exprops, NULL); + while (pair != NULL) { + const char *propname = nvpair_name(pair); + char *tods = strchr(propname, '@'); + if (tods == NULL) { + switch(nvpair_type(pair)) + { + case DATA_TYPE_BOOLEAN: + (void) nvlist_add_nvpair(gxprops, pair); + break; + case DATA_TYPE_STRING: + (void) nvlist_add_nvpair(goprops, pair); + break; + default: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "'%s' must be a string or boolean"), + propname); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + ret = -1; + goto error; + } + } else if (strcmp(dsname, tods+1) == 0) { + /* dataset specific property */ + *tods = '\0'; + switch(nvpair_type(pair)) + { + case DATA_TYPE_BOOLEAN: + (void) nvlist_add_boolean(dxprops, propname); + break; + case DATA_TYPE_STRING: + nvpair_value_string(pair, &strval); + (void) nvlist_add_string(doprops, propname, + strval); + break; + default: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "'%s' must be a string or boolean"), + propname); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + ret = -1; + goto error; + } + *tods = '@'; + } + pair = nvlist_next_nvpair(exprops, pair); + } + /* convert override properties e.g. strings to native */ + if ((vprops = zfs_valid_proplist(hdl, type, goprops, zoned, zhp, + errbuf)) == NULL) + goto error; + + nvlist_free(goprops); + goprops = vprops; + + if ((vprops = zfs_valid_proplist(hdl, type, doprops, zoned, zhp, + errbuf)) == NULL) + goto error; + + nvlist_free(doprops); + doprops = vprops; + + /* global - override / set properties */ + pair = nvlist_next_nvpair(goprops, NULL); + while (pair != NULL) { + const char *pname = nvpair_name(pair); + if (!nvlist_exists(gxprops, pname) && + !nvlist_exists(dxprops, pname)) { + if (flags->verbose) + (void) printf("%s %s property from %s\n", + nvlist_exists(nprops, pname) ? + "overriding" : "setting", pname, dsname); + (void) nvlist_add_nvpair(nprops, pair); + } + pair = nvlist_next_nvpair(goprops, pair); + } + + /* global - exclude properties */ + pair = nvlist_next_nvpair(gxprops, NULL); + while (pair != NULL) { + const char *pname = nvpair_name(pair); + if (!nvlist_exists(doprops, pname)) { + if (flags->verbose && nvlist_exists(nprops, pname)) + (void) printf("excluding %s property from %s\n", + pname, dsname); + + (void) nvlist_remove_all(nprops, pname); + } + pair = nvlist_next_nvpair(gxprops, pair); + } + + /* dataset - override / set properties */ + pair = nvlist_next_nvpair(doprops, NULL); + while (pair != NULL) { + const char *pname = nvpair_name(pair); + if (!nvlist_exists(dxprops, pname)) { + if (flags->verbose) + (void) printf("%s %s property from %s\n", + nvlist_exists(nprops, pname) ? + "overriding" : "setting", pname, dsname); + (void) nvlist_add_nvpair(nprops, pair); + } + + pair = nvlist_next_nvpair(doprops, pair); + + } + + /* dataset - exclude properties */ + pair = nvlist_next_nvpair(dxprops, NULL); + while (pair != NULL) { + const char *pname = nvpair_name(pair); + if (nvlist_exists(nprops, pname)) { + if (flags->verbose) + (void) printf("excluding %s property from %s\n", + pname, dsname); + + (void) nvlist_remove_all(nprops, pname); + } + + pair = nvlist_next_nvpair(dxprops, pair); + } + + *npropsp = nprops; + +error: + if (0 != ret) + nvlist_free(nprops); + nvlist_free(goprops); + nvlist_free(gxprops); + nvlist_free(doprops); + nvlist_free(dxprops); + return (ret); +} + /* * Restores a backup of tosnap from the file descriptor specified by infd. */ static int zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, - recvflags_t *flags, dmu_replay_record_t *drr, - dmu_replay_record_t *drr_noswap, const char *sendfs, - nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep) + recvflags_t *flags, nvlist_t *exprops, nvlist_t *limitds, + dmu_replay_record_t *drr, dmu_replay_record_t *drr_noswap, + const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl, + char **top_zfs, int cleanup_fd, uint64_t *action_handlep) { zfs_cmd_t zc = {"\0"}; time_t begin_time; - int ioctl_err, ioctl_errno, err; + int ioctl_err, ioctl_errno, err, skip; char *cp; struct drr_begin *drrb = &drr->drr_u.drr_begin; + char dsname[ZFS_MAXNAMELEN]; char errbuf[1024]; char prop_errbuf[1024]; const char *chopprefix; @@ -2664,8 +2833,10 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, uint64_t parent_snapguid = 0; prop_changelist_t *clp = NULL; nvlist_t *snapprops_nvlist = NULL; + nvlist_t *props = NULL; + nvlist_t *nprops = NULL; zprop_errflags_t prop_errflags; - boolean_t recursive; + boolean_t recursive, has_exprops; begin_time = time(NULL); @@ -2677,10 +2848,9 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (stream_avl != NULL) { char *snapname; + nvlist_t *snapprops; nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid, &snapname); - nvlist_t *props; - int ret; (void) nvlist_lookup_uint64(fs, "parentfromsnap", &parent_snapguid); @@ -2692,17 +2862,16 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, VERIFY(0 == nvlist_add_uint64(props, zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0)); } - ret = zcmd_write_src_nvlist(hdl, &zc, props); - if (err) + + if (err) { nvlist_free(props); + props = NULL; + } - if (0 == nvlist_lookup_nvlist(fs, "snapprops", &props)) { - VERIFY(0 == nvlist_lookup_nvlist(props, + if (0 == nvlist_lookup_nvlist(fs, "snapprops", &snapprops)) { + VERIFY(0 == nvlist_lookup_nvlist(snapprops, snapname, &snapprops_nvlist)); } - - if (ret != 0) - return (-1); } cp = NULL; @@ -2863,6 +3032,11 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, (void) strcpy(zc.zc_name, zc.zc_value); *strchr(zc.zc_name, '@') = '\0'; + (void) strcpy(dsname, drrb->drr_toname); + *strchr(dsname, '@') = '\0'; + + has_exprops = !nvlist_empty(exprops); + if (zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) { zfs_handle_t *zhp; @@ -2957,6 +3131,23 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, } newfs = B_TRUE; + if (has_exprops) { + /* Create an override set of properties if needed */ + uint64_t zoned = 0; + if (flags->isprefix && !flags->istail && !flags->dryrun) { + /* Check if we're zoned or not */ + if (check_parents(hdl, zc.zc_value, &zoned, B_FALSE, NULL) != 0) { + zcmd_free_nvlists(&zc); + return (-1); + } + } + + if (props_override(dsname, props, exprops, &nprops, flags, + hdl, ZFS_TYPE_DATASET, zoned, NULL, errbuf) != 0) { + zcmd_free_nvlists(&zc); + return (-1); + } + } } zc.zc_begin_record = drr_noswap->drr_u.drr_begin; @@ -2973,15 +3164,18 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, return (recv_skip(hdl, infd, flags->byteswap)); } + skip = limitds && !nvlist_exists(limitds, dsname); + if (flags->verbose) { (void) printf("%s %s stream of %s into %s\n", - flags->dryrun ? "would receive" : "receiving", + skip ? (flags->dryrun ? "would skip" : "skipping") : + (flags->dryrun ? "would receive" : "receiving"), drrb->drr_fromguid ? "incremental" : "full", drrb->drr_toname, zc.zc_value); (void) fflush(stdout); } - if (flags->dryrun) { + if (flags->dryrun || skip) { zcmd_free_nvlists(&zc); return (recv_skip(hdl, infd, flags->byteswap)); } @@ -2991,6 +3185,15 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, zc.zc_cleanup_fd = cleanup_fd; zc.zc_action_handle = *action_handlep; + if (nprops) { + if (zcmd_write_src_nvlist(hdl, &zc, nprops) != 0) { + nvlist_free(nprops); + return (-1); + } + nvlist_free(nprops); + } else if (props && zcmd_write_src_nvlist(hdl, &zc, props) != 0) + return (-1); + err = ioctl_err = zfs_ioctl(hdl, ZFS_IOC_RECV, &zc); ioctl_errno = errno; prop_errflags = (zprop_errflags_t)zc.zc_obj; @@ -3203,8 +3406,9 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, static int zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t *flags, - int infd, const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl, - char **top_zfs, int cleanup_fd, uint64_t *action_handlep) + int infd, nvlist_t *props, nvlist_t *limitds, const char *sendfs, + nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, + uint64_t *action_handlep) { int err; dmu_replay_record_t drr, drr_noswap; @@ -3296,13 +3500,14 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t *flags, sendfs = nonpackage_sendfs; } return (zfs_receive_one(hdl, infd, tosnap, flags, - &drr, &drr_noswap, sendfs, stream_nv, stream_avl, - top_zfs, cleanup_fd, action_handlep)); + props, limitds, &drr, &drr_noswap, sendfs, stream_nv, + stream_avl, top_zfs, cleanup_fd, action_handlep)); } else { assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_COMPOUNDSTREAM); return (zfs_receive_package(hdl, infd, tosnap, flags, - &drr, &zcksum, top_zfs, cleanup_fd, action_handlep)); + props, limitds, &drr, &zcksum, top_zfs, cleanup_fd, + action_handlep)); } } @@ -3314,7 +3519,7 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t *flags, */ int zfs_receive(libzfs_handle_t *hdl, const char *tosnap, recvflags_t *flags, - int infd, avl_tree_t *stream_avl) + int infd, nvlist_t *props, nvlist_t *limitds, avl_tree_t *stream_avl) { char *top_zfs = NULL; int err; @@ -3324,8 +3529,8 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, recvflags_t *flags, cleanup_fd = open(ZFS_DEV, O_RDWR); VERIFY(cleanup_fd >= 0); - err = zfs_receive_impl(hdl, tosnap, flags, infd, NULL, NULL, - stream_avl, &top_zfs, cleanup_fd, &action_handle); + err = zfs_receive_impl(hdl, tosnap, flags, infd, props, limitds, NULL, + NULL, stream_avl, &top_zfs, cleanup_fd, &action_handle); VERIFY(0 == close(cleanup_fd));