diff --git a/.travis.yml b/.travis.yml index 33112ea64643..db585eded680 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ before_install: # packages for tests - sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio - sudo apt-get install --yes -qq libgtest-dev cmake + # packages for debugging + - sudo apt-get install gdb # use gcc-6 by default - sudo unlink /usr/bin/gcc && sudo ln -s /usr/bin/gcc-6 /usr/bin/gcc - sudo unlink /usr/bin/g++ && sudo ln -s /usr/bin/g++-6 /usr/bin/g++ @@ -68,7 +70,6 @@ script: # run ztest and test supported zio backends # XXX enable the ztest when ticket #152 is fixed - if [ $ZFS_BUILD_TAGS = 0 ]; then - sudo mkdir /etc/zfs; export FIO_SRCDIR=$PWD/../fio; travis_wait 60 sudo bash ./tests/cbtest/script/test_uzfs.sh -T all || travis_terminate 1; else diff --git a/cmd/uzfs_test/uzfs_test.c b/cmd/uzfs_test/uzfs_test.c index 3057cb8ceb83..c5e50fba3298 100644 --- a/cmd/uzfs_test/uzfs_test.c +++ b/cmd/uzfs_test/uzfs_test.c @@ -654,7 +654,7 @@ open_pool(spa_t **spa) } void -open_ds(spa_t *spa, zvol_state_t **zv) +open_ds(spa_t *spa, char *ds, zvol_state_t **zv) { int err; err = uzfs_open_dataset(spa, ds, zv); @@ -662,6 +662,7 @@ open_ds(spa_t *spa, zvol_state_t **zv) printf("ds open errored.. %d\n", err); exit(1); } + uzfs_hold_dataset(*zv); } void @@ -671,14 +672,12 @@ unit_test_fn(void *arg) zvol_state_t *zv; kthread_t *reader1; kthread_t *writer[3]; - char name[MAXNAMELEN]; int i; kmutex_t mtx; kcondvar_t cv; int threads_done = 0; int num_threads = 0; uint64_t total_ios = 0; - zvol_info_t *zinfo = NULL; worker_args_t reader1_args; worker_args_t writer_args[3]; @@ -691,12 +690,7 @@ unit_test_fn(void *arg) } open_pool(&spa); - if (create == 1) { - open_ds(spa, &zv); - } else { - zinfo = uzfs_zinfo_lookup(ds); - zv = zinfo->zv; - } + open_ds(spa, ds, &zv); if (uzfs_test_id == 2) { reader1_args.zv = zv; @@ -741,16 +735,8 @@ unit_test_fn(void *arg) cv_destroy(&cv); mutex_destroy(&mtx); - - if (create == 1) { - uzfs_close_dataset(zv); - uzfs_close_pool(spa); - } else { - strlcpy(name, zinfo->name, MAXNAMELEN); - uzfs_zinfo_drop_refcnt(zinfo, 0); - uzfs_zinfo_destroy(name, NULL); - uzfs_close_pool(spa); - } + uzfs_close_dataset(zv); + uzfs_close_pool(spa); } void diff --git a/cmd/uzfs_test/uzfs_test_rebuilding.c b/cmd/uzfs_test/uzfs_test_rebuilding.c index 16638db2596b..56e67944dc38 100644 --- a/cmd/uzfs_test/uzfs_test_rebuilding.c +++ b/cmd/uzfs_test/uzfs_test_rebuilding.c @@ -534,8 +534,7 @@ replica_writer_thread(void *arg) } static void -open_pool_and_dataset(spa_t **spa, zvol_info_t **zinfo, char *pool_name, - char *ds_name) +open_pool_and_dataset(spa_t **spa, char *pool_name, char *ds_name) { int err; @@ -545,23 +544,12 @@ open_pool_and_dataset(spa_t **spa, zvol_info_t **zinfo, char *pool_name, printf("pool(%s) open errored.. %d\n", pool_name, err); exit(1); } - - *zinfo = uzfs_zinfo_lookup(ds_name); - if (*zinfo == NULL) { - uzfs_close_pool(*spa); - printf("failed to lookup dataset(%s)\n", ds_name); - exit(1); - } } static void -close_pool_and_dataset(spa_t *spa, zvol_info_t *zinfo) +close_pool_and_dataset(spa_t *spa, zvol_state_t *zvol) { - char name[MAXNAMELEN]; - - strlcpy(name, zinfo->name, MAXNAMELEN); - uzfs_zinfo_drop_refcnt(zinfo, 0); - uzfs_zinfo_destroy(name, NULL); + uzfs_close_dataset(zvol); uzfs_close_pool(spa); } @@ -579,7 +567,6 @@ uzfs_rebuild_test(void *arg) char *pooldup = strdup(pool); char *dsdup = strdup(ds); char *pool1, *pool2, *ds1, *ds2; - zvol_info_t *zinfo1, *zinfo2; printf("starting %s\n", test_info->name); pool1 = strtok(pooldup, ","); @@ -589,11 +576,11 @@ uzfs_rebuild_test(void *arg) if (!ds2) ds2 = ds1; - open_pool_and_dataset(&spa1, &zinfo1, pool1, ds1); - open_pool_and_dataset(&spa2, &zinfo2, pool2, ds2); + open_pool_and_dataset(&spa1, pool1, ds1); + open_pool_and_dataset(&spa2, pool2, ds2); - zvol1 = zinfo1->zv; - zvol2 = zinfo2->zv; + open_ds(spa1, ds1, &zvol1); + open_ds(spa2, ds2, &zvol2); while (n++ < test_iterations) { mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); @@ -671,8 +658,8 @@ uzfs_rebuild_test(void *arg) printf("%s pass:%d\n", test_info->name, n); } - close_pool_and_dataset(spa1, zinfo1); - close_pool_and_dataset(spa2, zinfo2); + close_pool_and_dataset(spa1, zvol1); + close_pool_and_dataset(spa2, zvol2); free(pooldup); free(dsdup); } diff --git a/cmd/uzfs_test/uzfs_test_sync.c b/cmd/uzfs_test/uzfs_test_sync.c index 3f8eabcd6c8f..8951c832171f 100644 --- a/cmd/uzfs_test/uzfs_test_sync.c +++ b/cmd/uzfs_test/uzfs_test_sync.c @@ -151,8 +151,6 @@ replay_fn(void *arg) { spa_t *spa; zvol_state_t *zv; - char name[MAXNAMELEN]; - zvol_info_t *zinfo = NULL; zfs_txg_timeout = 30; @@ -163,20 +161,10 @@ replay_fn(void *arg) } open_pool(&spa); - if (create == 1) - open_ds(spa, &zv); - else { - zinfo = uzfs_zinfo_lookup(ds); - zv = zinfo->zv; - } + open_ds(spa, ds, &zv); } else if (verify != 0) { open_pool(&spa); - if (create == 1) { - open_ds(spa, &zv); - } else { - zinfo = uzfs_zinfo_lookup(ds); - zv = zinfo->zv; - } + open_ds(spa, ds, &zv); } else { printf("exiting program..\n"); uzfs_fini(); @@ -188,16 +176,8 @@ replay_fn(void *arg) if (verify != 0) if (silent == 0) printf("verify error: %d\n", verify_err); - if (create == 1) { - uzfs_close_dataset(zv); - uzfs_close_pool(spa); - } else { - strlcpy(name, zinfo->name, MAXNAMELEN); - uzfs_zinfo_drop_refcnt(zinfo, 0); - uzfs_zinfo_destroy(name, NULL); - uzfs_close_pool(spa); - } - + uzfs_close_dataset(zv); + uzfs_close_pool(spa); if (verify_err) exit(verify_err); } diff --git a/cmd/uzfs_test/uzfs_zvol_zap.c b/cmd/uzfs_test/uzfs_zvol_zap.c index 86a527a9785b..7f1e59696434 100644 --- a/cmd/uzfs_test/uzfs_zvol_zap.c +++ b/cmd/uzfs_test/uzfs_zvol_zap.c @@ -115,7 +115,6 @@ uzfs_zvol_zap_operation(void *arg) { uzfs_test_info_t *test_info = (uzfs_test_info_t *)arg; int i = 0; - char name[MAXNAMELEN]; hrtime_t end, now; spa_t *spa; zvol_state_t *zvol; @@ -124,11 +123,9 @@ uzfs_zvol_zap_operation(void *arg) uint64_t txg1, txg2, txg3, txg4; struct timespec ts; int err1, err2; - zvol_info_t *zinfo = NULL; open_pool(&spa); - zinfo = uzfs_zinfo_lookup(ds); - zvol = zinfo->zv; + open_ds(spa, ds, &zvol); if (!zvol) { printf("couldn't find zvol\n"); uzfs_close_pool(spa); @@ -227,8 +224,6 @@ uzfs_zvol_zap_operation(void *arg) break; } - strlcpy(name, zinfo->name, MAXNAMELEN); - uzfs_zinfo_drop_refcnt(zinfo, 0); - uzfs_zinfo_destroy(name, NULL); + uzfs_close_dataset(zvol); uzfs_close_pool(spa); } diff --git a/cmd/uzfs_test/zrepl_utest.c b/cmd/uzfs_test/zrepl_utest.c index 36145467b240..4bbe966ce9a3 100644 --- a/cmd/uzfs_test/zrepl_utest.c +++ b/cmd/uzfs_test/zrepl_utest.c @@ -15,6 +15,9 @@ #include char *tgt_port = "6060"; +char *tgt_port1 = "99159"; +char *tgt_port2 = "99160"; +char *tgt_port3 = "99161"; char *ds1 = "ds1"; char *ds2 = "ds2"; char *ds3 = "ds3"; @@ -362,7 +365,6 @@ reader_thread(void *arg) static void writer_thread(void *arg) { - int i = 0; int sfd, sfd1; int count = 0; @@ -744,6 +746,40 @@ zrepl_utest(void *arg) } } +int +create_bind_listen_and_accept(const char *port, int bind_needed, + boolean_t nonblock) +{ + int sfd, rc, mgmt_fd; + socklen_t in_len; + struct sockaddr in_addr; + + sfd = rc = mgmt_fd = -1; + + sfd = create_and_bind(port, B_TRUE, B_FALSE); + if (sfd == -1) { + return (-1); + } + + rc = listen(sfd, 10); + if (rc == -1) { + printf("listen() failed with errno:%d\n", rc); + goto exit; + } + + in_len = sizeof (in_addr); + mgmt_fd = accept(sfd, &in_addr, &in_len); + if (mgmt_fd == -1) { + printf("Unable to accept\n"); + goto exit; + } + return (mgmt_fd); +exit: + if (sfd != -1) + close(sfd); + return (-1); +} + /* * Rebuilding downgraded replica test case. It covers following case: * =====Rebuild success case===== @@ -778,20 +814,19 @@ zrepl_rebuild_test(void *arg) { kmutex_t mtx; kcondvar_t cv; - int i, count, sfd, rc, mgmt_fd; + int i, count, rc; + int ds0_mgmt_fd, ds1_mgmt_fd, ds2_mgmt_fd, ds3_mgmt_fd; int ds0_io_sfd, ds1_io_sfd; int ds2_io_sfd, ds3_io_sfd; int threads_done = 0; int num_threads = 0; kthread_t *reader[2]; kthread_t *writer; - socklen_t in_len; mgmt_ack_t *p = NULL; mgmt_ack_t *mgmt_ack = NULL; mgmt_ack_t *mgmt_ack_ds1 = NULL; mgmt_ack_t *mgmt_ack_ds2 = NULL; mgmt_ack_t *mgmt_ack_ds3 = NULL; - struct sockaddr in_addr; zrepl_status_ack_t status_ack; worker_args_t writer_args, reader_args[2]; @@ -802,8 +837,10 @@ zrepl_rebuild_test(void *arg) ds = "ds0"; ds1 = "ds1"; - ds0_io_sfd = ds1_io_sfd = mgmt_fd = sfd = -1; + ds0_io_sfd = ds1_io_sfd = -1; ds2_io_sfd = ds3_io_sfd = -1; + ds0_mgmt_fd = ds1_mgmt_fd = -1; + ds2_mgmt_fd = ds3_mgmt_fd = -1; mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); cv_init(&cv, NULL, CV_DEFAULT, NULL); @@ -831,29 +868,32 @@ zrepl_rebuild_test(void *arg) reader_args[1].max_iops = max_iops / 2; reader_args[1].rebuild_test = B_TRUE; - sfd = create_and_bind(tgt_port, B_TRUE, B_FALSE); - if (sfd == -1) { + ds0_mgmt_fd = create_bind_listen_and_accept(tgt_port, B_TRUE, B_FALSE); + if (ds0_mgmt_fd == -1) { return; } - rc = listen(sfd, 10); - if (rc == -1) { - printf("listen() failed with errno:%d\n", rc); - goto exit; + ds1_mgmt_fd = create_bind_listen_and_accept(tgt_port1, B_TRUE, B_FALSE); + if (ds1_mgmt_fd == -1) { + return; } - printf("Listen was successful\n"); + ds2_mgmt_fd = create_bind_listen_and_accept(tgt_port2, B_TRUE, B_FALSE); + if (ds2_mgmt_fd == -1) { + return; + } - in_len = sizeof (in_addr); - mgmt_fd = accept(sfd, &in_addr, &in_len); - if (mgmt_fd == -1) { - printf("Unable to accept\n"); - goto exit; + ds3_mgmt_fd = create_bind_listen_and_accept(tgt_port3, B_TRUE, B_FALSE); + if (ds3_mgmt_fd == -1) { + return; } + + printf("Listen was successful\n"); + printf("Connection accepted from replica successfully\n"); /* Mgmt Handshake and IO-conn for replica ds0 */ - ds0_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds, mgmt_fd); + ds0_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds, ds0_mgmt_fd); if (ds0_io_sfd == -1) { goto exit; } @@ -861,26 +901,26 @@ zrepl_rebuild_test(void *arg) writer_args.sfd[0] = reader_args[0].sfd[0] = ds0_io_sfd; /* Mgmt Handshake and IO-conn for replica ds1 */ - ds1_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds1, mgmt_fd); + ds1_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds1, ds1_mgmt_fd); if (ds1_io_sfd == -1) { goto exit; } writer_args.sfd[1] = reader_args[1].sfd[0] = ds1_io_sfd; /* Mgmt Handshake and IO-conn for replica ds2 */ - ds2_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds2, mgmt_fd); + ds2_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds2, ds2_mgmt_fd); if (ds2_io_sfd == -1) { goto exit; } /* Mgmt Handshake and IO-conn for replica ds3 */ - ds3_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds3, mgmt_fd); + ds3_io_sfd = zrepl_utest_mgmt_hs_io_conn(ds3, ds3_mgmt_fd); if (ds3_io_sfd == -1) { goto exit; } /* Check status of replica ds0 */ - rc = zrepl_utest_get_replica_status(ds, mgmt_fd, &status_ack); + rc = zrepl_utest_get_replica_status(ds, ds0_mgmt_fd, &status_ack); if (rc == -1) { goto exit; } @@ -895,7 +935,7 @@ zrepl_rebuild_test(void *arg) strncpy(mgmt_ack->dw_volname, ds, sizeof (mgmt_ack->dw_volname)); strncpy(mgmt_ack->volname, "", sizeof (mgmt_ack->volname)); - rc = zrepl_utest_replica_rebuild_start(mgmt_fd, mgmt_ack, + rc = zrepl_utest_replica_rebuild_start(ds0_mgmt_fd, mgmt_ack, sizeof (mgmt_ack_t)); if (rc == -1) { goto exit; @@ -903,7 +943,7 @@ zrepl_rebuild_test(void *arg) } check_status: - rc = zrepl_utest_get_replica_status(ds, mgmt_fd, &status_ack); + rc = zrepl_utest_get_replica_status(ds, ds0_mgmt_fd, &status_ack); if (rc == -1) { goto exit; } @@ -943,7 +983,7 @@ zrepl_rebuild_test(void *arg) * and ip from healthy replica ds0. */ mgmt_ack_ds1 = umem_alloc(sizeof (mgmt_ack_t), UMEM_NOFAIL); - count = zrepl_utest_prepare_for_rebuild(ds, ds1, mgmt_fd, + count = zrepl_utest_prepare_for_rebuild(ds, ds1, ds0_mgmt_fd, mgmt_ack_ds1); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); @@ -954,7 +994,7 @@ zrepl_rebuild_test(void *arg) * Start rebuild process on downgraded replica ds1 * by sharing IP and rebuild_Port info with ds1. */ - rc = zrepl_utest_replica_rebuild_start(mgmt_fd, mgmt_ack_ds1, + rc = zrepl_utest_replica_rebuild_start(ds1_mgmt_fd, mgmt_ack_ds1, sizeof (mgmt_ack_t)); if (rc == -1) { goto exit; @@ -963,7 +1003,7 @@ zrepl_rebuild_test(void *arg) * Check rebuild status of of downgrade replica ds1. */ status_check: - count = zrepl_utest_get_replica_status(ds1, mgmt_fd, &status_ack); + count = zrepl_utest_get_replica_status(ds1, ds1_mgmt_fd, &status_ack); if (count == -1) { goto exit; } @@ -996,13 +1036,13 @@ zrepl_rebuild_test(void *arg) */ mgmt_ack_ds2 = umem_alloc(sizeof (mgmt_ack_t) * 2, UMEM_NOFAIL); p = mgmt_ack_ds2; - count = zrepl_utest_prepare_for_rebuild(ds, ds2, mgmt_fd, p); + count = zrepl_utest_prepare_for_rebuild(ds, ds2, ds0_mgmt_fd, p); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); goto exit; } p++; - count = zrepl_utest_prepare_for_rebuild(ds1, ds2, mgmt_fd, p); + count = zrepl_utest_prepare_for_rebuild(ds1, ds2, ds1_mgmt_fd, p); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); goto exit; @@ -1021,7 +1061,7 @@ zrepl_rebuild_test(void *arg) * Start rebuild process on downgraded replica ds2 * by sharing IP and rebuild_Port info with ds2. */ - rc = zrepl_utest_replica_rebuild_start(mgmt_fd, mgmt_ack_ds2, + rc = zrepl_utest_replica_rebuild_start(ds2_mgmt_fd, mgmt_ack_ds2, sizeof (mgmt_ack_t) * 2); if (rc == -1) { goto exit; @@ -1030,7 +1070,7 @@ zrepl_rebuild_test(void *arg) * Check rebuild status of ds2. */ status_check1: - count = zrepl_utest_get_replica_status(ds2, mgmt_fd, &status_ack); + count = zrepl_utest_get_replica_status(ds2, ds2_mgmt_fd, &status_ack); if (count == -1) { goto exit; } @@ -1052,19 +1092,20 @@ zrepl_rebuild_test(void *arg) mgmt_ack_ds3 = umem_alloc(sizeof (mgmt_ack_t) * 3, UMEM_NOFAIL); p = mgmt_ack_ds3; - count = zrepl_utest_prepare_for_rebuild(ds, ds3, mgmt_fd, p); + count = zrepl_utest_prepare_for_rebuild(ds, ds3, ds0_mgmt_fd, p); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); goto exit; } + p++; - count = zrepl_utest_prepare_for_rebuild(ds1, ds3, mgmt_fd, p); + count = zrepl_utest_prepare_for_rebuild(ds1, ds3, ds1_mgmt_fd, p); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); goto exit; } p++; - count = zrepl_utest_prepare_for_rebuild(ds2, ds3, mgmt_fd, p); + count = zrepl_utest_prepare_for_rebuild(ds2, ds3, ds2_mgmt_fd, p); if (count == -1) { printf("Prepare_for_rebuild: sending hdr failed\n"); goto exit; @@ -1089,7 +1130,7 @@ zrepl_rebuild_test(void *arg) * Start rebuild process on downgraded replica ds3 * by sharing IP and rebuild_Port info with ds3. */ - rc = zrepl_utest_replica_rebuild_start(mgmt_fd, mgmt_ack_ds3, + rc = zrepl_utest_replica_rebuild_start(ds3_mgmt_fd, mgmt_ack_ds3, sizeof (mgmt_ack_t) * 3); if (rc == -1) { goto exit; @@ -1098,7 +1139,7 @@ zrepl_rebuild_test(void *arg) * Check rebuild status of ds3. */ status_check2: - count = zrepl_utest_get_replica_status(ds3, mgmt_fd, &status_ack); + count = zrepl_utest_get_replica_status(ds3, ds3_mgmt_fd, &status_ack); if (count == -1) { goto exit; } @@ -1130,7 +1171,7 @@ zrepl_rebuild_test(void *arg) * Start rebuild process on downgraded replica ds3 * by sharing IP and rebuild_Port info with ds3. */ - rc = zrepl_utest_replica_rebuild_start(mgmt_fd, mgmt_ack_ds3, + rc = zrepl_utest_replica_rebuild_start(ds3_mgmt_fd, mgmt_ack_ds3, sizeof (mgmt_ack_t) * 3); if (rc == -1) { goto exit; @@ -1139,7 +1180,7 @@ zrepl_rebuild_test(void *arg) * Check rebuild status of ds3. */ status_check3: - count = zrepl_utest_get_replica_status(ds3, mgmt_fd, &status_ack); + count = zrepl_utest_get_replica_status(ds3, ds3_mgmt_fd, &status_ack); if (count == -1) { goto exit; } @@ -1151,11 +1192,17 @@ zrepl_rebuild_test(void *arg) printf("Replica:%s is healthy now\n", ds3); exit: - if (sfd != -1) - close(sfd); + if (ds0_mgmt_fd != -1) + close(ds0_mgmt_fd); + + if (ds1_mgmt_fd != -1) + close(ds1_mgmt_fd); + + if (ds2_mgmt_fd != -1) + close(ds2_mgmt_fd); - if (mgmt_fd != -1) - close(mgmt_fd); + if (ds3_mgmt_fd != -1) + close(ds3_mgmt_fd); if (ds0_io_sfd != -1) close(ds0_io_sfd); diff --git a/cmd/zrepl/mgmt_conn.c b/cmd/zrepl/mgmt_conn.c index 81c7540ec6b5..5f9231d3b650 100644 --- a/cmd/zrepl/mgmt_conn.c +++ b/cmd/zrepl/mgmt_conn.c @@ -33,9 +33,9 @@ #include #include -#include #include #include +#include #include "mgmt_conn.h" #include "data_conn.h" @@ -238,32 +238,6 @@ scan_conn_list(void) return (rc); } -/* - * Try to obtain controller address from zfs property. - */ -static int -get_controller_ip(objset_t *os, char *buf, int len) -{ - nvlist_t *props, *propval; - char *ip; - int error; - dsl_pool_t *dp = spa_get_dsl(os->os_spa); - - dsl_pool_config_enter(dp, FTAG); - error = dsl_prop_get_all(os, &props); - dsl_pool_config_exit(dp, FTAG); - if (error != 0) - return (error); - if (nvlist_lookup_nvlist(props, ZFS_PROP_TARGET_IP, &propval) != 0) - return (ENOENT); - if (nvlist_lookup_string(propval, ZPROP_VALUE, &ip) != 0) - return (EINVAL); - - strncpy(buf, ip, len); - nvlist_free(props); - return (0); -} - /* * This gets called whenever a new zinfo is created. We might need to create * a new mgmt connection to iscsi target in response to this event. @@ -271,7 +245,7 @@ get_controller_ip(objset_t *os, char *buf, int len) void zinfo_create_cb(zvol_info_t *zinfo, nvlist_t *create_props) { - char target_host[256]; + char target_host[MAXNAMELEN]; uint16_t target_port; uzfs_mgmt_conn_t *mgmt_conn, *new_mgmt_conn; zvol_state_t *zv = zinfo->zv; @@ -285,12 +259,13 @@ zinfo_create_cb(zvol_info_t *zinfo, nvlist_t *create_props) strncpy(target_host, ip, sizeof (target_host)); } else { /* get it from zvol properties */ - if (get_controller_ip(zv->zv_objset, target_host, - sizeof (target_host)) != 0) { + if (zv->zv_target_host[0] == 0) { /* in case of missing property take the default IP */ - strncpy(target_host, target_addr, sizeof (target_host)); + strncpy(target_host, "127.0.0.1", sizeof ("127.0.0.1")); target_port = TARGET_PORT; } + else + strncpy(target_host, zv->zv_target_host, MAXNAMELEN); } delim = strchr(target_host, ':'); @@ -511,6 +486,14 @@ uzfs_zvol_mgmt_do_handshake(uzfs_mgmt_conn_t *conn, zvol_io_hdr_t *hdrp, mgmt_ack.port = atoi((hdrp->opcode == ZVOL_OPCODE_PREPARE_FOR_REBUILD) ? REBUILD_IO_SERVER_PORT : IO_SERVER_PORT); mgmt_ack.pool_guid = spa_guid(zv->zv_spa); + + /* + * hold dataset during handshake if objset is NULL + * no critical section here as rebuild & handshake won't come at a time + */ + if (zv->zv_objset == NULL) + uzfs_hold_dataset(zv); + /* * We don't use fsid_guid because that one is not guaranteed * to stay the same (it is changed in case of conflicts). @@ -565,9 +548,9 @@ uzfs_zvol_rebuild_dw_replica_start(uzfs_mgmt_conn_t *conn, zvol_io_hdr_t *hdrp, } if (zinfo == NULL) { zinfo = uzfs_zinfo_lookup(mack->dw_volname); - if (zinfo == NULL) { - printf("Replica %s not found\n", - mack->dw_volname); + if ((zinfo == NULL) || (zinfo->mgmt_conn != conn)) { + printf("Replica %s not found or not matching " + "conn\n", mack->dw_volname); return (reply_error(conn, ZVOL_OP_STATUS_FAILED, hdrp->opcode, hdrp->io_seq, CS_INIT)); } @@ -593,6 +576,13 @@ uzfs_zvol_rebuild_dw_replica_start(uzfs_mgmt_conn_t *conn, zvol_io_hdr_t *hdrp, uzfs_zvol_set_rebuild_status(zinfo->zv, ZVOL_REBUILDING_IN_PROGRESS); } else { + if (strncmp(zinfo->name, mack->dw_volname, MAXNAMELEN) + != 0) { + printf("Replica %s not matching with zinfo" + " %s\n", mack->dw_volname, zinfo->name); + return (reply_error(conn, ZVOL_OP_STATUS_FAILED, + hdrp->opcode, hdrp->io_seq, CS_INIT)); + } uzfs_zinfo_take_refcnt(zinfo, B_FALSE); } @@ -653,7 +643,8 @@ process_message(uzfs_mgmt_conn_t *conn) strncpy(zvol_name, payload, payload_size); zvol_name[payload_size] = '\0'; - if ((zinfo = uzfs_zinfo_lookup(zvol_name)) == NULL) { + if (((zinfo = uzfs_zinfo_lookup(zvol_name)) == NULL) || + (zinfo->mgmt_conn != conn)) { fprintf(stderr, "Unknown zvol: %s\n", zvol_name); rc = reply_error(conn, ZVOL_OP_STATUS_FAILED, hdrp->opcode, hdrp->io_seq, CS_INIT); diff --git a/cmd/zrepl/zrepl.c b/cmd/zrepl/zrepl.c index 1169bd8a3c17..214c51e769a8 100644 --- a/cmd/zrepl/zrepl.c +++ b/cmd/zrepl/zrepl.c @@ -28,49 +28,6 @@ static void uzfs_zvol_io_ack_sender(void *arg); kthread_t *conn_accpt_thread; kthread_t *uzfs_timer_thread; kthread_t *mgmt_conn_thread; -char *pool_name = NULL; -struct in_addr addr = {0}; -int zrepl_import(int argc, char **argv); -int zrepl_start(int argc, char **argv); - -typedef struct zrepl_command { - const char *cmd_name; - int (*func)(int, char **); -} zrepl_cmd_t; - -static zrepl_cmd_t cmd_table[] = { - {"import", zrepl_import}, - {"start", zrepl_start}, - {NULL}, -}; - -#define NCMDS (sizeof (cmd_table) / sizeof (zrepl_cmd_t)) - -int -find_command(const char *cmd_name, int *index) -{ - for (int i = 0; i < NCMDS; i++) { - if (cmd_table[i].cmd_name == NULL) - continue; - if (strcmp(cmd_name, cmd_table[i].cmd_name) == 0) { - *index = i; - return (0); - } - } - return (1); -} - -void -help(void) -{ - /* - * XXX need to do better here - */ - - printf("zrepl command args ... \nwhere 'command' is one of:\n\n"); - printf("\t import [-t ip address)]\n"); - printf("\t start [-t ip address)]\n"); -} /* * Read header message from socket in safe manner, which is: first we read a @@ -773,114 +730,11 @@ uzfs_zrepl_close_log(void) closelog(); } -int -zrepl_import(int argc, char **argv) -{ - int c; - nvlist_t *config = NULL; - importargs_t importargs = {0}; - int error; - spa_t *spa; - nvlist_t *props = NULL; - - argc -= optind; - argv += optind; - - if (argc < 1) { - help(); - return (1); - } - - pool_name = argv[1]; - - while ((c = getopt(argc, argv, "t:")) != -1) { - switch (c) { - case 't': - if (inet_aton(optarg, &addr) == 0) { - fprintf(stderr, - "Invalid target address\n"); - help(); - return (1); - } - target_addr = optarg; - break; - default: - help(); - return (1); - } - } - - if (target_addr == NULL) { - help(); - return (1); - } - - fprintf(stdout, "import pool %s default target addr %s\n", pool_name, - target_addr); - libzfs_handle_t *hdl = libzfs_init(); - - importargs.scan = B_TRUE; - importargs.cachefile = NULL; - - if ((error = zpool_tryimport(hdl, pool_name, &config, &importargs)) - != 0) { - fprintf(stderr, "cannot import pool:%s, %s\n", pool_name, - libzfs_error_description(hdl)); - libzfs_fini(hdl); - return (1); - } - - if ((error = spa_import(pool_name, config, props, ZFS_IMPORT_NORMAL)) - != 0) { - fprintf(stderr, "failed import %s\n", strerror(error)); - return (1); - } - - libzfs_fini(hdl); - - if ((error = uzfs_open_pool(pool_name, &spa)) != 0) { - fprintf(stderr, "spa open failed %s\n ", strerror(error)); - return (1); - } - - return (0); -} - -int -zrepl_start(int argc, char **argv) -{ - - int c; - - while ((c = getopt(argc, argv, "t:")) != -1) { - switch (c) { - case 't': - if (inet_aton(optarg, &addr) == 0) { - fprintf(stderr, "Invalid target address\n"); - help(); - return (1); - } - target_addr = optarg; - break; - default: - help(); - return (1); - } - } - - if (target_addr == NULL) { - help(); - return (1); - } - - return (0); -} - void zrepl_svc_run(void) { mgmt_conn_thread = zk_thread_create(NULL, 0, - (thread_func_t)uzfs_zvol_mgmt_thread, target_addr, 0, NULL, + (thread_func_t)uzfs_zvol_mgmt_thread, NULL, 0, NULL, TS_RUN, 0, PTHREAD_CREATE_DETACHED); VERIFY3P(mgmt_conn_thread, !=, NULL); @@ -903,20 +757,6 @@ main(int argc, char **argv) { int rc; - int i = 0; - const char *cmd_name = NULL; - - if (argc < 2) { - help(); - return (1); - } - - cmd_name = argv[1]; - - if ((rc = find_command(cmd_name, &i)) != 0) { - help(); - return (1); - } if (getenv("CONFIG_LOAD_DISABLE") != NULL) { printf("disabled auto import (reading of zpool.cache)\n"); @@ -935,7 +775,6 @@ main(int argc, char **argv) return (-1); } - /* Ignore SIGPIPE signal */ signal(SIGPIPE, SIG_IGN); if (libuzfs_ioctl_init() < 0) { @@ -943,8 +782,6 @@ main(int argc, char **argv) goto initialize_error; } - if ((rc = cmd_table[i].func(argc, argv)) != 0) - goto initialize_error; zrepl_svc_run(); while (1) { sleep(5); diff --git a/include/sys/uzfs_zvol.h b/include/sys/uzfs_zvol.h index 42775853dbae..9bffb846220a 100644 --- a/include/sys/uzfs_zvol.h +++ b/include/sys/uzfs_zvol.h @@ -63,23 +63,24 @@ typedef struct zvol_rebuild_info { * The in-core state of each volume. */ struct zvol_state { - char zv_name[MAXNAMELEN]; /* name */ - uint64_t zv_volsize; /* advertised space */ - uint64_t zv_volblocksize; /* volume block size */ - objset_t *zv_objset; /* objset handle */ - zilog_t *zv_zilog; /* ZIL handle */ - dnode_t *zv_dn; /* dnode hold */ - zfs_rlock_t zv_range_lock; /* range lock */ - spa_t *zv_spa; /* spa */ - uint64_t zv_volmetablocksize; /* meta block size */ - uint64_t zv_volmetadatasize; /* volume meta data size */ + char zv_name[MAXNAMELEN]; /* name */ + uint64_t zv_volsize; /* advertised space */ + uint64_t zv_volblocksize; /* volume block size */ + objset_t *zv_objset; /* objset handle */ + zilog_t *zv_zilog; /* ZIL handle */ + dnode_t *zv_dn; /* dnode hold */ + zfs_rlock_t zv_range_lock; /* range lock */ + spa_t *zv_spa; /* spa */ + char zv_target_host[MAXNAMELEN]; /* target address */ + uint64_t zv_volmetablocksize; /* meta block size */ + uint64_t zv_volmetadatasize; /* volume meta data size */ /* * block size at which metadata is calculated. * This should not be greater than volblocksize */ uint64_t zv_metavolblocksize; - zvol_status_t zv_status; /* zvol status */ + zvol_status_t zv_status; /* zvol status */ zvol_rebuild_info_t rebuild_info; }; diff --git a/include/uzfs_mgmt.h b/include/uzfs_mgmt.h index 242bae0234c6..30d37bd2a4f0 100644 --- a/include/uzfs_mgmt.h +++ b/include/uzfs_mgmt.h @@ -37,16 +37,18 @@ extern int uzfs_zvol_create_meta(objset_t *os, uint64_t block_size, uint64_t meta_block_size, uint64_t meta_vol_block_size, dmu_tx_t *tx); extern int uzfs_open_dataset(spa_t *spa, const char *ds, zvol_state_t **zv); extern int uzfs_zvol_create_cb(const char *ds_name, void *n); +extern void uzfs_zvol_create_minors(spa_t *spa, const char *name, + boolean_t async); extern int uzfs_zvol_destroy_cb(const char *ds_name, void *n); extern uint64_t uzfs_synced_txg(zvol_state_t *zv); extern void uzfs_close_dataset(zvol_state_t *zv); extern void uzfs_close_pool(spa_t *spa); extern void uzfs_fini(void); extern uint64_t uzfs_random(uint64_t); +extern int uzfs_hold_dataset(zvol_state_t *zv); - -void uzfs_spa_init(spa_t *spa); -void uzfs_spa_fini(spa_t *spa); +extern void uzfs_spa_init(spa_t *spa); +extern void uzfs_spa_fini(spa_t *spa); int uzfs_pool_create(const char *name, char *path, spa_t **spa); #ifdef __cplusplus } diff --git a/include/uzfs_test.h b/include/uzfs_test.h index bf6d67e8081d..e1f80d782b4b 100644 --- a/include/uzfs_test.h +++ b/include/uzfs_test.h @@ -50,7 +50,7 @@ extern void replay_fn(void *arg); extern void setup_unit_test(void); extern void unit_test_create_pool_ds(void); extern void open_pool(spa_t **); -extern void open_ds(spa_t *, zvol_state_t **); +extern void open_ds(spa_t *, char *, zvol_state_t **); typedef struct worker_args { void *zv; diff --git a/include/zrepl_mgmt.h b/include/zrepl_mgmt.h index b5db6ec58503..5eb695bae390 100644 --- a/include/zrepl_mgmt.h +++ b/include/zrepl_mgmt.h @@ -38,7 +38,7 @@ extern "C" { #define uZFS_ZVOL_WORKERS_MAX 128 #define uZFS_ZVOL_WORKERS_DEFAULT 6 -#define ZFS_PROP_TARGET_IP "com.cloudbyte:targetip" +#define ZFS_PROP_TARGET_IP "io.openebs:targetip" #define REBUILD_IO_SERVER_PORT "3233" #define IO_SERVER_PORT "3232" diff --git a/lib/fio/README.md b/lib/fio/README.md index 11ffd6e83a1e..2292da9aa44d 100644 --- a/lib/fio/README.md +++ b/lib/fio/README.md @@ -22,7 +22,7 @@ In another window start zfs replica and instruct it to connect to "fio target" which is listening on loopback interface: ```bash -sudo /repos/zfs/cmd/zrepl/zrepl 127.0.0.1 +sudo /repos/zfs/cmd/zrepl/zrepl ``` At most after 5 secs `zrepl` will connect to fio and IO will start flowing. diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c index 6db160e94077..bc91e8964b34 100644 --- a/lib/libzpool/kernel.c +++ b/lib/libzpool/kernel.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1697,8 +1698,6 @@ __spl_pf_fstrans_check(void) return (0); } -void *zvol_tag = "zvol_tag"; - void zvol_create_minors(spa_t *spa, const char *name, boolean_t async) { diff --git a/lib/libzpool/uzfs_mgmt.c b/lib/libzpool/uzfs_mgmt.c index 060ed9d72cbb..2b4f4570367e 100644 --- a/lib/libzpool/uzfs_mgmt.c +++ b/lib/libzpool/uzfs_mgmt.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -280,6 +281,62 @@ uzfs_objset_create_cb(objset_t *new_os, void *arg, cred_t *cr, dmu_tx_t *tx) VERIFY(error == 0); } +/* + * own the dataset, hold node, open zilog + */ +int +uzfs_hold_dataset(zvol_state_t *zv) +{ + int error = -1; + objset_t *os; + error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, 1, zv, &os); + if (error) + goto free_ret; + + zv->zv_objset = os; + + error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); + if (error) { + dmu_objset_disown(zv->zv_objset, zv); +free_ret: + zv->zv_objset = NULL; + zv->zv_dn = NULL; + goto ret; + } + + zv->zv_zilog = zil_open(os, zvol_get_data); +ret: + return (error); +} + +/* + * Try to obtain controller address from zfs property. + */ +static int +get_controller_ip(objset_t *os, char *buf, int len) +{ + nvlist_t *props, *propval; + char *ip; + int error; + dsl_pool_t *dp = spa_get_dsl(os->os_spa); + + dsl_pool_config_enter(dp, FTAG); + error = dsl_prop_get_all(os, &props); + dsl_pool_config_exit(dp, FTAG); + if (error != 0) + return (error); + if (nvlist_lookup_nvlist(props, ZFS_PROP_TARGET_IP, &propval) != 0) { + nvlist_free(props); + return (ENOENT); + } + if (nvlist_lookup_string(propval, ZPROP_VALUE, &ip) != 0) { + nvlist_free(props); + return (EINVAL); + } + strncpy(buf, ip, len); + nvlist_free(props); + return (0); +} /* owns objset with name 'ds_name' in pool 'spa' */ static int @@ -303,6 +360,7 @@ uzfs_own_dataset(const char *ds_name, zvol_state_t **z) error = spa_open(ds_name, &spa, zv); if (error != 0) { kmem_free(zv, sizeof (zvol_state_t)); + zv = NULL; goto ret; } @@ -312,13 +370,23 @@ uzfs_own_dataset(const char *ds_name, zvol_state_t **z) strlcpy(zv->zv_name, ds_name, MAXNAMELEN); error = dmu_objset_own(ds_name, DMU_OST_ZVOL, 1, zv, &os); - if (error) + if (error != 0) goto free_ret; + zv->zv_objset = os; error = dmu_object_info(os, ZVOL_OBJ, &doi); - if (error) - goto disown_free; + if (error) { +disown_free: + dmu_objset_disown(zv->zv_objset, zv); +free_ret: + zfs_rlock_destroy(&zv->zv_range_lock); + spa_close(spa, zv); + kmem_free(zv, sizeof (zvol_state_t)); + zv = NULL; + goto ret; + } + block_size = doi.doi_data_block_size; error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &vol_size); @@ -344,19 +412,12 @@ uzfs_own_dataset(const char *ds_name, zvol_state_t **z) zv->zv_volmetadatasize = meta_data_size; zv->zv_metavolblocksize = meta_vol_block_size; - error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); - if (error) { -disown_free: - dmu_objset_disown(zv->zv_objset, zv); -free_ret: - spa_close(spa, zv); - zfs_rlock_destroy(&zv->zv_range_lock); - kmem_free(zv, sizeof (zvol_state_t)); - zv = NULL; - goto ret; - } + error = get_controller_ip(os, zv->zv_target_host, + sizeof (zv->zv_target_host)); + if (error != 0 && error != ENOENT) + goto disown_free; - zv->zv_zilog = zil_open(os, zvol_get_data); + error = 0; zv->zv_volblocksize = block_size; zv->zv_volsize = vol_size; @@ -370,8 +431,12 @@ uzfs_own_dataset(const char *ds_name, zvol_state_t **z) zil_replay(os, zv, zvol_replay_vector); } + dmu_objset_disown(zv->zv_objset, zv); + + zv->zv_objset = NULL; ret: *z = zv; + return (error); } @@ -432,12 +497,15 @@ uzfs_create_dataset(spa_t *spa, char *ds_name, uint64_t vol_size, int uzfs_zvol_create_cb(const char *ds_name, void *arg) { - zvol_state_t *zv = NULL; int error = -1; nvlist_t *nvprops = arg; + char *ip; - printf("ds_name %s\n", ds_name); + if (strrchr(ds_name, '@') != NULL) { + printf("no owning dataset for snapshots: %s\n", ds_name); + return (0); + } error = uzfs_own_dataset(ds_name, &zv); if (error) { @@ -445,6 +513,25 @@ uzfs_zvol_create_cb(const char *ds_name, void *arg) return (error); } + /* if zvol is being created, read target address from nvlist */ + if (nvprops != NULL) { + error = nvlist_lookup_string(nvprops, ZFS_PROP_TARGET_IP, &ip); + if (error == 0) + strncpy(zv->zv_target_host, ip, + sizeof (zv->zv_target_host)); + else { + printf("target IP address is not set for %s\n", + ds_name); + uzfs_close_dataset(zv); + return (error); + } + } else { + if (zv->zv_target_host[0] == '\0') { + printf("target IP address is NULL for %s\n", ds_name); + uzfs_close_dataset(zv); + return (EINVAL); + } + } if (uzfs_zinfo_init(zv, ds_name, nvprops) != 0) { printf("Failed in uzfs_zinfo_init\n"); return (error); @@ -453,6 +540,40 @@ uzfs_zvol_create_cb(const char *ds_name, void *arg) return (0); } +static void +uzfs_zvol_create_minors_impl(void *n) +{ + const char *name = (char *)n; + + dmu_objset_find((char *)name, uzfs_zvol_create_cb, NULL, + DS_FIND_CHILDREN); + + kmem_free(n, MAXNAMELEN); +} + +/* + * similar to zvol_create_minors which does + * - own dataset, zil replay, disown dataset + * for all children + */ +void +uzfs_zvol_create_minors(spa_t *spa, const char *name, boolean_t async) +{ + taskqid_t id; + char *pool_name; + + pool_name = (char *)kmem_zalloc(sizeof (char) * MAXNAMELEN, KM_SLEEP); + + strncpy(pool_name, name, MAXNAMELEN); + + if (strrchr(name, '@') == NULL) { + id = taskq_dispatch(spa->spa_zvol_taskq, + uzfs_zvol_create_minors_impl, (void *)pool_name, TQ_SLEEP); + if ((async == B_FALSE) && (id != TASKQID_INVALID)) + taskq_wait_id(spa->spa_zvol_taskq, id); + } +} + /* uZFS Zvol destroy call back function */ int uzfs_zvol_destroy_cb(const char *ds_name, void *arg) @@ -469,9 +590,12 @@ uzfs_zvol_destroy_cb(const char *ds_name, void *arg) void uzfs_close_dataset(zvol_state_t *zv) { - zil_close(zv->zv_zilog); - dnode_rele(zv->zv_dn, zv); - dmu_objset_disown(zv->zv_objset, zv); + if (zv->zv_zilog != NULL) + zil_close(zv->zv_zilog); + if (zv->zv_dn != NULL) + dnode_rele(zv->zv_dn, zv); + if (zv->zv_objset != NULL) + dmu_objset_disown(zv->zv_objset, zv); zfs_rlock_destroy(&zv->zv_range_lock); spa_close(zv->zv_spa, zv); kmem_free(zv, sizeof (zvol_state_t)); diff --git a/lib/libzpool/uzfs_rebuilding.c b/lib/libzpool/uzfs_rebuilding.c index 90a8dc8de0d3..d77a307031a4 100644 --- a/lib/libzpool/uzfs_rebuilding.c +++ b/lib/libzpool/uzfs_rebuilding.c @@ -90,6 +90,10 @@ get_snapshot_zv(zvol_state_t *zv, char *snap_name, zvol_state_t **snap_zv) } ret = uzfs_open_dataset(zv->zv_spa, dataset, snap_zv); + if (ret == 0) + ret = uzfs_hold_dataset(*snap_zv); + } else if (ret == 0) { + ret = uzfs_hold_dataset(*snap_zv); } if (ret != 0) { diff --git a/lib/libzpool/zrepl_mgmt.c b/lib/libzpool/zrepl_mgmt.c index 2fcf05918afe..697c2c4cdcfa 100644 --- a/lib/libzpool/zrepl_mgmt.c +++ b/lib/libzpool/zrepl_mgmt.c @@ -219,10 +219,11 @@ uzfs_zinfo_destroy(const char *name, spa_t *spa) } else { SLIST_FOREACH_SAFE(zinfo, &zvol_list, zinfo_next, zt) { - if (name == NULL || strcmp(zinfo->name, name) == 0 || - (strncmp(zinfo->name, name, namelen) == 0 && - (zinfo->name[namelen] == '/' || - zinfo->name[namelen] == '@'))) { + if (name == NULL || (strcmp(zinfo->name, name) == 0) || + ((strncmp(zinfo->name, name, namelen) == 0) && + zinfo->name[namelen] == '/' && + zinfo->name[namelen + 1] == '\0')) { + printf("destroying %s\n", zinfo->name); zv = zinfo->zv; uzfs_remove_zinfo_list(zinfo); uzfs_close_dataset(zv); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 16c71c9e70a4..d485d05c9c5e 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -3531,10 +3531,6 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, if (firstopen) { #ifdef _KERNEL zvol_create_minors(spa, spa_name(spa), B_TRUE); -#else - uzfs_spa_init(spa); - dmu_objset_find(spa_name(spa), uzfs_zvol_create_cb, NULL, - DS_FIND_CHILDREN); #endif } *spapp = spa; @@ -4452,8 +4448,11 @@ spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); +#ifdef _KERNEL zvol_create_minors(spa, pool, B_TRUE); - +#else + uzfs_zvol_create_minors(spa, pool, B_TRUE); +#endif mutex_exit(&spa_namespace_lock); return (0); @@ -4602,12 +4601,6 @@ spa_export_common(char *pool, int new_state, nvlist_t **oldconfig, #endif mutex_enter(&spa_namespace_lock); -#ifndef _KERNEL - if ((new_state == POOL_STATE_DESTROYED) || - (new_state == POOL_STATE_EXPORTED)) - uzfs_spa_fini(spa); -#endif - spa_close(spa, FTAG); if (spa->spa_state == POOL_STATE_UNINITIALIZED) @@ -4631,13 +4624,6 @@ spa_export_common(char *pool, int new_state, nvlist_t **oldconfig, (spa->spa_inject_ref != 0 && new_state != POOL_STATE_UNINITIALIZED)) { spa_async_resume(spa); - -#ifndef _KERNEL - if (((new_state == POOL_STATE_DESTROYED) || - (new_state == POOL_STATE_EXPORTED)) && - spa->spa_us == NULL) - uzfs_spa_init(spa); -#endif mutex_exit(&spa_namespace_lock); return (SET_ERROR(EBUSY)); } @@ -4652,13 +4638,6 @@ spa_export_common(char *pool, int new_state, nvlist_t **oldconfig, if (!force && new_state == POOL_STATE_EXPORTED && spa_has_active_shared_spare(spa)) { spa_async_resume(spa); - -#ifndef _KERNEL - if (((new_state == POOL_STATE_DESTROYED) || - (new_state == POOL_STATE_EXPORTED)) && - spa->spa_us == NULL) - uzfs_spa_init(spa); -#endif mutex_exit(&spa_namespace_lock); return (SET_ERROR(EXDEV)); } @@ -7147,9 +7126,6 @@ spa_evict_all(void) if (spa->spa_state != POOL_STATE_UNINITIALIZED) { spa_unload(spa); -#ifndef _KERNEL - uzfs_spa_fini(spa); -#endif spa_deactivate(spa); } spa_remove(spa); diff --git a/module/zfs/txg.c b/module/zfs/txg.c index aa840ddef543..5efc787937e0 100644 --- a/module/zfs/txg.c +++ b/module/zfs/txg.c @@ -113,6 +113,9 @@ static void txg_quiesce_thread(dsl_pool_t *dp); int zfs_txg_timeout = 5; /* max seconds worth of delta per txg */ +extern void uzfs_spa_init(spa_t *spa); +extern void uzfs_spa_fini(spa_t *spa); + /* * Prepare the txg subsystem. */ @@ -217,6 +220,13 @@ txg_sync_start(dsl_pool_t *dp) dp, 0, &p0, TS_RUN, defclsyspri); mutex_exit(&tx->tx_sync_lock); + +#ifndef _KERNEL + /* + * initialize uZFS spa + */ + uzfs_spa_init(dp->dp_spa); +#endif } static void @@ -265,6 +275,12 @@ txg_sync_stop(dsl_pool_t *dp) */ ASSERT(tx->tx_threads == 2); +#ifndef _KERNEL + /* + * tear down uZFS spa + */ + uzfs_spa_fini(dp->dp_spa); +#endif /* * We need to ensure that we've vacated the deferred space_maps. */ diff --git a/tests/cbtest/gtest/Makefile.am b/tests/cbtest/gtest/Makefile.am index 6d83bc8acab0..91fbd0cd5f64 100644 --- a/tests/cbtest/gtest/Makefile.am +++ b/tests/cbtest/gtest/Makefile.am @@ -4,14 +4,11 @@ DEFAULT_INCLUDES += \ -I$(top_srcdir)/include \ -I$(top_srcdir)/lib/libspl/include -sbin_PROGRAMS = test_uzfs test_zrepl_prot test_export +sbin_PROGRAMS = test_uzfs test_zrepl_prot test_uzfs_SOURCES = test_uzfs.cc test_zrepl_prot_SOURCES = test_zrepl_prot.cc gtest_utils.cc -test_export_SOURCES = \ - test_export.cc gtest_utils.cc - test_uzfs_LDADD = \ $(top_builddir)/lib/libnvpair/libnvpair.la \ $(top_builddir)/lib/libuutil/libuutil.la \ @@ -19,16 +16,6 @@ test_uzfs_LDADD = \ $(top_builddir)/lib/libzfs/libzfs.la \ $(top_builddir)/lib/libzfs_core/libzfs_core.la -test_export_LDADD = \ - $(top_builddir)/lib/libnvpair/libnvpair.la \ - $(top_builddir)/lib/libuutil/libuutil.la \ - $(top_builddir)/lib/libzpool/libzpool.la \ - $(top_builddir)/lib/libzfs/libzfs.la \ - $(top_builddir)/lib/libzfs_core/libzfs_core.la - -test_export_CXXFLAGS = -Wno-write-strings -std=c++11 -test_export_LDFLAGS = -pthread -lgtest -lgtest_main - test_uzfs_CXXFLAGS = -std=c++11 test_uzfs_LDFLAGS = -pthread -lgtest -lgtest_main test_zrepl_prot_LDFLAGS = -pthread -lgtest -lgtest_main diff --git a/tests/cbtest/gtest/test_export.cc b/tests/cbtest/gtest/test_export.cc deleted file mode 100644 index 0065814f17d1..000000000000 --- a/tests/cbtest/gtest/test_export.cc +++ /dev/null @@ -1,330 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DISK "/tmp/zrepl_test_disk.img" -#define POOL_NAME "test_export" - -using namespace GtestUtils; - -int -create_file(void) -{ - int fd; - - if ((fd = open(DISK, O_RDWR | O_CREAT | O_TRUNC, 0666)) < 0) - throw std::runtime_error("cant create disk image"); - - if (ftruncate(fd, 100 * 1024 * 1024) != 0) - throw std::runtime_error("cant truncate disk image"); - close(fd); - return (0); -} - -class CreatePool : public testing::Test { -public: - - spa_t *spa; - CreatePool() : zi(NULL), zv(NULL), spa(NULL) {} - - virtual void SetUp() override { - uzfs_init(); - libuzfs_ioctl_init(); - } - - - void createPool() { - /* destroy the pool if its auto loaded from cache */ - spa_destroy(pool_name); - - ASSERT_TRUE(uzfs_pool_create(pool_name, DISK, &spa ) == 0); - - /* implicitly open the pool */ - spaOpen(); - } - - void importPool() { - - char *spath[1]; - nvlist_t *config = NULL; - nvlist_t *props = NULL; - importargs_t args = {0}; - - /* - * we need to tell the import logic - * where to find our truncated file - * that it should use for import - */ - - args.path = spath; - args.path[0] = "/tmp"; - args.paths = 1; - - args.scan = B_TRUE; - args.cachefile = NULL; - - gzfs = libzfs_init(); - - ASSERT_TRUE(gzfs != NULL); - - error = zpool_tryimport(gzfs, pool_name, &config, &args); - ASSERT_TRUE(error == 0); - - error = spa_import(pool_name, config, props, ZFS_IMPORT_NORMAL); - ASSERT_TRUE(error == 0); - - spaOpen(); - - libzfs_fini(gzfs); - - } - - void zinfoLookupByName() { - - std::string zv_name(pool_name); - zv_name.append("/vol2"); - zi = NULL; - - zi = uzfs_zinfo_lookup(zv_name.c_str()); - ASSERT_FALSE(zi == NULL); - zi->refcnt--; - ASSERT_EQ(zi->refcnt, 1); - - } - - void createDataset() { - execCmd("zfs", std::string("create -V 10m -s ") + POOL_NAME + "/vol1"); - } - - void getZVolInfo() { - /* this bumps refcount */ - std::string zv_name = std::string(POOL_NAME) + "/vol1"; - zi = uzfs_zinfo_lookup(zv_name.c_str()); - ASSERT_FALSE(zi == NULL); - ASSERT_EQ(strcmp(zi->name, zv_name.c_str()), 0); - zi->refcnt--; - ASSERT_EQ(zi->refcnt, 1); - - } - - void writeSome() { - - char *buf = (char* ) malloc(4096); - memset(buf, 'j', 4096); - ASSERT_EQ(uzfs_write_data(zi->zv, buf, 0, 4096, NULL, B_FALSE), 0); - free(buf); - } - - void readSome() { - char *buf = (char* ) malloc(4096); - memset(buf, 'a', 4096); - ASSERT_EQ(uzfs_read_data(zi->zv, buf, 0, 4096, NULL), 0); - for (int i = 0; i < 4096; i++ ) - ASSERT_EQ(buf[i], 'j'); - free(buf); - - } - - void exportPool() { - spaClose(); - ASSERT_EQ(spa_export(pool_name, NULL, B_TRUE, B_FALSE), 0); - } - - - void finish() { - uzfs_fini(); - } - - void destroyPool() { - spaClose(); - ASSERT_EQ(spa_destroy(POOL_NAME), 0); - - } - - void destroyCLI() { - execCmd("zpool", std::string("destroy ") + POOL_NAME); - } - - - void exportCLI() { - execCmd("zpool", std::string("export ") + POOL_NAME); - } - - void spaOpen() { - spa_open(POOL_NAME, &spa, (void*)"GTEST"); - } - - void spaClose() { - spa_close(spa,(void*)"GTEST"); - } - - -private: - int error; - zvol_info_t *zi; - zvol_state_t *zv; - char *pool_name = POOL_NAME; - libzfs_handle_t *gzfs; - -}; - -TEST_F(CreatePool, exportTest) { - create_file(); - createPool(); - createDataset(); - - //fixme - getZVolInfo(); - writeSome(); - - readSome(); - exportPool(); - finish(); -} - -TEST_F(CreatePool, importTest) { - importPool(); - execCmd("zfs", std::string("create -V 10m -s ") + POOL_NAME + "/vol2"); - zinfoLookupByName(); - exportPool(); - finish(); -} - -TEST_F(CreatePool, DestroyTest) { - importPool(); - destroyPool(); - finish(); -} - -TEST_F(CreatePool, DestroyTestCLI) { - importPool(); - spaClose(); - execCmd("zpool", std::string("destroy ") + POOL_NAME); - finish(); -} - -TEST_F(CreatePool, recreatePoolDestroy) { - create_file(); - createPool(); - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "destroy " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - spaClose(); - destroyCLI(); - finish(); -} - -TEST_F(CreatePool, recreatePoolExport) { - create_file(); - createPool(); - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "destroy " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - spaClose(); - exportCLI(); - finish(); -} -TEST_F(CreatePool, ExportAllCLI) { - create_file(); - /* make sure there is no pool */ - spa_destroy(POOL_NAME); - std::stringstream ss; - - ss << "create " << POOL_NAME << " "<< DISK; - execCmd("zpool", ss.str()); - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "destroy " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - /* - * this tests if we shut down properly and release - * all zinfo_t's in the exort/destroy path - */ - - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - exportCLI(); - finish(); -} - -TEST_F(CreatePool, DestroyLeaveAllAllCLI) { - create_file(); - /* make sure there is no pool */ - spa_destroy(POOL_NAME); - std::stringstream ss; - - ss << "create " << POOL_NAME << " "<< DISK; - execCmd("zpool", ss.str()); - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - destroyCLI(); - finish(); -} - -TEST_F(CreatePool, DestroyLeaveOneAllCLI) { - create_file(); - /* make sure there is no pool */ - spa_destroy(POOL_NAME); - std::stringstream ss; - - ss << "create " << POOL_NAME << " "<< DISK; - execCmd("zpool", ss.str()); - for (int i = 0; i < 5; i++) { - std::stringstream ss; - ss << "create -V 10m -s " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - for (int i = 2; i < 5; i++) { - std::stringstream ss; - ss << "destroy " << POOL_NAME << "/vol" << i; - execCmd("zfs", ss.str()); - } - - destroyCLI(); - finish(); -} diff --git a/tests/cbtest/gtest/test_zrepl_prot.cc b/tests/cbtest/gtest/test_zrepl_prot.cc index 10ffca828565..fb06ccc2f22b 100644 --- a/tests/cbtest/gtest/test_zrepl_prot.cc +++ b/tests/cbtest/gtest/test_zrepl_prot.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,316 @@ static int ready_for_read(int fd, int timeout) { return ((rc > 0) ? 1 : 0); } +/* + * This fn does handshake for given volname, and fills host/IP + * res is the expected status of handshake + */ +static void do_handshake(std::string zvol_name, std::string &host, + uint16_t &port, int control_fd, int res) { + zvol_io_hdr_t hdr_out, hdr_in; + int rc; + mgmt_ack_t mgmt_ack; + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_HANDSHAKE; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = 0; + hdr_out.offset = 0; + hdr_out.len = zvol_name.length() + 1; + + rc = write(control_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_EQ(rc, sizeof (hdr_out)); + rc = write(control_fd, zvol_name.c_str(), hdr_out.len); + ASSERT_EQ(rc, hdr_out.len); + + rc = read(control_fd, &hdr_in, sizeof (hdr_in)); + ASSERT_EQ(rc, sizeof (hdr_in)); + EXPECT_EQ(hdr_in.version, REPLICA_VERSION); + EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_HANDSHAKE); + EXPECT_EQ(hdr_in.status, res); + EXPECT_EQ(hdr_in.io_seq, 0); + if (res == ZVOL_OP_STATUS_FAILED) + return; + ASSERT_EQ(hdr_in.len, sizeof (mgmt_ack)); + rc = read(control_fd, &mgmt_ack, sizeof (mgmt_ack)); + ASSERT_EQ(rc, sizeof (mgmt_ack)); + EXPECT_STREQ(mgmt_ack.volname, zvol_name.c_str()); + host = std::string(mgmt_ack.ip); + port = mgmt_ack.port; +} + +/* + * This fn does data conn for a host:ip and volume, and fills data fd + */ +void do_data_connection(int &data_fd, std::string host, uint16_t port, + std::string zvol_name) { + struct sockaddr_in addr; + zvol_io_hdr_t hdr_out; + int rc; + + data_fd = socket(AF_INET, SOCK_STREAM, 0); + ASSERT_TRUE(data_fd >= 0); + memset(&addr, 0, sizeof (addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + rc = inet_pton(AF_INET, host.c_str(), &addr.sin_addr); + ASSERT_TRUE(rc > 0); + rc = connect(data_fd, (struct sockaddr *)&addr, sizeof (addr)); + if (rc != 0) { + perror("connect"); + ASSERT_EQ(errno, 0); + } + + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_HANDSHAKE; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = 0; + hdr_out.offset = 0; + hdr_out.len = zvol_name.length() + 1; + + rc = write(data_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_EQ(rc, sizeof (hdr_out)); + rc = write(data_fd, zvol_name.c_str(), hdr_out.len); + ASSERT_EQ(rc, hdr_out.len); +} + +/* + * Send header for data write. Leave write of actual data to the caller. + * len is real length - including metadata headers. + */ +void write_data_start(int data_fd, int &ioseq, size_t offset, int len) { + zvol_io_hdr_t hdr_out; + int rc; + + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_WRITE; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = ++ioseq; + hdr_out.offset = offset; + hdr_out.len = len; + hdr_out.flags = 0; + + rc = write(data_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_EQ(rc, sizeof (hdr_out)); +} + +void write_data(int data_fd, int &ioseq, void *buf, size_t offset, int len, uint64_t io_num) { + struct zvol_io_rw_hdr write_hdr; + int rc; + + write_data_start(data_fd, ioseq, offset, sizeof (write_hdr) + len); + + write_hdr.len = len; + write_hdr.io_num = io_num; + rc = write(data_fd, &write_hdr, sizeof (write_hdr)); + ASSERT_EQ(rc, sizeof (write_hdr)); + rc = write(data_fd, buf, len); + ASSERT_EQ(rc, len); +} + + +/* + * Send command to read data and read reply header. Reading payload is + * left to the caller. + */ +void read_data_start(int data_fd, int &ioseq, size_t offset, int len, + zvol_io_hdr_t *hdr_inp, bool rebuild_flag=false) { + zvol_io_hdr_t hdr_out; + int rc; + + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_READ; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = ++ioseq; + hdr_out.offset = offset; + hdr_out.len = len; + hdr_out.flags = (rebuild_flag) ? ZVOL_OP_FLAG_REBUILD : 0; + + rc = write(data_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_EQ(rc, sizeof (hdr_out)); + rc = read(data_fd, hdr_inp, sizeof (*hdr_inp)); + ASSERT_EQ(rc, sizeof (*hdr_inp)); + ASSERT_EQ(hdr_inp->opcode, ZVOL_OPCODE_READ); + ASSERT_EQ(hdr_inp->io_seq, ioseq); + ASSERT_EQ(hdr_inp->offset, offset); +} + +/* + * Read 3 blocks of 4096 size at offset 0 + * Compares the io_num with expected value (hardcoded) and data + */ +void read_data_and_verify_resp(int data_fd, int &ioseq) { + zvol_io_hdr_t hdr_in; + struct zvol_io_rw_hdr read_hdr; + int rc; + struct zvol_io_rw_hdr write_hdr; + char buf[4096]; + int len = 4096; + + /* read all blocks at once and check IO nums */ + read_data_start(data_fd, ioseq, 0, 3 * sizeof (buf), &hdr_in); + ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); + ASSERT_EQ(hdr_in.len, 2 * sizeof (read_hdr) + 3 * sizeof (buf)); + + rc = read(data_fd, &read_hdr, sizeof (read_hdr)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (read_hdr)); + ASSERT_EQ(read_hdr.io_num, 123); + ASSERT_EQ(read_hdr.len, 2 * sizeof (buf)); + rc = read(data_fd, buf, sizeof (buf)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (buf)); + rc = verify_buf(buf, sizeof (buf), "cStor-data"); + ASSERT_EQ(rc, 0); + rc = read(data_fd, buf, sizeof (buf)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (buf)); + rc = verify_buf(buf, sizeof (buf), "cStor-data"); + ASSERT_EQ(rc, 0); + + rc = read(data_fd, &read_hdr, sizeof (read_hdr)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (read_hdr)); + ASSERT_EQ(read_hdr.io_num, 124); + ASSERT_EQ(read_hdr.len, sizeof (buf)); + rc = read(data_fd, buf, read_hdr.len); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, read_hdr.len); + rc = verify_buf(buf, sizeof (buf), "cStor-data"); + ASSERT_EQ(rc, 0); +} + +/* + * Writes two blocks of size 4096 with different io_num (hardcoded) at + * hardcoded offset + * Verifies the resp of write IO + */ +void write_two_chunks_and_verify_resp(int data_fd, int &ioseq, + size_t offset) { + zvol_io_hdr_t hdr_in; + struct zvol_io_rw_hdr read_hdr; + int rc; + struct zvol_io_rw_hdr write_hdr; + char buf[4096]; + int len = 4096; + /* write 1th data block */ + init_buf(buf, sizeof (buf), "cStor-data"); + + /* write two chunks with different IO nums in one request */ + write_data_start(data_fd, ioseq, sizeof (buf), + 2 * (sizeof (write_hdr) + sizeof (buf))); + + write_hdr.len = sizeof (buf); + write_hdr.io_num = 123; + rc = write(data_fd, &write_hdr, sizeof (write_hdr)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (write_hdr)); + rc = write(data_fd, buf, sizeof (buf)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (buf)); + + write_hdr.len = sizeof (buf); + write_hdr.io_num = 124; + rc = write(data_fd, &write_hdr, sizeof (write_hdr)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (write_hdr)); + rc = write(data_fd, buf, sizeof (buf)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (buf)); + + rc = read(data_fd, &hdr_in, sizeof (hdr_in)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (hdr_in)); + EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); + EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); + EXPECT_EQ(hdr_in.io_seq, ioseq); +} + +/* + * Writes data block of size 4096 at given offset and io_num + * Updates io_seq of volume + */ +void write_data_and_verify_resp(int data_fd, int &ioseq, size_t offset, + uint64_t io_num) { + zvol_io_hdr_t hdr_in; + struct zvol_io_rw_hdr read_hdr; + int rc; + struct zvol_io_rw_hdr write_hdr; + char buf[4096]; + int len = 4096; + /* write 1th data block */ + init_buf(buf, sizeof (buf), "cStor-data"); + + write_data(data_fd, ioseq, buf, offset, len, io_num); + + rc = read(data_fd, &hdr_in, sizeof (hdr_in)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (hdr_in)); + EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); + EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); + EXPECT_EQ(hdr_in.io_seq, ioseq); + EXPECT_EQ(hdr_in.offset, offset); + ASSERT_EQ(hdr_in.len, sizeof (buf) + sizeof (write_hdr)); +} + +void get_zvol_status(std::string zvol_name, int &ioseq, int control_fd, int state, int rebuild_status) +{ + zvol_io_hdr_t hdr_out, hdr_in; + struct zrepl_status_ack status; + int rc; + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_REPLICA_STATUS; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = ++ioseq; + hdr_out.offset = 0; + hdr_out.len = zvol_name.length() + 1; + hdr_out.flags = 0; + rc = write(control_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (hdr_out)); + rc = write(control_fd, zvol_name.c_str(), hdr_out.len); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, hdr_out.len); + rc = read(control_fd, &hdr_in, sizeof (hdr_in)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (hdr_in)); + EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_REPLICA_STATUS); + EXPECT_EQ(hdr_in.io_seq, ioseq); + EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); + EXPECT_EQ(hdr_in.len, sizeof (status)); + rc = read(control_fd, &status, sizeof (status)); + ASSERT_ERRNO("read", rc >= 0); + ASSERT_EQ(rc, sizeof (status)); + EXPECT_EQ(status.state, state); + EXPECT_EQ(status.rebuild_status, rebuild_status); +} + +void transition_zvol_to_online(int &ioseq, int control_fd, std::string zvol_name) +{ + zvol_io_hdr_t hdr_in, hdr_out; + struct mgmt_ack mgmt_ack; + int rc; + hdr_out.version = REPLICA_VERSION; + hdr_out.opcode = ZVOL_OPCODE_START_REBUILD; + hdr_out.status = ZVOL_OP_STATUS_OK; + hdr_out.io_seq = ++ioseq; + hdr_out.offset = 0; + hdr_out.len = sizeof (mgmt_ack); + hdr_out.flags = 0; + rc = write(control_fd, &hdr_out, sizeof (hdr_out)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (hdr_out)); + // Hack to tell the replica that it is the only replica + // -> rebuild will immediately finish + mgmt_ack.volname[0] = '\0'; + mgmt_ack.ip[0] = '\0'; + mgmt_ack.port = 0; + strncpy(mgmt_ack.dw_volname, zvol_name.c_str(), + sizeof (mgmt_ack.dw_volname)); + rc = write(control_fd, &mgmt_ack, sizeof (mgmt_ack)); + ASSERT_ERRNO("write", rc >= 0); + ASSERT_EQ(rc, sizeof (mgmt_ack)); +} + /* * zrepl program wrapper. * @@ -91,8 +402,7 @@ class Zrepl { } m_pid = fork(); if (m_pid == 0) { - execl(zrepl_path.c_str(), zrepl_path.c_str(), - "start", "-t", "127.0.0.1", NULL); + execl(zrepl_path.c_str(), zrepl_path.c_str(), NULL); } /* wait for zrepl to come up - is there a better way? */ while (i < 10) { @@ -221,11 +531,11 @@ class TestPool { } ~TestPool() { - try { +// try { execCmd("zpool", std::string("destroy -f ") + m_name); - } catch (std::runtime_error re) { - ; - } +// } catch (std::runtime_error re) { +// ; +// } unlink(m_path.c_str()); } @@ -246,6 +556,10 @@ class TestPool { m_path); } + void import() { + execCmd("zpool", std::string("import ") + m_name + " -d /tmp"); + } + void createZvol(std::string name, std::string arg = "") { execCmd("zfs", std::string("create -sV ") + std::to_string(ZVOL_SIZE) + @@ -272,7 +586,7 @@ class ZreplHandshakeTest : public testing::Test { m_pool = new TestPool("handshake"); m_zrepl->start(); m_pool->create(); - m_pool->createZvol("vol1"); + m_pool->createZvol("vol1", "-o io.openebs:targetip=127.0.0.1:6060"); m_zvol_name = m_pool->getZvolName("vol1"); } @@ -312,188 +626,129 @@ class ZreplDataTest : public testing::Test { /* Shared setup hook for all zrepl data tests - called just once */ static void SetUpTestCase() { zvol_io_hdr_t hdr_out, hdr_in; - mgmt_ack_t mgmt_ack; - Target target; - m_pool = new TestPool("handshake"); + Target target1, target2; + m_pool1 = new TestPool("ihandshake"); + m_pool2 = new TestPool("handshake"); m_zrepl = new Zrepl(); int rc; m_zrepl->start(); - m_pool->create(); - m_pool->createZvol("vol1"); - m_zvol_name = m_pool->getZvolName("vol1"); + m_pool1->create(); + m_pool1->createZvol("vol1", "-o io.openebs:targetip=127.0.0.1:6060"); + m_zvol_name1 = m_pool1->getZvolName("vol1"); - rc = target.listen(); + rc = target1.listen(); ASSERT_GE(rc, 0); - m_control_fd = target.accept(-1); - ASSERT_GE(m_control_fd, 0); + m_control_fd1 = target1.accept(-1); + ASSERT_GE(m_control_fd1, 0); + + do_handshake(m_zvol_name1, m_host1, m_port1, m_control_fd1, + ZVOL_OP_STATUS_OK); + m_zrepl->kill(); + + m_zrepl->start(); + m_pool1->import(); + m_control_fd1 = target1.accept(-1); + ASSERT_GE(m_control_fd1, 0); - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_HANDSHAKE; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = 0; - hdr_out.offset = 0; - hdr_out.len = m_zvol_name.length() + 1; - - rc = write(m_control_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_EQ(rc, sizeof (hdr_out)); - rc = write(m_control_fd, m_zvol_name.c_str(), hdr_out.len); - ASSERT_EQ(rc, hdr_out.len); - - rc = read(m_control_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.version, REPLICA_VERSION); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_HANDSHAKE); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, 0); - ASSERT_EQ(hdr_in.len, sizeof (mgmt_ack)); - rc = read(m_control_fd, &mgmt_ack, sizeof (mgmt_ack)); - ASSERT_EQ(rc, sizeof (mgmt_ack)); - EXPECT_STREQ(mgmt_ack.volname, m_zvol_name.c_str()); - m_host = std::string(mgmt_ack.ip); - m_port = mgmt_ack.port; + do_handshake(m_zvol_name1, m_host1, m_port1, m_control_fd1, + ZVOL_OP_STATUS_OK); + + m_pool2->create(); + m_pool2->createZvol("vol1", "-o io.openebs:targetip=127.0.0.1:12345"); + m_zvol_name2 = m_pool1->getZvolName("vol1"); + + rc = target2.listen(12345); + ASSERT_GE(rc, 0); + m_control_fd2 = target2.accept(-1); + ASSERT_GE(m_control_fd2, 0); + + do_handshake(m_zvol_name2, m_host2, m_port2, m_control_fd2, + ZVOL_OP_STATUS_FAILED); + + m_zvol_name2 = m_pool2->getZvolName("vol1"); + do_handshake(m_zvol_name2, m_host2, m_port2, m_control_fd2, + ZVOL_OP_STATUS_OK); } static void TearDownTestCase() { - delete m_pool; - if (m_control_fd >= 0) - close(m_control_fd); + m_pool1->destroyZvol("vol1"); + m_pool2->destroyZvol("vol1"); + delete m_pool1; + delete m_pool2; + if (m_control_fd1 >= 0) + close(m_control_fd1); + if (m_control_fd2 >= 0) + close(m_control_fd2); delete m_zrepl; } ZreplDataTest() { - m_data_fd = -1; - m_ioseq = 0; + m_data_fd1 = -1; + m_data_fd2 = -1; + m_ioseq1 = 0; + m_ioseq2 = 0; } /* * Create data connection and send handshake msg for the zvol. */ virtual void SetUp() override { - struct sockaddr_in addr; - zvol_io_hdr_t hdr_out; - int rc; - - m_data_fd = socket(AF_INET, SOCK_STREAM, 0); - ASSERT_TRUE(m_data_fd >= 0); - memset(&addr, 0, sizeof (addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(m_port); - rc = inet_pton(AF_INET, m_host.c_str(), &addr.sin_addr); - ASSERT_TRUE(rc > 0); - rc = connect(m_data_fd, (struct sockaddr *)&addr, sizeof (addr)); - if (rc != 0) { - perror("connect"); - ASSERT_EQ(errno, 0); - } - - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_HANDSHAKE; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = 0; - hdr_out.offset = 0; - hdr_out.len = m_zvol_name.length() + 1; - - rc = write(m_data_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_EQ(rc, sizeof (hdr_out)); - rc = write(m_data_fd, m_zvol_name.c_str(), hdr_out.len); - ASSERT_EQ(rc, hdr_out.len); + do_data_connection(m_data_fd1, m_host1, m_port1, m_zvol_name1); + do_data_connection(m_data_fd2, m_host2, m_port2, m_zvol_name2); } virtual void TearDown() override { int rc, val; - if (m_data_fd >= 0) { + if (m_data_fd1 >= 0) { /* * We have to wait for the other end to close the * connection, because the next test case could * initiate a new connection before this one is * fully closed and cause a handshake error. */ - shutdown(m_data_fd, SHUT_WR); - rc = read(m_data_fd, &val, sizeof (val)); + shutdown(m_data_fd1, SHUT_WR); + rc = read(m_data_fd1, &val, sizeof (val)); ASSERT_EQ(rc, 0); - close(m_data_fd); + close(m_data_fd1); + } + if (m_data_fd2 >= 0) { + shutdown(m_data_fd2, SHUT_WR); + rc = read(m_data_fd2, &val, sizeof (val)); + ASSERT_EQ(rc, 0); + close(m_data_fd2); } } - /* - * Send header for data write. Leave write of actual data to the caller. - * len is real length - including metadata headers. - */ - void write_data_start(size_t offset, int len) { - zvol_io_hdr_t hdr_out; - int rc; - - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_WRITE; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; - hdr_out.offset = offset; - hdr_out.len = len; - hdr_out.flags = 0; - - rc = write(m_data_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_EQ(rc, sizeof (hdr_out)); - } - - void write_data(void *buf, size_t offset, int len, uint64_t io_num) { - struct zvol_io_rw_hdr write_hdr; - int rc; - - write_data_start(offset, sizeof (write_hdr) + len); - - write_hdr.len = len; - write_hdr.io_num = io_num; - rc = write(m_data_fd, &write_hdr, sizeof (write_hdr)); - ASSERT_EQ(rc, sizeof (write_hdr)); - rc = write(m_data_fd, buf, len); - ASSERT_EQ(rc, len); - } - - /* - * Send command to read data and read reply header. Evaluating reply - * and reading payload is left to the caller. - */ - void read_data_start(size_t offset, int len, zvol_io_hdr_t *hdr_inp, - bool rebuild_flag=false) - { - zvol_io_hdr_t hdr_out; - int rc; - - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_READ; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; - hdr_out.offset = offset; - hdr_out.len = len; - hdr_out.flags = (rebuild_flag) ? ZVOL_OP_FLAG_REBUILD : 0; - - rc = write(m_data_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_EQ(rc, sizeof (hdr_out)); - rc = read(m_data_fd, hdr_inp, sizeof (*hdr_inp)); - ASSERT_EQ(rc, sizeof (*hdr_inp)); - ASSERT_EQ(hdr_inp->opcode, ZVOL_OPCODE_READ); - ASSERT_EQ(hdr_inp->io_seq, m_ioseq); - ASSERT_EQ(hdr_inp->offset, offset); - } - - static int m_control_fd; - static uint16_t m_port; - static std::string m_host; + static int m_control_fd1; + static int m_control_fd2; + static uint16_t m_port1; + static uint16_t m_port2; + static std::string m_host1; + static std::string m_host2; static Zrepl *m_zrepl; - static TestPool *m_pool; - static std::string m_zvol_name; - - int m_data_fd; - int m_ioseq; + static TestPool *m_pool1; + static TestPool *m_pool2; + static std::string m_zvol_name1; + static std::string m_zvol_name2; + + int m_data_fd1; + int m_data_fd2; + int m_ioseq1; + int m_ioseq2; }; -int ZreplDataTest::m_control_fd = -1; -uint16_t ZreplDataTest::m_port = 0; -std::string ZreplDataTest::m_host = ""; -std::string ZreplDataTest::m_zvol_name = ""; -TestPool *ZreplDataTest::m_pool = nullptr; +int ZreplDataTest::m_control_fd1 = -1; +uint16_t ZreplDataTest::m_port1 = 0; +std::string ZreplDataTest::m_host1 = ""; +std::string ZreplDataTest::m_zvol_name1 = ""; +TestPool *ZreplDataTest::m_pool1 = nullptr; +int ZreplDataTest::m_control_fd2 = -1; +uint16_t ZreplDataTest::m_port2 = 0; +std::string ZreplDataTest::m_host2 = ""; +std::string ZreplDataTest::m_zvol_name2 = ""; +TestPool *ZreplDataTest::m_pool2 = nullptr; Zrepl *ZreplDataTest::m_zrepl = nullptr; TEST_F(ZreplHandshakeTest, HandshakeOk) { @@ -630,83 +885,13 @@ TEST_F(ZreplHandshakeTest, UnknownOpcode) { * and test that read returns two metadata chunks. */ TEST_F(ZreplDataTest, WriteAndReadBlocksWithIonum) { - zvol_io_hdr_t hdr_in; - struct zvol_io_rw_hdr read_hdr; - int rc; - struct zvol_io_rw_hdr write_hdr; - char buf[4096]; - - /* write 1th data block */ - init_buf(buf, sizeof (buf), "cStor-data"); - write_data(buf, 0, sizeof (buf), 123); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); - EXPECT_EQ(hdr_in.offset, 0); - ASSERT_EQ(hdr_in.len, sizeof (buf) + sizeof (write_hdr)); - - /* write two chunks with different IO nums in one request */ - write_data_start(sizeof (buf), 2 * (sizeof (write_hdr) + sizeof (buf))); - - write_hdr.len = sizeof (buf); - write_hdr.io_num = 123; - rc = write(m_data_fd, &write_hdr, sizeof (write_hdr)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (write_hdr)); - rc = write(m_data_fd, buf, sizeof (buf)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (buf)); - - write_hdr.len = sizeof (buf); - write_hdr.io_num = 124; - rc = write(m_data_fd, &write_hdr, sizeof (write_hdr)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (write_hdr)); - rc = write(m_data_fd, buf, sizeof (buf)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (buf)); - - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); - - /* read all blocks at once and check IO nums */ - read_data_start(0, 3 * sizeof (buf), &hdr_in); - ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - ASSERT_EQ(hdr_in.len, 2 * sizeof (read_hdr) + 3 * sizeof (buf)); - - rc = read(m_data_fd, &read_hdr, sizeof (read_hdr)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (read_hdr)); - ASSERT_EQ(read_hdr.io_num, 123); - ASSERT_EQ(read_hdr.len, 2 * sizeof (buf)); - rc = read(m_data_fd, buf, sizeof (buf)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (buf)); - rc = verify_buf(buf, sizeof (buf), "cStor-data"); - ASSERT_EQ(rc, 0); - rc = read(m_data_fd, buf, sizeof (buf)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (buf)); - rc = verify_buf(buf, sizeof (buf), "cStor-data"); - ASSERT_EQ(rc, 0); + write_data_and_verify_resp(m_data_fd1, m_ioseq1, 0, 123); + write_two_chunks_and_verify_resp(m_data_fd1, m_ioseq1, 4096); + read_data_and_verify_resp(m_data_fd1, m_ioseq1); - rc = read(m_data_fd, &read_hdr, sizeof (read_hdr)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (read_hdr)); - ASSERT_EQ(read_hdr.io_num, 124); - ASSERT_EQ(read_hdr.len, sizeof (buf)); - rc = read(m_data_fd, buf, read_hdr.len); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, read_hdr.len); - rc = verify_buf(buf, sizeof (buf), "cStor-data"); - ASSERT_EQ(rc, 0); + write_data_and_verify_resp(m_data_fd2, m_ioseq2, 0, 123); + write_two_chunks_and_verify_resp(m_data_fd2, m_ioseq2, 4096); + read_data_and_verify_resp(m_data_fd2, m_ioseq2); } /* Read two blocks without metadata from the end of zvol */ @@ -718,16 +903,16 @@ TEST_F(ZreplDataTest, ReadBlockWithoutMeta) { size_t offset = ZVOL_SIZE - 2 * sizeof (buf); for (int i = 0; i < 2; i++) { - read_data_start(offset, sizeof (buf), &hdr_in); + read_data_start(m_data_fd1, m_ioseq1, offset, sizeof (buf), &hdr_in); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); ASSERT_EQ(hdr_in.len, sizeof (read_hdr) + sizeof (buf)); - rc = read(m_data_fd, &read_hdr, sizeof (read_hdr)); + rc = read(m_data_fd1, &read_hdr, sizeof (read_hdr)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (read_hdr)); ASSERT_EQ(read_hdr.io_num, 0); ASSERT_EQ(read_hdr.len, sizeof (buf)); - rc = read(m_data_fd, buf, read_hdr.len); + rc = read(m_data_fd1, buf, read_hdr.len); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, read_hdr.len); offset += sizeof (buf); @@ -744,32 +929,32 @@ TEST_F(ZreplDataTest, WriteAndSync) { int rc; init_buf(buf, sizeof (buf), "cStor-data"); - write_data(buf, 0, sizeof (buf), ++m_ioseq); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); + write_data(m_data_fd1, m_ioseq1, buf, 0, sizeof (buf), ++m_ioseq1); + rc = read(m_data_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); hdr_out.version = REPLICA_VERSION; hdr_out.opcode = ZVOL_OPCODE_SYNC; hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; + hdr_out.io_seq = ++m_ioseq1; hdr_out.offset = 0; hdr_out.len = 0; hdr_out.flags = 0; - rc = write(m_data_fd, &hdr_out, sizeof (hdr_out)); + rc = write(m_data_fd1, &hdr_out, sizeof (hdr_out)); ASSERT_ERRNO("write", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_out)); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); + rc = read(m_data_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_SYNC); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); EXPECT_EQ(hdr_in.offset, 0); EXPECT_EQ(hdr_in.len, 0); } @@ -781,21 +966,21 @@ TEST_F(ZreplDataTest, UnknownOpcode) { hdr_out.version = REPLICA_VERSION; hdr_out.opcode = (zvol_op_code_t) 255; hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; + hdr_out.io_seq = ++m_ioseq1; hdr_out.offset = 0; hdr_out.len = 0; hdr_out.flags = 0; - rc = write(m_control_fd, &hdr_out, sizeof (hdr_out)); + rc = write(m_control_fd1, &hdr_out, sizeof (hdr_out)); ASSERT_ERRNO("write", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_out)); - rc = read(m_control_fd, &hdr_in, sizeof (hdr_in)); + rc = read(m_control_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, 255); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); } TEST_F(ZreplDataTest, ReadInvalidOffset) { @@ -803,12 +988,12 @@ TEST_F(ZreplDataTest, ReadInvalidOffset) { int rc; // unaligned offset - read_data_start(33, 4096, &hdr_in); + read_data_start(m_data_fd1, m_ioseq1, 33, 4096, &hdr_in); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); ASSERT_EQ(hdr_in.len, 0); // offset past the end of zvol - read_data_start(ZVOL_SIZE + 4096, 4096, &hdr_in); + read_data_start(m_data_fd1, m_ioseq1, ZVOL_SIZE + 4096, 4096, &hdr_in); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); ASSERT_EQ(hdr_in.len, 0); } @@ -818,12 +1003,12 @@ TEST_F(ZreplDataTest, ReadInvalidLength) { int rc; // unaligned length - read_data_start(0, 4097, &hdr_in); + read_data_start(m_data_fd1, m_ioseq1, 0, 4097, &hdr_in); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); ASSERT_EQ(hdr_in.len, 0); // length past the end of zvol - read_data_start(ZVOL_SIZE - 4096, 2 * 4096, &hdr_in); + read_data_start(m_data_fd1, m_ioseq1, ZVOL_SIZE - 4096, 2 * 4096, &hdr_in); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); ASSERT_EQ(hdr_in.len, 0); } @@ -835,21 +1020,21 @@ TEST_F(ZreplDataTest, WriteInvalidOffset) { // Writing last block of zvol should succeed init_buf(buf, sizeof (buf), "cStor-data"); - write_data(buf, ZVOL_SIZE - sizeof (buf), sizeof (buf), 333); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); + write_data(m_data_fd1, m_ioseq1, buf, ZVOL_SIZE - sizeof (buf), sizeof (buf), 333); + rc = read(m_data_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); // Writing past the end of zvol should fail - write_data(buf, ZVOL_SIZE, sizeof (buf), 334); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); + write_data(m_data_fd1, m_ioseq1, buf, ZVOL_SIZE, sizeof (buf), 334); + rc = read(m_data_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); } @@ -860,12 +1045,12 @@ TEST_F(ZreplDataTest, WriteInvalidLength) { init_buf(buf, sizeof (buf), "cStor-data"); - write_data(buf, ZVOL_SIZE - 4096, sizeof (buf), 334); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); + write_data(m_data_fd1, m_ioseq1, buf, ZVOL_SIZE - 4096, sizeof (buf), 334); + rc = read(m_data_fd1, &hdr_in, sizeof (hdr_in)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (hdr_in)); EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); + EXPECT_EQ(hdr_in.io_seq, m_ioseq1); EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_FAILED); } @@ -883,116 +1068,41 @@ TEST_F(ZreplDataTest, RebuildFlag) { int rc; /* write a data block with known ionum */ - init_buf(buf, sizeof (buf), "cStor-data"); - write_data(buf, 0, sizeof (buf), 654); - rc = read(m_data_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_WRITE); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); - EXPECT_EQ(hdr_in.offset, 0); - ASSERT_EQ(hdr_in.len, sizeof (buf) + sizeof (write_hdr)); + write_data_and_verify_resp(m_data_fd1, m_ioseq1, 0, 654); /* Get zvol status before rebuild */ - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_REPLICA_STATUS; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; - hdr_out.offset = 0; - hdr_out.len = m_zvol_name.length() + 1; - hdr_out.flags = 0; - rc = write(m_control_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_out)); - rc = write(m_control_fd, m_zvol_name.c_str(), hdr_out.len); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, hdr_out.len); - rc = read(m_control_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_REPLICA_STATUS); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.len, sizeof (status)); - rc = read(m_control_fd, &status, sizeof (status)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (status)); - EXPECT_EQ(status.state, ZVOL_STATUS_DEGRADED); - EXPECT_EQ(status.rebuild_status, ZVOL_REBUILDING_INIT); - + get_zvol_status(m_zvol_name1, m_ioseq1, m_control_fd1, ZVOL_STATUS_DEGRADED, ZVOL_REBUILDING_INIT); /* transition the zvol to online state */ - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_START_REBUILD; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; - hdr_out.offset = 0; - hdr_out.len = sizeof (mgmt_ack); - hdr_out.flags = 0; - rc = write(m_control_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_out)); - // Hack to tell the replica that it is the only replica - // -> rebuild will immediately finish - mgmt_ack.volname[0] = '\0'; - mgmt_ack.ip[0] = '\0'; - mgmt_ack.port = 0; - strncpy(mgmt_ack.dw_volname, m_zvol_name.c_str(), - sizeof (mgmt_ack.dw_volname)); - rc = write(m_control_fd, &mgmt_ack, sizeof (mgmt_ack)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (mgmt_ack)); + transition_zvol_to_online(m_ioseq1, m_control_fd1, m_zvol_name1); + + sleep(5); /* Get zvol status after rebuild */ - hdr_out.version = REPLICA_VERSION; - hdr_out.opcode = ZVOL_OPCODE_REPLICA_STATUS; - hdr_out.status = ZVOL_OP_STATUS_OK; - hdr_out.io_seq = ++m_ioseq; - hdr_out.offset = 0; - hdr_out.len = m_zvol_name.length() + 1; - hdr_out.flags = 0; - rc = write(m_control_fd, &hdr_out, sizeof (hdr_out)); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_out)); - rc = write(m_control_fd, m_zvol_name.c_str(), hdr_out.len); - ASSERT_ERRNO("write", rc >= 0); - ASSERT_EQ(rc, hdr_out.len); - rc = read(m_control_fd, &hdr_in, sizeof (hdr_in)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (hdr_in)); - EXPECT_EQ(hdr_in.opcode, ZVOL_OPCODE_REPLICA_STATUS); - EXPECT_EQ(hdr_in.io_seq, m_ioseq); - EXPECT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); - EXPECT_EQ(hdr_in.len, sizeof (status)); - rc = read(m_control_fd, &status, sizeof (status)); - ASSERT_ERRNO("read", rc >= 0); - ASSERT_EQ(rc, sizeof (status)); - EXPECT_EQ(status.state, ZVOL_STATUS_HEALTHY); - EXPECT_EQ(status.rebuild_status, ZVOL_REBUILDING_DONE); + get_zvol_status(m_zvol_name1, m_ioseq1, m_control_fd1, ZVOL_STATUS_HEALTHY, ZVOL_REBUILDING_DONE); /* read the block without rebuild flag */ - read_data_start(0, sizeof (buf), &hdr_in, false); + read_data_start(m_data_fd1, m_ioseq1, 0, sizeof (buf), &hdr_in, false); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); ASSERT_EQ(hdr_in.len, sizeof (read_hdr) + sizeof (buf)); - rc = read(m_data_fd, &read_hdr, sizeof (read_hdr)); + rc = read(m_data_fd1, &read_hdr, sizeof (read_hdr)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (read_hdr)); ASSERT_EQ(read_hdr.io_num, 0); ASSERT_EQ(read_hdr.len, sizeof (buf)); - rc = read(m_data_fd, buf, sizeof (buf)); + rc = read(m_data_fd1, buf, sizeof (buf)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (buf)); /* read the block with rebuild flag */ - read_data_start(0, sizeof (buf), &hdr_in, true); + read_data_start(m_data_fd1, m_ioseq1, 0, sizeof (buf), &hdr_in, true); ASSERT_EQ(hdr_in.status, ZVOL_OP_STATUS_OK); ASSERT_EQ(hdr_in.len, sizeof (read_hdr) + sizeof (buf)); - rc = read(m_data_fd, &read_hdr, sizeof (read_hdr)); + rc = read(m_data_fd1, &read_hdr, sizeof (read_hdr)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (read_hdr)); ASSERT_EQ(read_hdr.io_num, 654); ASSERT_EQ(read_hdr.len, sizeof (buf)); - rc = read(m_data_fd, buf, sizeof (buf)); + rc = read(m_data_fd1, buf, sizeof (buf)); ASSERT_ERRNO("read", rc >= 0); ASSERT_EQ(rc, sizeof (buf)); } @@ -1020,8 +1130,8 @@ TEST(TargetIPTest, CreateAndDestroy) { zrepl.start(); pool.create(); - pool.createZvol("implicit1"); - pool.createZvol("explicit1", "-o com.cloudbyte:targetip=127.0.0.1:12345"); + pool.createZvol("implicit1", "-o io.openebs:targetip=127.0.0.1:6060"); + pool.createZvol("explicit1", "-o io.openebs:targetip=127.0.0.1:12345"); zrepl.kill(); rc = targetImpl.listen(); @@ -1030,15 +1140,16 @@ TEST(TargetIPTest, CreateAndDestroy) { ASSERT_GE(rc, 0); zrepl.start(); + pool.import(); // two new connections (one for each target) - fdImpl = targetImpl.accept(5); + fdImpl = targetImpl.accept(50); ASSERT_GE(fdImpl, 0); - fdExpl = targetExpl.accept(5); + fdExpl = targetExpl.accept(50); ASSERT_GE(fdExpl, 0); - pool.createZvol("implicit2"); - pool.createZvol("explicit2", "-o com.cloudbyte:targetip=127.0.0.1:12345"); + pool.createZvol("implicit2", "-o io.openebs:targetip=127.0.0.1:6060"); + pool.createZvol("explicit2", "-o io.openebs:targetip=127.0.0.1:12345"); // no new connections rc = targetImpl.accept(5); @@ -1087,7 +1198,7 @@ TEST(TargetIPTest, Reconnect) { zrepl.start(); pool.create(); - pool.createZvol("reconnect"); + pool.createZvol("reconnect", "-o io.openebs:targetip=127.0.0.1:6060"); // First we test that zrepl connects even if it could not connect // first couple of times after start diff --git a/tests/cbtest/script/test_uzfs.sh b/tests/cbtest/script/test_uzfs.sh index d245ed130c69..9b1d19de2d65 100755 --- a/tests/cbtest/script/test_uzfs.sh +++ b/tests/cbtest/script/test_uzfs.sh @@ -23,10 +23,9 @@ fi ZPOOL="$SRC_PATH/cmd/zpool/zpool" ZFS="$SRC_PATH/cmd/zfs/zfs" ZDB="$SRC_PATH/cmd/zdb/zdb" -ZREPL="$SRC_PATH/cmd/zrepl/zrepl start -t 127.0.0.1" +ZREPL="$SRC_PATH/cmd/zrepl/zrepl" GTEST_UZFS="$SRC_PATH/tests/cbtest/gtest/test_uzfs" GTEST_ZREPL_PROT="$SRC_PATH/tests/cbtest/gtest/test_zrepl_prot" -GTEST_EXPORT="$SRC_PATH/tests/cbtest/gtest/test_export" ZTEST="$SRC_PATH/cmd/ztest/ztest" UZFS_TEST="$SRC_PATH/cmd/uzfs_test/uzfs_test" UZFS_TEST_SYNC_SH="$SRC_PATH/cmd/uzfs_test/uzfs_test_sync.sh" @@ -114,6 +113,17 @@ stop_zrepl() fi } +wait_for_pids() +{ + for p in "$@"; do + wait $p + status=$? + if [ $status -ne 0 ] && [ $status -ne 127 ]; then + exit 1 + fi + done +} + create_disk() { for disk in "$@"; do @@ -170,6 +180,8 @@ run_zvol_tests() log_must datasetexists $src_pool/$src_vol log_must check_prop $src_pool/$src_vol type volume + log_must $ZFS create -V $VOLSIZE -o io.openebs:targetip=127.0.0.1:6060 $src_pool/$src_vol"_1" + # test volume properties log_must $ZFS get all $src_pool/$src_vol > /dev/null log_must $ZFS list $src_pool/$src_vol > /dev/null @@ -187,6 +199,8 @@ run_zvol_tests() log_must $ZFS set sync=always $src_pool/$src_vol log_must check_prop "$src_pool/$src_vol" sync always + log_must check_prop "$src_pool/$src_vol""_1" io.openebs:targetip 127.0.0.1:6060 + # dump some data #log_must dump_data @@ -214,6 +228,7 @@ run_zvol_tests() log_must $ZFS destroy -r $dst_pool/$dst_vol log_must $ZFS list -t all $src_pool/$src_vol > /dev/null log_must $ZFS destroy -r $src_pool/$src_vol + log_must $ZFS destroy -r $src_pool/$src_vol"_1" # test snap destroy log_must $ZFS create -s -V $VOLSIZE $src_pool/$src_vol @@ -413,6 +428,26 @@ destroy_pool() return $? } +# $1 - pool name +import_pool() +{ + pool=$1 + + if [ -z $pool ]; then + echo "No pool name given." + return 1 + fi + + if poolexists "$pool" ; then + echo "Pool already imported. ($pool)" + return 1 + else + log_must $ZPOOL import $pool -d $TMPDIR + fi + + return $? +} + # $1 - pool name export_pool() { @@ -570,7 +605,6 @@ test_mirror_pool() # test pool creation create_disk $src_pool_disk_a $src_pool_disk_b create_disk $dst_pool_disk_a $dst_pool_disk_b - # test pool creation log_must $ZPOOL create -f $src_pool mirror \ -o cachefile="$TMPDIR/zpool_$src_pool.cache" \ @@ -712,8 +746,8 @@ run_fio_test() log_must $ZPOOL create -f $fio_pool \ -o cachefile="$TMPDIR/zpool_$fio_pool.cache" \ "$TMPDIR/fio_disk1.img" - log_must $ZFS create -sV $VOLSIZE -o volblocksize=4k $fio_pool/vol1 - log_must $ZFS create -sV $VOLSIZE -o volblocksize=4k $fio_pool/vol2 + log_must $ZFS create -sV $VOLSIZE -o volblocksize=4k -o io.openebs:targetip=127.0.0.1:6060 $fio_pool/vol1 + log_must $ZFS create -sV $VOLSIZE -o volblocksize=4k -o io.openebs:targetip=127.0.0.1:6060 $fio_pool/vol2 cat >$TMPDIR/test.fio <> ~/.bash_rc +sysctl -p + start_zrepl if [ $test_type == "all" ]; then START=$(date +%s.%N) @@ -1218,3 +1260,6 @@ else execute_test $test_type fi stop_zrepl + +ls -ltr +