[TA3148] feat(snap_rebuild): doing rebuild from AFS after ALL_SNAP_DO…

…NE (#122)

Signed-off-by: Vishnu Itta <[email protected]>

include/mgmt_conn.h

@@ -106,7 +106,7 @@ void zinfo_create_cb(zvol_info_t *zinfo, nvlist_t *create_props);
 void zinfo_destroy_cb(zvol_info_t *zinfo);
 void uzfs_zvol_mgmt_thread(void *arg);
 int finish_async_tasks(void);
-
+int uzfs_zinfo_rebuild_from_clone(zvol_info_t *zinfo);
 int uzfs_zvol_create_snapshot_update_zap(zvol_info_t *zinfo,
     char *snapname, uint64_t snapshot_io_num);
 int uzfs_get_snap_zv_ionum(zvol_info_t *, uint64_t, zvol_state_t **);

include/sys/uzfs_zvol.h

@@ -82,6 +82,8 @@ struct zvol_state {
 	 * This should not be greater than volblocksize
 	 */
 	uint64_t zv_metavolblocksize;
+
+	/* Don't use status directly. Use getter/setter of zvol_info */
 	zvol_status_t zv_status;		/* zvol status */
 	kmutex_t rebuild_mtx;
 	zvol_rebuild_info_t rebuild_info;
@@ -94,7 +96,9 @@ typedef struct zvol_state zvol_state_t;
 #define	UZFS_IO_READ_FAIL	2
 #define	UZFS_IO_MREAD_FAIL	3
 
+#define	ZINFO_IS_DEGRADED(zinfo)	(ZVOL_IS_DEGRADED(zinfo->main_zv))
 #define	ZVOL_IS_DEGRADED(zv)	(zv->zv_status == ZVOL_STATUS_DEGRADED)
+
 #define	ZVOL_IS_REBUILDING(zv)		\
 	((zv->rebuild_info.zv_rebuild_status == ZVOL_REBUILDING_SNAP) || \
 	(zv->rebuild_info.zv_rebuild_status == ZVOL_REBUILDING_AFS))

include/zrepl_mgmt.h

@@ -210,16 +210,19 @@ extern zvol_info_t *uzfs_zinfo_lookup(const char *name);
 extern void uzfs_zinfo_replay_zil_all(void);
 extern int uzfs_zinfo_destroy(const char *ds_name, spa_t *spa);
 int uzfs_zvol_get_last_committed_io_no(zvol_state_t *, char *, uint64_t *);
-void uzfs_zvol_store_last_committed_healthy_io_no(zvol_info_t *zinfo,
+void uzfs_zinfo_store_last_committed_healthy_io_no(zvol_info_t *zinfo,
     uint64_t io_seq);
-void uzfs_zvol_store_last_committed_degraded_io_no(zvol_info_t *zv,
+void uzfs_zinfo_store_last_committed_degraded_io_no(zvol_info_t *zv,
     uint64_t io_seq);
 extern int set_socket_keepalive(int sfd);
 extern int create_and_bind(const char *port, int bind_needed,
     boolean_t nonblocking);
 int uzfs_zvol_name_compare(zvol_info_t *zv, const char *name);
 void shutdown_fds_related_to_zinfo(zvol_info_t *zinfo);
 
+extern void uzfs_zinfo_set_status(zvol_info_t *zinfo, zvol_status_t status);
+extern zvol_status_t uzfs_zinfo_get_status(zvol_info_t *zinfo);
+
 /*
  * API to drop refcnt on zinfo. If refcnt
  * dropped to zero then free zinfo.

include/zrepl_prot.h

@@ -99,7 +99,9 @@ struct zvol_io_hdr {
 	uint8_t 	flags;
 	uint8_t 	padding[3];
 	union {
+		/* IOnum as sent from target */
 		uint64_t	io_seq;
+		/* IOnum from which rebuild need to be done */
 		uint64_t	checkpointed_io_seq;
 	};
 	/* only used for read/write */

lib/libzpool/uzfs_io.c

@@ -406,6 +406,7 @@ uzfs_zvol_get_status(zvol_state_t *zv)
 {
 	return (zv->zv_status);
 }
+
 void
 uzfs_zvol_set_rebuild_status(zvol_state_t *zv, zvol_rebuild_status_t status)
 {

lib/libzpool/zrepl_mgmt.c

@@ -442,7 +442,7 @@ uzfs_zvol_store_kv_pair(zvol_state_t *zv, char *key,
 }
 
 void
-uzfs_zvol_store_last_committed_degraded_io_no(zvol_info_t *zinfo,
+uzfs_zinfo_store_last_committed_degraded_io_no(zvol_info_t *zinfo,
     uint64_t io_seq)
 {
 	uzfs_zvol_store_kv_pair(zinfo->main_zv,
@@ -455,7 +455,7 @@ uzfs_zvol_store_last_committed_degraded_io_no(zvol_info_t *zinfo,
  * Updates in-memory committed io_num.
  */
 void
-uzfs_zvol_store_last_committed_healthy_io_no(zvol_info_t *zinfo,
+uzfs_zinfo_store_last_committed_healthy_io_no(zvol_info_t *zinfo,
     uint64_t io_seq)
 {
 	if (io_seq == 0)
@@ -471,3 +471,15 @@ uzfs_zvol_store_last_committed_healthy_io_no(zvol_info_t *zinfo,
 	    HEALTHY_IO_SEQNUM, io_seq);
 	pthread_mutex_unlock(&zinfo->zinfo_ionum_mutex);
 }
+
+void
+uzfs_zinfo_set_status(zvol_info_t *zinfo, zvol_status_t status)
+{
+	uzfs_zvol_set_status(zinfo->main_zv, status);
+}
+
+zvol_status_t
+uzfs_zinfo_get_status(zvol_info_t *zinfo)
+{
+	return (uzfs_zvol_get_status(zinfo->main_zv));
+}

lib/libzrepl/data_conn.c

@@ -484,6 +484,7 @@ uzfs_zvol_rebuild_dw_replica(void *arg)
 	rebuild_thread_arg_t *rebuild_args = arg;
 	struct sockaddr_in replica_ip;
 
+	int		start_rebuild_from_clone = 0;
 	int		rc = 0;
 	int		sfd = -1;
 	uint64_t	offset = 0;
@@ -608,7 +609,7 @@ uzfs_zvol_rebuild_dw_replica(void *arg)
 		hdr.status = ZVOL_OP_STATUS_OK;
 		hdr.version = REPLICA_VERSION;
 		hdr.opcode = ZVOL_OPCODE_REBUILD_STEP;
-		hdr.io_seq = checkpointed_ionum;
+		hdr.checkpointed_io_seq = checkpointed_ionum;
 		hdr.offset = offset;
 		if ((offset + zvol_rebuild_step_size) >
 		    ZVOL_VOLUME_SIZE(zvol_state))
@@ -644,14 +645,16 @@ uzfs_zvol_rebuild_dw_replica(void *arg)
 
 		if (hdr.opcode == ZVOL_OPCODE_REBUILD_STEP_DONE) {
 			offset += zvol_rebuild_step_size;
-			LOG_DEBUG("ZVOL_OPCODE_REBUILD_STEP_DONE received");
+			LOG_DEBUG("ZVOL_OPCODE_REBUILD_STEP_DONE received on "
+			    "fd: %d", sfd);
 			goto next_step;
 		}
 
 		if (hdr.opcode == ZVOL_OPCODE_REBUILD_SNAP_DONE)
 			goto next_step;
 
 		if (hdr.opcode == ZVOL_OPCODE_REBUILD_ALL_SNAP_DONE) {
+			LOG_DEBUG("Received ALL_SNAP_DONE on fd: %d", sfd);
 			/* All snapshots has been transferred */
 			all_snap_done = B_TRUE;
 			/*
@@ -669,11 +672,29 @@ uzfs_zvol_rebuild_dw_replica(void *arg)
 			 * Multiple rebuild ops going on in parallel,
 			 * one of them might have changed rebuild state
 			 */
-			if (uzfs_zvol_get_rebuild_status(zinfo->main_zv) !=
-			    ZVOL_REBUILDING_AFS)
+			zvol_rebuild_status_t state;
+			state = uzfs_zvol_get_rebuild_status(zinfo->main_zv);
+			if (state != ZVOL_REBUILDING_AFS) {
+				ASSERT(state == ZVOL_REBUILDING_SNAP);
 				uzfs_zvol_set_rebuild_status(zinfo->main_zv,
 				    ZVOL_REBUILDING_AFS);
+				if (start_rebuild_from_clone == 0)
+					start_rebuild_from_clone = 1;
+			}
 			mutex_exit(&zinfo->main_zv->rebuild_mtx);
+
+			if (start_rebuild_from_clone == 1) {
+				start_rebuild_from_clone = 2;
+				rc = uzfs_zinfo_rebuild_from_clone(zinfo);
+				if (rc != 0) {
+					LOG_ERR("Rebuild from clone for vol %s "
+					    "failed", zinfo->name);
+					rc = -1;
+					goto exit;
+				}
+				LOG_INFO("Rebuild started from clone for vol "
+				    "%s", zinfo->name);
+			}
 			continue;
 		}
 		ASSERT((hdr.opcode == ZVOL_OPCODE_READ) &&
@@ -742,10 +763,10 @@ uzfs_zvol_rebuild_dw_replica(void *arg)
 }
 
 #define	STORE_LAST_COMMITTED_HEALTHY_IO_NO	\
-    uzfs_zvol_store_last_committed_healthy_io_no
+    uzfs_zinfo_store_last_committed_healthy_io_no
 
 #define	STORE_LAST_COMMITTED_DEGRADED_IO_NO	\
-    uzfs_zvol_store_last_committed_degraded_io_no
+    uzfs_zinfo_store_last_committed_degraded_io_no
 
 void
 uzfs_zvol_timer_thread(void)
@@ -1268,7 +1289,7 @@ uzfs_zvol_rebuild_scanner(void *arg)
 
 		case ZVOL_OPCODE_REBUILD_STEP:
 
-			metadata.io_num = hdr.io_seq;
+			metadata.io_num = hdr.checkpointed_io_seq;
 			rebuild_req_offset = hdr.offset;
 			rebuild_req_len = hdr.len;
 
@@ -1291,16 +1312,26 @@ uzfs_zvol_rebuild_scanner(void *arg)
 				}
 			}
 
-			ASSERT((snap_zv == NULL) && (all_snap_done == B_FALSE));
-
-			if ((snap_zv == NULL) && (all_snap_done == B_FALSE)) {
-				uzfs_zvol_send_zio_cmd(zinfo, &hdr,
-				    ZVOL_OPCODE_REBUILD_ALL_SNAP_DONE,
-				    fd, NULL, 0, 0);
-				all_snap_done = B_TRUE;
+			zvol_state_t *zv = zinfo->main_zv;
+			if (snap_zv == NULL) {
+			/*
+			 * which means there is no user snapshot of given
+			 * io_num, but, TODO: we need to make sure that there
+			 * are no ongoing snapshots.
+			 */
+				if (all_snap_done == B_FALSE) {
+					uzfs_zvol_send_zio_cmd(zinfo, &hdr,
+					    ZVOL_OPCODE_REBUILD_ALL_SNAP_DONE,
+					    fd, NULL, 0, 0);
+					all_snap_done = B_TRUE;
+				}
+				if (ZINFO_IS_DEGRADED(zinfo))
+					zv = zinfo->clone_zv;
+			} else {
+				ASSERT(all_snap_done == B_FALSE);
 			}
 
-			rc = uzfs_get_io_diff(zinfo->main_zv, &metadata,
+			rc = uzfs_get_io_diff(zv, &metadata,
 			    snap_zv, uzfs_zvol_rebuild_scanner_callback,
 			    rebuild_req_offset, rebuild_req_len, &warg);
 			if (rc != 0) {
@@ -1658,14 +1689,14 @@ open_zvol(int fd, zvol_info_t **zinfopp)
 	zv = zinfo->main_zv;
 	ASSERT3P(zv, !=, NULL);
 
-	ASSERT3P(zv->zv_status, ==, ZVOL_STATUS_DEGRADED);
+	ASSERT3P(uzfs_zinfo_get_status(zinfo), ==, ZVOL_STATUS_DEGRADED);
 	ASSERT3P(zv->rebuild_info.zv_rebuild_status, ==, ZVOL_REBUILDING_INIT);
 
-	if ((zv->zv_status != ZVOL_STATUS_DEGRADED) ||
+	if ((uzfs_zinfo_get_status(zinfo) != ZVOL_STATUS_DEGRADED) ||
 	    ((zv->rebuild_info.zv_rebuild_status != ZVOL_REBUILDING_INIT) &&
 	    (zv->rebuild_info.zv_rebuild_status != ZVOL_REBUILDING_FAILED))) {
 		LOG_ERR("as status for %s is %d or rebuild status is %d",
-		    open_data.volname, zv->zv_status,
+		    open_data.volname, uzfs_zinfo_get_status(zinfo),
 		    zv->rebuild_info.zv_rebuild_status);
 		hdr.status = ZVOL_OP_STATUS_FAILED;
 		(void) pthread_mutex_unlock(&zinfo->zinfo_mutex);