Merge remote-tracking branch 'upstream/master' into grom72/restore_pa…

…rtition_w_900s Priority: 2 Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki <[email protected]>
daos-stack · Aug 21, 2024 · 740a16e · 740a16e
2 parents 0b5263b + c4c3a54
commit 740a16e
Show file tree

Hide file tree

Showing 33 changed files with 1,272 additions and 1,117 deletions.
diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,9 @@
+daos (2.7.100-5) unstable; urgency=medium
+  [ Michael MacDonald ]
+  * Add libdaos_self_test.so to client package
+
+ -- Michael MacDonald <[email protected]>  Thu, 15 Aug 2024 12:00:00 -0500
+
 daos (2.7.100-4) unstable; urgency=medium
   [ Jerome Soumagne ]
   * Bump mercury version to 2.4.0rc4

diff --git a/debian/daos-client.install b/debian/daos-client.install
@@ -12,6 +12,7 @@ usr/lib64/libioil.so
 usr/lib64/libpil4dfs.so
 usr/lib64/libdaos.so.*
 usr/lib64/libdaos_cmd_hdlrs.so
+usr/lib64/libdaos_self_test.so
 usr/lib64/python3.8/site-packages/pydaos/*.py
 usr/lib64/python3.8/site-packages/pydaos/pydaos_shim.so
 usr/lib64/python3.8/site-packages/pydaos/raw/*.py

diff --git a/docs/user/filesystem.md b/docs/user/filesystem.md
@@ -719,7 +719,7 @@ These are two command line options to control the DFuse process itself.
 | **Command line option** | **Description**           |
 | ----------------------- | ------------------------- |
 | --disable-caching       | Disables all caching      |
-| --disable-wb-caching    | Disables write-back cache |
+| --disable-wb-cache      | Disables write-back cache |
 
 These will affect all containers accessed via DFuse, regardless of any container attributes.
 

diff --git a/src/bio/bio_internal.h b/src/bio/bio_internal.h
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2018-2023 Intel Corporation.
+ * (C) Copyright 2018-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -359,9 +359,9 @@ struct bio_blobstore {
 	 * layer, teardown procedure needs be postponed.
 	 */
 	int			 bb_holdings;
-	/* Flags indicating blobstore load/unload is in-progress */
-	unsigned		 bb_loading:1,
-				 bb_unloading:1;
+	unsigned		 bb_loading:1,		/* Blobstore is loading */
+				 bb_unloading:1,	/* Blobstore is unloading */
+				 bb_faulty_done:1;	/* Faulty reaction is done */
 };
 
 /* Per-xstream blobstore */
@@ -650,6 +650,7 @@ uint64_t default_wal_sz(uint64_t meta_sz);
 /* bio_recovery.c */
 int bio_bs_state_transit(struct bio_blobstore *bbs);
 int bio_bs_state_set(struct bio_blobstore *bbs, enum bio_bs_state new_state);
+void trigger_faulty_reaction(struct bio_blobstore *bbs);
 
 /* bio_device.c */
 int fill_in_traddr(struct bio_dev_info *b_info, char *dev_name);

diff --git a/src/bio/bio_monitor.c b/src/bio/bio_monitor.c
@@ -728,17 +728,54 @@ is_bbs_faulty(struct bio_blobstore *bbs)
 void
 auto_faulty_detect(struct bio_blobstore *bbs)
 {
-	int	rc;
+	struct smd_dev_info	*dev_info;
+	int			 rc;
+
+	/* The in-memory device is already in FAULTY state */
+	if (bbs->bb_state == BIO_BS_STATE_FAULTY)
+		return;
 
-	if (bbs->bb_state != BIO_BS_STATE_NORMAL)
+	/* To make things simpler, don't detect faulty in SETUP phase */
+	if (bbs->bb_state == BIO_BS_STATE_SETUP)
 		return;
 
 	if (!is_bbs_faulty(bbs))
 		return;
 
-	rc = bio_bs_state_set(bbs, BIO_BS_STATE_FAULTY);
-	if (rc)
-		D_ERROR("Failed to set FAULTY state. "DF_RC"\n", DP_RC(rc));
+	/*
+	 * The device might have been unplugged before marked as FAULTY, and the bbs is
+	 * already in teardown.
+	 */
+	if (bbs->bb_state != BIO_BS_STATE_NORMAL) {
+		/* Faulty reaction is already successfully performed */
+		if (bbs->bb_faulty_done)
+			return;
+
+		rc = smd_dev_get_by_id(bbs->bb_dev->bb_uuid, &dev_info);
+		if (rc) {
+			DL_ERROR(rc, "Get device info "DF_UUID" failed.",
+				 DP_UUID(bbs->bb_dev->bb_uuid));
+			return;
+		}
+
+		/* The device is already marked as FAULTY */
+		if (dev_info->sdi_state == SMD_DEV_FAULTY) {
+			smd_dev_free_info(dev_info);
+			trigger_faulty_reaction(bbs);
+			return;
+		}
+		smd_dev_free_info(dev_info);
+
+		rc = smd_dev_set_state(bbs->bb_dev->bb_uuid, SMD_DEV_FAULTY);
+		if (rc)
+			DL_ERROR(rc, "Set device state failed.");
+		else
+			trigger_faulty_reaction(bbs);
+	} else {
+		rc = bio_bs_state_set(bbs, BIO_BS_STATE_FAULTY);
+		if (rc)
+			DL_ERROR(rc, "Failed to set FAULTY state.");
+	}
 
 	if (rc == 0)
 		ras_notify_eventf(RAS_DEVICE_SET_FAULTY, RAS_TYPE_INFO, RAS_SEV_NOTICE, NULL, NULL,

diff --git a/src/bio/bio_recovery.c b/src/bio/bio_recovery.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2018-2023 Intel Corporation.
+ * (C) Copyright 2018-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -53,10 +53,19 @@ on_faulty(struct bio_blobstore *bbs)
 	rc = ract_ops->faulty_reaction(tgt_ids, tgt_cnt);
 	if (rc < 0)
 		D_ERROR("Faulty reaction failed. "DF_RC"\n", DP_RC(rc));
+	else if (rc == 0)
+		bbs->bb_faulty_done = 1;
 
 	return rc;
 }
 
+void
+trigger_faulty_reaction(struct bio_blobstore *bbs)
+{
+	D_ASSERT(!bbs->bb_faulty_done);
+	on_faulty(bbs);
+}
+
 static void
 teardown_xs_bs(void *arg)
 {
@@ -460,9 +469,10 @@ bio_bs_state_set(struct bio_blobstore *bbs, enum bio_bs_state new_state)
 }
 
 int
-bio_xsctxt_health_check(struct bio_xs_context *xs_ctxt)
+bio_xsctxt_health_check(struct bio_xs_context *xs_ctxt, bool log_err, bool update)
 {
 	struct bio_xs_blobstore	*bxb;
+	struct media_error_msg	*mem;
 	enum smd_dev_type	 st;
 
 	/* sys xstream in pmem mode doesn't have NVMe context */
@@ -475,8 +485,21 @@ bio_xsctxt_health_check(struct bio_xs_context *xs_ctxt)
 		if (!bxb || !bxb->bxb_blobstore)
 			continue;
 
-		if (bxb->bxb_blobstore->bb_state != BIO_BS_STATE_NORMAL)
+		if (bxb->bxb_blobstore->bb_state != BIO_BS_STATE_NORMAL) {
+			if (log_err && bxb->bxb_blobstore->bb_state != BIO_BS_STATE_SETUP) {
+				D_ALLOC_PTR(mem);
+				if (mem == NULL) {
+					D_ERROR("Failed to allocate media error msg.\n");
+					return -DER_NVME_IO;
+				}
+
+				mem->mem_err_type = update ? MET_WRITE : MET_READ;
+				mem->mem_bs = bxb->bxb_blobstore;
+				mem->mem_tgt_id = xs_ctxt->bxc_tgt_id;
+				spdk_thread_send_msg(owner_thread(mem->mem_bs), bio_media_error, mem);
+			}
 			return -DER_NVME_IO;
+		}
 	}
 
 	return 0;
@@ -492,7 +515,7 @@ is_reint_ready(struct bio_blobstore *bbs)
 		xs_ctxt = bbs->bb_xs_ctxts[i];
 
 		D_ASSERT(xs_ctxt != NULL);
-		if (bio_xsctxt_health_check(xs_ctxt))
+		if (bio_xsctxt_health_check(xs_ctxt, false, false))
 			return false;
 	}
 	return true;

diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c
@@ -1705,6 +1705,18 @@ bio_nvme_ctl(unsigned int cmd, void *arg)
 	return rc;
 }
 
+static inline void
+reset_media_errors(struct bio_blobstore *bbs)
+{
+	struct nvme_stats	*dev_stats = &bbs->bb_dev_health.bdh_health_state;
+
+	dev_stats->bio_read_errs = 0;
+	dev_stats->bio_write_errs = 0;
+	dev_stats->bio_unmap_errs = 0;
+	dev_stats->checksum_errs = 0;
+	bbs->bb_faulty_done = 0;
+}
+
 void
 setup_bio_bdev(void *arg)
 {
@@ -1736,6 +1748,7 @@ setup_bio_bdev(void *arg)
 		goto out;
 	}
 
+	reset_media_errors(bbs);
 	rc = bio_bs_state_set(bbs, BIO_BS_STATE_SETUP);
 	D_ASSERT(rc == 0);
 out:

diff --git a/src/cart/crt_rpc.c b/src/cart/crt_rpc.c
@@ -965,16 +965,20 @@ uri_lookup_cb(const struct crt_cb_info *cb_info)
 retry:
 
 	if (rc != 0) {
-		if (chained_rpc_priv->crp_ul_retry++ < MAX_URI_LOOKUP_RETRIES) {
-			rc = crt_issue_uri_lookup_retry(lookup_rpc->cr_ctx,
-							grp_priv,
-							ul_in->ul_rank,
-							ul_in->ul_tag,
-							chained_rpc_priv);
-			D_GOTO(out, rc);
+		/* PROTO_QUERY will be retried by the caller, no need to retry URI lookups */
+		if (chained_rpc_priv->crp_pub.cr_opc != CRT_OPC_PROTO_QUERY) {
+			if (chained_rpc_priv->crp_ul_retry++ < MAX_URI_LOOKUP_RETRIES) {
+				rc = crt_issue_uri_lookup_retry(lookup_rpc->cr_ctx, grp_priv,
+								ul_in->ul_rank, ul_in->ul_tag,
+								chained_rpc_priv);
+				D_GOTO(out, rc);
+			} else {
+				D_ERROR("URI lookups exceeded %d retries\n",
+					chained_rpc_priv->crp_ul_retry);
+			}
 		} else {
-			D_ERROR("URI lookups exceeded %d retries\n",
-				chained_rpc_priv->crp_ul_retry);
+			DL_INFO(rc, "URI_LOOKUP for (%d:%d) failed during PROTO_QUERY",
+				ul_in->ul_rank, ul_in->ul_tag);
 		}
 	}
 

diff --git a/src/client/dfuse/pil4dfs/int_dfs.c b/src/client/dfuse/pil4dfs/int_dfs.c
@@ -940,13 +940,6 @@ child_hdlr(void)
 		DL_WARN(rc, "daos_eq_lib_init() failed in child process");
 	daos_dti_reset();
 	td_eqh = main_eqh = DAOS_HDL_INVAL;
-	if (d_eq_count_max > 0) {
-		rc = daos_eq_create(&td_eqh);
-		if (rc)
-			DL_WARN(rc, "daos_eq_create() failed");
-		else
-			main_eqh = td_eqh;
-	}
 	context_reset = true;
 }
 

diff --git a/src/common/lru.c b/src/common/lru.c
@@ -216,7 +216,12 @@ daos_lru_ref_hold(struct daos_lru_cache *lcache, void *key,
 		llink = link2llink(link);
 		D_ASSERT(llink->ll_evicted == 0);
 		if (llink->ll_evicting) {
-			daos_lru_ref_release(lcache, llink);
+			/**
+			 * Avoid calling `lru_hop_rec_decref()` at this point
+			 * to prevent `wakeup()` from being invoked twice.
+			 */
+			D_ASSERT(llink->ll_ref > 1);
+			llink->ll_ref--;
 			D_GOTO(out, rc = -DER_SHUTDOWN);
 		}
 		/* remove busy item from LRU */

diff --git a/src/dtx/dtx_rpc.c b/src/dtx/dtx_rpc.c
@@ -1220,6 +1220,9 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
 	/* Handle the entries whose leaders are on current server. */
 	d_list_for_each_entry_safe(dsp, tmp, &self, dsp_link) {
 		struct dtx_entry	dte;
+		struct dtx_entry	*pdte = &dte;
+		struct dtx_cos_key	 dck;
+
 
 		d_list_del(&dsp->dsp_link);
 
@@ -1228,13 +1231,31 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
 		dte.dte_refs = 1;
 		dte.dte_mbs = dsp->dsp_mbs;
 
+		if (for_io) {
+			rc = vos_dtx_check(cont->sc_hdl, &dsp->dsp_xid, NULL, NULL, NULL, false);
+			switch(rc) {
+			case DTX_ST_COMMITTABLE:
+				dck.oid = dsp->dsp_oid;
+				dck.dkey_hash = dsp->dsp_dkey_hash;
+				rc = dtx_commit(cont, &pdte, &dck, 1);
+				if (rc < 0 && rc != -DER_NONEXIST && for_io)
+					d_list_add_tail(&dsp->dsp_link, cmt_list);
+				else
+					dtx_dsp_free(dsp);
+				continue;
+			case DTX_ST_COMMITTED:
+			case -DER_NONEXIST: /* Aborted */
+				dtx_dsp_free(dsp);
+				continue;
+			default:
+				break;
+			}
+		}
+
 		rc = dtx_status_handle_one(cont, &dte, dsp->dsp_oid, dsp->dsp_dkey_hash,
 					   dsp->dsp_epoch, NULL, NULL);
 		switch (rc) {
 		case DSHR_NEED_COMMIT: {
-			struct dtx_entry	*pdte = &dte;
-			struct dtx_cos_key	 dck;
-
 			dck.oid = dsp->dsp_oid;
 			dck.dkey_hash = dsp->dsp_dkey_hash;
 			rc = dtx_commit(cont, &pdte, &dck, 1);

diff --git a/src/gurt/errno.c b/src/gurt/errno.c
@@ -1,15 +1,13 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
 /**
  * This file is part of GURT.
  */
-#include <string.h>
-
+#include <stdio.h>
 #include <daos_errno.h>
-#include <gurt/debug.h>
 #include <gurt/list.h>
 #include <gurt/common.h>
 

diff --git a/src/include/daos_srv/bio.h b/src/include/daos_srv/bio.h
@@ -486,11 +486,13 @@ void bio_xsctxt_free(struct bio_xs_context *ctxt);
  * Health check on the per-xstream NVMe context
  *
  * \param[in] xs_ctxt	Per-xstream NVMe context
+ * \param[in] log_err	Log media error if the device is not healthy
+ * \param[in] update	The check is called for an update operation or not
  *
  * \returns		0:		NVMe context is healthy
  *			-DER_NVME_IO:	NVMe context is faulty
  */
-int bio_xsctxt_health_check(struct bio_xs_context *xs_ctxt);
+int bio_xsctxt_health_check(struct bio_xs_context *xs_ctxt, bool log_err, bool update);
 
 /**
  * NVMe poller to poll NVMe I/O completions.

diff --git a/src/include/gurt/list.h b/src/include/gurt/list.h
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2022 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -242,8 +242,7 @@ d_list_splice_init(d_list_t *list, d_list_t *head)
  * \param[in] type	the type of the struct this is embedded in.
  * \param[in] member	the member name of the list within the struct.
  */
-#define d_list_entry(ptr, type, member) \
-	((type *)((char *)(ptr)-(char *)(&((type *)0)->member)))
+#define d_list_entry(ptr, type, member) ((type *)((char *)(ptr)-offsetof(type, member)))
 
 #define d_list_pop_entry(list, type, member)			\
 	({							\

diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c
@@ -5655,7 +5655,8 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc)
 			   1 /* start, [0] is for current engine */, ocpi->ocpi_disp_width,
 			   &exec_arg.coll_cur);
 
-	rc = dtx_leader_begin(ioc.ioc_vos_coh, &odm->odm_xid, &epoch, 1, version,
+	rc = dtx_leader_begin(ioc.ioc_vos_coh, &odm->odm_xid, &epoch,
+			      dcts[0].dct_shards[dmi->dmi_tgt_id].dcs_nr, version,
 			      &ocpi->ocpi_oid, NULL /* dti_cos */, 0 /* dti_cos_cnt */,
 			      NULL /* tgts */, exec_arg.coll_cur.grp_nr /* tgt_cnt */,
 			      dtx_flags, odm->odm_mbs, dce, &dlh);

diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py
@@ -128,7 +128,10 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False):
             f"bzip2 -z {fuse_root_dir}/lib.a",
             f"chmod u-r {fuse_root_dir}/lib.a.bz2",
             'fio --readwrite=randwrite --name=test --size="2M" --directory '
-            f'{fuse_root_dir}/ --bs=1M --numjobs="4" --ioengine=psync '
+            f'{fuse_root_dir}/ --bs=1M --numjobs="4" --ioengine=psync --thread=0'
+            "--group_reporting --exitall_on_error --continue_on_error=none",
+            'fio --readwrite=randwrite --name=test --size="2M" --directory '
+            f'{fuse_root_dir}/ --bs=1M --numjobs="4" --ioengine=psync --thread=1'
             "--group_reporting --exitall_on_error --continue_on_error=none",
             'fio --readwrite=randwrite --name=test --size="2M" --directory '
             f'{fuse_root_dir}/ --bs=1M --numjobs="1" --ioengine=libaio --iodepth=16'

diff --git a/src/tests/ftest/soak/harassers.yaml b/src/tests/ftest/soak/harassers.yaml
@@ -108,6 +108,7 @@ soak_harassers:
   enable_intercept_lib: false
   enable_remote_logging: false
   enable_scrubber: false
+  enable_rebuild_logmasks: false
 # Commandline parameters
 # Benchmark and application params
 # IOR params -a DFS and -a MPIIO