Merge branch 'master' into lxz/peer_check_fix

Required-githooks: true

.github/workflows/rpm-build-and-test.yml

@@ -5,10 +5,10 @@ env:
   #       build is done on the lowest version and test on the highest with a "sanity test"
   #       stage done on all versions in the list ecept the highest
   EL8_BUILD_VERSION: 8.6
-  EL8_VERSION: 8
+  EL8_VERSION: 8.8
   EL9_BUILD_VERSION: 9
   EL9_VERSION: 9
-  LEAP15_VERSION: 15.4
+  LEAP15_VERSION: 15.5
 
 on:
   workflow_dispatch:

ci/functional/test_main.sh

@@ -45,7 +45,7 @@ test_cluster() {
         FIRST_NODE=${first_node}                        \
         TEST_RPMS=${TEST_RPMS}                          \
         NODELIST=${tnodes}                              \
-        BUILD_URL=\"$BUILD_URL\"                        \
+        BUILD_URL=\"${BUILD_URL:-Unknown in GHA}\"      \
         STAGE_NAME=\"$STAGE_NAME\"                      \
         $(cat ci/functional/test_main_prep_node.sh)"
 }

src/cart/crt_iv.c

@@ -3508,8 +3508,8 @@ crt_iv_update_internal(crt_iv_namespace_t ivns, uint32_t class_id,
 
 		D_GOTO(exit, rc);
 	} else {
-		DL_CDEBUG(rc == -DER_NONEXIST || rc == -DER_NOTLEADER, DLOG_INFO, DLOG_ERR, rc,
-			  "ivo_on_update failed");
+		DL_CDEBUG(rc == -DER_NONEXIST || rc == -DER_NOTLEADER || rc == -DER_BUSY,
+			  DLOG_INFO, DLOG_ERR, rc, "ivo_on_update failed");
 
 		update_comp_cb(ivns, class_id, iv_key, NULL,
 			       iv_value, rc, cb_arg);

src/cart/crt_rpc.c

@@ -264,7 +264,8 @@ crt_opc_decode(crt_opcode_t crt_opc, char **module_name, char **opc_name)
 /* Redefining X macro allows to reuse existing lists */
 #define X(a, ...)                                                                                  \
 	case a:                                                                                    \
-		opc = #a;
+		opc = #a;                                                                          \
+		break;
 
 	/* Next find the opcode name if available for the module  */
 	if (cart_module) {

src/client/dfs/dfs.c

@@ -6686,7 +6686,7 @@ oit_mark_cb(dfs_t *dfs, dfs_obj_t *parent, const char name[], void *args)
 	}
 
 	/** open the entry name and get the oid */
-	rc = dfs_lookup_rel(dfs, parent, name, O_RDONLY, &obj, NULL, NULL);
+	rc = dfs_lookup_rel(dfs, parent, name, O_RDONLY | O_NOFOLLOW, &obj, NULL, NULL);
 	if (rc) {
 		D_ERROR("dfs_lookup_rel() of %s failed: %d\n", name, rc);
 		return rc;

src/container/oid_iv.c

@@ -31,6 +31,7 @@ struct oid_iv_entry {
 	struct oid_iv_range	rg;
 	/** protect the entry */
 	ABT_mutex		lock;
+	void			*current_req;
 };
 
 /** Priv data in the iv layer */
@@ -130,7 +131,14 @@ oid_iv_ent_update(struct ds_iv_entry *ns_entry, struct ds_iv_key *iv_key,
 	D_ASSERT(priv != NULL);
 
 	entry = ns_entry->iv_value.sg_iovs[0].iov_buf;
-	ABT_mutex_lock(entry->lock);
+	rc = ABT_mutex_trylock(entry->lock);
+	/* For retry requests, from _iv_op(), the lock may not be released
+	 * in some cases.
+	 */
+	if (rc == ABT_ERR_MUTEX_LOCKED && entry->current_req != src)
+		return -DER_BUSY;
+
+	entry->current_req = src;
 	avail = &entry->rg;
 
 	oids = src->sg_iovs[0].iov_buf;

src/control/cmd/daos/filesystem.go

@@ -291,7 +291,6 @@ type fsCheckCmd struct {
 
 	FsckFlags FsCheckFlag `long:"flags" short:"f" description:"comma-separated flags: print, remove, relink, verify, evict"`
 	DirName   string      `long:"dir-name" short:"n" description:"directory name under lost+found to store leaked oids (a timestamp dir would be created if this is not specified)"`
-	Evict     bool        `long:"evict" short:"e" description:"evict all open handles on the container"`
 }
 
 func (cmd *fsCheckCmd) Execute(_ []string) error {

src/control/lib/control/pool.go

@@ -1365,7 +1365,7 @@ func processNVMeSpaceStats(log debugLogger, filterRank filterRankFn, nvmeControl
 		for _, smdDevice := range controller.SmdDevices {
 			if !smdDevice.Roles.IsEmpty() && (smdDevice.Roles.OptionBits&storage.BdevRoleData) == 0 {
 				log.Debugf("Skipping SMD device %s (rank %d, ctrlr %s) not used for storing data",
-					smdDevice.UUID, smdDevice.Rank, controller.PciAddr, smdDevice.Rank)
+					smdDevice.UUID, smdDevice.Rank, controller.PciAddr)
 				continue
 			}
 
@@ -1377,7 +1377,7 @@ func processNVMeSpaceStats(log debugLogger, filterRank filterRankFn, nvmeControl
 
 			if !filterRank(smdDevice.Rank) {
 				log.Debugf("Skipping SMD device %s (rank %d, ctrlr %s) not in ranklist",
-					smdDevice.UUID, smdDevice.Rank, controller.PciAddr, smdDevice.Rank)
+					smdDevice.UUID, smdDevice.Rank, controller.PciAddr)
 				continue
 			}
 

src/engine/server_iv.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2017-2023 Intel Corporation.
+ * (C) Copyright 2017-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -1053,7 +1053,7 @@ _iv_op(struct ds_iv_ns *ns, struct ds_iv_key *key, d_sg_list_t *value,
 retry:
 	rc = iv_op_internal(ns, key, value, sync, shortcut, opc);
 	if (retry && !ns->iv_stop &&
-	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER)) {
+	    (daos_rpc_retryable_rc(rc) || rc == -DER_NOTLEADER || rc == -DER_BUSY)) {
 		if (rc == -DER_NOTLEADER && key->rank != (d_rank_t)(-1) &&
 		    sync && (sync->ivs_mode == CRT_IV_SYNC_LAZY ||
 			     sync->ivs_mode == CRT_IV_SYNC_EAGER)) {
@@ -1070,7 +1070,7 @@ _iv_op(struct ds_iv_ns *ns, struct ds_iv_key *key, d_sg_list_t *value,
 		 * but in-flight fetch request return IVCB_FORWARD, then queued RPC will
 		 * reply IVCB_FORWARD.
 		 */
-		D_WARN("ns %u retry for class %d opc %d rank %u/%u: " DF_RC "\n", ns->iv_ns_id,
+		D_INFO("ns %u retry for class %d opc %d rank %u/%u: " DF_RC "\n", ns->iv_ns_id,
 		       key->class_id, opc, key->rank, ns->iv_master_rank, DP_RC(rc));
 		/* sleep 1sec and retry */
 		dss_sleep(1000);

src/gurt/misc.c

@@ -25,8 +25,6 @@
 #include <gurt/common.h>
 #include <gurt/atomic.h>
 
-#define UINT64_MAX_STR "18446744073709551615"
-
 /* state buffer for DAOS rand and srand calls, NOT thread safe */
 static struct drand48_data randBuffer = {0};
 
@@ -951,18 +949,17 @@ d_rank_range_list_free(d_rank_range_list_t *range_list)
 }
 
 static inline bool
-dis_unsigned_str(char *str)
+dis_signed_str(char *str)
 {
-	char *eos;
-
-	if (str == NULL || str[0] == '\0')
-		return false;
+	char  *eos;
+	size_t str_size;
 
-	eos = str + (sizeof(UINT64_MAX_STR) - 1);
-	while (str != eos && *str != '\0' && *str >= '0' && *str <= '9')
+	str_size = strlen(str);
+	eos      = str + str_size;
+	while (str != eos && *str != '-' && (*str < '0' || *str > '9'))
 		++str;
 
-	return *str == '\0';
+	return *str == '-';
 }
 
 static inline bool
@@ -1214,40 +1211,68 @@ d_getenv_char(const char *name, char *char_val)
 }
 
 static int
-d_getenv_ull(unsigned long long *val, const char *name)
+d_getenv_ull(unsigned long long *val, const char *name, size_t val_size)
 {
 	char              *env;
+	char              *env_tmp = NULL;
 	char              *endptr;
-	unsigned long long tmp;
+	unsigned long long val_tmp;
 	int                rc;
 
 	assert(val != NULL);
 	assert(name != NULL);
+	assert(val_size <= sizeof(unsigned long long));
 
 	d_env_rwlock_rdlock();
 	env = getenv(name);
 	if (env == NULL) {
 		rc = -DER_NONEXIST;
+		d_env_rwlock_unlock();
 		goto out;
 	}
 
-	if (!dis_unsigned_str(env)) {
-		rc = -DER_INVAL;
+	/* DAOS-14896 NOTES:
+	 * - Duplicate env to reduce data race condition with external libraries not using the DAOS
+	 *   thread safe environment variables management API.
+	 * - Use of strdup() as there is no limit to environment variable size.
+	 */
+	env_tmp = strdup(env);
+	if (env_tmp == NULL) {
+		rc = -DER_NOMEM;
+		d_env_rwlock_unlock();
 		goto out;
 	}
+	d_env_rwlock_unlock();
 
-	errno = 0;
-	tmp   = strtoull(env, &endptr, 0);
-	if (errno != 0 || endptr == env || *endptr != '\0') {
+	errno   = 0;
+	val_tmp = strtoull(env_tmp, &endptr, 10);
+	if (errno != 0 || endptr == env_tmp || *endptr != '\0') {
 		rc = -DER_INVAL;
 		goto out;
 	}
 
-	*val = tmp;
+	if (val_size != sizeof(unsigned long long)) {
+		const unsigned long long val_max   = (1ull << val_size * 8) - 1;
+		const bool               is_signed = dis_signed_str(env_tmp);
+
+		if (is_signed)
+			val_tmp = ~val_tmp;
+		if (val_tmp > val_max || (is_signed && val_tmp >= val_max)) {
+			rc = -DER_INVAL;
+			goto out;
+		}
+		if (is_signed) {
+			val_tmp = ~val_tmp;
+			val_tmp <<= (sizeof(unsigned long long) - val_size) * 8;
+			val_tmp >>= (sizeof(unsigned long long) - val_size) * 8;
+		}
+	}
+
+	*val = val_tmp;
 	rc   = -DER_SUCCESS;
 
 out:
-	d_env_rwlock_unlock();
+	free(env_tmp);
 
 	return rc;
 }
@@ -1269,21 +1294,29 @@ d_getenv_uint(const char *name, unsigned *uint_val)
 	assert(uint_val != NULL);
 	assert(name != NULL);
 
-	rc = d_getenv_ull(&tmp, name);
+	rc = d_getenv_ull(&tmp, name, sizeof(unsigned));
 	if (rc != -DER_SUCCESS)
 		return rc;
 
-#if UINT_MAX != ULLONG_MAX
-	assert(sizeof(unsigned) < sizeof(unsigned long long));
-	if (tmp > UINT_MAX) {
-		return -DER_INVAL;
-	}
-#endif
-
 	*uint_val = (unsigned)tmp;
 	return -DER_SUCCESS;
 }
 
+/**
+ * get an unsigned integer type environment variables.
+ *
+ * \param[in]		name		name of the environment variable.
+ * \param[in,out]	uint_val	returned value of the ENV. Will not change the original
+ *					value if ENV is not set or set as a non-integer value.
+ * \return				0 on success, a negative value on error.
+ * \deprecated				d_getenv_int() is deprecated, please use d_getenv_uint().
+ */
+int
+d_getenv_int(const char *name, unsigned *uint_val)
+{
+	return d_getenv_uint(name, uint_val);
+}
+
 /**
  * get a 32bits unsigned integer type environment variables
  *
@@ -1301,17 +1334,10 @@ d_getenv_uint32_t(const char *name, uint32_t *uint32_val)
 	assert(uint32_val != NULL);
 	assert(name != NULL);
 
-	rc = d_getenv_ull(&tmp, name);
+	rc = d_getenv_ull(&tmp, name, sizeof(uint32_t));
 	if (rc != -DER_SUCCESS)
 		return rc;
 
-#if UINT32_MAX != ULLONG_MAX
-	assert(sizeof(uint32_t) < sizeof(unsigned long long));
-	if (tmp > UINT32_MAX) {
-		return -DER_INVAL;
-	}
-#endif
-
 	*uint32_val = (uint32_t)tmp;
 	return -DER_SUCCESS;
 }
@@ -1333,17 +1359,10 @@ d_getenv_uint64_t(const char *name, uint64_t *uint64_val)
 	assert(uint64_val != NULL);
 	assert(name != NULL);
 
-	rc = d_getenv_ull(&tmp, name);
+	rc = d_getenv_ull(&tmp, name, sizeof(uint64_t));
 	if (rc != -DER_SUCCESS)
 		return rc;
 
-#if UINT64_MAX != ULLONG_MAX
-	assert(sizeof(uint64_t) < sizeof(unsigned long long));
-	if (tmp > UINT64_MAX) {
-		return -DER_INVAL;
-	}
-#endif
-
 	*uint64_val = (uint64_t)tmp;
 	return -DER_SUCCESS;
 }