DAOS-15476 client: Better protocol query handling

Pick from all possible ranks for protocol query handling

Backports the following patches, last one fixes the issue but
prior two are benign and make the backport work without conflict

DAOS-13717 client: use d_rand() to generate random number (#12418)
DAOS-15120 crt: Misc d_rand-related fixes (#13738)
DAOS-15476 client: do not use rsvc for proto query rank selection (#14131)

Required-githooks: true

Change-Id: Icd83ae9307a2dfc928ca524af2954131b161157d
Signed-off-by: Mohamad Chaarawi <[email protected]>
Signed-off-by: Wang Shilong <[email protected]>
Signed-off-by: Li Wei <[email protected]>
Signed-off-by: Jeff Olivier <[email protected]>

src/cart/crt_init.c

@@ -9,7 +9,6 @@
  */
 #include <malloc.h>
 #include <sys/mman.h>
-#include <sys/time.h>
 #include <sys/resource.h>
 #include "crt_internal.h"
 
@@ -74,6 +73,7 @@ crt_lib_init(void)
 {
 	int		rc;
 	uint64_t	start_rpcid;
+	struct timespec	now;
 
 	rc = D_RWLOCK_INIT(&crt_gdata.cg_rwlock, NULL);
 	D_ASSERT(rc == 0);
@@ -88,7 +88,9 @@ crt_lib_init(void)
 	crt_gdata.cg_inited = 0;
 	crt_gdata.cg_primary_prov = CRT_PROV_OFI_SOCKETS;
 
-	d_srand(d_timeus_secdiff(0) + getpid());
+	rc = d_gettime(&now);
+	D_ASSERTF(rc == 0, "d_gettime: " DF_RC "\n", DP_RC(rc));
+	d_srand(now.tv_sec * 1000 * 1000 * 1000 + now.tv_nsec + getpid());
 	start_rpcid = ((uint64_t)d_rand()) << 32;
 
 	crt_gdata.cg_rpcid = start_rpcid;
@@ -611,12 +613,14 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info)
 int
 crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt)
 {
-	char          *provider, *provider_env;
-	char          *interface, *interface_env;
-	char          *domain, *domain_env;
-	char          *auth_key, *auth_key_env;
-	struct timeval now;
-	unsigned int   seed;
+	char          *provider;
+	char          *provider_env = NULL;
+	char          *interface;
+	char          *interface_env = NULL;
+	char          *domain;
+	char          *domain_env = NULL;
+	char          *auth_key;
+	char          *auth_key_env = NULL;
 	char          *path;
 	bool           server;
 	int            rc            = 0;
@@ -684,11 +688,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt)
 
 	D_RWLOCK_WRLOCK(&crt_gdata.cg_rwlock);
 	if (crt_gdata.cg_inited == 0) {
-		/* feed a seed for pseudo-random number generator */
-		gettimeofday(&now, NULL);
-		seed = (unsigned int)(now.tv_sec * 1000000 + now.tv_usec);
-		d_srand(seed);
-
 		crt_gdata.cg_server = server;
 		crt_gdata.cg_auto_swim_disable =
 			(flags & CRT_FLAG_BIT_AUTO_SWIM_DISABLE) ? 1 : 0;

src/client/api/rpc.c

@@ -1,13 +1,12 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
 #define D_LOGFAC	DD_FAC(client)
 
 #include <daos/rpc.h>
 #include <daos/event.h>
-#include <daos/rsvc.h>
 #include <daos/mgmt.h>
 
 static void
@@ -96,14 +95,14 @@ daos_rpc_send_wait(crt_rpc_t *rpc)
 }
 
 struct rpc_proto {
-	struct rsvc_client	cli;
-	crt_endpoint_t		ep;
-	int			version;
-	int			rc;
-	bool			completed;
-	crt_opcode_t		base_opc;
-	uint32_t		*ver_array;
-	uint32_t		 array_size;
+	int            nr_ranks;
+	crt_endpoint_t ep;
+	int            version;
+	int            rc;
+	bool           completed;
+	crt_opcode_t   base_opc;
+	uint32_t      *ver_array;
+	uint32_t       array_size;
 };
 
 static void
@@ -113,16 +112,10 @@ query_cb(struct crt_proto_query_cb_info *cb_info)
 	int			 rc;
 
 	if (daos_rpc_retryable_rc(cb_info->pq_rc)) {
-		rc = rsvc_client_choose(&rproto->cli, &rproto->ep);
-		if (rc) {
-			D_ERROR("rsvc_client_choose() failed: "DF_RC"\n", DP_RC(rc));
-			rproto->rc = rc;
-			rproto->completed = true;
-		}
-
-		rc = crt_proto_query_with_ctx(&rproto->ep, rproto->base_opc,
-					      rproto->ver_array, rproto->array_size,
-					      query_cb, rproto, daos_get_crt_ctx());
+		rproto->ep.ep_rank = (rproto->ep.ep_rank + 1) % rproto->nr_ranks;
+		rc = crt_proto_query_with_ctx(&rproto->ep, rproto->base_opc, rproto->ver_array,
+					      rproto->array_size, query_cb, rproto,
+					      daos_get_crt_ctx());
 		if (rc) {
 			D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc));
 			rproto->rc = rc;
@@ -141,8 +134,7 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int
 	struct dc_mgmt_sys	*sys;
 	struct rpc_proto	*rproto = NULL;
 	crt_context_t		 ctx = daos_get_crt_ctx();
-	int			 rc;
-	int			 num_ranks;
+	int                      rc;
 	int			 i;
 
 	rc = dc_mgmt_sys_attach(NULL, &sys);
@@ -155,39 +147,34 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int
 	if (rproto == NULL)
 		D_GOTO(out_detach, rc = -DER_NOMEM);
 
-	rc = rsvc_client_init(&rproto->cli, sys->sy_info.ms_ranks);
-	if (rc) {
-		D_ERROR("rsvc_client_init() failed: "DF_RC"\n", DP_RC(rc));
-		D_GOTO(out_free, rc);
-	}
-
-	num_ranks = dc_mgmt_net_get_num_srv_ranks();
-	rproto->ep.ep_rank = rand() % num_ranks;
+	/** select a random rank to issue the proto query rpc to */
+	rproto->nr_ranks   = dc_mgmt_net_get_num_srv_ranks();
+	rproto->ep.ep_rank = d_rand() % rproto->nr_ranks;
+	rproto->ep.ep_tag  = 0;
 	rproto->ver_array = ver_array;
 	rproto->array_size = count;
-	rproto->ep.ep_grp = sys->sy_group;
-	rproto->ep.ep_tag = 0;
+	rproto->ep.ep_grp  = sys->sy_group;
 	rproto->base_opc = base_opc;
 
 	rc = crt_proto_query_with_ctx(&rproto->ep, base_opc,
 				      ver_array, count, query_cb, rproto, ctx);
 	if (rc) {
 		D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc));
-		D_GOTO(out_rsvc, rc);
+		D_GOTO(out_free, rc);
 	}
 
 	while (!rproto->completed) {
 		rc = crt_progress(ctx, 0);
 		if (rc && rc != -DER_TIMEDOUT) {
 			D_ERROR("failed to progress CART context: %d\n", rc);
-			D_GOTO(out_rsvc, rc);
+			D_GOTO(out_free, rc);
 		}
 	}
 
 	if (rproto->rc != -DER_SUCCESS) {
 		rc = rproto->rc;
 		D_ERROR("crt_proto_query()failed: "DF_RC"\n", DP_RC(rc));
-		D_GOTO(out_rsvc, rc);
+		D_GOTO(out_free, rc);
 	}
 	rc = 0;
 
@@ -201,8 +188,6 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int
 	} else {
 		*ret_ver = rproto->version;
 	}
-out_rsvc:
-	rsvc_client_fini(&rproto->cli);
 out_free:
 	D_FREE(rproto);
 out_detach:

src/common/rsvc.c

@@ -91,7 +91,7 @@ rsvc_client_choose(struct rsvc_client *client, crt_endpoint_t *ep)
 		chosen = client->sc_leader_index;
 	} else {
 		if (client->sc_next < 0)
-			client->sc_next = d_randn(client->sc_ranks->rl_nr);
+			client->sc_next = d_rand() % client->sc_ranks->rl_nr;
 		chosen = client->sc_next;
 		/* The hintless search is a round robin of all replicas. */
 		client->sc_next++;
@@ -148,7 +148,7 @@ rsvc_client_process_error(struct rsvc_client *client, int rc,
 			 * search.
 			 */
 			D_DEBUG(DB_MD, "give up leader rank %u\n", ep->ep_rank);
-			client->sc_next = d_randn(client->sc_ranks->rl_nr);
+			client->sc_next = d_rand() % client->sc_ranks->rl_nr;
 			if (client->sc_next == leader_index) {
 				client->sc_next++;
 				client->sc_next %= client->sc_ranks->rl_nr;

src/gurt/misc.c

@@ -498,7 +498,7 @@ d_rank_list_shuffle(d_rank_list_t *rank_list)
 		return;
 
 	for (i = 0; i < rank_list->rl_nr; i++) {
-		j = rand() % rank_list->rl_nr;
+		j = d_rand() % rank_list->rl_nr;
 		tmp = rank_list->rl_ranks[i];
 		rank_list->rl_ranks[i] = rank_list->rl_ranks[j];
 		rank_list->rl_ranks[j] = tmp;

src/include/gurt/common.h

@@ -74,7 +74,6 @@ extern "C" {
 
 void d_srand(long int);
 long int d_rand(void);
-long int d_randn(long int n);
 
 /* memory allocating macros */
 void  d_free(void *);

src/pool/cli.c

@@ -1827,9 +1827,7 @@ choose_map_refresh_rank(struct map_refresh_arg *arg)
 
 	if (arg->mra_i == -1) {
 		/* Let i be a random integer in [0, n). */
-		i = ((double)rand() / RAND_MAX) * n;
-		if (i == n)
-			i = 0;
+		i = d_rand() % n;
 	} else {
 		/* Continue the round robin. */
 		i = arg->mra_i;

src/tests/ftest/erasurecode/rebuild_disabled_single.yaml

@@ -15,6 +15,7 @@ setup:
 server_config:
   name: daos_server
   engines_per_host: 2
+  crt_timeout: 10
   engines:
     0:
       pinned_numa_node: 0