From 5da76e9003cdfbb1aceb1786aa8df080b14e80ac Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Mon, 19 Jun 2023 22:36:59 +0800 Subject: [PATCH] DAOS-15476 client: Better protocol query handling Pick from all possible ranks for protocol query handling Backports the following patches, last one fixes the issue but prior two are benign and make the backport work without conflict DAOS-13717 client: use d_rand() to generate random number (#12418) DAOS-15120 crt: Misc d_rand-related fixes (#13738) DAOS-15476 client: do not use rsvc for proto query rank selection (#14131) Required-githooks: true Change-Id: Icd83ae9307a2dfc928ca524af2954131b161157d Signed-off-by: Mohamad Chaarawi Signed-off-by: Wang Shilong Signed-off-by: Li Wei Signed-off-by: Jeff Olivier --- src/cart/crt_init.c | 25 ++++---- src/client/api/rpc.c | 59 +++++++------------ src/common/rsvc.c | 4 +- src/gurt/misc.c | 2 +- src/include/gurt/common.h | 1 - src/pool/cli.c | 4 +- .../erasurecode/rebuild_disabled_single.yaml | 1 + 7 files changed, 39 insertions(+), 57 deletions(-) diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index df243b1dce9..32c005a5569 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -9,7 +9,6 @@ */ #include #include -#include #include #include "crt_internal.h" @@ -74,6 +73,7 @@ crt_lib_init(void) { int rc; uint64_t start_rpcid; + struct timespec now; rc = D_RWLOCK_INIT(&crt_gdata.cg_rwlock, NULL); D_ASSERT(rc == 0); @@ -88,7 +88,9 @@ crt_lib_init(void) crt_gdata.cg_inited = 0; crt_gdata.cg_primary_prov = CRT_PROV_OFI_SOCKETS; - d_srand(d_timeus_secdiff(0) + getpid()); + rc = d_gettime(&now); + D_ASSERTF(rc == 0, "d_gettime: " DF_RC "\n", DP_RC(rc)); + d_srand(now.tv_sec * 1000 * 1000 * 1000 + now.tv_nsec + getpid()); start_rpcid = ((uint64_t)d_rand()) << 32; crt_gdata.cg_rpcid = start_rpcid; @@ -611,12 +613,14 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info) int crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) { - char *provider, *provider_env; - char *interface, *interface_env; - char *domain, *domain_env; - char *auth_key, *auth_key_env; - struct timeval now; - unsigned int seed; + char *provider; + char *provider_env = NULL; + char *interface; + char *interface_env = NULL; + char *domain; + char *domain_env = NULL; + char *auth_key; + char *auth_key_env = NULL; char *path; bool server; int rc = 0; @@ -684,11 +688,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) D_RWLOCK_WRLOCK(&crt_gdata.cg_rwlock); if (crt_gdata.cg_inited == 0) { - /* feed a seed for pseudo-random number generator */ - gettimeofday(&now, NULL); - seed = (unsigned int)(now.tv_sec * 1000000 + now.tv_usec); - d_srand(seed); - crt_gdata.cg_server = server; crt_gdata.cg_auto_swim_disable = (flags & CRT_FLAG_BIT_AUTO_SWIM_DISABLE) ? 1 : 0; diff --git a/src/client/api/rpc.c b/src/client/api/rpc.c index 1b8cd7229e8..eeff4e0be0e 100644 --- a/src/client/api/rpc.c +++ b/src/client/api/rpc.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -7,7 +7,6 @@ #include #include -#include #include static void @@ -96,14 +95,14 @@ daos_rpc_send_wait(crt_rpc_t *rpc) } struct rpc_proto { - struct rsvc_client cli; - crt_endpoint_t ep; - int version; - int rc; - bool completed; - crt_opcode_t base_opc; - uint32_t *ver_array; - uint32_t array_size; + int nr_ranks; + crt_endpoint_t ep; + int version; + int rc; + bool completed; + crt_opcode_t base_opc; + uint32_t *ver_array; + uint32_t array_size; }; static void @@ -113,16 +112,10 @@ query_cb(struct crt_proto_query_cb_info *cb_info) int rc; if (daos_rpc_retryable_rc(cb_info->pq_rc)) { - rc = rsvc_client_choose(&rproto->cli, &rproto->ep); - if (rc) { - D_ERROR("rsvc_client_choose() failed: "DF_RC"\n", DP_RC(rc)); - rproto->rc = rc; - rproto->completed = true; - } - - rc = crt_proto_query_with_ctx(&rproto->ep, rproto->base_opc, - rproto->ver_array, rproto->array_size, - query_cb, rproto, daos_get_crt_ctx()); + rproto->ep.ep_rank = (rproto->ep.ep_rank + 1) % rproto->nr_ranks; + rc = crt_proto_query_with_ctx(&rproto->ep, rproto->base_opc, rproto->ver_array, + rproto->array_size, query_cb, rproto, + daos_get_crt_ctx()); if (rc) { D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc)); rproto->rc = rc; @@ -141,8 +134,7 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int struct dc_mgmt_sys *sys; struct rpc_proto *rproto = NULL; crt_context_t ctx = daos_get_crt_ctx(); - int rc; - int num_ranks; + int rc; int i; rc = dc_mgmt_sys_attach(NULL, &sys); @@ -155,39 +147,34 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int if (rproto == NULL) D_GOTO(out_detach, rc = -DER_NOMEM); - rc = rsvc_client_init(&rproto->cli, sys->sy_info.ms_ranks); - if (rc) { - D_ERROR("rsvc_client_init() failed: "DF_RC"\n", DP_RC(rc)); - D_GOTO(out_free, rc); - } - - num_ranks = dc_mgmt_net_get_num_srv_ranks(); - rproto->ep.ep_rank = rand() % num_ranks; + /** select a random rank to issue the proto query rpc to */ + rproto->nr_ranks = dc_mgmt_net_get_num_srv_ranks(); + rproto->ep.ep_rank = d_rand() % rproto->nr_ranks; + rproto->ep.ep_tag = 0; rproto->ver_array = ver_array; rproto->array_size = count; - rproto->ep.ep_grp = sys->sy_group; - rproto->ep.ep_tag = 0; + rproto->ep.ep_grp = sys->sy_group; rproto->base_opc = base_opc; rc = crt_proto_query_with_ctx(&rproto->ep, base_opc, ver_array, count, query_cb, rproto, ctx); if (rc) { D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc)); - D_GOTO(out_rsvc, rc); + D_GOTO(out_free, rc); } while (!rproto->completed) { rc = crt_progress(ctx, 0); if (rc && rc != -DER_TIMEDOUT) { D_ERROR("failed to progress CART context: %d\n", rc); - D_GOTO(out_rsvc, rc); + D_GOTO(out_free, rc); } } if (rproto->rc != -DER_SUCCESS) { rc = rproto->rc; D_ERROR("crt_proto_query()failed: "DF_RC"\n", DP_RC(rc)); - D_GOTO(out_rsvc, rc); + D_GOTO(out_free, rc); } rc = 0; @@ -201,8 +188,6 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int } else { *ret_ver = rproto->version; } -out_rsvc: - rsvc_client_fini(&rproto->cli); out_free: D_FREE(rproto); out_detach: diff --git a/src/common/rsvc.c b/src/common/rsvc.c index 65e9eb50087..5ec33f0f6ee 100644 --- a/src/common/rsvc.c +++ b/src/common/rsvc.c @@ -91,7 +91,7 @@ rsvc_client_choose(struct rsvc_client *client, crt_endpoint_t *ep) chosen = client->sc_leader_index; } else { if (client->sc_next < 0) - client->sc_next = d_randn(client->sc_ranks->rl_nr); + client->sc_next = d_rand() % client->sc_ranks->rl_nr; chosen = client->sc_next; /* The hintless search is a round robin of all replicas. */ client->sc_next++; @@ -148,7 +148,7 @@ rsvc_client_process_error(struct rsvc_client *client, int rc, * search. */ D_DEBUG(DB_MD, "give up leader rank %u\n", ep->ep_rank); - client->sc_next = d_randn(client->sc_ranks->rl_nr); + client->sc_next = d_rand() % client->sc_ranks->rl_nr; if (client->sc_next == leader_index) { client->sc_next++; client->sc_next %= client->sc_ranks->rl_nr; diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 01e2d609942..f09f059cc55 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -498,7 +498,7 @@ d_rank_list_shuffle(d_rank_list_t *rank_list) return; for (i = 0; i < rank_list->rl_nr; i++) { - j = rand() % rank_list->rl_nr; + j = d_rand() % rank_list->rl_nr; tmp = rank_list->rl_ranks[i]; rank_list->rl_ranks[i] = rank_list->rl_ranks[j]; rank_list->rl_ranks[j] = tmp; diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 52d232e2e8a..783c2790f59 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -74,7 +74,6 @@ extern "C" { void d_srand(long int); long int d_rand(void); -long int d_randn(long int n); /* memory allocating macros */ void d_free(void *); diff --git a/src/pool/cli.c b/src/pool/cli.c index 89f7eb256a1..4258ce9084e 100644 --- a/src/pool/cli.c +++ b/src/pool/cli.c @@ -1827,9 +1827,7 @@ choose_map_refresh_rank(struct map_refresh_arg *arg) if (arg->mra_i == -1) { /* Let i be a random integer in [0, n). */ - i = ((double)rand() / RAND_MAX) * n; - if (i == n) - i = 0; + i = d_rand() % n; } else { /* Continue the round robin. */ i = arg->mra_i; diff --git a/src/tests/ftest/erasurecode/rebuild_disabled_single.yaml b/src/tests/ftest/erasurecode/rebuild_disabled_single.yaml index 8a2294753bc..0be117b134f 100644 --- a/src/tests/ftest/erasurecode/rebuild_disabled_single.yaml +++ b/src/tests/ftest/erasurecode/rebuild_disabled_single.yaml @@ -15,6 +15,7 @@ setup: server_config: name: daos_server engines_per_host: 2 + crt_timeout: 10 engines: 0: pinned_numa_node: 0