DAOS-15914 cart: add D_MRECV_BUF env var to control number of multi-r…

…ecv buffers #14683 Signed-off-by: Jerome Soumagne <[email protected]> Signed-off-by: Dalton Bohning <[email protected]>
daos-stack · Jul 18, 2024 · a2c6a46 · a2c6a46
1 parent 9a59e37
commit a2c6a46
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 8 deletions.
diff --git a/debian/control b/debian/control
@@ -10,7 +10,7 @@ Build-Depends: debhelper (>= 10),
                libopenmpi-dev,
                libssl-dev,
                libyaml-dev,
-               libmercury-dev (>= 2.3.1-1),
+               libmercury-dev (>= 2.4.0~rc3-1),
                scons,
                uuid-dev,
                pkg-config,

diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -900,8 +900,9 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 
 	init_info.request_post_init = crt_gdata.cg_post_init;
 	init_info.request_post_incr = crt_gdata.cg_post_incr;
+	init_info.multi_recv_op_max = crt_gdata.cg_mrecv_buf;
 
-	hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info);
+	hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info);
 	if (hg_class == NULL) {
 		D_ERROR("Could not initialize HG class.\n");
 		D_GOTO(out, rc = -DER_HG);

diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -30,6 +30,7 @@
 /** default values for init / incr to prepost handles */
 #define CRT_HG_POST_INIT        (512)
 #define CRT_HG_POST_INCR        (512)
+#define CRT_HG_MRECV_BUF        (16)
 
 struct crt_rpc_priv;
 struct crt_common_hdr;

diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c
@@ -54,6 +54,7 @@ static const char      *crt_env_names[] = {
     "D_QUOTA_RPCS",
     "D_POST_INIT",
     "D_POST_INCR",
+    "D_MRECV_BUF",
     "DAOS_SIGNAL_REGISTER",
     "D_CLIENT_METRICS_ENABLE",
     "D_CLIENT_METRICS_RETAIN",
@@ -274,7 +275,8 @@ static int data_init(int server, crt_init_options_t *opt)
 	uint32_t	is_secondary;
 	char		ucx_ib_fork_init = 0;
 	uint32_t        post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR;
-	int		rc = 0;
+	unsigned int    mrecv_buf = CRT_HG_MRECV_BUF;
+	int             rc        = 0;
 
 	D_DEBUG(DB_ALL, "initializing crt_gdata...\n");
 
@@ -288,6 +290,8 @@ static int data_init(int server, crt_init_options_t *opt)
 	crt_gdata.cg_post_init = post_init;
 	d_getenv_uint32_t("D_POST_INCR", &post_incr);
 	crt_gdata.cg_post_incr = post_incr;
+	d_getenv_uint("D_MRECV_BUF", &mrecv_buf);
+	crt_gdata.cg_mrecv_buf = mrecv_buf;
 
 	is_secondary = 0;
 	/* Apply CART-890 workaround for server side only */

diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h
@@ -94,6 +94,7 @@ struct crt_gdata {
 	/** Hints to mercury for request post init (ignored for clients) */
 	uint32_t                 cg_post_init;
 	uint32_t                 cg_post_incr;
+	unsigned int             cg_mrecv_buf;
 
 	/** global timeout value (second) for all RPCs */
 	uint32_t		cg_timeout;

diff --git a/utils/build.config b/utils/build.config
@@ -9,12 +9,12 @@ ISAL = v2.30.0
 ISAL_CRYPTO = v2.23.0
 SPDK = v22.01.2
 OFI = v1.19.1
-MERCURY = v2.3.1
+MERCURY = v2.4.0rc3
 PROTOBUFC = v1.3.3
 UCX=v1.14.1
 
 [patch_versions]
 spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff
 ofi=https://github.com/ofiwg/libfabric/commit/d827c6484cc5bf67dfbe395890e258860c3f0979.diff
-mercury=https://raw.githubusercontent.com/daos-stack/mercury/857f1d5d2ca72d4c1b8d7be5e7fd26d6292b495f/na_ucx_am_send_retry.patch,https://github.com/mercury-hpc/mercury/commit/b8c26fd86281f3b0883c31bd2d0cb467a12b860d.diff,https://github.com/mercury-hpc/mercury/commit/a35589c3d1134d9c80640e78247e210162ac4a3c.diff,https://github.com/mercury-hpc/mercury/commit/fa4abbb6273d975b2ef17ac4e561fd4255d384db.diff
+mercury=https://github.com/mercury-hpc/mercury/commit/ac0a2902c86b96d81473d3a844a54b5097b5d2cc.diff
 fuse=https://github.com/libfuse/libfuse/commit/c9905341ea34ff9acbc11b3c53ba8bcea35eeed8.diff
diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec
@@ -3,7 +3,7 @@
 %define agent_svc_name daos_agent.service
 %define sysctl_script_name 10-daos_server.conf
 
-%global mercury_version 2.3.1-1%{?dist}
+%global mercury_version 2.4.0~rc3-1%{?dist}
 %global libfabric_version 1.15.1-1
 %global __python %{__python3}