fabric: Add mr_count field to domain attribute

include/rdma/fabric.h

@@ -351,6 +351,7 @@ struct fi_domain_attr {
 	uint8_t			*auth_key;
 	size_t 			auth_key_size;
 	size_t			max_err_data;
+	size_t			mr_cnt;
 };
 
 struct fi_fabric_attr {

man/fi_domain.3.md

@@ -133,6 +133,7 @@ struct fi_domain_attr {
 	uint8_t               *auth_key;
 	size_t                auth_key_size;
 	size_t                max_err_data;
+	size_t                mr_cnt;
 };
 ```
 
@@ -638,6 +639,16 @@ opened with API version 1.5 or greater.
   a completion or event queue error.  This value corresponds to the
   err_data_size field in struct fi_cq_err_entry and struct fi_eq_err_entry.
 
+## Memory Regions Count (mr_cnt)
+
+The optimal number of memory regions supported by the domain.  The mr_cnt
+value may be a fixed value of the maximum number of MRs supported by the
+underlying hardware, or may be a dynamic value, based on the default
+attributes of the domain, such as the supported memory registration modes.
+Applications can set the mr_cnt on input to fi_getinfo, in order to
+indicate their memory registration requirements.  Doing so may allow the
+provider to optimize any memory registration cache or lookup tables.
+
 # RETURN VALUE
 
 Returns 0 on success. On error, a negative value corresponding to fabric

prov/bgq/src/fi_bgq_domain.c

@@ -275,6 +275,7 @@ int fi_bgq_alloc_default_domain_attr(struct fi_domain_attr **domain_attr)
 	attr->max_ep_stx_ctx	= attr->max_ep_tx_ctx;
 	attr->max_ep_srx_ctx	= 0;	/* TODO - reserve some mu reception fifos for use as shared receive context ... how to address? */
 	attr->mr_iov_limit	= 1;
+	attr->mr_cnt		= 1 << (attr->mr_key_size << 3);
 
 	*domain_attr = attr;
 

prov/gni/src/gnix_fabric.c

@@ -253,6 +253,7 @@ static struct fi_info *_gnix_allocinfo(void)
 	gnix_info->domain_attr->mr_iov_limit = 1;
 	gnix_info->domain_attr->caps = GNIX_DOM_CAPS;
 	gnix_info->domain_attr->mode = 0;
+	gnix_info->domain_attr->mr_cnt = gnix_default_user_registration_limit;
 
 	gnix_info->next = NULL;
 	gnix_info->addr_format = FI_ADDR_GNI;

prov/mlx/src/mlx.h

@@ -69,6 +69,7 @@ extern "C" {
 #define FI_MLX_FABRIC_NAME "mlx"
 #define FI_MLX_DEFAULT_INJECT_SIZE 1024
 #define FI_MLX_DEF_CQ_SIZE (1024)
+#define FI_MLX_DEF_MR_CNT (1 << 16)
 
 #define FI_MLX_VERSION_MINOR 4
 #define FI_MLX_VERSION_MAJOR 1

prov/mlx/src/mlx_init.c

@@ -83,6 +83,7 @@ struct fi_domain_attr mlx_domain_attrs = {
 	.rx_ctx_cnt = 1,
 	.max_ep_tx_ctx = 1,
 	.max_ep_rx_ctx = 1,
+	.mr_cnt = FI_MLX_DEF_MR_CNT;
 };
 
 struct fi_rx_attr mlx_rx_attrs = {

prov/netdir/src/netdir.h

@@ -51,10 +51,10 @@ extern "C" {
 #define OFI_ND_MAJOR_VERSION 1
 #define OFI_ND_MINOR_VERSION 5
 
-#define ND_MSG_IOV_LIMIT (256)
-#define ND_MSG_INTERNAL_IOV_LIMIT (512)
-
-#define ND_EP_MAX_CM_DATA_SIZE (256)
+#define ND_MSG_IOV_LIMIT		(256)
+#define ND_MSG_INTERNAL_IOV_LIMIT	(512)
+#define ND_EP_MAX_CM_DATA_SIZE		(256)
+#define OFI_ND_MAX_MR_CNT		(1 << 16)
 
 #define OFI_ND_PRIMARY_CAPS					\
 	FI_MSG | FI_RMA | FI_SEND | FI_RECV |			\

prov/netdir/src/netdir_init.c

@@ -115,6 +115,7 @@ static int ofi_nd_adapter_cb(const ND2_ADAPTER_INFO *adapter, const char *name)
 	info->domain_attr->mr_mode = OFI_MR_BASIC_MAP | FI_MR_LOCAL;
 	info->domain_attr->cq_cnt = (size_t)adapter->MaxCompletionQueueDepth;
 	info->domain_attr->mr_iov_limit = ND_MSG_IOV_LIMIT;
+	info->domain_attr->mr_cnt = OFI_ND_MAX_MR_CNT;
 
 	info->fabric_attr->name = strdup(ofi_nd_prov_name);
 	info->fabric_attr->prov_version = FI_VERSION(OFI_ND_MAJOR_VERSION, OFI_ND_MINOR_VERSION);

prov/psm/src/psmx_init.c

@@ -558,6 +558,7 @@ static int psmx_getinfo(uint32_t version, const char *node, const char *service,
 	psmx_info->domain_attr->mr_iov_limit = 65535;
 	psmx_info->domain_attr->caps = PSMX_DOM_CAPS;
 	psmx_info->domain_attr->mode = 0;
+	psmx_info->domain_attr->mr_cnt = 10000;
 
 	psmx_info->next = NULL;
 	psmx_info->caps = (hints && hints->caps) ? hints->caps : caps;

prov/psm2/src/psmx2_init.c

@@ -619,6 +619,7 @@ static int psmx2_getinfo(uint32_t version, const char *node,
 	psmx2_info->domain_attr->mr_iov_limit = 65535;
 	psmx2_info->domain_attr->caps = PSMX2_DOM_CAPS;
 	psmx2_info->domain_attr->mode = 0;
+	psmx_info->domain_attr->mr_cnt = 10000;
 
 	psmx2_info->next = NULL;
 	psmx2_info->caps = caps;

prov/sockets/include/sock.h

@@ -79,6 +79,7 @@
 #define SOCK_EP_MAX_ATOMIC_SZ (4096)
 #define SOCK_EP_MAX_CTX_BITS (16)
 #define SOCK_EP_MSG_PREFIX_SZ (0)
+#define SOCK_DOMAIN_MR_CNT (65535)
 
 #define SOCK_PE_POLL_TIMEOUT (100000)
 #define SOCK_PE_MAX_ENTRIES (128)

prov/sockets/src/sock_dom.c

@@ -64,6 +64,7 @@ const struct fi_domain_attr sock_domain_attr = {
 	.cntr_cnt = SOCK_EP_MAX_CNTR_CNT,
 	.mr_iov_limit = SOCK_EP_MAX_IOV_LIMIT,
 	.max_err_data = SOCK_MAX_ERR_CQ_EQ_DATA_SZ,
+	.mr_cnt = SOCK_DOMAIN_MR_CNT,
 };
 
 int sock_verify_domain_attr(uint32_t version, struct fi_domain_attr *attr)
@@ -162,6 +163,9 @@ int sock_verify_domain_attr(uint32_t version, struct fi_domain_attr *attr)
 	if (attr->max_err_data > sock_domain_attr.max_err_data)
 		return -FI_ENODATA;
 
+	if (attr->mr_cnt > sock_domain_attr.mr_cnt)
+		return -FI_ENODATA;
+
 	return 0;
 }
 

prov/util/src/util_attr.c

@@ -540,6 +540,18 @@ int ofi_check_domain_attr(const struct fi_provider *prov, uint32_t api_version,
 		return -FI_ENODATA;
 	}
 
+	if (user_attr->max_err_data > prov_attr->max_err_data) {
+		FI_INFO(prov, FI_LOG_CORE, "Max err data too large");
+		FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, max_err_data);
+		return -FI_ENODATA;
+	}
+
+	if (user_attr->mr_cnt > prov_attr->mr_cnt) {
+		FI_INFO(prov, FI_LOG_CORE, "MR count too large");
+		FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, mr_cnt);
+		return -FI_ENODATA;
+	}
+
 	return 0;
 }
 

prov/verbs/src/verbs_info.c

@@ -601,6 +601,7 @@ static int fi_ibv_get_device_attrs(struct ibv_context *ctx, struct fi_info *info
 	info->domain_attr->rx_ctx_cnt 		= MIN(info->domain_attr->rx_ctx_cnt, device_attr.max_qp);
 	info->domain_attr->max_ep_tx_ctx 	= device_attr.max_qp;
 	info->domain_attr->max_ep_rx_ctx 	= device_attr.max_qp;
+	info->domain_attr->mr_cnt		= device_attr.max_mr;
 	if (info->ep_attr->type == FI_EP_RDM)
 		info->domain_attr->cntr_cnt	= device_attr.max_qp * 4;
 

src/fi_tostr.c

@@ -482,6 +482,19 @@ static void fi_tostr_domain_attr(char *buf, const struct fi_domain_attr *attr,
 	strcatf(buf, "%s%smax_ep_stx_ctx: %zd\n", prefix, TAB, attr->max_ep_stx_ctx);
 	strcatf(buf, "%s%smax_ep_srx_ctx: %zd\n", prefix, TAB, attr->max_ep_srx_ctx);
 	strcatf(buf, "%s%scntr_cnt: %zd\n", prefix, TAB, attr->cntr_cnt);
+	strcatf(buf, "%s%smr_iov_limit: %zd\n", prefix, TAB, attr->mr_iov_limit);
+
+	strcatf(buf, "%scaps: [ ", TAB);
+	fi_tostr_caps(buf, attr->caps);
+	strcatf(buf, " ]\n");
+
+	strcatf(buf, "%smode: [ ", TAB);
+	fi_tostr_mode(buf, attr->mode);
+	strcatf(buf, " ]\n");
+
+	strcatf(buf, "%s%sauth_key_size: %zd\n", prefix, TAB, attr->auth_key_size);
+	strcatf(buf, "%s%smax_err_data: %zd\n", prefix, TAB, attr->max_err_data);
+	strcatf(buf, "%s%smr_cnt: %zd\n", prefix, TAB, attr->mr_cnt);
 }
 
 static void fi_tostr_fabric_attr(char *buf, const struct fi_fabric_attr *attr,