diff --git a/prov/usnic/src/usdf.h b/prov/usnic/src/usdf.h index bde14b08544..841b78901b7 100644 --- a/prov/usnic/src/usdf.h +++ b/prov/usnic/src/usdf.h @@ -76,6 +76,9 @@ extern struct fi_provider usdf_ops; #define USDF_EVENT_FLAG_ERROR (1ULL << 62) #define USDF_EVENT_FLAG_FREE_BUF (1ULL << 63) +/* usdf domain capability: no loopback */ +#define USDF_DOM_CAPS (FI_REMOTE_COMM) + /* * TAILQ stuff that should exist */ diff --git a/prov/usnic/src/usdf_ep_dgram.c b/prov/usnic/src/usdf_ep_dgram.c index c74c646acb0..0e48e899151 100644 --- a/prov/usnic/src/usdf_ep_dgram.c +++ b/prov/usnic/src/usdf_ep_dgram.c @@ -413,6 +413,7 @@ static const struct fi_ep_attr dgram_dflt_ep_attr = { }; static const struct fi_domain_attr dgram_dflt_domain_attr = { + .caps = USDF_DOM_CAPS, .threading = FI_THREAD_ENDPOINT, .control_progress = FI_PROGRESS_AUTO, .data_progress = FI_PROGRESS_MANUAL, @@ -490,14 +491,8 @@ int usdf_dgram_fill_dom_attr(uint32_t version, struct fi_info *hints, if (ret < 0) return -FI_ENODATA; - if (!hints || !hints->domain_attr) { - /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. - * Hence, if a version less than v1.5 was requested, return the prior default: - * FI_MR_BASIC. */ - if (FI_VERSION_LT(version, FI_VERSION(1,5))) - defaults.mr_mode = FI_MR_BASIC; + if (!hints || !hints->domain_attr) goto out; - } switch (hints->domain_attr->threading) { case FI_THREAD_UNSPEC: @@ -540,11 +535,31 @@ int usdf_dgram_fill_dom_attr(uint32_t version, struct fi_info *hints, return -FI_ENODATA; } - if (ofi_check_mr_mode(version, defaults.mr_mode, - hints->domain_attr->mr_mode)) + switch (hints->domain_attr->caps) { + case FI_REMOTE_COMM: + if (FI_VERSION_LT(version, FI_VERSION(1, 5))) + return -FI_ENODATA; + case 0: + break; + default: + USDF_WARN_SYS(DOMAIN, + "invalid domain capabilities\n"); return -FI_ENODATA; + } - defaults.mr_mode = hints->domain_attr->mr_mode; + switch (hints->domain_attr->mr_mode) { + case FI_MR_UNSPEC: + /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. + * Hence, if a version less than v1.5 was requested, return the prior default: + * FI_MR_BASIC. */ + if (FI_VERSION_LT(version, FI_VERSION(1,5))) + defaults.mr_mode = FI_MR_BASIC; + break; + default : + if (ofi_check_mr_mode(version, defaults.mr_mode, hints->domain_attr->mr_mode)) + return -FI_ENODATA; + defaults.mr_mode = hints->domain_attr->mr_mode; + } out: *fi->domain_attr = defaults; diff --git a/prov/usnic/src/usdf_ep_msg.c b/prov/usnic/src/usdf_ep_msg.c index 4c7314417d2..9e2e0f46b56 100644 --- a/prov/usnic/src/usdf_ep_msg.c +++ b/prov/usnic/src/usdf_ep_msg.c @@ -112,6 +112,7 @@ static const struct fi_ep_attr msg_dflt_ep_attr = { }; static const struct fi_domain_attr msg_dflt_domain_attr = { + .caps = USDF_DOM_CAPS, .threading = FI_THREAD_UNSPEC, .control_progress = FI_PROGRESS_AUTO, .data_progress = FI_PROGRESS_MANUAL, @@ -175,14 +176,8 @@ int usdf_msg_fill_dom_attr(uint32_t version, struct fi_info *hints, if (ret < 0) return -FI_ENODATA; - if (!hints || !hints->domain_attr) { - /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. - * Hence, if a version less than v1.5 was requested, return the prior default: - * FI_MR_BASIC. */ - if (FI_VERSION_LT(version, FI_VERSION(1,5))) - defaults.mr_mode = FI_MR_BASIC; + if (!hints || !hints->domain_attr) goto out; - } /* how to handle fi_thread_fid, fi_thread_completion, etc? */ @@ -220,11 +215,32 @@ int usdf_msg_fill_dom_attr(uint32_t version, struct fi_info *hints, return -FI_ENODATA; } - if (ofi_check_mr_mode(version, defaults.mr_mode, - hints->domain_attr->mr_mode)) + switch (hints->domain_attr->caps) { + case FI_REMOTE_COMM: + if (FI_VERSION_LT(version, FI_VERSION(1, 5))) + return -FI_ENODATA; + defaults.caps = FI_REMOTE_COMM; + case 0: + break; + default: + USDF_WARN_SYS(DOMAIN, + "invalid domain capabilities\n"); return -FI_ENODATA; + } - defaults.mr_mode = hints->domain_attr->mr_mode; + switch (hints->domain_attr->mr_mode) { + case FI_MR_UNSPEC: + /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. + * Hence, if a version less than v1.5 was requested, return the prior default: + * FI_MR_BASIC. */ + if (FI_VERSION_LT(version, FI_VERSION(1,5))) + defaults.mr_mode = FI_MR_BASIC; + break; + default : + if (ofi_check_mr_mode(version, defaults.mr_mode, hints->domain_attr->mr_mode)) + return -FI_ENODATA; + defaults.mr_mode = hints->domain_attr->mr_mode; + } out: *fi->domain_attr = defaults; diff --git a/prov/usnic/src/usdf_ep_rdm.c b/prov/usnic/src/usdf_ep_rdm.c index 0b62929ea64..770a97f18f3 100644 --- a/prov/usnic/src/usdf_ep_rdm.c +++ b/prov/usnic/src/usdf_ep_rdm.c @@ -115,6 +115,7 @@ static const struct fi_ep_attr rdm_dflt_ep_attr = { }; static const struct fi_domain_attr rdm_dflt_domain_attr = { + .caps = USDF_DOM_CAPS, .threading = FI_THREAD_ENDPOINT, .control_progress = FI_PROGRESS_AUTO, .data_progress = FI_PROGRESS_MANUAL, @@ -179,14 +180,8 @@ int usdf_rdm_fill_dom_attr(uint32_t version, struct fi_info *hints, if (ret < 0) return -FI_ENODATA; - if (!hints || !hints->domain_attr) { - /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. - * Hence, if a version less than v1.5 was requested, return the prior default: - * FI_MR_BASIC. */ - if (FI_VERSION_LT(version, FI_VERSION(1,5))) - defaults.mr_mode = FI_MR_BASIC; + if (!hints || !hints->domain_attr) goto out; - } /* how to handle fi_thread_fid, fi_thread_completion, etc? */ @@ -224,11 +219,31 @@ int usdf_rdm_fill_dom_attr(uint32_t version, struct fi_info *hints, return -FI_ENODATA; } - if (ofi_check_mr_mode(version, defaults.mr_mode, - hints->domain_attr->mr_mode)) + switch (hints->domain_attr->caps) { + case FI_REMOTE_COMM: + if (FI_VERSION_LT(version, FI_VERSION(1, 5))) + return -FI_ENODATA; + case 0: + break; + default: + USDF_WARN_SYS(DOMAIN, + "invalid domain capabilities\n"); return -FI_ENODATA; + } - defaults.mr_mode = hints->domain_attr->mr_mode; + switch (hints->domain_attr->mr_mode) { + case FI_MR_UNSPEC: + /* mr_mode behavior changed in version 1.5 from a single flag to mode bits. + * Hence, if a version less than v1.5 was requested, return the prior default: + * FI_MR_BASIC. */ + if (FI_VERSION_LT(version, FI_VERSION(1,5))) + defaults.mr_mode = FI_MR_BASIC; + break; + default : + if (ofi_check_mr_mode(version, defaults.mr_mode, hints->domain_attr->mr_mode)) + return -FI_ENODATA; + defaults.mr_mode = hints->domain_attr->mr_mode; + } out: *fi->domain_attr = defaults;