Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ch4/ofi: Add psm3 capability set #5864

Merged
merged 2 commits into from
Apr 29, 2022
Merged

ch4/ofi: Add psm3 capability set #5864

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e785543
ch4/ofi: Create PSM3 Capability set
nitbhat Feb 23, 2022
fa162c4
ch4/ofi: Add psm3 to subconfigure
nitbhat Feb 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/mpid/ch4/netmod/ofi/globals.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,32 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
.major_version = MPIDI_OFI_MAJOR_VERSION_PSM2,
.minor_version = MPIDI_OFI_MINOR_VERSION_PSM2}
,
{ /* psm3 */
.enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_PSM3,
.enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3,
.enable_shared_contexts = MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3,
.enable_mr_virt_address = MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3,
.enable_mr_allocated = MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3,
.enable_mr_prov_key = MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3,
.enable_tagged = MPIDI_OFI_ENABLE_TAGGED_PSM3,
.enable_am = MPIDI_OFI_ENABLE_AM_PSM3,
.enable_rma = MPIDI_OFI_ENABLE_RMA_PSM3,
.enable_atomics = MPIDI_OFI_ENABLE_ATOMICS_PSM3,
.enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3,
.enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3,
.enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3,
.num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_PSM3,
.num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3,
.max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_PSM3,
.max_endpoints_bits = MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3,
.fetch_atomic_iovecs = MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3,
.context_bits = MPIDI_OFI_CONTEXT_BITS_PSM3,
.source_bits = MPIDI_OFI_SOURCE_BITS_PSM3,
.tag_bits = MPIDI_OFI_TAG_BITS_PSM3,
.counter_wait_objects = MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3,
.major_version = MPIDI_OFI_MAJOR_VERSION_PSM3,
.minor_version = MPIDI_OFI_MINOR_VERSION_PSM3}
,
{ /* sockets */
.enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS,
.enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS,
Expand Down
70 changes: 69 additions & 1 deletion src/mpid/ch4/netmod/ofi/ofi_capability_sets.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ enum {
MPIDI_OFI_SET_NUMBER_DEFAULT = 0,
MPIDI_OFI_SET_NUMBER_MINIMAL,
MPIDI_OFI_SET_NUMBER_PSM2,
MPIDI_OFI_SET_NUMBER_PSM3,
MPIDI_OFI_SET_NUMBER_SOCKETS,
MPIDI_OFI_SET_NUMBER_BGQ,
MPIDI_OFI_SET_NUMBER_CXI,
Expand All @@ -46,6 +47,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
return MPIDI_OFI_SET_NUMBER_DEFAULT;
} else if (!strcmp("psm2", set_name)) {
return MPIDI_OFI_SET_NUMBER_PSM2;
} else if (!strcmp("psm3", set_name)) {
return MPIDI_OFI_SET_NUMBER_PSM3;
} else if (!strcmp("sockets", set_name)) {
return MPIDI_OFI_SET_NUMBER_SOCKETS;
} else if (!strcmp("bgq", set_name)) {
Expand Down Expand Up @@ -176,6 +179,71 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
#define MPIDI_OFI_CONTEXT_STRUCTS 1
#endif

#define MPIDI_OFI_ENABLE_AV_TABLE_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_MAX_ENDPOINTS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_SCALABLE
#define MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_BITS_SCALABLE
#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_TAGGED_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_RMA_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_AM_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_ATOMICS_PSM3 MPIDI_OFI_ENABLE_RMA_PSM3
#define MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3 1
#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_HMEM_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_NUM_AM_BUFFERS_PSM3 MPIDI_OFI_MAX_NUM_AM_BUFFERS
#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3 (0)
#define MPIDI_OFI_CONTEXT_BITS_PSM3 (20)
#define MPIDI_OFI_SOURCE_BITS_PSM3 (0)
#define MPIDI_OFI_TAG_BITS_PSM3 (31)
#define MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_MAJOR_VERSION_PSM3 1
#define MPIDI_OFI_MINOR_VERSION_PSM3 6

#ifdef MPIDI_CH4_OFI_USE_SET_PSM3
#define MPIDI_OFI_SET_NUMBER MPIDI_OFI_SET_NUMBER_PSM3
#define MPIDI_OFI_ENABLE_AV_TABLE MPIDI_OFI_ENABLE_AV_TABLE_PSM3
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3
#define MPIDI_OFI_MAX_ENDPOINTS MPIDI_OFI_MAX_ENDPOINTS_PSM3
#define MPIDI_OFI_MAX_ENDPOINTS_BITS MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3
#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS MPIDI_OFI_global.settings.enable_shared_contexts /* Always controlled by CVAR */
#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3
#define MPIDI_OFI_ENABLE_MR_ALLOCATED MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3
#define MPIDI_OFI_ENABLE_MR_PROV_KEY MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3
#define MPIDI_OFI_ENABLE_TAGGED MPIDI_OFI_ENABLE_TAGGED_PSM3
#define MPIDI_OFI_ENABLE_AM MPIDI_OFI_ENABLE_AM_PSM3
#define MPIDI_OFI_ENABLE_RMA MPIDI_OFI_ENABLE_RMA_PSM3
#define MPIDI_OFI_ENABLE_ATOMICS MPIDI_OFI_ENABLE_ATOMICS_PSM3
#define MPIDI_OFI_FETCH_ATOMIC_IOVECS MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3
#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3
#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3
#define MPIDI_OFI_ENABLE_PT2PT_NOPACK MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3
#define MPIDI_OFI_ENABLE_HMEM MPIDI_OFI_ENABLE_HMEM_PSM3
#define MPIDI_OFI_NUM_AM_BUFFERS MPIDI_OFI_NUM_AM_BUFFERS_PSM3
#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3
#define MPIDI_OFI_PROTOCOL_MASK (0x00E0000000000000ULL)
#define MPIDI_OFI_CONTEXT_MASK (0x000FFFFF00000000ULL)
#define MPIDI_OFI_SOURCE_MASK (0x0000000000000000ULL) /* PSM3 does support immediate data
* so this field is zeroed */
#define MPIDI_OFI_TAG_MASK (0x000000007FFFFFFFULL)
#define MPIDI_OFI_CONTEXT_BITS MPIDI_OFI_CONTEXT_BITS_PSM3
#define MPIDI_OFI_SOURCE_BITS MPIDI_OFI_SOURCE_BITS_PSM3
#define MPIDI_OFI_TAG_BITS MPIDI_OFI_TAG_BITS_PSM3
#define MPIDI_OFI_COUNTER_WAIT_OBJECTS MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3
#define MPIDI_OFI_SYNC_SEND_ACK (0x0010000000000000ULL)
#define MPIDI_OFI_SYNC_SEND (0x0020000000000000ULL)
#define MPIDI_OFI_DYNPROC_SEND (0x0040000000000000ULL)
#define MPIDI_OFI_HUGE_SEND (0x0080000000000000ULL)
#define MPIDI_OFI_MAJOR_VERSION MPIDI_OFI_MAJOR_VERSION_PSM3
#define MPIDI_OFI_MINOR_VERSION MPIDI_OFI_MINOR_VERSION_PSM3
#define MPIDI_OFI_CONTEXT_STRUCTS 1
#endif

#define MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS MPIDI_OFI_ON
#define MPIDI_OFI_MAX_ENDPOINTS_SOCKETS MPIDI_OFI_MAX_ENDPOINTS_SCALABLE
Expand Down Expand Up @@ -686,7 +754,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
#else
/* MPIDI_CH4_OFI_USE_SET_RUNTIME not defined */
#if !defined(MPIDI_OFI_MAX_ENDPOINTS_BITS)
#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set ***
#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,PSM3,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set ***
#endif

#define MPIDI_OFI_ENABLE_RUNTIME_CHECKS MPIDI_OFI_OFF
Expand Down
12 changes: 11 additions & 1 deletion src/mpid/ch4/netmod/ofi/subconfigure.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[
if test "$no_providers" = "no" ; then
enable_psm="no"
enable_psm2="no"
enable_psm3="no"
enable_sockets="no"
enable_verbs="no"
enable_usnic="no"
Expand All @@ -71,6 +72,7 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[
else
enable_psm="yes"
enable_psm2="yes"
enable_psm3="yes"
enable_sockets="yes"
enable_verbs="yes"
enable_usnic="yes"
Expand Down Expand Up @@ -100,6 +102,9 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[
"psm2" | "opa")
enable_psm2="yes"
;;
"psm3")
enable_psm3="yes"
;;
"sockets")
enable_sockets="yes"
;;
Expand Down Expand Up @@ -189,6 +194,10 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[
AC_DEFINE([MPIDI_CH4_OFI_USE_SET_PSM2], [1], [Define to use PSM2 capability set])
enable_psm2="yes"
;;
"psm3")
AC_DEFINE([MPIDI_CH4_OFI_USE_SET_PSM3], [1], [Define to use PSM3 capability set])
enable_psm3="yes"
;;
"sockets")
AC_DEFINE([MPIDI_CH4_OFI_USE_SET_SOCKETS], [1], [Define to use sockets capability set])
enable_sockets="yes"
Expand Down Expand Up @@ -273,12 +282,13 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[
if test "$with_libfabric" = "embedded" ; then
ofi_embedded="yes"
AC_MSG_NOTICE([CH4 OFI Netmod: Using an embedded libfabric])
ofi_subdir_args="--enable-embedded --disable-psm3"
ofi_subdir_args="--enable-embedded"

prov_config=""
if test "x${netmod_args}" != "x" ; then
prov_config="$prov_config --enable-psm=${enable_psm}"
prov_config="$prov_config --enable-psm2=${enable_psm2}"
prov_config="$prov_config --enable-psm3=${enable_psm3}"
prov_config="$prov_config --enable-sockets=${enable_sockets}"
prov_config="$prov_config --enable-verbs=${enable_verbs}"
prov_config="$prov_config --enable-usnic=${enable_usnic}"
Expand Down