From e7855438883f2adc3456ac9356cc30fd3577f5b1 Mon Sep 17 00:00:00 2001 From: Nitin Bhat Date: Wed, 23 Feb 2022 09:54:12 -0600 Subject: [PATCH 1/2] ch4/ofi: Create PSM3 Capability set --- src/mpid/ch4/netmod/ofi/globals.c | 26 +++++++ src/mpid/ch4/netmod/ofi/ofi_capability_sets.h | 70 ++++++++++++++++++- 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/mpid/ch4/netmod/ofi/globals.c b/src/mpid/ch4/netmod/ofi/globals.c index 5e9ff636024..b48628c4b0e 100644 --- a/src/mpid/ch4/netmod/ofi/globals.c +++ b/src/mpid/ch4/netmod/ofi/globals.c @@ -101,6 +101,32 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] = .major_version = MPIDI_OFI_MAJOR_VERSION_PSM2, .minor_version = MPIDI_OFI_MINOR_VERSION_PSM2} , + { /* psm3 */ + .enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_PSM3, + .enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3, + .enable_shared_contexts = MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3, + .enable_mr_virt_address = MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3, + .enable_mr_allocated = MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3, + .enable_mr_prov_key = MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3, + .enable_tagged = MPIDI_OFI_ENABLE_TAGGED_PSM3, + .enable_am = MPIDI_OFI_ENABLE_AM_PSM3, + .enable_rma = MPIDI_OFI_ENABLE_RMA_PSM3, + .enable_atomics = MPIDI_OFI_ENABLE_ATOMICS_PSM3, + .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3, + .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3, + .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3, + .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_PSM3, + .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3, + .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_PSM3, + .max_endpoints_bits = MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3, + .fetch_atomic_iovecs = MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3, + .context_bits = MPIDI_OFI_CONTEXT_BITS_PSM3, + .source_bits = MPIDI_OFI_SOURCE_BITS_PSM3, + .tag_bits = MPIDI_OFI_TAG_BITS_PSM3, + .counter_wait_objects = MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3, + .major_version = MPIDI_OFI_MAJOR_VERSION_PSM3, + .minor_version = MPIDI_OFI_MINOR_VERSION_PSM3} + , { /* sockets */ .enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS, .enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS, diff --git a/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h b/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h index 0a5171e8184..4cb64857205 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h +++ b/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h @@ -22,6 +22,7 @@ enum { MPIDI_OFI_SET_NUMBER_DEFAULT = 0, MPIDI_OFI_SET_NUMBER_MINIMAL, MPIDI_OFI_SET_NUMBER_PSM2, + MPIDI_OFI_SET_NUMBER_PSM3, MPIDI_OFI_SET_NUMBER_SOCKETS, MPIDI_OFI_SET_NUMBER_BGQ, MPIDI_OFI_SET_NUMBER_CXI, @@ -46,6 +47,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name) return MPIDI_OFI_SET_NUMBER_DEFAULT; } else if (!strcmp("psm2", set_name)) { return MPIDI_OFI_SET_NUMBER_PSM2; + } else if (!strcmp("psm3", set_name)) { + return MPIDI_OFI_SET_NUMBER_PSM3; } else if (!strcmp("sockets", set_name)) { return MPIDI_OFI_SET_NUMBER_SOCKETS; } else if (!strcmp("bgq", set_name)) { @@ -176,6 +179,71 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name) #define MPIDI_OFI_CONTEXT_STRUCTS 1 #endif +#define MPIDI_OFI_ENABLE_AV_TABLE_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_MAX_ENDPOINTS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_SCALABLE +#define MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_BITS_SCALABLE +#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_ENABLE_TAGGED_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_RMA_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_AM_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_ATOMICS_PSM3 MPIDI_OFI_ENABLE_RMA_PSM3 +#define MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3 1 +#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_ENABLE_HMEM_PSM3 MPIDI_OFI_OFF +#define MPIDI_OFI_NUM_AM_BUFFERS_PSM3 MPIDI_OFI_MAX_NUM_AM_BUFFERS +#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3 (0) +#define MPIDI_OFI_CONTEXT_BITS_PSM3 (20) +#define MPIDI_OFI_SOURCE_BITS_PSM3 (0) +#define MPIDI_OFI_TAG_BITS_PSM3 (31) +#define MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3 MPIDI_OFI_ON +#define MPIDI_OFI_MAJOR_VERSION_PSM3 1 +#define MPIDI_OFI_MINOR_VERSION_PSM3 6 + +#ifdef MPIDI_CH4_OFI_USE_SET_PSM3 +#define MPIDI_OFI_SET_NUMBER MPIDI_OFI_SET_NUMBER_PSM3 +#define MPIDI_OFI_ENABLE_AV_TABLE MPIDI_OFI_ENABLE_AV_TABLE_PSM3 +#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3 +#define MPIDI_OFI_MAX_ENDPOINTS MPIDI_OFI_MAX_ENDPOINTS_PSM3 +#define MPIDI_OFI_MAX_ENDPOINTS_BITS MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3 +#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS MPIDI_OFI_global.settings.enable_shared_contexts /* Always controlled by CVAR */ +#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3 +#define MPIDI_OFI_ENABLE_MR_ALLOCATED MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3 +#define MPIDI_OFI_ENABLE_MR_PROV_KEY MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3 +#define MPIDI_OFI_ENABLE_TAGGED MPIDI_OFI_ENABLE_TAGGED_PSM3 +#define MPIDI_OFI_ENABLE_AM MPIDI_OFI_ENABLE_AM_PSM3 +#define MPIDI_OFI_ENABLE_RMA MPIDI_OFI_ENABLE_RMA_PSM3 +#define MPIDI_OFI_ENABLE_ATOMICS MPIDI_OFI_ENABLE_ATOMICS_PSM3 +#define MPIDI_OFI_FETCH_ATOMIC_IOVECS MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3 +#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3 +#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3 +#define MPIDI_OFI_ENABLE_PT2PT_NOPACK MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3 +#define MPIDI_OFI_ENABLE_HMEM MPIDI_OFI_ENABLE_HMEM_PSM3 +#define MPIDI_OFI_NUM_AM_BUFFERS MPIDI_OFI_NUM_AM_BUFFERS_PSM3 +#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3 +#define MPIDI_OFI_PROTOCOL_MASK (0x00E0000000000000ULL) +#define MPIDI_OFI_CONTEXT_MASK (0x000FFFFF00000000ULL) +#define MPIDI_OFI_SOURCE_MASK (0x0000000000000000ULL) /* PSM3 does support immediate data + * so this field is zeroed */ +#define MPIDI_OFI_TAG_MASK (0x000000007FFFFFFFULL) +#define MPIDI_OFI_CONTEXT_BITS MPIDI_OFI_CONTEXT_BITS_PSM3 +#define MPIDI_OFI_SOURCE_BITS MPIDI_OFI_SOURCE_BITS_PSM3 +#define MPIDI_OFI_TAG_BITS MPIDI_OFI_TAG_BITS_PSM3 +#define MPIDI_OFI_COUNTER_WAIT_OBJECTS MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3 +#define MPIDI_OFI_SYNC_SEND_ACK (0x0010000000000000ULL) +#define MPIDI_OFI_SYNC_SEND (0x0020000000000000ULL) +#define MPIDI_OFI_DYNPROC_SEND (0x0040000000000000ULL) +#define MPIDI_OFI_HUGE_SEND (0x0080000000000000ULL) +#define MPIDI_OFI_MAJOR_VERSION MPIDI_OFI_MAJOR_VERSION_PSM3 +#define MPIDI_OFI_MINOR_VERSION MPIDI_OFI_MINOR_VERSION_PSM3 +#define MPIDI_OFI_CONTEXT_STRUCTS 1 +#endif + #define MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS MPIDI_OFI_ON #define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS MPIDI_OFI_ON #define MPIDI_OFI_MAX_ENDPOINTS_SOCKETS MPIDI_OFI_MAX_ENDPOINTS_SCALABLE @@ -686,7 +754,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name) #else /* MPIDI_CH4_OFI_USE_SET_RUNTIME not defined */ #if !defined(MPIDI_OFI_MAX_ENDPOINTS_BITS) -#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set *** +#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,PSM3,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set *** #endif #define MPIDI_OFI_ENABLE_RUNTIME_CHECKS MPIDI_OFI_OFF From fa162c44674c4804555245fc9e10fba957c70485 Mon Sep 17 00:00:00 2001 From: Nitin Bhat Date: Wed, 23 Feb 2022 09:54:32 -0600 Subject: [PATCH 2/2] ch4/ofi: Add psm3 to subconfigure --- src/mpid/ch4/netmod/ofi/subconfigure.m4 | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/mpid/ch4/netmod/ofi/subconfigure.m4 b/src/mpid/ch4/netmod/ofi/subconfigure.m4 index ccb011ecf55..bc1761be6cf 100644 --- a/src/mpid/ch4/netmod/ofi/subconfigure.m4 +++ b/src/mpid/ch4/netmod/ofi/subconfigure.m4 @@ -51,6 +51,7 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[ if test "$no_providers" = "no" ; then enable_psm="no" enable_psm2="no" + enable_psm3="no" enable_sockets="no" enable_verbs="no" enable_usnic="no" @@ -71,6 +72,7 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[ else enable_psm="yes" enable_psm2="yes" + enable_psm3="yes" enable_sockets="yes" enable_verbs="yes" enable_usnic="yes" @@ -100,6 +102,9 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[ "psm2" | "opa") enable_psm2="yes" ;; + "psm3") + enable_psm3="yes" + ;; "sockets") enable_sockets="yes" ;; @@ -189,6 +194,10 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[ AC_DEFINE([MPIDI_CH4_OFI_USE_SET_PSM2], [1], [Define to use PSM2 capability set]) enable_psm2="yes" ;; + "psm3") + AC_DEFINE([MPIDI_CH4_OFI_USE_SET_PSM3], [1], [Define to use PSM3 capability set]) + enable_psm3="yes" + ;; "sockets") AC_DEFINE([MPIDI_CH4_OFI_USE_SET_SOCKETS], [1], [Define to use sockets capability set]) enable_sockets="yes" @@ -273,12 +282,13 @@ AM_COND_IF([BUILD_CH4_NETMOD_OFI],[ if test "$with_libfabric" = "embedded" ; then ofi_embedded="yes" AC_MSG_NOTICE([CH4 OFI Netmod: Using an embedded libfabric]) - ofi_subdir_args="--enable-embedded --disable-psm3" + ofi_subdir_args="--enable-embedded" prov_config="" if test "x${netmod_args}" != "x" ; then prov_config="$prov_config --enable-psm=${enable_psm}" prov_config="$prov_config --enable-psm2=${enable_psm2}" + prov_config="$prov_config --enable-psm3=${enable_psm3}" prov_config="$prov_config --enable-sockets=${enable_sockets}" prov_config="$prov_config --enable-verbs=${enable_verbs}" prov_config="$prov_config --enable-usnic=${enable_usnic}"