Skip to content

Commit

Permalink
ch4/ofi: Create PSM3 Capability set
Browse files Browse the repository at this point in the history
  • Loading branch information
@nitbhat
nitbhat committed Apr 27, 2022
1 parent c86cbbc commit 265cf4e
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 1 deletion.
26 changes: 26 additions & 0 deletions src/mpid/ch4/netmod/ofi/globals.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,32 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
.major_version = MPIDI_OFI_MAJOR_VERSION_PSM2,
.minor_version = MPIDI_OFI_MINOR_VERSION_PSM2}
,
{ /* psm3 */
.enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_PSM3,
.enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3,
.enable_shared_contexts = MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3,
.enable_mr_virt_address = MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3,
.enable_mr_allocated = MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3,
.enable_mr_prov_key = MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3,
.enable_tagged = MPIDI_OFI_ENABLE_TAGGED_PSM3,
.enable_am = MPIDI_OFI_ENABLE_AM_PSM3,
.enable_rma = MPIDI_OFI_ENABLE_RMA_PSM3,
.enable_atomics = MPIDI_OFI_ENABLE_ATOMICS_PSM3,
.enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3,
.enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3,
.enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3,
.num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_PSM3,
.num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3,
.max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_PSM3,
.max_endpoints_bits = MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3,
.fetch_atomic_iovecs = MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3,
.context_bits = MPIDI_OFI_CONTEXT_BITS_PSM3,
.source_bits = MPIDI_OFI_SOURCE_BITS_PSM3,
.tag_bits = MPIDI_OFI_TAG_BITS_PSM3,
.counter_wait_objects = MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3,
.major_version = MPIDI_OFI_MAJOR_VERSION_PSM3,
.minor_version = MPIDI_OFI_MINOR_VERSION_PSM3}
,
{ /* sockets */
.enable_av_table = MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS,
.enable_scalable_endpoints = MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS,
Expand Down
70 changes: 69 additions & 1 deletion src/mpid/ch4/netmod/ofi/ofi_capability_sets.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ enum {
MPIDI_OFI_SET_NUMBER_DEFAULT = 0,
MPIDI_OFI_SET_NUMBER_MINIMAL,
MPIDI_OFI_SET_NUMBER_PSM2,
MPIDI_OFI_SET_NUMBER_PSM3,
MPIDI_OFI_SET_NUMBER_SOCKETS,
MPIDI_OFI_SET_NUMBER_BGQ,
MPIDI_OFI_SET_NUMBER_CXI,
Expand All @@ -46,6 +47,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
return MPIDI_OFI_SET_NUMBER_DEFAULT;
} else if (!strcmp("psm2", set_name)) {
return MPIDI_OFI_SET_NUMBER_PSM2;
} else if (!strcmp("psm3", set_name)) {
return MPIDI_OFI_SET_NUMBER_PSM3;
} else if (!strcmp("sockets", set_name)) {
return MPIDI_OFI_SET_NUMBER_SOCKETS;
} else if (!strcmp("bgq", set_name)) {
Expand Down Expand Up @@ -176,6 +179,71 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
#define MPIDI_OFI_CONTEXT_STRUCTS 1
#endif

#define MPIDI_OFI_ENABLE_AV_TABLE_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_MAX_ENDPOINTS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_SCALABLE
#define MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3 MPIDI_OFI_MAX_ENDPOINTS_BITS_SCALABLE
#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_TAGGED_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_RMA_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_AM_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_ATOMICS_PSM3 MPIDI_OFI_ENABLE_RMA_PSM3
#define MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3 1
#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_HMEM_PSM3 MPIDI_OFI_OFF
#define MPIDI_OFI_NUM_AM_BUFFERS_PSM3 MPIDI_OFI_MAX_NUM_AM_BUFFERS
#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3 (0)
#define MPIDI_OFI_CONTEXT_BITS_PSM3 (20)
#define MPIDI_OFI_SOURCE_BITS_PSM3 (0)
#define MPIDI_OFI_TAG_BITS_PSM3 (31)
#define MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3 MPIDI_OFI_ON
#define MPIDI_OFI_MAJOR_VERSION_PSM3 1
#define MPIDI_OFI_MINOR_VERSION_PSM3 6

#ifdef MPIDI_CH4_OFI_USE_SET_PSM3
#define MPIDI_OFI_SET_NUMBER MPIDI_OFI_SET_NUMBER_PSM3
#define MPIDI_OFI_ENABLE_AV_TABLE MPIDI_OFI_ENABLE_AV_TABLE_PSM3
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_PSM3
#define MPIDI_OFI_MAX_ENDPOINTS MPIDI_OFI_MAX_ENDPOINTS_PSM3
#define MPIDI_OFI_MAX_ENDPOINTS_BITS MPIDI_OFI_MAX_ENDPOINTS_BITS_PSM3
#define MPIDI_OFI_ENABLE_SHARED_CONTEXTS MPIDI_OFI_global.settings.enable_shared_contexts /* Always controlled by CVAR */
#define MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS MPIDI_OFI_ENABLE_MR_VIRT_ADDRESS_PSM3
#define MPIDI_OFI_ENABLE_MR_ALLOCATED MPIDI_OFI_ENABLE_MR_ALLOCATED_PSM3
#define MPIDI_OFI_ENABLE_MR_PROV_KEY MPIDI_OFI_ENABLE_MR_PROV_KEY_PSM3
#define MPIDI_OFI_ENABLE_TAGGED MPIDI_OFI_ENABLE_TAGGED_PSM3
#define MPIDI_OFI_ENABLE_AM MPIDI_OFI_ENABLE_AM_PSM3
#define MPIDI_OFI_ENABLE_RMA MPIDI_OFI_ENABLE_RMA_PSM3
#define MPIDI_OFI_ENABLE_ATOMICS MPIDI_OFI_ENABLE_ATOMICS_PSM3
#define MPIDI_OFI_FETCH_ATOMIC_IOVECS MPIDI_OFI_FETCH_ATOMIC_IOVECS_PSM3
#define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3
#define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3
#define MPIDI_OFI_ENABLE_PT2PT_NOPACK MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3
#define MPIDI_OFI_ENABLE_HMEM MPIDI_OFI_ENABLE_HMEM_PSM3
#define MPIDI_OFI_NUM_AM_BUFFERS MPIDI_OFI_NUM_AM_BUFFERS_PSM3
#define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3
#define MPIDI_OFI_PROTOCOL_MASK (0x00E0000000000000ULL)
#define MPIDI_OFI_CONTEXT_MASK (0x000FFFFF00000000ULL)
#define MPIDI_OFI_SOURCE_MASK (0x0000000000000000ULL) /* PSM3 does support immediate data
* so this field is zeroed */
#define MPIDI_OFI_TAG_MASK (0x000000007FFFFFFFULL)
#define MPIDI_OFI_CONTEXT_BITS MPIDI_OFI_CONTEXT_BITS_PSM3
#define MPIDI_OFI_SOURCE_BITS MPIDI_OFI_SOURCE_BITS_PSM3
#define MPIDI_OFI_TAG_BITS MPIDI_OFI_TAG_BITS_PSM3
#define MPIDI_OFI_COUNTER_WAIT_OBJECTS MPIDI_OFI_COUNTER_WAIT_OBJECTS_PSM3
#define MPIDI_OFI_SYNC_SEND_ACK (0x0010000000000000ULL)
#define MPIDI_OFI_SYNC_SEND (0x0020000000000000ULL)
#define MPIDI_OFI_DYNPROC_SEND (0x0040000000000000ULL)
#define MPIDI_OFI_HUGE_SEND (0x0080000000000000ULL)
#define MPIDI_OFI_MAJOR_VERSION MPIDI_OFI_MAJOR_VERSION_PSM3
#define MPIDI_OFI_MINOR_VERSION MPIDI_OFI_MINOR_VERSION_PSM3
#define MPIDI_OFI_CONTEXT_STRUCTS 1
#endif

#define MPIDI_OFI_ENABLE_AV_TABLE_SOCKETS MPIDI_OFI_ON
#define MPIDI_OFI_ENABLE_SCALABLE_ENDPOINTS_SOCKETS MPIDI_OFI_ON
#define MPIDI_OFI_MAX_ENDPOINTS_SOCKETS MPIDI_OFI_MAX_ENDPOINTS_SCALABLE
Expand Down Expand Up @@ -686,7 +754,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
#else
/* MPIDI_CH4_OFI_USE_SET_RUNTIME not defined */
#if !defined(MPIDI_OFI_MAX_ENDPOINTS_BITS)
#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set ***
#error *** One of MPIDI_CH4_OFI_USE_SET_{e.g. PSM2,PSM3,SOCKETS,BGQ,VERBS_RXM,RXM,GNI or RUNTIME} must be set ***
#endif

#define MPIDI_OFI_ENABLE_RUNTIME_CHECKS MPIDI_OFI_OFF
Expand Down

0 comments on commit 265cf4e

Please sign in to comment.