Merge pull request #6098 from hzhou/2207_hmem

ch4/ofi: fix enable_hmem checking Approved-by: Ken Raffenetti
pmodels · Aug 1, 2022 · 3391a84 · 3391a84
2 parents 118fd08 + e06d942
commit 3391a84
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 1 deletion.
diff --git a/src/mpid/ch4/netmod/ofi/globals.c b/src/mpid/ch4/netmod/ofi/globals.c
@@ -38,6 +38,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_DEFAULT,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_DEFAULT,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_DEFAULT,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_DEFAULT,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_DEFAULT,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_DEFAULT,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_DEFAULT,
@@ -65,6 +66,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_MINIMAL,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_MINIMAL,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_MINIMAL,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_MINIMAL,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_MINIMAL,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_MINIMAL,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_MINIMAL,
@@ -92,6 +94,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM2,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM2,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM2,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_PSM2,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_PSM2,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM2,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_PSM2,
@@ -119,6 +122,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_PSM3,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_PSM3,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_PSM3,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_PSM3,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_PSM3,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_PSM3,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_PSM3,
@@ -146,6 +150,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_SOCKETS,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_SOCKETS,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_SOCKETS,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_SOCKETS,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_SOCKETS,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_SOCKETS,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_SOCKETS,
@@ -173,6 +178,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_BGQ,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_BGQ,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_BGQ,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_BGQ,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_BGQ,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_BGQ,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_BGQ,
@@ -200,6 +206,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_CXI,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_CXI,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_CXI,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_CXI,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_CXI,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_CXI,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_CXI,
@@ -227,6 +234,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_VERBS_RXM,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_VERBS_RXM,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_VERBS_RXM,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_VERBS_RXM,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_VERBS_RXM,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_VERBS_RXM,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_VERBS_RXM,
@@ -254,6 +262,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_RXM,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_RXM,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_RXM,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_RXM,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_RXM,
      .num_optimized_memory_regions = MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_RXM,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_RXM,
@@ -281,6 +290,7 @@ MPIDI_OFI_capabilities_t MPIDI_OFI_caps_list[MPIDI_OFI_NUM_SETS] =
      .enable_data_auto_progress = MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_GNI,
      .enable_control_auto_progress = MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_GNI,
      .enable_pt2pt_nopack = MPIDI_OFI_ENABLE_PT2PT_NOPACK_GNI,
+     .enable_hmem = MPIDI_OFI_ENABLE_HMEM_GNI,
      .num_am_buffers = MPIDI_OFI_NUM_AM_BUFFERS_GNI,
      .max_endpoints = MPIDI_OFI_MAX_ENDPOINTS_GNI,
      .max_endpoints_bits = MPIDI_OFI_MAX_ENDPOINTS_BITS_GNI,

diff --git a/src/mpid/ch4/netmod/ofi/init_settings.c b/src/mpid/ch4/netmod/ofi/init_settings.c
@@ -263,6 +263,7 @@ void MPIDI_OFI_init_settings(MPIDI_OFI_capabilities_t * p_settings, const char *
     MPIDI_OFI_global.settings.require_rdm = 1;
     UPDATE_SETTING_BY_CAP(num_optimized_memory_regions,
                           MPIR_CVAR_CH4_OFI_NUM_OPTIMIZED_MEMORY_REGIONS);
+    UPDATE_SETTING_BY_CAP(enable_hmem, MPIR_CVAR_CH4_OFI_ENABLE_HMEM);
 }
 
 #define CHECK_CAP(setting, cond_bad) \

diff --git a/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h b/src/mpid/ch4/netmod/ofi/ofi_capability_sets.h
@@ -400,7 +400,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_get_set_number(const char *set_name)
 #define MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS_CXI     MPIDI_OFI_OFF
 #define MPIDI_OFI_ENABLE_CONTROL_AUTO_PROGRESS_CXI  MPIDI_OFI_OFF
 #define MPIDI_OFI_ENABLE_PT2PT_NOPACK_CXI           MPIDI_OFI_ON
-#define MPIDI_OFI_ENABLE_HMEM_CXI                   MPIDI_OFI_OFF
+#define MPIDI_OFI_ENABLE_HMEM_CXI                   MPIDI_OFI_ON
 #define MPIDI_OFI_NUM_AM_BUFFERS_CXI                MPIDI_OFI_MAX_NUM_AM_BUFFERS
 #define MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS_CXI  (0) /* This will be added to the CXI provider as
                                                          * RMA support is fixed. */

diff --git a/src/mpid/ch4/netmod/ofi/ofi_init.c b/src/mpid/ch4/netmod/ofi/ofi_init.c
@@ -192,6 +192,16 @@ categories :
       description : >-
         If true, enable iovec for pt2pt.
 
+    - name        : MPIR_CVAR_CH4_OFI_ENABLE_HMEM
+      category    : CH4_OFI
+      type        : int
+      default     : -1
+      class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_LOCAL
+      description : >-
+        If true, enable OFI HMEM support.
+
     - name        : MPIR_CVAR_CH4_OFI_CONTEXT_ID_BITS
       category    : CH4_OFI
       type        : int