prov/efa: Fall back to zero sl when non-zero sl qp creation failed

When driver doesn't support non-zero sl, the qp creation will fail. In this case, fallback to zero sl qp. Signed-off-by: Shi Jin <[email protected]>
shijin-aws · Nov 6, 2024 · 1247a68 · 1247a68
1 parent a03c5a7
commit 1247a68
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -186,12 +186,29 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
 #if HAVE_EFADV_SL
-		if (tclass == FI_TC_LOW_LATENCY)
-			efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
-#endif
+		efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
+		if (tclass == FI_TC_LOW_LATENCY) {
+			efa_attr.sl = EFA_QP_LOW_LATENCY_SERVICE_LEVEL;
+			(*qp)->ibv_qp = efadv_create_qp_ex(
+				init_attr_ex->pd->context, init_attr_ex, &efa_attr,
+				sizeof(struct efadv_qp_init_attr));
+			if (!(*qp)->ibv_qp) {
+				EFA_INFO(FI_LOG_EP_CTRL, "ibv_create_qp failed with sl %u, errno: %d. Retrying with default sl. \n", efa_attr.sl, errno);
+				efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
+				(*qp)->ibv_qp = efadv_create_qp_ex(
+					init_attr_ex->pd->context, init_attr_ex, &efa_attr,
+					sizeof(struct efadv_qp_init_attr));
+			}
+		} else {
+			(*qp)->ibv_qp = efadv_create_qp_ex(
+				init_attr_ex->pd->context, init_attr_ex, &efa_attr,
+				sizeof(struct efadv_qp_init_attr));
+		}
+#else
 		(*qp)->ibv_qp = efadv_create_qp_ex(
 			init_attr_ex->pd->context, init_attr_ex, &efa_attr,
 			sizeof(struct efadv_qp_init_attr));
+#endif
 	}
 
 	if (!(*qp)->ibv_qp) {

diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h
@@ -12,7 +12,8 @@
 #include "ofi_util.h"
 #include "rdm/efa_rdm_protocol.h"
 
-#define EFA_QP_DEFAULT_SERVICE_LEVEL 8
+#define EFA_QP_DEFAULT_SERVICE_LEVEL 0
+#define EFA_QP_LOW_LATENCY_SERVICE_LEVEL 8
 
 struct efa_qp {
 	struct ibv_qp *ibv_qp;