prov/cxi: Add FI_CXI_DISABLE_DMABUF* env variable to disable dmabuf

NETCASSINI-4994

Signed-off-by: Chuck Fossen <[email protected]>

man/fi_cxi.7.md

@@ -446,6 +446,15 @@ faults but requires all buffers to be backed by physical memory. Copy-on-write
 semantics are broken when using pinned memory. See the Fork section for more
 information.
 
+The CXI provider supports DMABUF for device memory registration. If the ROCR
+and CUDA libraries support it, the CXI provider will default to use DMA-buf.
+There may be situations with CUDA that may double the BAR consumption.
+Until this is fixed in the CUDA stack, the environment variable
+*FI_CXI_DISABLE_DMABUF_CUDA* can be used to fall back to the nivida
+peer-to-peer interface.
+Also, *FI_CXI_DISABLE_DMABUF_ROCR* can be used to fall back to the amdgpu
+peer-to-peer interface.
+
 ## Translation Cache
 
 Mapping a buffer for use by the NIC is an expensive operation. To avoid this

man/man7/fi_cxi.7

@@ -528,6 +528,16 @@ Using Pinned mode avoids any overhead due to network page faults but
 requires all buffers to be backed by physical memory.
 Copy-on-write semantics are broken when using pinned memory.
 See the Fork section for more information.
+.PP
+The CXI provider supports DMABUF for device memory registration. If
+the ROCR and CUDA libraries support it, the CXI provider will default
+to use DMA-buf.
+There may be situations with CUDA that may double the BAR consumption.
+Until this is fixed in the CUDA stack, the environment variable
+*FI_CXI_DISABLE_DMABUF_CUDA* can be used to fall back to the nivida
+peer-to-peer interface.
+Also, *FI_CXI_DISABLE_DMABUF_ROCR* can be used to fall back to the
+amdgpu peer-to-peer interface.
 .SS Translation Cache
 .PP
 Mapping a buffer for use by the NIC is an expensive operation.

prov/cxi/include/cxip.h

@@ -241,6 +241,8 @@ struct cxip_environment {
 	int force_odp;
 	int ats;
 	int iotlb;
+	int disable_dmabuf_cuda;
+	int disable_dmabuf_rocr;
 	enum cxip_ats_mlock_mode ats_mlock_mode;
 
 	/* Messaging */

prov/cxi/src/cxip_info.c

@@ -649,6 +649,8 @@ struct cxip_environment cxip_env = {
 	.force_odp = false,
 	.ats = false,
 	.iotlb = true,
+	.disable_dmabuf_cuda = false,
+	.disable_dmabuf_rocr = false,
 	.ats_mlock_mode = CXIP_ATS_MLOCK_ALL,
 	.fork_safe_requested = false,
 	.rx_match_mode = CXIP_PTLTE_DEFAULT_MODE,
@@ -804,6 +806,18 @@ static void cxip_env_init(void)
 			"Enables the NIC IOTLB (default %d).", cxip_env.iotlb);
 	fi_param_get_bool(&cxip_prov, "iotlb", &cxip_env.iotlb);
 
+	fi_param_define(&cxip_prov, "disable_dmabuf_cuda", FI_PARAM_BOOL,
+			"Disables the DMABUF interface for CUDA (default %d).",
+			cxip_env.disable_dmabuf_cuda);
+	fi_param_get_bool(&cxip_prov, "disable_dmabuf_cuda",
+			  &cxip_env.disable_dmabuf_cuda);
+
+	fi_param_define(&cxip_prov, "disable_dmabuf_rocr", FI_PARAM_BOOL,
+			"Disables the DMABUF interface for ROCR (default %d).",
+			cxip_env.disable_dmabuf_rocr);
+	fi_param_get_bool(&cxip_prov, "disable_dmabuf_rocr",
+			  &cxip_env.disable_dmabuf_rocr);
+
 	fi_param_define(&cxip_prov, "ats_mlock_mode", FI_PARAM_STRING,
 			"Sets ATS mlock mode (off | all).");
 	fi_param_get_str(&cxip_prov, "ats_mlock_mode", &param_str);

prov/cxi/src/cxip_iomm.c

@@ -28,6 +28,12 @@ static int cxip_dmabuf_hints(enum fi_hmem_iface iface, void *iov_base,
 		return -FI_ENOSYS;
 	}
 
+	if (iface == FI_HMEM_CUDA && cxip_env.disable_dmabuf_cuda)
+		return FI_SUCCESS;
+
+	if (iface == FI_HMEM_ROCR && cxip_env.disable_dmabuf_rocr)
+		return FI_SUCCESS;
+
 	ret = ofi_hmem_get_base_addr(iface, iov_base, len, (void*)&base, &size);
 	if (ret)
 		return ret;