Resolving conflicts

Resolving conflicts

Makefile

@@ -102,6 +102,24 @@ else
   endif
 endif
 
+ifeq ($(NO_MULTI_GPU), 1)
+  $(info Multi-GPU (OpenMPI + NCCL) is manually disabled)
+else
+  # Detect if running on macOS or Linux
+  ifeq ($(shell uname), Darwin)
+    $(warning Multi-GPU on CUDA on Darwin is not supported, skipping OpenMPI + NCCL support)
+  else ifeq ($(shell [ -d /usr/lib/x86_64-linux-gnu/openmpi/lib/ ] && [ -d /usr/lib/x86_64-linux-gnu/openmpi/include/ ] && echo "exists"), exists)
+    $(info OpenMPI found, adding support)
+    NVCC_INCLUDES += -I/usr/lib/x86_64-linux-gnu/openmpi/include
+    NVCC_LDFLAGS += -L/usr/lib/x86_64-linux-gnu/openmpi/lib/
+    NVCC_LDLIBS += -lmpi -lnccl
+    NVCC_FLAGS += -DMULTI_GPU
+  else
+    $(warning OpenMPI is not found, disabling multi-GPU support)
+    $(warning On Linux you can try install OpenMPI with `sudo apt install openmpi-bin openmpi-doc libopenmpi-dev`)
+  endif
+endif
+
 # Force the rebuild of targets
 .PHONY: all train_gpt2 test_gpt2 train_gpt2cu test_gpt2cu train_gpt2fp32cu test_gpt2fp32cu