Skip to content

Commit

Permalink
simplify Hip.cmake and prepare for the future ROCm versions (#2106)
Browse files Browse the repository at this point in the history 
Summary: Pull Request resolved: #2106

Reviewed By: q10

Differential Revision: D50938407

Pulled By: shintaro-iwasaki

fbshipit-source-id: 59598292b1caceeccaddf5c19f3639766d5a09d6
  • Loading branch information
@liligwu @facebook-github-bot
liligwu authored and facebook-github-bot committed Nov 3, 2023
1 parent cbb3130 commit df8d189
Showing 1 changed file with 82 additions and 140 deletions.
222 changes: 82 additions & 140 deletions fbgemm_gpu/cmake/Hip.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,125 +6,79 @@

set(FBGEMM_HAVE_HIP FALSE)

IF(NOT DEFINED ENV{ROCM_PATH})
SET(ROCM_PATH /opt/rocm)
ELSE()
SET(ROCM_PATH $ENV{ROCM_PATH})
ENDIF()
if(NOT DEFINED ENV{ROCM_INCLUDE_DIRS})
set(ROCM_INCLUDE_DIRS ${ROCM_PATH}/include)
if(NOT DEFINED ENV{ROCM_PATH})
set(ROCM_PATH /opt/rocm)
else()
set(ROCM_INCLUDE_DIRS $ENV{ROCM_INCLUDE_DIRS})
set(ROCM_PATH $ENV{ROCM_PATH})
endif()
# HIP_PATH
IF(NOT DEFINED ENV{HIP_PATH})
SET(HIP_PATH ${ROCM_PATH}/hip)
ELSE()
SET(HIP_PATH $ENV{HIP_PATH})
ENDIF()

IF(NOT EXISTS ${HIP_PATH})
return()
ENDIF()

# HCC_PATH
IF(NOT DEFINED ENV{HCC_PATH})
SET(HCC_PATH ${ROCM_PATH}/hcc)
ELSE()
SET(HCC_PATH $ENV{HCC_PATH})
ENDIF()

# HSA_PATH
IF(NOT DEFINED ENV{HSA_PATH})
SET(HSA_PATH ${ROCM_PATH}/hsa)
ELSE()
SET(HSA_PATH $ENV{HSA_PATH})
ENDIF()

# ROCBLAS_PATH
IF(NOT DEFINED ENV{ROCBLAS_PATH})
SET(ROCBLAS_PATH ${ROCM_PATH}/rocblas)
ELSE()
SET(ROCBLAS_PATH $ENV{ROCBLAS_PATH})
ENDIF()

# ROCSPARSE_PATH
IF(NOT DEFINED ENV{ROCSPARSE_PATH})
SET(ROCSPARSE_PATH ${ROCM_PATH}/rocsparse)
ELSE()
SET(ROCSPARSE_PATH $ENV{ROCSPARSE_PATH})
ENDIF()

# ROCFFT_PATH
IF(NOT DEFINED ENV{ROCFFT_PATH})
SET(ROCFFT_PATH ${ROCM_PATH}/rocfft)
ELSE()
SET(ROCFFT_PATH $ENV{ROCFFT_PATH})
ENDIF()

# HIPSPARSE_PATH
IF(NOT DEFINED ENV{HIPSPARSE_PATH})
SET(HIPSPARSE_PATH ${ROCM_PATH}/hipsparse)
ELSE()
SET(HIPSPARSE_PATH $ENV{HIPSPARSE_PATH})
ENDIF()

# THRUST_PATH
IF(NOT DEFINED ENV{THRUST_PATH})
SET(THRUST_PATH ${ROCM_PATH}/include)
ELSE()
SET(THRUST_PATH $ENV{THRUST_PATH})
ENDIF()

# HIPRAND_PATH
IF(NOT DEFINED ENV{HIPRAND_PATH})
SET(HIPRAND_PATH ${ROCM_PATH}/hiprand)
ELSE()
SET(HIPRAND_PATH $ENV{HIPRAND_PATH})
ENDIF()

# ROCRAND_PATH
IF(NOT DEFINED ENV{ROCRAND_PATH})
SET(ROCRAND_PATH ${ROCM_PATH}/rocrand)
ELSE()
SET(ROCRAND_PATH $ENV{ROCRAND_PATH})
ENDIF()

# MIOPEN_PATH
IF(NOT DEFINED ENV{MIOPEN_PATH})
SET(MIOPEN_PATH ${ROCM_PATH}/miopen)
ELSE()
SET(MIOPEN_PATH $ENV{MIOPEN_PATH})
ENDIF()
macro(torch_hip_get_arch_list store_var)
if(DEFINED ENV{PYTORCH_ROCM_ARCH})
set(_TMP $ENV{PYTORCH_ROCM_ARCH})
else()
# Use arch of installed GPUs as default
execute_process(COMMAND "rocm_agent_enumerator" COMMAND bash "-c" "grep -v gfx000 | sort -u | xargs | tr -d '\n'"
RESULT_VARIABLE ROCM_AGENT_ENUMERATOR_RESULT
OUTPUT_VARIABLE ROCM_ARCH_INSTALLED)
if(NOT ROCM_AGENT_ENUMERATOR_RESULT EQUAL 0)
message(FATAL_ERROR " Could not detect ROCm arch for GPUs on machine. Result: '${ROCM_AGENT_ENUMERATOR_RESULT}'")
endif()
set(_TMP ${ROCM_ARCH_INSTALLED})
endif()
string(REPLACE " " ";" ${store_var} "${_TMP}")
endmacro()

# Add HIP to the CMAKE Module Path
set(CMAKE_MODULE_PATH ${HIP_PATH}/cmake ${CMAKE_MODULE_PATH})
torch_hip_get_arch_list(PYTORCH_ROCM_ARCH)
if(PYTORCH_ROCM_ARCH STREQUAL "")
message(FATAL_ERROR "No GPU arch specified for ROCm build. Please use PYTORCH_ROCM_ARCH environment variable to specify GPU archs to build for.")
endif()
message("Building FBGEMM for GPU arch: ${PYTORCH_ROCM_ARCH}")

# Disable Asserts In Code (Can't use asserts on HIP stack.)
ADD_DEFINITIONS(-DNDEBUG)
ADD_DEFINITIONS(-DUSE_ROCM)
ADD_DEFINITIONS(-D__HIP_PLATFORM_AMD__)

IF(NOT DEFINED ENV{PYTORCH_ROCM_ARCH})
SET(FBGEMM_ROCM_ARCH gfx900;gfx906;gfx908;gfx90a)
ELSE()
SET(FBGEMM_ROCM_ARCH $ENV{PYTORCH_ROCM_ARCH})
ENDIF()
# Add HIP to the CMAKE Module Path
set(CMAKE_MODULE_PATH ${ROCM_PATH}/lib/cmake/hip ${CMAKE_MODULE_PATH})

macro(find_package_and_print_version PACKAGE_NAME)
find_package("${PACKAGE_NAME}" ${ARGN})
message("${PACKAGE_NAME} VERSION: ${${PACKAGE_NAME}_VERSION}")
endmacro()

# Find the HIP Package
find_package(HIP)
find_package_and_print_version(HIP 1.0)

IF(HIP_FOUND)
if(HIP_FOUND)
set(FBGEMM_HAVE_HIP TRUE)
set(FOUND_ROCM_VERSION_H FALSE)

if(EXISTS ${ROCM_PATH}/.info/version-dev)
# ROCM < 4.5, we don't have the header api file, use flat file
file(READ "${ROCM_PATH}/.info/version-dev" ROCM_VERSION_DEV_RAW)
message("\n***** ROCm version from ${ROCM_PATH}/.info/version-dev ****\n")
endif()

set(PROJECT_RANDOM_BINARY_DIR "${PROJECT_BINARY_DIR}")
set(file "${PROJECT_BINARY_DIR}/detect_rocm_version.cc")

# Find ROCM version for checks
# ROCM 5.0 and later will have header api for version management
if(EXISTS ${ROCM_INCLUDE_DIRS}/rocm_version.h)

set(PROJECT_RANDOM_BINARY_DIR "${PROJECT_BINARY_DIR}")
set(file "${PROJECT_BINARY_DIR}/detect_rocm_version.cc")
set(FOUND_ROCM_VERSION_H TRUE)
file(WRITE ${file} ""
"#include <rocm_version.h>\n"
)
elseif(EXISTS ${ROCM_INCLUDE_DIRS}/rocm-core/rocm_version.h)
set(FOUND_ROCM_VERSION_H TRUE)
file(WRITE ${file} ""
"#include <rocm-core/rocm_version.h>\n"
)
else()
message("********************* rocm_version.h couldnt be found ******************\n")
endif()

if(FOUND_ROCM_VERSION_H)
file(APPEND ${file} ""
"#include <cstdio>\n"

"#ifndef ROCM_VERSION_PATCH\n"
Expand Down Expand Up @@ -184,37 +138,27 @@ IF(HIP_FOUND)

message("\n***** Library versions from cmake find_package *****\n")

# As of ROCm 5.1.x, all *.cmake files are under /opt/rocm/lib/cmake/<package>
if(ROCM_VERSION_DEV VERSION_GREATER_EQUAL "5.1.0")
set(hip_DIR ${HIP_PATH}/lib/cmake/hip)
set(hsa-runtime64_DIR ${ROCM_PATH}/lib/cmake/hsa-runtime64)
set(AMDDeviceLibs_DIR ${ROCM_PATH}/lib/cmake/AMDDeviceLibs)
set(amd_comgr_DIR ${ROCM_PATH}/lib/cmake/amd_comgr)
set(rocrand_DIR ${ROCM_PATH}/lib/cmake/rocrand)
set(hiprand_DIR ${ROCM_PATH}/lib/cmake/hiprand)
set(rocblas_DIR ${ROCM_PATH}/lib/cmake/rocblas)
set(miopen_DIR ${ROCM_PATH}/lib/cmake/miopen)
set(rocfft_DIR ${ROCM_PATH}/lib/cmake/rocfft)
set(hipfft_DIR ${ROCM_PATH}/lib/cmake/hipfft)
set(hipsparse_DIR ${ROCM_PATH}/lib/cmake/hipsparse)
set(rccl_DIR ${ROCM_PATH}/lib/cmake/rccl)
set(rocprim_DIR ${ROCM_PATH}/lib/cmake/rocprim)
set(hipcub_DIR ${ROCM_PATH}/lib/cmake/hipcub)
set(rocthrust_DIR ${ROCM_PATH}/lib/cmake/rocthrust)
set(ROCclr_DIR ${ROCM_PATH}/rocclr/lib/cmake/rocclr)
set(ROCRAND_INCLUDE ${ROCM_PATH}/include)
set(ROCM_SMI_INCLUDE ${ROCM_PATH}/include)
else()
message(FATAL_ERROR "\n***** The minimal ROCm version is 5.1.0 but have ${ROCM_VERSION_DEV} installed *****\n")
endif()
set(CMAKE_HCC_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
set(CMAKE_HCC_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})

find_package(hip REQUIRED)
find_package(rocblas REQUIRED)
find_package(hipfft REQUIRED)
find_package(hiprand REQUIRED)
find_package(rocrand REQUIRED)
find_package(hipsparse REQUIRED)
find_package(rocprim REQUIRED)
find_package_and_print_version(hip REQUIRED)
find_package_and_print_version(hsa-runtime64 REQUIRED)
find_package_and_print_version(amd_comgr REQUIRED)
find_package_and_print_version(rocrand REQUIRED)
find_package_and_print_version(hiprand REQUIRED)
find_package_and_print_version(hipblas REQUIRED)
find_package_and_print_version(rocblas REQUIRED)
find_package_and_print_version(miopen REQUIRED)
find_package_and_print_version(rocfft REQUIRED)
find_package_and_print_version(hipsparse REQUIRED)
find_package_and_print_version(rccl)
find_package_and_print_version(rocprim REQUIRED)
find_package_and_print_version(hipcub REQUIRED)
find_package_and_print_version(rocthrust REQUIRED)
find_package_and_print_version(hipsolver REQUIRED)

# Enabling HIP language support
enable_language(HIP)

if(HIP_COMPILER STREQUAL clang)
set(hip_library_name amdhip64)
Expand All @@ -223,9 +167,11 @@ IF(HIP_FOUND)
endif()
message("HIP library name: ${hip_library_name}")

set(CMAKE_HCC_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
set(CMAKE_HCC_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
FIND_LIBRARY(FBGEMM_HIP_HCC_LIBRARIES ${hip_library_name} HINTS ${HIP_PATH}/lib)
# TODO: hip_hcc has an interface include flag "-hc" which is only
# recognizable by hcc, but not gcc and clang. Right now in our
# setup, hcc is only used for linking, but it should be used to
# compile the *_hip.cc files as well.
find_library(FBGEMM_HIP_HCC_LIBRARIES ${hip_library_name} HINTS ${ROCM_PATH}/lib)

list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_OPERATORS__=1)
# list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_CONVERSIONS__=1)
Expand All @@ -242,7 +188,7 @@ IF(HIP_FOUND)
# host linker to link.
list(APPEND HIP_HCC_FLAGS -fno-gpu-rdc)
list(APPEND HIP_HCC_FLAGS -Wno-defaulted-function-deleted)
foreach(fbgemm_rocm_arch ${FBGEMM_ROCM_ARCH})
foreach(fbgemm_rocm_arch ${PYTORCH_ROCM_ARCH})
list(APPEND HIP_HCC_FLAGS --offload-arch=${fbgemm_rocm_arch})
endforeach()

Expand All @@ -252,8 +198,4 @@ IF(HIP_FOUND)
hip_include_directories(${FBGEMM_HIP_INCLUDE} ${ROCRAND_INCLUDE} ${ROCM_SMI_INCLUDE})

list (APPEND CMAKE_PREFIX_PATH ${HIP_PATH} ${ROCM_PATH})
set(CMAKE_MODULE_PATH ${HIP_PATH}/cmake ${CMAKE_MODULE_PATH})

ELSE()
message("Not able to find HIP installation.")
ENDIF()
endif()

0 comments on commit df8d189

Please sign in to comment.