Skip to content

Commit

Permalink
OMPT Target Offload Support (#411)
Browse files Browse the repository at this point in the history
* Improve OMPT support
* Add openMP target example to testing
* Update GitHub workflows
* Update Timemory submodule to use ROCm/timemory rather than NERSC/timemory
* Fix issue with Perfetto
  • Loading branch information
jrmadsen authored Oct 30, 2024
1 parent c8ae2e0 commit 5db537c
Show file tree
Hide file tree
Showing 19 changed files with 880 additions and 134 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/opensuse.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ jobs:
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.21.4' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
- name: Configure Env
run:
Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/redhat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,12 @@ jobs:
run: |
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell
yum install -y epel-release && yum install -y 'dnf-command(config-manager)' && crb enable && \
yum install -y curl dpkg-devel numactl-devel openmpi-devel papi-devel python3-pip texinfo wget which zlib-devel &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.21.4' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
- name: Install ROCm Packages
if: ${{ matrix.rocm-version > 0 }}
Expand Down
30 changes: 15 additions & 15 deletions .github/workflows/ubuntu-focal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,13 @@ jobs:
add-apt-repository -y ppa:ubuntu-toolchain-r/test &&
apt-get update &&
apt-get upgrade -y &&
apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
apt-get install -y autoconf bison build-essential clang environment-modules libiberty-dev libmpich-dev libtool m4 mpich python3-pip texinfo ${{ matrix.compiler }} &&
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.16.3' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
apt-get -y --purge autoremove &&
apt-get -y clean &&
/opt/conda/bin/conda clean -y -a
Expand Down Expand Up @@ -275,13 +275,13 @@ jobs:
wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&
echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
apt-get update &&
apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
apt-get install -y autoconf bison build-essential clang curl hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libfabric-dev libnuma1 libomp-dev libopenmpi-dev libpapi-dev libtool libudev1 m4 openmpi-bin python3-pip rccl-dev rocm-dev rocm-smi-lib rocm-utils rocprofiler-dev roctracer-dev texinfo ${{ matrix.compiler }} &&
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.21.4' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
apt-get -y --purge autoremove &&
apt-get -y clean &&
/opt/conda/bin/conda clean -y -a
Expand Down Expand Up @@ -436,14 +436,14 @@ jobs:
max_attempts: 5
command: |
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
sudo apt-get install -y autoconf bison build-essential clang environment-modules gettext libomp-dev libtool m4 python3-pip texinfo ${{ matrix.compiler }} ${{ matrix.deps }} &&
if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.16.3' &&
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
sudo apt-get -y --purge autoremove &&
sudo apt-get -y clean
Expand Down Expand Up @@ -591,13 +591,13 @@ jobs:
max_attempts: 5
command: |
apt-get update &&
apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
apt-get install -y autoconf bison build-essential clang environment-modules gcc g++ libmpich-dev libomp-dev libtool m4 mpich python3-pip texinfo &&
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.24.1' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
apt-get -y --purge autoremove &&
apt-get -y clean &&
/opt/conda/bin/conda clean -y -a
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/ubuntu-jammy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,13 @@ jobs:
apt-get update &&
apt-get install -y software-properties-common &&
apt-get upgrade -y &&
apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
apt-get install -y autoconf bison build-essential clang environment-modules libfabric-dev libiberty-dev libomp-dev libopenmpi-dev libtool m4 openmpi-bin python3-pip texinfo ${{ matrix.compiler }} &&
wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
chmod +x /opt/trace_processor/bin/trace_processor_shell &&
python3 -m pip install --upgrade pip &&
python3 -m pip install numpy perfetto dataclasses &&
python3 -m pip install 'cmake==3.21.4' &&
for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
python3 -m pip install --upgrade numpy perfetto dataclasses &&
python3 -m pip install --upgrade cmake &&
for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
- name: Install ROCm Packages
timeout-minutes: 25
Expand Down
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[submodule "external/timemory"]
path = external/timemory
url = https://github.com/NERSC/timemory.git
url = https://github.com/ROCm/timemory.git
[submodule "external/perfetto"]
path = external/perfetto
url = https://github.com/google/perfetto.git
Expand Down
2 changes: 1 addition & 1 deletion cmake/Packages.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ mark_as_advanced(TIMEMORY_PROJECT_NAME)
omnitrace_checkout_git_submodule(
RELATIVE_PATH external/timemory
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
REPO_URL https://github.com/NERSC/timemory.git
REPO_URL https://github.com/ROCm/timemory.git
REPO_BRANCH omnitrace)

omnitrace_save_variables(
Expand Down
2 changes: 2 additions & 0 deletions examples/openmp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,5 @@ if(OMNITRACE_INSTALL_EXAMPLES)
DESTINATION bin
COMPONENT omnitrace-examples)
endif()

add_subdirectory(target)
108 changes: 108 additions & 0 deletions examples/openmp/target/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#
#
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)

# try to find a compatible HIP version
if(ROCmVersion_MAJOR_VERSION AND ROCmVersion_MAJOR_VERSION GREATER_EQUAL 6)
find_package(hip ${ROCmVersion_MAJOR_VERSION}.0.0)
else()
find_package(hip)
endif()

if(NOT hip_FOUND)
message(WARNING "ROCm >= 5.6 not found. Skipping OpenMP target example.")
return()
elseif(hip_FOUND AND hip_VERSION VERSION_LESS 5.6.0)
message(
WARNING
"ROCm >= 5.6 not found (found ${hip_VERSION}). Skipping OpenMP target example."
)
return()
endif()

if(NOT OMP_TARGET_COMPILER)
find_program(
amdclangpp_EXECUTABLE
NAMES amdclang++
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
PATH_SUFFIXES bin llvm/bin)
mark_as_advanced(amdclangpp_EXECUTABLE)

if(amdclangpp_EXECUTABLE)
set(OMP_TARGET_COMPILER
"${amdclangpp_EXECUTABLE}"
CACHE FILEPATH "OpenMP target compiler")
else()
message(WARNING "OpenMP target compiler not found. Skipping this example.")
return()
endif()
endif()

project(omnitrace-example-openmp-target-lib LANGUAGES CXX)

set(CMAKE_BUILD_TYPE "RelWithDebInfo")

set(DEFAULT_GPU_TARGETS
"gfx900"
"gfx906"
"gfx908"
"gfx90a"
"gfx940"
"gfx941"
"gfx942"
"gfx1030"
"gfx1010"
"gfx1100"
"gfx1101"
"gfx1102")

set(GPU_TARGETS
"${DEFAULT_GPU_TARGETS}"
CACHE STRING "GPU targets to compile for")

find_package(Threads REQUIRED)

add_library(openmp-target-lib SHARED)
target_sources(openmp-target-lib PRIVATE library.cpp)
target_link_libraries(openmp-target-lib PUBLIC Threads::Threads)
target_compile_options(openmp-target-lib PRIVATE -fopenmp -ggdb)
target_link_options(openmp-target-lib PUBLIC -fopenmp)

foreach(_TARGET ${GPU_TARGETS})
target_compile_options(openmp-target-lib PRIVATE --offload-arch=${_TARGET})
target_link_options(openmp-target-lib PUBLIC --offload-arch=${_TARGET})
endforeach()

message(STATUS "Using OpenMP target compiler: ${OMP_TARGET_COMPILER}")

get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER} PATH)
get_filename_component(OMP_TARGET_COMPILER_DIR ${OMP_TARGET_COMPILER_DIR} PATH)

message(STATUS "Using OpemMP target compiler directory: ${OMP_TARGET_COMPILER_DIR}")

if(NOT EXISTS ${OMP_TARGET_COMPILER_DIR}/llvm/lib)
message(FATAL_ERROR "${OMP_TARGET_COMPILER_DIR}/llvm/lib does not exist")
endif()
set_target_properties(
openmp-target-lib
PROPERTIES BUILD_RPATH
"${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib"
OUTPUT_NAME "openmp-target"
POSITION_INDEPENDENT_CODE ON)

omnitrace_custom_compilation(TARGET openmp-target-lib COMPILER ${OMP_TARGET_COMPILER})

add_executable(openmp-target)
target_sources(openmp-target PRIVATE main.cpp)
target_link_libraries(openmp-target PRIVATE openmp-target-lib)
target_compile_options(openmp-target PRIVATE -ggdb)

set_target_properties(
openmp-target
PROPERTIES BUILD_RPATH
"${OMP_TARGET_COMPILER_DIR}/llvm/lib:${OMP_TARGET_COMPILER_DIR}/lib"
POSITION_INDEPENDENT_CODE ON)

omnitrace_custom_compilation(TARGET openmp-target COMPILER ${OMP_TARGET_COMPILER})
Loading

0 comments on commit 5db537c

Please sign in to comment.