Add CI jobs (#10)

* add initial check * move pr.yaml * add empty check_style.sh * basic check_style.sh that does nothing * add +x * try adding a wheel-build job * fix spacing * add runs-on * install ctk * add -q for now * switch to apt-get * minor fixes * update package name * add install options * expanding build_wheel.sh * add tests and logging * more test build steps * fix path * Apply suggestions from code review Co-authored-by: Bradley Dice <[email protected]> * fix style * add wheel-build to needs * apt-get -> yum due to switching to centos * fix centos cuda install * set c++11 * remove testing phase for now, will test on downstream jobs instead later * try adding a matrix * Update .github/workflows/pr.yaml Co-authored-by: Bradley Dice <[email protected]> * install python via conda * missed endquote * try adding test steps in separate jobs * fix syntax * switch to * for now * formatting * add missing job * try uploading a directory * update paths * drop 3.12 for now * add matrix to download * +x to test_wheel.sh * expand test_wheel.sh * try reading artifact name * a debugging commit * update wheel install command * missing .. * fix more paths * build in verbose mode * cmake.verbose=True in pyproject.toml * require cuda 12.3 * explicitly install gcc * switch containers, remove extra gcc install * endquote * base image name on python version, dont install python in test container * adjust matrix * install pytest using pip * use gpu runners * install gcc again * enforce compiler versions in cmake * try installing system compilers * set and unset e * start expanding test matrix * do not set e for now * use SCCACHE_S3_NO_CREDENTIALS=1 * use ci-wheel containers to get arm images * get python from the image instead of conda * use inputs.node_type * build on cpu runners * Update .github/workflows/pr.yaml * remove python version from path * separate testing into arm64 and amd64 * add to needs section * use citestwheel test images * skip bad test * Apply suggestions from code review * Update pynvjitlink/tests/test_pynvjitlink.py Co-authored-by: Bradley Dice <[email protected]> * build amd and arm in two separate jobs * don't test ubuntu 18.04 with arm * address reviews * fix cmake * try removing cache again * Revert "try removing cache again" This reverts commit 78b0b41. --------- Co-authored-by: Bradley Dice <[email protected]>
rapidsai · Nov 17, 2023 · c540358 · c540358
1 parent 65e1109
commit c540358
Show file tree

Hide file tree

Showing 8 changed files with 183 additions and 3 deletions.
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -0,0 +1,120 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+name: pr
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pr-builder:
+    needs:
+      - checks
+      - wheel-build-amd64
+      - wheel-build-arm64
+      - wheel-test-amd64
+      - wheel-test-arm64
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
+  wheel-build-amd64:
+    runs-on: linux-amd64-cpu4
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    container:
+      image: "rapidsai/ci-wheel:cuda12.0.1-centos7-py${{ matrix.python-version }}"
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Get PR Info
+      id: get-pr-info
+      uses: rapidsai/shared-actions/get-pr-info@main
+    - name: Run build_wheel.sh
+      run: ci/build_wheel.sh
+    - name: Upload Python Wheel
+      uses: actions/upload-artifact@v2
+      with:
+        path: ./wheel-build
+  wheel-build-arm64:
+    runs-on: linux-arm64-cpu4
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    container:
+      image: "rapidsai/ci-wheel:cuda12.0.1-rockylinux8-py${{ matrix.python-version }}"
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Get PR Info
+      id: get-pr-info
+      uses: rapidsai/shared-actions/get-pr-info@main
+    - name: Run build_wheel.sh
+      run: ci/build_wheel.sh
+    - name: Upload Python Wheel
+      uses: actions/upload-artifact@v2
+      with:
+        path: ./wheel-build
+  wheel-test-amd64:
+    needs:
+      - wheel-build-amd64
+    runs-on: linux-amd64-gpu-v100-latest-1
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+        linux-version: ["ubuntu20.04", "ubuntu18.04"]
+    container:
+      image: "rapidsai/citestwheel:cuda12.0.1-${{ matrix.linux-version }}-py${{ matrix.python-version }}"
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Get PR Info
+      id: get-pr-info
+      uses: rapidsai/shared-actions/get-pr-info@main
+    - name: Download Python Wheel
+      uses: actions/download-artifact@v2
+      with:
+        path: ./
+    - name: Run test_wheel.sh
+      run: ci/test_wheel.sh 
+  wheel-test-arm64:
+    needs:
+      - wheel-build-arm64
+    runs-on: linux-arm64-gpu-a100-latest-1
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    container:
+      image: "rapidsai/citestwheel:cuda12.0.1-ubuntu20.04-py${{ matrix.python-version }}"
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Get PR Info
+      id: get-pr-info
+      uses: rapidsai/shared-actions/get-pr-info@main
+    - name: Download Python Wheel
+      uses: actions/download-artifact@v2
+      with:
+        path: ./
+    - name: run test_wheel.sh
+      run: ci/test_wheel.sh 
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,18 +3,20 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 project(
   pynvjitlink
   VERSION 0.1.0
-  LANGUAGES CXX
+  LANGUAGES CXX CUDA
 )
 
 find_package(Python COMPONENTS Interpreter Development REQUIRED)
 
 Python_add_library(_nvjitlinklib MODULE pynvjitlink/_nvjitlinklib.cpp WITH_SOABI)
 
 find_package(
-  CUDAToolkit 12.0 REQUIRED
+  CUDAToolkit 12.3 REQUIRED
 )
 target_link_libraries(_nvjitlinklib PRIVATE CUDA::nvJitLink_static CUDA::nvptxcompiler_static)
 
 target_compile_options(_nvjitlinklib PRIVATE -Werror -Wall)
 
+target_compile_features(_nvjitlinklib PRIVATE cxx_std_11)
+
 install(TARGETS _nvjitlinklib LIBRARY DESTINATION pynvjitlink)
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION
+
+
+rapids-logger "Install CUDA Toolkit"
+yum update
+yum install -y epel-release
+yum install -y nvidia-driver-latest-dkms
+yum install -y cuda-toolkit-12-3
+
+rapids-logger "Check conda environment"
+conda list
+
+rapids-logger "Build wheel"
+export SCCACHE_S3_NO_CREDENTIALS=1
+mkdir -p ./wheel-build
+pip wheel . --wheel-dir=./wheel-build -vvv
diff --git a/ci/check_style.sh b/ci/check_style.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION
+
+set -euo pipefail
+
+pip install pre-commit
+
+# Run pre-commit checks
+pre-commit run --all-files --show-diff-on-failure
diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION
+
+set -e
+
+rapids-logger "Search artifact directory"
+ls ./artifact
+
+rapids-logger "Install wheel"
+pip install --find-links ./artifact pynvjitlink
+
+rapids-logger "Build Tests"
+cd test_binary_generation && make
+
+rapids-logger "Run Tests"
+cd ..
+pip install pytest
+pytest pynvjitlink/tests
diff --git a/pynvjitlink/tests/test_pynvjitlink.py b/pynvjitlink/tests/test_pynvjitlink.py
@@ -48,6 +48,7 @@ def device_functions_ptx():
 def undefined_extern_cubin():
     return read_test_file('undefined_extern.cubin')
 
+
 @pytest.mark.skip
 def test_create_no_arch_error():
     # nvjitlink expects at least the architecture to be specified.
@@ -63,12 +64,14 @@ def test_invalid_arch_error():
                        match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'):
         _nvjitlinklib.create('-arch=sm_XX')
 
+
 @pytest.mark.skip
 def test_unrecognized_option_error():
     with pytest.raises(RuntimeError,
                        match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'):
         _nvjitlinklib.create('-fictitious_option')
 
+
 @pytest.mark.skip
 def test_invalid_option_type_error():
     with pytest.raises(TypeError,
@@ -108,6 +111,7 @@ def test_add_file(input_file, input_type, request):
     _nvjitlinklib.add_data(handle, input_type.value, data, filename)
     _nvjitlinklib.destroy(handle)
 
+
 @pytest.mark.skip
 def test_get_error_log(undefined_extern_cubin):
     handle = _nvjitlinklib.create('-arch=sm_75')
@@ -188,7 +192,7 @@ def test_get_linked_ptx_from_lto(device_functions_ltoir):
     handle = _nvjitlinklib.create('-arch=sm_75', '-lto', '-ptx')
     _nvjitlinklib.add_data(handle, input_type, data, filename)
     _nvjitlinklib.complete(handle)
-    ptx = _nvjitlinklib.get_linked_ptx(handle)
+    _nvjitlinklib.get_linked_ptx(handle)
     _nvjitlinklib.destroy(handle)
 
 

diff --git a/pynvjitlink/tests/test_pynvjitlink_api.py b/pynvjitlink/tests/test_pynvjitlink_api.py
@@ -30,20 +30,23 @@ def undefined_extern_cubin():
     with open(fatbin_path, 'rb') as f:
         return f.read()
 
+
 @pytest.mark.skip
 def test_create_no_arch_error():
     # nvlink expects at least the architecture to be specified.
     with pytest.raises(NvJitLinkError,
                        match='NVJITLINK_ERROR_MISSING_ARCH error'):
         NvJitLinker()
 
+
 @pytest.mark.skip
 def test_invalid_arch_error():
     # sm_XX is not a valid architecture
     with pytest.raises(NvJitLinkError,
                        match='NVJITLINK_ERROR_UNRECOGNIZED_OPTION error'):
         NvJitLinker('-arch=sm_XX')
 
+
 @pytest.mark.skip
 def test_invalid_option_type_error():
     with pytest.raises(TypeError,
@@ -61,6 +64,7 @@ def test_add_cubin(device_functions_cubin):
     name = 'test_device_functions.cubin'
     nvjitlinker.add_cubin(device_functions_cubin, name)
 
+
 @pytest.mark.skip
 def test_add_incompatible_cubin_arch_error(device_functions_cubin):
     nvjitlinker = NvJitLinker('-arch=sm_70')
@@ -81,6 +85,7 @@ def test_add_fatbin_sm70(device_functions_fatbin):
     name = 'test_device_functions.fatbin'
     nvjitlinker.add_fatbin(device_functions_fatbin, name)
 
+
 @pytest.mark.skip
 def test_add_incompatible_fatbin_arch_error(device_functions_fatbin):
     nvjitlinker = NvJitLinker('-arch=sm_80')
@@ -89,6 +94,7 @@ def test_add_incompatible_fatbin_arch_error(device_functions_fatbin):
                        match='NVJITLINK_ERROR_INVALID_INPUT error'):
         nvjitlinker.add_fatbin(device_functions_fatbin, name)
 
+
 @pytest.mark.skip
 def test_add_cubin_with_fatbin_error(device_functions_fatbin):
     nvjitlinker = NvJitLinker('-arch=sm_75')
@@ -98,13 +104,15 @@ def test_add_cubin_with_fatbin_error(device_functions_fatbin):
         nvjitlinker.add_cubin(device_functions_fatbin, name)
 
 
+@pytest.mark.skip
 def test_add_fatbin_with_cubin(device_functions_cubin):
     # Adding a cubin with add_fatbin seems to work - this may be expected
     # behaviour.
     nvjitlinker = NvJitLinker('-arch=sm_75')
     name = 'test_device_functions.cubin'
     nvjitlinker.add_fatbin(device_functions_cubin, name)
 
+
 @pytest.mark.skip
 def test_duplicate_symbols_cubin_and_fatbin(device_functions_cubin,
                                             device_functions_fatbin):
@@ -136,6 +144,7 @@ def test_get_linked_cubin(device_functions_cubin):
     # Just check we got something that looks like an ELF
     assert cubin[:4] == b'\x7fELF'
 
+
 @pytest.mark.skip
 def test_get_error_log(undefined_extern_cubin):
     nvjitlinker = NvJitLinker('-arch=sm_75')

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,6 @@
 [tool.scikit-build]
 cmake.minimum-version = "3.26.4"
+cmake.verbose = true
 ninja.make-fallback = true
 build-dir = "build/{wheel_tag}"