From 1c86afa7bc49a33117f6a0c35c1b9fdb951f3943 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 16:43:23 -0500 Subject: [PATCH] Full CI support for public builds + switch to use cibuildwheel (#267) * switch to cibuildwheel + some cleanups * try setting up Python manually * comment out un-needed code + propagate python-version * fix: need to check out first * only build natively; add -v; ensure targeting manylinux * single quotes * restore env setup to get artifact dir * fix artifact dir * fix artifact name * restore & fix artifact name again * build on all platforms! * fix CIBW_BUILD for windows * fix typo * no quotes for wildcard matching * move CIBW_BUILD logic to script * fix win runner name * try to find where pwsh is * try to escape * continue hunting.. * try to overwrite shell * try to install ps * be explicit about shell (why?) * only build for win 64 bits * try to install msvc * install msvc ourselves * fix typo * skip custom cl ver check * install to standard location * try to locate Python include path * switch to public windows runner for now * windows image does not have sudo * pwd on Windows Bash does not use Windows path format * cover all Python versions! * add quotes * align the python version installed via GHA vs used at build time * fix constraint syntax * check if setup-python is causing interference * fix typo * apply a WAR on Linux * fix unbound var * detect Python path after it's installed (by CIBW) * try CIBW_BEFORE_ALL_LINUX * try to restore the pre-py-span setup... * reduce build matrix to experiment with cuda.bindings builds * fix parenthesis * use abs path * defer CUDA_PATH setting * use CIBW_ENVIRONMENT to pass env var * fetch cuda-profiler-api headers * only rely on redist * allow wheel repair to fix the triplet tags * restore full build matrix! * fix wget on Windows; pass PARALLEL_LEVEL to CIBW * switch from wget to curl * windows needs unzip not tar * mv -> rsync * git bash has no wget or rsync... * ensure win-style path on win * convert abs path * debug * another level down * check if it is a race condition on win * clean up unused (but still relevant) code * clean up unused (but still relevant) code - cont'd * consolidate with PYTHON_VERSION_FORMATTED --- .github/actions/build/action.yml | 108 ++++++++++++----------- .github/actions/setup/action.yml | 112 +++++++++++++++++++----- .github/workflows/ci-gh.yml | 13 ++- .github/workflows/gh-build-and-test.yml | 21 +++-- .github/workflows/gh-build.yml | 13 ++- 5 files changed, 178 insertions(+), 89 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 952fb9cd..e1552ae8 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -13,73 +13,79 @@ inputs: host-platform: required: true type: string - use-container: - required: true - type: boolean - docker-image: - type: string - required: true upload-enabled: required: true type: boolean - python-version: - required: true - type: string runs: using: composite steps: + - name: Build cuda.core wheel + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: ${{ env.CIBW_BUILD }} + CIBW_ARCHS_LINUX: "native" + CIBW_BUILD_VERBOSITY: 1 + # # ensure Python.h & co can be found + # CIBW_BEFORE_BUILD_WINDOWS: > + # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE + with: + package-dir: ./cuda_core/ + output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - if: ${{ inputs.use-container }} - name: Build (in container) - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - - docker run \ - -e AWS_REGION \ - -e AWS_SESSION_TOKEN \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e GITHUB_TOKEN \ - -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \ - -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \ - -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \ - -e USE_CUDA="$USE_CUDA" \ - -e REPO_DIR="$REPO_DIR" \ - -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \ - -e PYTHON_VERSION="$PYTHON_VERSION" \ - -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \ - -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \ - -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \ - --rm "${{ inputs.docker-image }}" \ - /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}" - - - if: ${{ !inputs.use-container }} - name: Build (without container) - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}" - - - name: Display structure of the bindings artifacts folder (post build) + - name: List the cuda.core artifacts directory shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }} - ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }} + if [[ "${{ inputs.host-platform }}" == win* ]]; then + export CHOWN=chown + else + export CHOWN="sudo chown" + fi + $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - name: Upload bindings build artifacts + - name: Upload cuda.core build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.BINDINGS_ARTIFACT_NAME }} - path: ${{ env.BINDINGS_ARTIFACTS_DIR }} + name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} + path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl + if-no-files-found: error + overwrite: 'true' + + - name: Build cuda.bindings wheel + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: ${{ env.CIBW_BUILD }} + CIBW_ARCHS_LINUX: "native" + CIBW_BUILD_VERBOSITY: 1 + CIBW_ENVIRONMENT_LINUX: > + CUDA_PATH="$(realpath ./cuda_toolkit)" + PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} + CIBW_ENVIRONMENT_WINDOWS: > + CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))" + # PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} + # # ensure Python.h & co can be found + # CIBW_BEFORE_BUILD_WINDOWS: > + # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE + with: + package-dir: ./cuda_bindings/ + output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: Display structure of the core artifacts folder (post build) + - name: List the cuda.bindings artifacts directory shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }} - ls -lahR ${{ env.CORE_ARTIFACTS_DIR }} + if [[ "${{ inputs.host-platform }}" == win* ]]; then + export CHOWN=chown + else + export CHOWN="sudo chown" + fi + $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: Upload core build artifacts + - name: Upload cuda.bindings build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT_NAME }} - path: ${{ env.CORE_ARTIFACTS_DIR }} + name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} + path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl + if-no-files-found: error + overwrite: 'true' diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index c2a8407c..e00cf27f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -22,34 +22,106 @@ inputs: python-version: required: true type: string + cuda-version: + required: true + type: string runs: using: composite steps: - - name: Set REPO_DIR and Dump environment + # WAR: setup-python is not relocatable... + # see https://github.com/actions/setup-python/issues/871 + - name: Set up Python ${{ inputs.python-version }} + if: ${{ startsWith(inputs.host-platform, 'linux') }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up MSVC + if: ${{ startsWith(inputs.host-platform, 'win') }} + uses: ilammy/msvc-dev-cmd@v1 + + - name: Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} run: | - echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV env - - name: Set environment variables + - name: Get CUDA components shell: bash --noprofile --norc -xeuo pipefail {0} run: | + CUDA_PATH="./cuda_toolkit" + mkdir $CUDA_PATH - WITH_TESTS_STR='' - if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then - WITH_TESTS_STR='-with_tests' + # The binary archives (redist) are guaranteed to be updated as part of the release posting. + CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" + CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json" + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then + CTK_SUBDIR="linux-x86_64" + elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + CTK_SUBDIR="linux-sbsa" + fi + function extract() { + tar -xvf $1 -C $CUDA_PATH --strip-components=1 + } + elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then + CTK_SUBDIR="windows-x86_64" + function extract() { + _TEMP_DIR_=$(mktemp -d) + unzip $1 -d $_TEMP_DIR_ + cp -r $_TEMP_DIR_/*/* $CUDA_PATH + rm -rf $_TEMP_DIR_ + } fi + function populate_cuda_path() { + # take the component name as a argument + function download() { + curl -kLSs $1 -o $2 + } + CTK_COMPONENT=$1 + CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | + python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" + CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" + CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" + download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME + extract $CTK_COMPONENT_COMPONENT_FILENAME + rm $CTK_COMPONENT_COMPONENT_FILENAME + } - TARGET_PLATFORM='linux-64' - if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + # Get headers and shared libraries in place + populate_cuda_path cuda_nvcc + populate_cuda_path cuda_cudart + populate_cuda_path cuda_nvrtc + populate_cuda_path cuda_profiler_api + ls -l $CUDA_PATH + + # Note: the headers will be copied into the cibuildwheel manylinux container, + # so setting the CUDA_PATH env var here is meaningless. + + - name: Set environment variables + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + # TODO: just align host-platform names with TARGET_PLATFORM... + if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then + TARGET_PLATFORM='linux-64' + elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then TARGET_PLATFORM='linux-aarch64' + elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then + TARGET_PLATFORM='win-64' fi - BUILD_MODE="${{ inputs.build-mode }}" - BUILD_MODE_STR="" - [ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}" + PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*" + REPO_DIR=$(pwd) + elif [[ "${{ inputs.host-platform }}" == win* ]]; then + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64" + PWD=$(pwd) + REPO_DIR=$(cygpath -w $PWD) + fi + BUILD_MODE="${{ inputs.build-mode }}" if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then # We upload release versions in the default folder. PKG_DIR="${TARGET_PLATFORM}" @@ -57,16 +129,14 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') - - echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV - echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV - echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV - echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV - echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV + echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV + echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV + echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV - echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV - echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV + echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index d38cb8e3..1975c3b5 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -18,6 +18,8 @@ jobs: matrix: host-platform: - linux-x64 + - linux-aarch64 + - win-x64 target-device: - gpu build-mode: @@ -25,8 +27,14 @@ jobs: upload-enabled: - false python-version: - #TODO cover the whole python and cuda matrix - - 3.12 + - "3.12" + - "3.11" + - "3.10" + - "3.9" + cuda-version: + # Note: this is for build-time only; the test-time matrix needs to be + # defined separately. + - "12.6.2" uses: ./.github/workflows/gh-build-and-test.yml with: @@ -36,4 +44,5 @@ jobs: build-type: ci upload-enabled: ${{ matrix.upload-enabled }} python-version: ${{ matrix.python-version }} + cuda-version: ${{ matrix.cuda-version }} secrets: inherit diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index ffc6f959..a9a711d4 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -1,16 +1,16 @@ on: workflow_call: inputs: - host-platform: + target-device: type: string required: true - target-device: + build-type: type: string required: true - build-mode: + host-platform: type: string required: true - build-type: + build-mode: type: string required: true upload-enabled: @@ -19,6 +19,10 @@ on: python-version: type: string required: true + cuda-version: + type: string + required: true + jobs: build: if: ${{ github.repository_owner == 'nvidia' }} @@ -28,13 +32,14 @@ jobs: client-repo: ${{ github.event.repository.name }} target-device: ${{ inputs.target-device }} runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') || - (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') }} + (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') || + (inputs.host-platform == 'win-x64' && 'windows-2019') }} + # (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }} build-type: ${{ inputs.build-type }} - use-container: ${{ inputs.host-platform == 'linux-x64' || - inputs.host-platform == 'linux-aarch64'}} host-platform: ${{ inputs.host-platform }} - dependencies-file: "" build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} + cuda-version: ${{ inputs.cuda-version }} + dependencies-file: "" secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index c60e0c2a..7a9f03ce 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -16,9 +16,6 @@ on: required: true type: string description: One of ci / release - use-container: - required: true - type: boolean host-platform: required: true type: string @@ -35,10 +32,13 @@ on: python-version: required: true type: string + cuda-version: + required: true + type: string jobs: build: - name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, CMake build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}", Use container=${{ inputs.use-container }} ) + name: Build (${{ inputs.host-platform }}, ${{ inputs.build-type }}, ${{ inputs.build-mode }}, Python "${{ inputs.python-version }}") permissions: id-token: write # This is required for configure-aws-credentials @@ -52,7 +52,7 @@ jobs: with: fetch-depth: 0 - - name: Setup + - name: Set up build environment uses: ./.github/actions/setup with: client-repo: ${{ inputs.client-repo }} @@ -62,6 +62,7 @@ jobs: build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} + cuda-version: ${{ inputs.cuda-version }} - name: Call build action uses: ./.github/actions/build @@ -69,6 +70,4 @@ jobs: build-type: ${{ inputs.build-type }} target-device: "${{ inputs.target-device }}" host-platform: ${{ inputs.host-platform }} - use-container: ${{ inputs.use-container }} - docker-image: "condaforge/miniforge3:latest" upload-enabled: ${{ inputs.upload-enabled }}