Add CI scripts to build conda packages and run tests

.github/CODEOWNERS

@@ -0,0 +1,21 @@
+#cpp code owners
+cpp/                              @rapidsai/ucxx-cpp-codeowners
+
+#python code owners
+python/                           @rapidsai/ucxx-python-codeowners
+
+#cmake code owners
+cpp/CMakeLists.txt                @rapidsai/ucxx-cmake-codeowners
+cpp/python/CMakeLists.txt         @rapidsai/ucxx-cmake-codeowners
+cpp/examples/CMakeLists.txt       @rapidsai/ucxx-cmake-codeowners
+cpp/benchmarks/CMakeLists.txt     @rapidsai/ucxx-cmake-codeowners
+cpp/tests/CMakeLists.txt          @rapidsai/ucxx-cmake-codeowners
+python/CMakeLists.txt             @rapidsai/ucxx-cmake-codeowners
+python/ucxx/_lib/CMakeLists.txt   @rapidsai/ucxx-cmake-codeowners
+fetch_rapids.cmake                @rapidsai/ucxx-cmake-codeowners
+**/cmake/                         @rapidsai/ucxx-cmake-codeowners
+
+#build/ops code owners
+.github/                          @rapidsai/ops-codeowners
+ci/                               @rapidsai/ops-codeowners
+conda/                            @rapidsai/ops-codeowners

.github/workflows/build.yaml

@@ -0,0 +1,46 @@
+name: build
+
+on:
+  push:
+    branches:
+      - "branch-*"
+    tags:
+      - v[0-9].[0-9][0-9].[0-9][0-9]
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      date:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+      build_type:
+        type: string
+        default: nightly
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  conda-cpp-build:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      matrix_filter: map(select(.ARCH != "arm64"))
+  upload-conda:
+    needs: [conda-cpp-build]
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}

.github/workflows/pr.yaml

@@ -0,0 +1,46 @@
+name: pr
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pr-builder:
+    needs:
+      - checks
+      - conda-cpp-build
+      - conda-cpp-tests
+      - conda-python-tests
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+  checks:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
+  conda-cpp-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      matrix_filter: map(select(.ARCH != "arm64"))
+  conda-cpp-tests:
+    needs: conda-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      matrix_filter: map(select(.ARCH != "arm64"))
+  conda-python-tests:
+    needs: conda-cpp-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      matrix_filter: map(select(.ARCH != "arm64"))

.github/workflows/test.yaml

@@ -0,0 +1,34 @@
+name: test
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        required: true
+        type: string
+      date:
+        required: true
+        type: string
+      sha:
+        required: true
+        type: string
+
+jobs:
+  conda-cpp-tests:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      matrix_filter: map(select(.ARCH != "arm64"))
+  conda-python-tests:
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      matrix_filter: map(select(.ARCH != "arm64"))

.pre-commit-config.yaml

@@ -43,6 +43,11 @@ repos:
                 # language: system
                 files: \.(h|cpp)$
                 # exclude: path/to/myfile.h
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.5.1
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
 
 
 

ci/build_cpp.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES.
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -euo pipefail
+
+source rapids-env-update
+
+export CMAKE_GENERATOR=Ninja
+
+rapids-print-env
+
+rapids-logger "Begin C++ and Python builds"
+
+rapids-mamba-retry mambabuild \
+  conda/recipes/ucxx
+
+rapids-upload-conda-to-s3 cpp

ci/check_style.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES.
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -euo pipefail
+
+rapids-logger "Create checks conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key checks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n checks
+conda activate checks
+
+# Run pre-commit checks
+pre-commit run --hook-stage manual --all-files --show-diff-on-failure

ci/release/update-version.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+########################
+# ucxx Version Updater #
+########################
+
+## Usage
+# bash update-version.sh <new_version>
+
+
+# Format is Major.Minor.Patch - no leading 'v' or trailing 'a'
+# Example: 0.30.00
+NEXT_FULL_TAG=$1
+
+# Get current version
+CURRENT_TAG=$(git tag | grep -xE 'v[0-9\.]+' | sort --version-sort | tail -n 1 | tr -d 'v')
+CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}')
+CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}')
+CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}')
+CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
+
+#Get <major>.<minor> for next version
+NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
+NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
+NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
+
+# Get RAPIDS version associated w/ ucx-py version
+NEXT_RAPIDS_VERSION="$(curl -sL https://version.gpuci.io/ucx-py/${NEXT_SHORT_TAG})"
+
+
+echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
+
+# Inplace sed replace; workaround for Linux and Mac
+function sed_runner() {
+    sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
+}
+
+sed_runner "s/cudf=.*/cudf=${NEXT_RAPIDS_VERSION}/g" dependencies.yaml
+
+for FILE in .github/workflows/*.yaml; do
+  sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_RAPIDS_VERSION}/g" "${FILE}"
+done

ci/test_cpp.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES.
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -euo pipefail
+
+source "$(dirname "$0")/test_utils.sh"
+
+rapids-logger "Create test conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key test_cpp \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n test
+conda activate test
+
+rapids-print-env
+
+print_system_stats
+
+BINARY_PATH=${CONDA_PREFIX}/bin
+
+run_tests() {
+  CMD_LINE="UCX_TCP_CM_REUSEADDR=y timeout 10m ${BINARY_PATH}/gtests/libucxx/UCXX_TEST --gtest_filter=-*DelayedSubmission*ProgressTagMulti*:ListenerTest.CloseCallback:ListenerTest.IsAlive:ListenerTest.RaiseOnError"
+
+  rapids-logger "Running: \n  - ${CMD_LINE}"
+
+  UCX_TCP_CM_REUSEADDR=y timeout 10m ${BINARY_PATH}/gtests/libucxx/UCXX_TEST --gtest_filter=-*DelayedSubmission*ProgressTagMulti*:ListenerTest.CloseCallback:ListenerTest.IsAlive:ListenerTest.RaiseOnError
+}
+
+run_benchmark() {
+  PROGRESS_MODE=$1
+
+  # UCX_TCP_CM_REUSEADDR=y to be able to bind immediately to the same port before
+  # `TIME_WAIT` timeout
+  CMD_LINE_SERVER="UCX_TCP_CM_REUSEADDR=y timeout 1m ${BINARY_PATH}/benchmarks/libucxx/ucxx_perftest -s 8388608 -r -n 20 -m ${PROGRESS_MODE}"
+  CMD_LINE_CLIENT="timeout 10m ${BINARY_PATH}/benchmarks/libucxx/ucxx_perftest -s 8388608 -r -n 20 -m ${PROGRESS_MODE} 127.0.0.1"
+
+  rapids-logger "Running: \n  - ${CMD_LINE_SERVER}\n  - ${CMD_LINE_CLIENT}"
+  UCX_TCP_CM_REUSEADDR=y timeout 1m ${BINARY_PATH}/benchmarks/libucxx/ucxx_perftest -s 8388608 -r -n 20 -m ${PROGRESS_MODE} &
+  sleep 1
+  timeout 1m ${BINARY_PATH}/benchmarks/libucxx/ucxx_perftest -s 8388608 -r -n 20 -m ${PROGRESS_MODE} 127.0.0.1
+}
+
+run_example() {
+  PROGRESS_MODE=$1
+
+  # UCX_TCP_CM_REUSEADDR=y to be able to bind immediately to the same port before
+  # `TIME_WAIT` timeout
+  CMD_LINE="UCX_TCP_CM_REUSEADDR=y timeout 1m ${BINARY_PATH}/examples/libucxx/ucxx_example_basic -m ${PROGRESS_MODE}"
+
+  rapids-logger "Running: \n  - ${CMD_LINE}"
+  UCX_TCP_CM_REUSEADDR=y timeout 1m ${BINARY_PATH}/examples/libucxx/ucxx_example_basic -m ${PROGRESS_MODE}
+}
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+
+rapids-mamba-retry install \
+  --channel "${CPP_CHANNEL}" \
+  libucxx libucxx-examples libucxx-tests
+
+print_ucx_config
+
+rapids-logger "Run tests with conda package"
+run_tests
+
+# run_cpp_benchmark PROGRESS_MODE
+run_benchmark   polling
+run_benchmark   blocking
+run_benchmark   thread-polling
+run_benchmark   thread-blocking
+run_benchmark   wait
+
+# run_cpp_example PROGRESS_MODE
+run_example   polling
+run_example   blocking
+run_example   thread-polling
+run_example   thread-blocking
+run_example   wait