Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Forward-merge branch-24.12 into branch-25.02 #570

Merged
merged 1 commit into from
Dec 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 7 additions & 13 deletions ci/build_docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,22 @@ set -euo pipefail
rapids-logger "Create test conda environment"
. /opt/conda/etc/profile.d/conda.sh

export RAPIDS_VERSION="$(rapids-version)"
rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-dependency-file-generator \
--output conda \
--file-key docs \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n docs
rapids-mamba-retry env create -yq -f env.yaml -n docs
conda activate docs

rapids-print-env

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"kvikio=${RAPIDS_VERSION}" \
"libkvikio=${RAPIDS_VERSION}"

export RAPIDS_DOCS_DIR="$(mktemp -d)"

rapids-logger "Build CPP docs"
Expand Down
3 changes: 3 additions & 0 deletions ci/release/update-version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@ function sed_runner() {
echo "${NEXT_FULL_TAG}" > VERSION

DEPENDENCIES=(
kvikio
libkvikio
libkvikio-tests
rapids-dask-dependency
)
for DEP in "${DEPENDENCIES[@]}"; do
for FILE in dependencies.yaml conda/environments/*.yaml; do
Expand Down
15 changes: 5 additions & 10 deletions ci/test_cpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,28 @@ set -euo pipefail

. /opt/conda/etc/profile.d/conda.sh

RAPIDS_VERSION="$(rapids-version)"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

rapids-logger "Generate C++ testing dependencies"
rapids-dependency-file-generator \
--output conda \
--file-key test_cpp \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
--prepend-channel "${CPP_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n test
rapids-mamba-retry env create -qy -f env.yaml -n test

# Temporarily allow unbound variables for conda activation.
set +u
conda activate test
set -u

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
mkdir -p "${RAPIDS_TESTS_DIR}"
SUITEERROR=0

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
"libkvikio=${RAPIDS_VERSION}" \
"libkvikio-tests=${RAPIDS_VERSION}"

rapids-logger "Check GPU usage"
nvidia-smi

Expand Down
20 changes: 7 additions & 13 deletions ci/test_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../

. /opt/conda/etc/profile.d/conda.sh

RAPIDS_VERSION="$(rapids-version)"
rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

rapids-logger "Generate Python testing dependencies"
rapids-dependency-file-generator \
--output conda \
--file-key test_python \
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
--matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
--prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
| tee env.yaml

rapids-mamba-retry env create --yes -f env.yaml -n test
rapids-mamba-retry env create -qy -f env.yaml -n test

# Temporarily allow unbound variables for conda activation.
set +u
conda activate test
set -u

rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
"libkvikio=${RAPIDS_VERSION}" \
"kvikio=${RAPIDS_VERSION}"

rapids-logger "Check GPU usage"
nvidia-smi

Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-aarch64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-aarch64=11.*
- libcurl>=7.87.0
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==24.12.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-x86_64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-64=11.*
- libcufile-dev=1.4.0.31
Expand All @@ -35,6 +34,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==24.12.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-125_arch-aarch64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-aarch64=11.*
- libcufile-dev
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==24.12.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-125_arch-x86_64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- cupy>=12.0.0
- cxx-compiler
- cython>=3.0.0
- dask>=2022.05.2
- doxygen=1.9.1
- gcc_linux-64=11.*
- libcufile-dev
Expand All @@ -33,6 +32,7 @@ dependencies:
- python>=3.10,<3.13
- rangehttpserver
- rapids-build-backend>=0.3.0,<0.4.0.dev0
- rapids-dask-dependency==24.12.*,>=0.0.0a0
- scikit-build-core>=0.10.0
- sphinx
- sphinx-click
Expand Down
17 changes: 15 additions & 2 deletions conda/recipes/kvikio/conda_build_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ c_compiler_version:
cxx_compiler_version:
- 11

cmake_version:
- ">=3.26.4,!=3.30.0"

cuda_compiler:
- cuda-nvcc

Expand All @@ -16,8 +19,18 @@ c_stdlib:
c_stdlib_version:
- "2.17"

cmake_version:
- ">=3.26.4,!=3.30.0"
# The CTK libraries below are missing from the conda-forge::cudatoolkit package
# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
# and the "*_run_*" version specifiers correspond to `11.x` packages.

cuda11_libcufile_host_version:
- "1.4.0.31"

cuda11_libcufile_run_version:
- ">=1.0.0.82,<=1.4.0.31"

libcurl_version:
- "==7.87.0"

nvcomp_version:
- "=4.1.0.6"
12 changes: 11 additions & 1 deletion conda/recipes/kvikio/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ build:
- {{ compiler('cuda11') }}
{% else %}
- {{ compiler('cuda') }}
- cuda-cudart-dev
- libcufile-dev # [linux]
{% endif %}

requirements:
Expand All @@ -58,13 +60,17 @@ requirements:
- cython >=3.0.0
{% if cuda_major == "11" %}
- cudatoolkit
- libcufile {{ cuda11_libcufile_run_version }} # [linux64]
{% else %}
- cuda-cudart-dev
- libcufile-dev # [linux]
{% endif %}
- cuda-version ={{ cuda_version }}
- nvcomp {{ nvcomp_version }}
- rapids-build-backend >=0.3.0,<0.4.0.dev0
- scikit-build-core >=0.10.0
- libkvikio ={{ version }}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}
run:
- python
- numpy >=1.23,<3.0a0
Expand All @@ -76,6 +82,10 @@ requirements:
- {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
{% if cuda_major == "11" %}
- cudatoolkit
- libcufile {{ cuda11_libcufile_run_version }} # [linux64]
{% else %}
- cuda-cudart
- libcufile # [linux]
{% endif %}

test:
Expand Down
3 changes: 3 additions & 0 deletions conda/recipes/libkvikio/conda_build_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ cuda11_libcufile_host_version:

cuda11_libcufile_run_version:
- ">=1.0.0.82,<=1.4.0.31"

libcurl_version:
- "==7.87.0"
4 changes: 2 additions & 2 deletions conda/recipes/libkvikio/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ requirements:
{% else %}
- libcufile-dev # [linux]
{% endif %}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}

outputs:
- name: libkvikio
Expand All @@ -75,7 +75,7 @@ outputs:
- cmake {{ cmake_version }}
host:
- cuda-version ={{ cuda_version }}
- libcurl==7.87.0
- libcurl {{ libcurl_version }}
run:
- {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
{% if cuda_major == "11" %}
Expand Down
17 changes: 16 additions & 1 deletion dependencies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@ files:
output: none
includes:
- cuda_version
- test_libkvikio
- test_cpp
test_python:
output: none
includes:
- cuda_version
- py_version
- test_kvikio
- test_python
checks:
output: none
Expand All @@ -42,6 +44,7 @@ files:
- cuda_version
- docs
- py_version
- test_kvikio
py_build_kvikio:
output: pyproject
pyproject_dir: python/kvikio
Expand Down Expand Up @@ -358,6 +361,18 @@ dependencies:
# See https://github.com/zarr-developers/numcodecs/pull/475
- numcodecs !=0.12.0
- packaging
test_libkvikio:
common:
- output_types: conda
packages:
- libkvikio==24.12.*,>=0.0.0a0
- libkvikio-tests==24.12.*,>=0.0.0a0
test_kvikio:
common:
- output_types: conda
packages:
- libkvikio==24.12.*,>=0.0.0a0
- kvikio==24.12.*,>=0.0.0a0
test_cpp:
common:
- output_types: conda
Expand All @@ -367,7 +382,7 @@ dependencies:
common:
- output_types: [conda, requirements, pyproject]
packages:
- &dask dask>=2022.05.2
- rapids-dask-dependency==24.12.*,>=0.0.0a0
- pytest
- pytest-cov
- rangehttpserver
Expand Down
2 changes: 1 addition & 1 deletion python/kvikio/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ classifiers = [
test = [
"boto3>=1.21.21",
"cuda-python>=11.7.1,<12.0a0,<=11.8.3",
"dask>=2022.05.2",
"moto[server]>=4.0.8",
"pytest",
"pytest-cov",
"rangehttpserver",
"rapids-dask-dependency==24.12.*,>=0.0.0a0",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

[project.urls]
Expand Down
Loading