diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ade4a1c..396ff859 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: arch: "amd64" branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 56871079..ca2c646f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -33,7 +33,7 @@ jobs: docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: build_type: pull-request node_type: "gpu-v100-latest-1" diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 703e2a66..210d6547 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -25,7 +25,7 @@ rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ libucxx -export RAPIDS_VERSION_NUMBER="24.02" +export RAPIDS_VERSION_NUMBER="24.04" export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index be4c643c..a9009056 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -20,8 +20,9 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} NEXT_RAPIDS_VERSION="$(curl -sL https://version.gpuci.io/ucx-py/${NEXT_SHORT_TAG})" # Need to distutils-normalize the versions for some use cases -NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_VERSION}'))") -echo "Next tag is ${NEXT_SHORT_TAG_PEP440}" +NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") +NEXT_RAPIDS_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_VERSION}'))") +echo "Next tag is ${NEXT_RAPIDS_SHORT_TAG_PEP440}" echo "Preparing release: $NEXT_FULL_TAG" @@ -44,6 +45,7 @@ sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/distributed # bump RAPIDS libs sed_runner "/- librmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml sed_runner "/- rmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml +sed_runner "/- rapids-dask-dependency =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml # doxyfile update sed_runner 's/PROJECT_NUMBER = .*/PROJECT_NUMBER = '${NEXT_FULL_TAG}'/g' cpp/doxygen/Doxyfile @@ -54,12 +56,26 @@ DEPENDENCIES=( dask-cudf librmm rmm + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml; do - sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*/g" ${FILE}; + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*/g" "${FILE}" done - sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml; + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml +done + +UCXX_DEPENDENCIES=( + ucxx + distributed-ucxx +) +for DEP in "${UCXX_DEPENDENCIES[@]}"; do + for FILE in dependencies.yaml; do + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}" + done + sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml + sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml done # rapids-cmake version @@ -68,4 +84,4 @@ sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_RAPIDS_VERSION}\/RAPID for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_RAPIDS_VERSION}/g" "${FILE}" done -sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_SHORT_TAG}\"/g" ci/build_docs.sh +sed_runner "s/RAPIDS_VERSION_NUMBER=\".*/RAPIDS_VERSION_NUMBER=\"${NEXT_RAPIDS_VERSION}\"/g" ci/build_docs.sh diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 74d24d65..f6c90e38 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -18,10 +18,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask - dask-cuda==24.4.* - dask-cudf==24.4.* -- distributed - doxygen=1.9.1 - fmt>=10.1.1,<11 - gmock>=1.13.0 @@ -39,8 +37,11 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - python>=3.9,<3.11 +- rapids-dask-dependency==24.4.* - rmm==24.4.* - scikit-build-core>=0.7.0 +- setuptools>=64.0.0 - spdlog>=1.12.0,<1.13 +- tomli - ucx name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index da9243f6..be4fb7be 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -18,10 +18,8 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask - dask-cuda==24.4.* - dask-cudf==24.4.* -- distributed - doxygen=1.9.1 - fmt>=10.1.1,<11 - gmock>=1.13.0 @@ -39,8 +37,11 @@ dependencies: - pytest-asyncio - pytest-rerunfailures - python>=3.9,<3.11 +- rapids-dask-dependency==24.4.* - rmm==24.4.* - scikit-build-core>=0.7.0 +- setuptools>=64.0.0 - spdlog>=1.12.0,<1.13 +- tomli - ucx name: all_cuda-120_arch-x86_64 diff --git a/conda/recipes/ucxx/meta.yaml b/conda/recipes/ucxx/meta.yaml index da9f5ce6..2484fdde 100644 --- a/conda/recipes/ucxx/meta.yaml +++ b/conda/recipes/ucxx/meta.yaml @@ -271,8 +271,7 @@ outputs: - tomli run: - python * *_cpython - - dask >=2023.9.2 - - distributed >=2023.9.2 + - rapids-dask-dependency =24.04 - {{ pin_subpackage('ucxx', max_pin='x.x') }} test: imports: diff --git a/cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile index 76ce2ab6..89e943a4 100644 --- a/cpp/doxygen/Doxyfile +++ b/cpp/doxygen/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = libucxx # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.36.00 +PROJECT_NUMBER = 0.37.00 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/dependencies.yaml b/dependencies.yaml index 4038b7a7..a35cd12d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,17 +7,21 @@ files: arch: [x86_64] includes: - build_cpp - - build_python + - build_python_ucxx + - build_python_distributed_ucxx - checks - cuda - cuda_version - dev - docs - py_version - - run_python + - run_python_ucxx + - run_python_distributed_ucxx - test_cpp - - test_python + - test_python_ucxx + - test_python_distributed_ucxx - depends_on_cupy + - depends_on_rmm - depends_on_cudf test_cpp: output: none @@ -29,7 +33,8 @@ files: includes: - cuda_version - py_version - - test_python + - test_python_ucxx + - test_python_distributed_ucxx - depends_on_cupy - depends_on_cudf checks: @@ -42,30 +47,55 @@ files: includes: - docs - py_version - py_build: + py_build_ucxx: output: pyproject pyproject_dir: python extras: table: build-system includes: - - build_python - py_run: + - build_python_ucxx + - depends_on_rmm + py_run_ucxx: output: pyproject pyproject_dir: python extras: table: project includes: - - run_python - py_test: + - run_python_ucxx + - depends_on_rmm + py_test_ucxx: output: pyproject pyproject_dir: python extras: table: project.optional-dependencies key: test includes: - - test_python + - test_python_ucxx - depends_on_cupy - depends_on_cudf + py_build_distributed_ucxx: + output: pyproject + pyproject_dir: python/distributed-ucxx + extras: + table: build-system + includes: + - build_python_distributed_ucxx + py_run_distributed_ucxx: + output: pyproject + pyproject_dir: python/distributed-ucxx + extras: + table: project + includes: + - run_python_distributed_ucxx + - depends_on_ucxx + py_test_distributed_ucxx: + output: pyproject + pyproject_dir: python/distributed-ucxx + extras: + table: project.optional-dependencies + key: test + includes: + - test_python_distributed_ucxx channels: - rapidsai - rapidsai-nightly @@ -86,11 +116,8 @@ dependencies: - librmm==24.4.* - ninja - spdlog>=1.12.0,<1.13 - build_python: + build_python_ucxx: common: - - output_types: [conda] - packages: - - &rmm_conda rmm==24.4.* - output_types: [conda, requirements, pyproject] packages: - *cmake_ver @@ -102,16 +129,12 @@ dependencies: - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 - specific: - - output_types: [requirements, pyproject] - matrices: - - matrix: {cuda: "12.*"} - packages: - - rmm-cu12==24.4.* - - matrix: {cuda: "11.*"} - packages: - - rmm-cu11==24.4.* - - {matrix: null, packages: [*rmm_conda]} + build_python_distributed_ucxx: + common: + - output_types: [conda, requirements, pyproject] + packages: + - setuptools>=64.0.0 + - tomli checks: common: - output_types: [conda, requirements] @@ -190,15 +213,21 @@ dependencies: - matrix: packages: - python>=3.9,<3.11 - run_python: + run_python_ucxx: common: - output_types: [conda, requirements, pyproject] packages: - - numpy>=1.21 + - &numpy numpy>=1.21 - pynvml>=11.4.1 - output_types: [conda] packages: - ucx + run_python_distributed_ucxx: + common: + - output_types: [conda, requirements, pyproject] + packages: + - &numba numba>=0.57.1 + - rapids-dask-dependency==24.4.* test_cpp: common: - output_types: conda @@ -206,22 +235,27 @@ dependencies: - *cmake_ver - *gtest - *gmock - test_python: + test_python_ucxx: common: - output_types: [conda, requirements, pyproject] packages: - cloudpickle - - dask - - distributed - - numba>=0.57.1 + - *numba - pytest - pytest-asyncio - pytest-rerunfailures + - rapids-dask-dependency==24.4.* + test_python_distributed_ucxx: + common: + - output_types: [conda, requirements, pyproject] + packages: + - *numpy + - pytest depends_on_cupy: common: - output_types: conda packages: - - &cupy_conda cupy>=12.0.0 + - cupy>=12.0.0 specific: - output_types: [requirements, pyproject] matrices: @@ -229,9 +263,29 @@ dependencies: packages: - cupy-cuda12x>=12.0.0 - matrix: {cuda: "11.*"} - packages: + packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - - {matrix: null, packages: [*cupy_conda]} + - {matrix: null, packages: *cupy_packages_cu11} + depends_on_rmm: + common: + - output_types: conda + packages: + - &rmm_conda rmm==24.4.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - rmm-cu12==24.4.* + - matrix: {cuda: "11.*"} + packages: + - rmm-cu11==24.4.* + - {matrix: null, packages: [*rmm_conda]} depends_on_cudf: common: - output_types: conda @@ -240,7 +294,6 @@ dependencies: - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file - # This index is needed for rmm, cubinlinker, ptxcompiler. - --extra-index-url=https://pypi.nvidia.com - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple specific: @@ -253,3 +306,23 @@ dependencies: packages: - cudf-cu11==24.4.* - {matrix: null, packages: [*cudf_conda]} + depends_on_ucxx: + common: + - output_types: conda + packages: + - &ucxx_conda ucxx==0.37.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - ucxx-cu12==0.37.* + - matrix: {cuda: "11.*"} + packages: + - ucxx-cu11==0.37.* + - {matrix: null, packages: [*ucxx_conda]} diff --git a/python/distributed-ucxx/pyproject.toml b/python/distributed-ucxx/pyproject.toml index c815f900..a6fce545 100644 --- a/python/distributed-ucxx/pyproject.toml +++ b/python/distributed-ucxx/pyproject.toml @@ -2,8 +2,8 @@ build-backend = "setuptools.build_meta" requires = [ "setuptools>=64.0.0", - "tomli ; python_version < '3.11'", -] + "tomli", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] name = "distributed-ucxx" @@ -16,10 +16,10 @@ authors = [ license = { text = "Apache-2.0" } requires-python = ">=3.8" dependencies = [ - "dask >=2023.9.2", - "distributed >=2023.9.2", - "numba >=0.54", -] + "numba>=0.57.1", + "rapids-dask-dependency==24.4.*", + "ucxx==0.37.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -39,9 +39,9 @@ docs = [ "sphinx-rtd-theme>=0.5.1", ] test = [ + "numpy>=1.21", "pytest", - "numpy", -] +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/ucxx" diff --git a/python/pyproject.toml b/python/pyproject.toml index 6e197655..1b30cdc6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -24,6 +24,7 @@ requires-python = ">=3.9" dependencies = [ "numpy>=1.21", "pynvml>=11.4.1", + "rmm==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -41,13 +42,12 @@ classifiers = [ test = [ "cloudpickle", "cudf==24.4.*", - "cupy>=12.0.0", - "dask", - "distributed", + "cupy-cuda11x>=12.0.0", "numba>=0.57.1", "pytest", "pytest-asyncio", "pytest-rerunfailures", + "rapids-dask-dependency==24.4.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls]