Merge branch 'branch-0.41' into tb

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ concurrency:
 jobs:
   conda-python-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,15 +37,15 @@ jobs:
   upload-conda:
     needs: [conda-python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -55,7 +55,7 @@ jobs:
   wheel-publish:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -18,34 +18,34 @@ jobs:
       - wheel-build
       - wheel-tests
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12
   conda-python-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12
     with:
       build_type: pull-request
       container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
   wheel-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
     with:
       build_type: pull-request
       script: ci/build_wheel.sh
   wheel-tests:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
     with:
       build_type: pull-request
       script: ci/test_wheel.sh

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -25,7 +25,7 @@ jobs:
       container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
   wheel-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
               - id: verify-alpha-spec
                 args:
                   - --fix
-                  - --rapids-version=24.10
+                  - --rapids-version=24.12
       - repo: https://github.com/rapidsai/dependency-file-generator
         rev: v1.13.11
         hooks:

VERSION

@@ -1 +1 @@
-0.40.0
+0.41.0

ci/release/update-version.sh

@@ -27,8 +27,8 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 NEXT_RAPIDS_SHORT_TAG="$(curl -sL https://version.gpuci.io/ucx-py/${NEXT_SHORT_TAG})"
 
 # Need to distutils-normalize the versions for some use cases
-NEXT_RAPIDS_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_RAPIDS_SHORT_TAG}'))")
-NEXT_RAPIDS_FULL_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_FULL_TAG}'))")
+NEXT_RAPIDS_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_RAPIDS_SHORT_TAG}'))")
+NEXT_RAPIDS_FULL_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_FULL_TAG}'))")
 
 echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
 
@@ -42,9 +42,9 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" pyproject.toml
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" pyproject.toml
 done
 
 for FILE in .github/workflows/*.yaml; do

ci/test_python.sh

@@ -9,6 +9,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
+UCX_PY_VERSION="$(head -1 ./VERSION)"
+
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_python \
@@ -50,7 +52,7 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-mamba-retry install \
   --channel "${PYTHON_CHANNEL}" \
-  ucx-py
+  "ucx-py=${UCX_PY_VERSION}"
 
 rapids-logger "Run tests with conda package"
 run_tests

dependencies.yaml

@@ -225,20 +225,20 @@ dependencies:
           - pytest-rerunfailures
       - output_types: [conda]
         packages:
-          - &cudf_unsuffixed cudf==24.10.*,>=0.0.0a0
+          - &cudf_unsuffixed cudf==24.12.*,>=0.0.0a0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix:
               cuda: "12.*"
               cuda_suffixed: "true"
             packages:
-              - cudf-cu12==24.10.*,>=0.0.0a0
+              - cudf-cu12==24.12.*,>=0.0.0a0
           - matrix:
               cuda: "11.*"
               cuda_suffixed: "true"
             packages:
-              - cudf-cu11==24.10.*,>=0.0.0a0
+              - cudf-cu11==24.12.*,>=0.0.0a0
           - matrix:
             packages:
               - *cudf_unsuffixed

docs/source/ucx-debug.rst

@@ -33,7 +33,7 @@ For example, we can check if UCX has been built correctly with ``RDMA`` and if i
     #define HAVE_DECL_RDMA_ESTABLISH  1
     #define HAVE_DECL_RDMA_INIT_QP_ATTR 1
     #define HAVE_RDMACM_QP_LESS       1
-    #define UCX_CONFIGURE_FLAGS       "--disable-logging --disable-debug --disable-assertions --disable-params-check --prefix=/gpfs/fs1/user/miniconda3/envs/ucx-dev --with-sysroot --enable-cma --enable-mt --with-gnu-ld --with-rdmacm --with-verbs --with-cuda=/gpfs/fs1/SHARE/Utils/CUDA/10.2.89.0_440.33.01"
+    #define UCX_CONFIGURE_FLAGS       "--disable-logging --disable-debug --disable-assertions --disable-params-check --prefix=/gpfs/fs1/user/miniforge/envs/ucx-dev --with-sysroot --enable-cma --enable-mt --with-gnu-ld --with-rdmacm --with-verbs --with-cuda=/gpfs/fs1/SHARE/Utils/CUDA/10.2.89.0_440.33.01"
     #define uct_MODULES               ":cuda:ib:rdmacm:cma"
 
 

pyproject.toml

@@ -48,7 +48,7 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "cloudpickle",
-    "cudf==24.10.*,>=0.0.0a0",
+    "cudf==24.12.*,>=0.0.0a0",
     "cupy-cuda12x>=12.0.0",
     "dask",
     "distributed",

ucp/_libs/arr.pxd

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved.
 # See file LICENSE for terms.
 
 # cython: language_level=3
@@ -24,3 +24,6 @@ cdef class Array:
     cpdef bint _f_contiguous(self)
     cpdef bint _contiguous(self)
     cpdef Py_ssize_t _nbytes(self)
+
+
+cpdef Array asarray(obj)

ucp/_libs/arr.pyi

@@ -1,7 +1,12 @@
-from typing import Tuple
+# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved.
+# See file LICENSE for terms.
 
-class Array:
-    def __init__(self, obj: object): ...
+from typing import Generic, Tuple, TypeVar
+
+T = TypeVar("T")
+
+class Array(Generic[T]):
+    def __init__(self, obj: T): ...
     @property
     def c_contiguous(self) -> bool: ...
     @property
@@ -14,3 +19,9 @@ class Array:
     def shape(self) -> Tuple[int]: ...
     @property
     def strides(self) -> Tuple[int]: ...
+    @property
+    def cuda(self) -> bool: ...
+    @property
+    def obj(self) -> T: ...
+
+def asarray(obj) -> Array: ...

ucp/_libs/arr.pyx

@@ -1,16 +1,15 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved.
 # See file LICENSE for terms.
 
 # cython: language_level=3
 
 
-from cpython.array cimport array, newarrayobject
 from cpython.buffer cimport PyBuffer_IsContiguous
+from cpython.mem cimport PyMem_Free, PyMem_Malloc
 from cpython.memoryview cimport (
     PyMemoryView_FromObject,
     PyMemoryView_GET_BUFFER,
 )
-from cpython.object cimport PyObject
 from cpython.ref cimport Py_INCREF
 from cpython.tuple cimport PyTuple_New, PyTuple_SET_ITEM
 from cython cimport (
@@ -20,6 +19,7 @@ from cython cimport (
     nonecheck,
     wraparound,
 )
+from cython.view cimport array
 from libc.stdint cimport uintptr_t
 from libc.string cimport memcpy
 
@@ -62,13 +62,14 @@ cdef dict itemsize_mapping = {
 }
 
 
-cdef array array_Py_ssize_t = array("q")
+cdef sizeof_Py_ssize_t = sizeof(Py_ssize_t)
 
 
-cdef inline Py_ssize_t[::1] new_Py_ssize_t_array(Py_ssize_t n):
-    return newarrayobject(
-        (<PyObject*>array_Py_ssize_t).ob_type, n, array_Py_ssize_t.ob_descr
-    )
+cdef Py_ssize_t[::1] new_Py_ssize_t_array(Py_ssize_t n):
+    cdef array a = array((n,), sizeof_Py_ssize_t, b"q", "c", False)
+    a.data = <char*>PyMem_Malloc(n * sizeof(Py_ssize_t))
+    a.callback_free_data = PyMem_Free
+    return a
 
 
 @auto_pickle(False)
@@ -245,7 +246,7 @@ cdef class Array:
 cdef inline bint _c_contiguous(Py_ssize_t itemsize,
                                Py_ssize_t ndim,
                                Py_ssize_t[::1] shape_mv,
-                               Py_ssize_t[::1] strides_mv) nogil:
+                               Py_ssize_t[::1] strides_mv) noexcept nogil:
     cdef Py_ssize_t i, s
     if strides_mv is not None:
         s = itemsize
@@ -263,7 +264,7 @@ cdef inline bint _c_contiguous(Py_ssize_t itemsize,
 cdef inline bint _f_contiguous(Py_ssize_t itemsize,
                                Py_ssize_t ndim,
                                Py_ssize_t[::1] shape_mv,
-                               Py_ssize_t[::1] strides_mv) nogil:
+                               Py_ssize_t[::1] strides_mv) noexcept nogil:
     cdef Py_ssize_t i, s
     if strides_mv is not None:
         s = itemsize
@@ -279,7 +280,7 @@ cdef inline bint _f_contiguous(Py_ssize_t itemsize,
 cdef inline bint _contiguous(Py_ssize_t itemsize,
                              Py_ssize_t ndim,
                              Py_ssize_t[::1] shape_mv,
-                             Py_ssize_t[::1] strides_mv) nogil:
+                             Py_ssize_t[::1] strides_mv) noexcept nogil:
     cdef bint r = _c_contiguous(itemsize, ndim, shape_mv, strides_mv)
     if not r:
         r = _f_contiguous(itemsize, ndim, shape_mv, strides_mv)
@@ -292,8 +293,27 @@ cdef inline bint _contiguous(Py_ssize_t itemsize,
 @wraparound(False)
 cdef inline Py_ssize_t _nbytes(Py_ssize_t itemsize,
                                Py_ssize_t ndim,
-                               Py_ssize_t[::1] shape_mv) nogil:
+                               Py_ssize_t[::1] shape_mv) noexcept nogil:
     cdef Py_ssize_t i, nbytes = itemsize
     for i in range(ndim):
         nbytes *= shape_mv[i]
     return nbytes
+
+
+cpdef Array asarray(obj):
+    """Coerce other objects to ``Array``. No-op for existing ``Array``s.
+
+    Parameters
+    ----------
+    obj: object
+        Object exposing the Python buffer protocol or ``__cuda_array_interface__``.
+
+    Returns
+    -------
+    array: Array
+        An instance of the ``Array`` class.
+    """
+    if isinstance(obj, Array):
+        return <Array>obj
+    else:
+        return Array(obj)