[REVIEW] Add duration types support in cudf(python/cython)

CHANGELOG.md

@@ -70,6 +70,7 @@
 - PR #5845 Add support for `mask_to_bools`
 - PR #5851 Add support for `Index.sort_values`
 - PR #5859 Add conversion form `fixed_point` to `bool`
+- PR #5781 Add duration types support in cudf(python/cython)
 - PR #5815 LIST Support for ColumnVector
 
 ## Improvements

cpp/src/binaryop/jit/code/operation.cpp

@@ -134,10 +134,23 @@ const char* operation =
     };
 
     struct TrueDiv {
-        template <typename TypeOut, typename TypeLhs, typename TypeRhs>
+        template <typename TypeOut, typename TypeLhs, typename TypeRhs,
+                    enable_if_t<(!is_duration_v<TypeLhs>)>* = nullptr>
         static TypeOut operate(TypeLhs x, TypeRhs y) {
             return (static_cast<double>(x) / static_cast<double>(y));
         }
+
+        template <typename TypeOut, typename TypeLhs, typename TypeRhs,
+                  enable_if_t<(is_duration_v<TypeLhs>)>* = nullptr>
+        static TypeOut operate(TypeLhs x, TypeRhs y) {
+            return DurationDivide<TypeOut>(x, y);
+        }
+
+        template <typename TypeOut, typename TypeLhs, typename TypeRhs,
+                  enable_if_t<(is_integral_v<TypeRhs> || is_duration_v<TypeRhs>)>* = nullptr>
+        static TypeOut DurationDivide(TypeLhs x, TypeRhs y) {
+            return x / y;
+        }
     };
 
     struct RTrueDiv {
@@ -221,6 +234,14 @@ const char* operation =
             double y1 = static_cast<double>(y);
             return fmod(fmod(x1, y1) + y1, y1);
         }
+
+        template <typename TypeOut,
+                  typename TypeLhs,
+                  typename TypeRhs,
+                  enable_if_t<(is_duration_v<TypeLhs> && is_duration_v<TypeOut>)>* = nullptr>
+        static TypeOut operate(TypeLhs x, TypeRhs y) {
+            return ((x % y) + y) % y;
+        }
     };
 
     struct RPyMod {

docs/cudf/source/api.rst

@@ -169,6 +169,14 @@ DatetimeIndex
     :inherited-members:
     :exclude-members: deserialize, serialize, device_deserialize, device_serialize, host_deserialize, host_serialize, tolist, to_list
 
+TimedeltaIndex
+--------------
+.. currentmodule:: cudf.core.index
+.. autoclass:: TimedeltaIndex
+    :members:
+    :inherited-members:
+    :exclude-members: deserialize, serialize, device_deserialize, device_serialize, host_deserialize, host_serialize, tolist, to_list
+
 Categories
 ----------
 .. currentmodule:: cudf.core.column.categorical

python/cudf/cudf/__init__.py

@@ -24,6 +24,7 @@
     MultiIndex,
     RangeIndex,
     Series,
+    TimedeltaIndex,
     UInt8Index,
     UInt16Index,
     UInt32Index,
@@ -34,6 +35,7 @@
 from cudf.core.dtypes import CategoricalDtype
 from cudf.core.groupby import Grouper
 from cudf.core.ops import (
+    add,
     arccos,
     arcsin,
     arctan,
@@ -44,10 +46,13 @@
     logical_and,
     logical_not,
     logical_or,
+    multiply,
     remainder,
     sin,
     sqrt,
+    subtract,
     tan,
+    true_divide,
 )
 from cudf.core.reshape import concat, get_dummies, melt, merge_sorted
 from cudf.core.series import isclose

python/cudf/cudf/_lib/cpp/scalar/scalar.pxd

@@ -35,6 +35,16 @@ cdef extern from "cudf/scalar/scalar.hpp" namespace "cudf" nogil:
         int32_t ticks_since_epoch_32 "ticks_since_epoch"() except +
         T value() except +
 
+    cdef cppclass duration_scalar[T](scalar):
+        duration_scalar() except +
+        duration_scalar(duration_scalar other) except +
+        duration_scalar(int64_t value) except +
+        duration_scalar(int64_t value, bool is_valid) except +
+        duration_scalar(int32_t value) except +
+        duration_scalar(int32_t value, bool is_valid) except +
+        int64_t ticks "count"() except +
+        T value() except +
+
     cdef cppclass string_scalar(scalar):
         string_scalar() except +
         string_scalar(string st) except +

python/cudf/cudf/_lib/cpp/strings/convert/convert_durations.pxd

@@ -0,0 +1,19 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+
+from cudf._lib.cpp.column.column cimport column
+from cudf._lib.cpp.column.column_view cimport column_view
+from cudf._lib.cpp.types cimport data_type
+
+from libcpp.memory cimport unique_ptr
+from libcpp.string cimport string
+
+cdef extern from "cudf/strings/convert/convert_durations.hpp" namespace \
+        "cudf::strings" nogil:
+    cdef unique_ptr[column] to_durations(
+        const column_view & strings_col,
+        data_type duration_type,
+        const string & format) except +
+
+    cdef unique_ptr[column] from_durations(
+        const column_view & durations,
+        const string & format) except +

python/cudf/cudf/_lib/cpp/types.pxd

@@ -68,6 +68,10 @@ cdef extern from "cudf/types.hpp" namespace "cudf" nogil:
         STRING "cudf::type_id::STRING"
         LIST "cudf::type_id::LIST"
         NUM_TYPE_IDS "cudf::type_id::NUM_TYPE_IDS"
+        DURATION_SECONDS "cudf::type_id::DURATION_SECONDS"
+        DURATION_MILLISECONDS "cudf::type_id::DURATION_MILLISECONDS"
+        DURATION_MICROSECONDS "cudf::type_id::DURATION_MICROSECONDS"
+        DURATION_NANOSECONDS "cudf::type_id::DURATION_NANOSECONDS"
 
     cdef cppclass data_type:
         data_type() except +

python/cudf/cudf/_lib/cpp/wrappers/durations.pxd

@@ -0,0 +1,10 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+
+from libc.stdint cimport int64_t
+
+
+cdef extern from "cudf/wrappers/durations.hpp" namespace "cudf" nogil:
+    ctypedef int64_t duration_s
+    ctypedef int64_t duration_ms
+    ctypedef int64_t duration_us
+    ctypedef int64_t duration_ns

python/cudf/cudf/_lib/scalar.pxd

@@ -1,6 +1,7 @@
 # Copyright (c) 2020, NVIDIA CORPORATION.
 
 from libcpp.memory cimport unique_ptr
+from libcpp cimport bool
 
 from cudf._lib.cpp.scalar.scalar cimport scalar
 
@@ -10,3 +11,5 @@ cdef class Scalar:
 
     @staticmethod
     cdef Scalar from_unique_ptr(unique_ptr[scalar] ptr)
+
+    cpdef bool is_valid(Scalar s)

python/cudf/cudf/_lib/scalar.pyx

@@ -16,7 +16,9 @@ from libc.stdint cimport (
 from libcpp.memory cimport unique_ptr
 from libcpp cimport bool
 
-from cudf._lib.types import np_to_cudf_types, cudf_to_np_types
+import cudf
+from cudf._lib.types import cudf_to_np_types, duration_unit_map
+from cudf._lib.types import datetime_unit_map
 from cudf._lib.types cimport underlying_type_t_type_id
 from cudf._lib.move cimport move
 
@@ -26,10 +28,17 @@ from cudf._lib.cpp.wrappers.timestamps cimport (
     timestamp_us,
     timestamp_ns
 )
+from cudf._lib.cpp.wrappers.durations cimport(
+    duration_s,
+    duration_ms,
+    duration_us,
+    duration_ns
+)
 from cudf._lib.cpp.scalar.scalar cimport (
     scalar,
     numeric_scalar,
     timestamp_scalar,
+    duration_scalar,
     string_scalar
 )
 cimport cudf._lib.cpp.types as libcudf_types
@@ -51,9 +60,9 @@ cdef class Scalar:
         dtype : dtype
             A NumPy dtype.
         """
-        from cudf.utils.dtypes import to_cudf_compatible_scalar
 
-        value = to_cudf_compatible_scalar(value, dtype=dtype)
+        value = cudf.utils.dtypes.to_cudf_compatible_scalar(value, dtype=dtype)
+
         valid = value is not None
 
         if dtype is None:
@@ -74,11 +83,14 @@ cdef class Scalar:
             _set_datetime64_from_np_scalar(
                 self.c_value, value, dtype, valid
             )
+        elif pd.api.types.is_timedelta64_dtype(dtype):
+            _set_timedelta64_from_np_scalar(
+                self.c_value, value, dtype, valid
+            )
         else:
             raise ValueError(
-                "Cannot convert value of type {} to cudf scalar".format(
-                    type(value).__name__
-                )
+                f"Cannot convert value of type "
+                f"{type(value).__name__} to cudf scalar"
             )
 
     @property
@@ -101,11 +113,19 @@ cdef class Scalar:
             return _get_np_scalar_from_numeric(self.c_value)
         elif pd.api.types.is_datetime64_dtype(self.dtype):
             return _get_np_scalar_from_timestamp64(self.c_value)
+        elif pd.api.types.is_timedelta64_dtype(self.dtype):
+            return _get_np_scalar_from_timedelta64(self.c_value)
         else:
             raise ValueError(
                 "Could not convert cudf::scalar to a Python value"
             )
 
+    cpdef bool is_valid(self):
+        """
+        Returns if the Scalar is valid or not(i.e., <NA>).
+        """
+        return self.c_value.get()[0].is_valid()
+
     def __repr__(self):
         if self.value is None:
             return f"Scalar({self.value}, {self.dtype.__repr__()})"
@@ -128,8 +148,8 @@ cdef _set_string_from_np_string(unique_ptr[scalar]& s, value, bool valid=True):
 
 
 cdef _set_numeric_from_np_scalar(unique_ptr[scalar]& s,
-                                 value,
-                                 dtype,
+                                 object value,
+                                 object dtype,
                                  bool valid=True):
     value = value if valid else 0
     if dtype == "int8":
@@ -155,14 +175,16 @@ cdef _set_numeric_from_np_scalar(unique_ptr[scalar]& s,
     elif dtype == "bool":
         s.reset(new numeric_scalar[bool](<bool>value, valid))
     else:
-        raise ValueError("dtype not supported: {}".format(dtype))
+        raise ValueError(f"dtype not supported: {dtype}")
 
 
 cdef _set_datetime64_from_np_scalar(unique_ptr[scalar]& s,
-                                    value,
-                                    dtype,
+                                    object value,
+                                    object dtype,
                                     bool valid=True):
+
     value = value if valid else 0
+
     if dtype == "datetime64[s]":
         s.reset(
             new timestamp_scalar[timestamp_s](<int64_t>np.int64(value), valid)
@@ -180,8 +202,33 @@ cdef _set_datetime64_from_np_scalar(unique_ptr[scalar]& s,
             new timestamp_scalar[timestamp_ns](<int64_t>np.int64(value), valid)
         )
     else:
-        raise ValueError("dtype not supported: {}".format(dtype))
+        raise ValueError(f"dtype not supported: {dtype}")
+
+cdef _set_timedelta64_from_np_scalar(unique_ptr[scalar]& s,
+                                     object value,
+                                     object dtype,
+                                     bool valid=True):
 
+    value = value if valid else 0
+
+    if dtype == "timedelta64[s]":
+        s.reset(
+            new duration_scalar[duration_s](<int64_t>np.int64(value), valid)
+        )
+    elif dtype == "timedelta64[ms]":
+        s.reset(
+            new duration_scalar[duration_ms](<int64_t>np.int64(value), valid)
+        )
+    elif dtype == "timedelta64[us]":
+        s.reset(
+            new duration_scalar[duration_us](<int64_t>np.int64(value), valid)
+        )
+    elif dtype == "timedelta64[ns]":
+        s.reset(
+            new duration_scalar[duration_ns](<int64_t>np.int64(value), valid)
+        )
+    else:
+        raise ValueError(f"dtype not supported: {dtype}")
 
 cdef _get_py_string_from_string(unique_ptr[scalar]& s):
     if not s.get()[0].is_valid():
@@ -236,28 +283,69 @@ cdef _get_np_scalar_from_timestamp64(unique_ptr[scalar]& s):
             (
                 <timestamp_scalar[timestamp_ms]*> s_ptr
             )[0].ticks_since_epoch_64(),
-            "s"
+            datetime_unit_map[<underlying_type_t_type_id>(cdtype.id())]
         )
     elif cdtype.id() == libcudf_types.TIMESTAMP_MILLISECONDS:
         return np.datetime64(
             (
                 <timestamp_scalar[timestamp_ms]*> s_ptr
             )[0].ticks_since_epoch_64(),
-            "ms"
+            datetime_unit_map[<underlying_type_t_type_id>(cdtype.id())]
         )
     elif cdtype.id() == libcudf_types.TIMESTAMP_MICROSECONDS:
         return np.datetime64(
             (
                 <timestamp_scalar[timestamp_ms]*> s_ptr
             )[0].ticks_since_epoch_64(),
-            "us"
+            datetime_unit_map[<underlying_type_t_type_id>(cdtype.id())]
         )
     elif cdtype.id() == libcudf_types.TIMESTAMP_NANOSECONDS:
         return np.datetime64(
             (
                 <timestamp_scalar[timestamp_ms]*> s_ptr
             )[0].ticks_since_epoch_64(),
-            "ns"
+            datetime_unit_map[<underlying_type_t_type_id>(cdtype.id())]
+        )
+    else:
+        raise ValueError("Could not convert cudf::scalar to numpy scalar")
+
+
+cdef _get_np_scalar_from_timedelta64(unique_ptr[scalar]& s):
+
+    cdef scalar* s_ptr = s.get()
+
+    if not s_ptr[0].is_valid():
+        return None
+
+    cdef libcudf_types.data_type cdtype = s_ptr[0].type()
+
+    if cdtype.id() == libcudf_types.DURATION_SECONDS:
+        return np.timedelta64(
+            (
+                <duration_scalar[duration_s]*> s_ptr
+            )[0].ticks(),
+            duration_unit_map[<underlying_type_t_type_id>(cdtype.id())]
+        )
+    elif cdtype.id() == libcudf_types.DURATION_MILLISECONDS:
+        return np.timedelta64(
+            (
+                <duration_scalar[duration_ms]*> s_ptr
+            )[0].ticks(),
+            duration_unit_map[<underlying_type_t_type_id>(cdtype.id())]
+        )
+    elif cdtype.id() == libcudf_types.DURATION_MICROSECONDS:
+        return np.timedelta64(
+            (
+                <duration_scalar[duration_us]*> s_ptr
+            )[0].ticks(),
+            duration_unit_map[<underlying_type_t_type_id>(cdtype.id())]
+        )
+    elif cdtype.id() == libcudf_types.DURATION_NANOSECONDS:
+        return np.timedelta64(
+            (
+                <duration_scalar[duration_ns]*> s_ptr
+            )[0].ticks(),
+            duration_unit_map[<underlying_type_t_type_id>(cdtype.id())]
         )
     else:
         raise ValueError("Could not convert cudf::scalar to numpy scalar")
@@ -270,4 +358,4 @@ def as_scalar(val, dtype=None):
         else:
             return Scalar(val.value, dtype)
     else:
-        return Scalar(val, dtype)
+        return Scalar(value=val, dtype=dtype)