Core type system (#697)

* Rename _Dtype to Dtype, as it's now going to be a user-facing type

* Refactor type codes as part of the prep for the core type system

* Cython module for type codes and rename float types

* Start using data type objects

* Bind C++ data type classes to Python

* Doxygen for data type classes

* Finish a complete loop for the new type system

* Remaining logic for custom reduction operators

* Alignment for struct types

* Clean up abstractions for legate data interface

* Implement a proper class hierarchy for dtypes in Python

* Replace a hacky tuple type for scalars with a proper array type

* Fix for string scalars

* Add legate data interface for stores

* Fix tutorial examples

* Fix for installed header files

* IO example doesn't require pyarrow

* Address comments from @bryevdv

* Fix for CPU-only build

* Address comments from @jjwilke

examples/hello/editable-install.sh

@@ -1,5 +1,5 @@
 legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'`
 echo "Using Legate at $legate_root"
 cmake -S . -B build -D legate_core_ROOT=$legate_root
-cmake --build build
+cmake --build build --parallel 8
 python -m pip install -e .

examples/hello/src/square.cc

@@ -25,14 +25,14 @@ class SquareTask : public Task<SquareTask, SQUARE> {
   {
     legate::Store& output = context.outputs().at(0);
     // Best-practice to validate the store types
-    assert(output.code() == FLOAT_LT);
+    assert(output.code() == legate::Type::Code::FLOAT32);
     assert(output.dim() == 1);
     legate::Rect<1> output_shape = output.shape<1>();
     auto out                     = output.write_accessor<float, 1>();
 
     legate::Store& input = context.inputs().at(0);
     // Best-practice to validate the store types
-    assert(input.code() == FLOAT_LT);
+    assert(input.code() == legate::Type::Code::FLOAT32);
     assert(input.dim() == 1);
     legate::Rect<1> input_shape = input.shape<1>();  // should be a 1-Dim array
     auto in                     = input.read_accessor<float, 1>();

examples/hello/src/sum.cc

@@ -45,7 +45,7 @@ class SumTask : public Task<SumTask, SUM> {
     legate::Store& output = context.reductions().at(0);
     auto sum              = output.reduce_accessor<Reduce, true, 1>();
     // Best-practice is to validate types
-    assert(output.code() == FLOAT_LT);
+    assert(output.code() == legate::Type::Code::FLOAT32);
     assert(output.dim() == 1);
     assert(output.shape<1>() == legate::Rect<1>(0, 0));
     sum.reduce(0, total);

examples/io/editable-install.sh

@@ -17,5 +17,5 @@
 legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'`
 echo "Using Legate at $legate_root"
 cmake -S . -B build -D legate_core_ROOT=$legate_root
-cmake --build build -j 8
+cmake --build build --parallel 8
 python -m pip install -e .

examples/io/legateio/legateio.py

@@ -18,10 +18,8 @@
 from enum import IntEnum
 from typing import Any, Optional
 
-import pyarrow as pa
-
 import legate.core.types as ty
-from legate.core import Array, Rect, Store, get_legate_runtime
+from legate.core import Array, Field, Rect, Store, get_legate_runtime
 
 from .library import user_context as context, user_lib
 
@@ -39,7 +37,7 @@ class LegateIOOpCode(IntEnum):
 # We use this mapping to create an array based on the type information
 # from a dataset's header file.
 _CODES_TO_DTYPES = dict(
-    (context.type_system[t].code, t)
+    (t.code, t)
     for t in (
         ty.bool_,
         ty.int8,
@@ -67,7 +65,7 @@ class IOArray:
     data interface.
     """
 
-    def __init__(self, store: Store, dtype: pa.lib.DataType) -> None:
+    def __init__(self, store: Store, dtype: ty.Dtype) -> None:
         self._store = store
         self._dtype = dtype
 
@@ -86,7 +84,7 @@ def from_legate_data_interface(data: dict[str, Any]) -> "IOArray":
 
         assert data["version"] == 1
         # For now, we assume that there's only one field in the container
-        field: pa.Field = next(iter(data["data"]))
+        field: Field = next(iter(data["data"]))
         stores = data["data"][field].stores()
 
         # We only support non-nullable arrays
@@ -105,7 +103,7 @@ def __legate_data_interface__(self) -> dict[str, Any]:
         array = Array(self._dtype, [None, self._store])
 
         # Create a field metadata to populate the data field
-        field = pa.field("Legate IO Array", self._dtype, nullable=False)
+        field = Field("Legate IO Array", self._dtype)
 
         return {
             "version": 1,
@@ -179,7 +177,7 @@ def to_even_tiles(self, path: str, tile_shape: tuple[int, ...]) -> None:
         task.execute()
 
 
-def read_file(filename: str, dtype: pa.lib.DataType) -> IOArray:
+def read_file(filename: str, dtype: ty.Dtype) -> IOArray:
     """
     Reads a file into an IOArray.
 
@@ -221,7 +219,7 @@ def read_file(filename: str, dtype: pa.lib.DataType) -> IOArray:
 
 
 def read_file_parallel(
-    filename: str, dtype: pa.lib.DataType, parallelism: Optional[int] = None
+    filename: str, dtype: ty.Dtype, parallelism: Optional[int] = None
 ) -> IOArray:
     """
     Reads a file into an IOArray using multiple tasks.

examples/io/setup.py

@@ -56,5 +56,5 @@
     ),
     include_package_data=True,
     zip_safe=False,
-    install_requires=["cunumeric", "pyarrow>=5"],
+    install_requires=["cunumeric"],
 )

examples/io/src/read_even_tiles.cc

@@ -28,7 +28,7 @@ namespace legateio {
 namespace {
 
 struct read_fn {
-  template <legate::LegateTypeCode CODE, int32_t DIM>
+  template <legate::Type::Code CODE, int32_t DIM>
   void operator()(legate::Store& output, const fs::path& path)
   {
     using VAL = legate::legate_type_of<CODE>;

examples/io/src/read_file.cc

@@ -24,7 +24,7 @@ namespace legateio {
 namespace {
 
 struct read_fn {
-  template <legate::LegateTypeCode CODE>
+  template <legate::Type::Code CODE>
   void operator()(legate::Store& output,
                   const std::string& filename,
                   int64_t my_id,
@@ -40,7 +40,7 @@ struct read_fn {
     in.read(reinterpret_cast<char*>(&code), sizeof(int64_t));
     in.read(reinterpret_cast<char*>(&size), sizeof(size_t));
 
-    if (static_cast<legate::LegateTypeCode>(code) != CODE) {
+    if (static_cast<legate::Type::Code>(code) != CODE) {
       logger.error() << "Type mismatch: " << CODE << " != " << code;
       LEGATE_ABORT;
     }

examples/io/src/read_uneven_tiles.cc

@@ -28,7 +28,7 @@ namespace legateio {
 namespace {
 
 struct read_fn {
-  template <legate::LegateTypeCode CODE, int32_t DIM>
+  template <legate::Type::Code CODE, int32_t DIM>
   void operator()(legate::Store& output, const fs::path& path)
   {
     using VAL = legate::legate_type_of<CODE>;

examples/io/src/util.cc

@@ -19,8 +19,8 @@
 #include "legateio.h"
 #include "util.h"
 
+#include "core/type/type_traits.h"
 #include "core/utilities/dispatch.h"
-#include "core/utilities/type_traits.h"
 
 namespace fs = std::filesystem;
 
@@ -29,7 +29,7 @@ namespace legateio {
 namespace {
 
 struct write_fn {
-  template <legate::LegateTypeCode CODE, int32_t DIM>
+  template <legate::Type::Code CODE, int32_t DIM>
   void operator()(const legate::Store& store, const fs::path& path)
   {
     using VAL = legate::legate_type_of<CODE>;

examples/io/src/write_even_tiles.cc

@@ -30,7 +30,7 @@ namespace legateio {
 namespace {
 
 void write_header(std::ofstream& out,
-                  legate::LegateTypeCode type_code,
+                  legate::Type::Code type_code,
                   const legate::Span<const int32_t>& shape,
                   const legate::Span<const int32_t>& tile_shape)
 {

examples/io/src/write_file.cc

@@ -26,7 +26,7 @@ namespace legateio {
 namespace {
 
 struct write_fn {
-  template <legate::LegateTypeCode CODE>
+  template <legate::Type::Code CODE>
   void operator()(const legate::Store& input, const std::string& filename)
   {
     using VAL = legate::legate_type_of<CODE>;

examples/io/src/write_uneven_tiles.cc

@@ -33,7 +33,7 @@ struct header_write_fn {
   template <int32_t DIM>
   void operator()(std::ofstream& out,
                   const legate::Domain& launch_domain,
-                  legate::LegateTypeCode type_code)
+                  legate::Type::Code type_code)
   {
     legate::Rect<DIM> rect(launch_domain);
     auto extents = rect.hi - rect.lo + legate::Point<DIM>::ONES();

legate/core/__init__.py

@@ -87,7 +87,9 @@
 from .context import Annotation, track_provenance
 from .legate import (
     Array,
+    Field,
     Library,
+    Table,
 )
 from .runtime import (
     get_legate_runtime,
@@ -98,6 +100,7 @@
 from .store import Store
 
 from .types import (
+    array_type,
     bool_,
     int8,
     int16,
@@ -112,44 +115,8 @@
     float64,
     complex64,
     complex128,
+    struct_type,
+    Dtype,
     ReductionOp,
 )
 from .io import CustomSplit, TiledSplit, ingest
-
-# Import the PyArrow type system
-from pyarrow import (
-    DataType,
-    DictionaryType,
-    ListType,
-    MapType,
-    StructType,
-    UnionType,
-    TimestampType,
-    Time32Type,
-    Time64Type,
-    FixedSizeBinaryType,
-    Decimal128Type,
-    Field,
-    Schema,
-    null,
-    time32,
-    time64,
-    timestamp,
-    date32,
-    date64,
-    binary,
-    string,
-    utf8,
-    large_binary,
-    large_string,
-    large_utf8,
-    decimal128,
-    list_,
-    large_list,
-    map_,
-    struct,
-    dictionary,
-    field,
-    schema,
-    from_numpy_dtype,
-)

legate/core/_legion/space.py

@@ -17,7 +17,7 @@
 from typing import TYPE_CHECKING, Any, Optional, Union
 
 from .. import ffi, legion
-from ..types import _Dtype
+from ..types import Dtype
 from .env import LEGATE_MAX_FIELDS
 from .field import FieldID
 from .future import Future
@@ -248,7 +248,7 @@ def allocate_field(
                 + str(LEGATE_MAX_FIELDS)
                 + " in field space"
             )
-        if isinstance(size_or_type, _Dtype):
+        if isinstance(size_or_type, Dtype):
             return self.allocate_field_dtype(size_or_type, field_id=field_id)
         elif isinstance(size_or_type, Future):
             return self.allocate_field_from_future(

legate/core/_lib/CMakeLists.txt

@@ -14,7 +14,7 @@
 # limitations under the License.
 #=============================================================================
 
-set(cython_sources context.pyx)
+set(cython_sources context.pyx types.pyx)
 set(linked_libraries legate::core)
 
 rapids_cython_create_modules(