Merge remote-tracking branch 'origin/main' into db/sc-50707/aarch64

.github/workflows/ci-tiledb-from-source.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       libtiledb_ref:
-        default: dev
+        default: main
         type: string
       libtiledb_version:
         type: string
@@ -14,11 +14,11 @@ jobs:
   build_libtiledb:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout TileDB Core ${{ inputs.libtiledb_ref || 'dev' }}
+      - name: Checkout TileDB Core ${{ inputs.libtiledb_ref || 'main' }}
         uses: actions/checkout@v4
         with:
           repository: TileDB-Inc/TileDB
-          ref: ${{ inputs.libtiledb_ref || 'dev' }}
+          ref: ${{ inputs.libtiledb_ref || 'main' }}
 
       - name: Configure TileDB
         run: |

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
     env:
       MACOSX_DEPLOYMENT_TARGET: "11"
     steps:
-      - name: Checkout TileDB-Py `dev`
+      - name: Checkout TileDB-Py `main`
         uses: actions/checkout@v4
 
       - name: Setup MSVC toolset (VS 2022)

.github/workflows/daily-test-build-numpy.yml

@@ -47,7 +47,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "11"
       VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
-      - name: Checkout TileDB-Py `dev`
+      - name: Checkout TileDB-Py `main`
         uses: actions/checkout@v4
 
       - name: Setup MSVC toolset (VS 2022)
@@ -85,7 +85,7 @@ jobs:
       - name: Print env
         run: printenv
 
-      - name: Checkout TileDB-Py `dev`
+      - name: Checkout TileDB-Py `main`
         uses: actions/checkout@v3
 
       - name: Build TileDB-Py

.github/workflows/daily-test-build.yml

@@ -65,7 +65,7 @@ jobs:
         run: brew install pkg-config
         if: matrix.os == 'macos-13' || matrix.os == 'macos-14'
 
-      - name: Checkout TileDB-Py `dev`
+      - name: Checkout TileDB-Py `main`
         uses: actions/checkout@v4
 
       - name: Build and install TileDB-Py and dependencies

.github/workflows/daily-tests.yml

@@ -10,14 +10,14 @@ jobs:
   ci1:
     uses: ./.github/workflows/daily-test-build.yml
     with:
-      libtiledb_version: '2.26.0'
+      libtiledb_version: '2.27.0'
 
   ci2:
     uses: ./.github/workflows/daily-test-build.yml
     with:
-      libtiledb_version: '2.25.0'
+      libtiledb_version: '2.26.0'
 
   ci3:
     uses: ./.github/workflows/daily-test-build-numpy.yml
     with:
-      libtiledb_version: '2.25.0'
+      libtiledb_version: '2.26.0'

.gitignore

@@ -7,9 +7,6 @@ __pycache__/
 *.so
 *.dylib
 
-# Cython extensions
-*.cpp
-
 # Distribution / packaging
 .Python
 env/

CMakeLists.txt

@@ -46,8 +46,8 @@ if (NOT TileDB_FOUND)
         message(STATUS "Downloading TileDB default version ...")
         # Download latest release
         fetch_prebuilt_tiledb(
-                VERSION 2.26.2
-                RELLIST_HASH SHA256=86c19d7c5246cb18e370a4272cead63ea84bd651789842e618de4d57d4510522
+                VERSION 2.27.0
+                RELLIST_HASH SHA256=8056514b1949cdab19405376e32e299578491a6d3e953321d12d761f94dc19b9
         )
     endif()
     find_package(TileDB REQUIRED)

CONTRIBUTING.md

@@ -10,7 +10,7 @@ Thanks for your interest in TileDB-Py. The notes below give some pointers for fi
 - Please paste code blocks with triple backquotes (```) so that github will format it nicely. See [GitHub's guide on Markdown](https://guides.github.com/features/mastering-markdown) for more formatting tricks.
 
 ## Contributing Code
-*By contributing code to TileDB-Py, you are agreeing to release it under the [MIT License](https://github.com/TileDB-Inc/TileDB/tree/dev/LICENSE).*
+*By contributing code to TileDB-Py, you are agreeing to release it under the [MIT License](https://github.com/TileDB-Inc/TileDB/tree/main/LICENSE).*
 
 ### Contribution Workflow
 
@@ -45,4 +45,4 @@ Thanks for your interest in TileDB-Py. The notes below give some pointers for fi
 - Make changes locally, then rebuild with `python setup.py develop [--tiledb=<>]`
 - Make sure to run `pytest` to verify changes against tests (add new tests where applicable).
   - Execute the tests as `pytest tiledb` from the top-level directory or `pytest` in the `tiledb/` directory.
-- Please submit [pull requests](https://help.github.com/en/desktop/contributing-to-projects/creating-a-pull-request) against the default [`dev` branch of TileDB-Py](https://github.com/TileDB-Inc/TileDB-Py/tree/dev)
+- Please submit [pull requests](https://help.github.com/en/desktop/contributing-to-projects/creating-a-pull-request) against the default [`main` branch of TileDB-Py](https://github.com/TileDB-Inc/TileDB-Py/tree/main)

HISTORY.md

@@ -1,3 +1,31 @@
+# Release 0.33.1
+
+## Packaging Notes
+This version eliminates the Cython dependency, and communication with the TileDB Arrays library (libtiledb) is now handled exclusively with pybind11.
+
+## Improvements
+
+* Remove Cython Dependency by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2118
+* Factor _write_array out of Cython by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2115
+* Factor write_direct_dense out of Cython by @kounelisagis and @nguyenv in https://github.com/TileDB-Inc/TileDB-Py/pull/2109
+* Remove indexing.pyx by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2108
+* Move Query to pure Python by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2106
+* Merge Metadata and GroupMetadata into one Python class by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2099
+* Move _setitem_impl_sparse to Python by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2092
+* Move Array to pure Python by @kounelisagis, @nguyenv, and @teo-tsirpanis in https://github.com/TileDB-Inc/TileDB-Py/pull/2076
+
+# Release 0.33.0
+
+* TileDB-Py 0.33.0 includes TileDB Embedded [2.27.0](https://github.com/TileDB-Inc/TileDB/releases/tag/2.27.0)
+
+## Improvements
+
+* Fix `stats_dump` broken return value type by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2104
+* Enable builds for Python 3.13 by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2107
+* Implement serialization functionality for `Config` and `VFS` objects by @kounelisagis in https://github.com/TileDB-Inc/TileDB-Py/pull/2110
+
+**Full Changelog**: https://github.com/TileDB-Inc/TileDB-Py/compare/0.32.5...0.33.0
+
 # Release 0.32.5
 
 * TileDB-Py 0.32.5 includes TileDB Embedded [2.26.2](https://github.com/TileDB-Inc/TileDB/releases/tag/2.26.2)

README.md

@@ -1,8 +1,8 @@
-<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/dev/doc/source/_static/[email protected]" alt="TileDB logo" width="400"></a>
+<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/main/doc/source/_static/[email protected]" alt="TileDB logo" width="400"></a>
 
 
-[![Build Status](https://dev.azure.com/TileDB-Inc/CI/_apis/build/status/TileDB-Inc.TileDB-Py?branchName=dev)](https://dev.azure.com/TileDB-Inc/CI/_build/latest?definitionId=1&branchName=dev)
-![](https://raw.githubusercontent.com/TileDB-Inc/TileDB/dev/doc/anaconda.svg?sanitize=true)[![Anaconda download count badge](https://anaconda.org/conda-forge/TileDB-Py/badges/downloads.svg)](https://anaconda.org/conda-forge/TileDB-Py)
+[![Build Status](https://dev.azure.com/TileDB-Inc/CI/_apis/build/status/TileDB-Inc.TileDB-Py?branchName=main)](https://dev.azure.com/TileDB-Inc/CI/_build/latest?definitionId=1&branchName=main)
+![](https://raw.githubusercontent.com/TileDB-Inc/TileDB/main/doc/anaconda.svg?sanitize=true)[![Anaconda download count badge](https://anaconda.org/conda-forge/TileDB-Py/badges/downloads.svg)](https://anaconda.org/conda-forge/TileDB-Py)
 
 
 # TileDB-Py
@@ -35,6 +35,6 @@ Dataframes functionality (`tiledb.from_pandas`, `Array.df[]`) requires [Pandas](
 
 # Contributing
 
-We welcome contributions, please see [`CONTRIBUTING.md`](https://github.com/TileDB-Inc/TileDB-Py/blob/dev/CONTRIBUTING.md) for suggestions and
+We welcome contributions, please see [`CONTRIBUTING.md`](https://github.com/TileDB-Inc/TileDB-Py/blob/main/CONTRIBUTING.md) for suggestions and
 development-build instructions. For larger features, please open an issue to discuss goals and
 approach in order to ensure a smooth PR integration and review process.

doc/source/python-api.rst

@@ -119,7 +119,7 @@ Domain
 Array
 -----
 
-.. autoclass:: tiledb.libtiledb.Array
+.. autoclass:: tiledb.Array
    :members:
 
 .. autofunction:: tiledb.consolidate
@@ -142,7 +142,7 @@ Sparse Array
 Query
 ---------------
 
-.. autoclass:: tiledb.libtiledb.Query
+.. autoclass:: tiledb.Query
    :members:
 
 Query Condition

pyproject.toml

@@ -2,7 +2,6 @@
 requires = [
     "scikit-build-core",
     "pybind11",
-    "Cython>=3",
     "numpy>=2.0.1"
 ]
 build-backend = "scikit_build_core.build"

tiledb/CMakeLists.txt

@@ -1,33 +1,3 @@
-# Cython
-
-add_custom_command(
-  OUTPUT libtiledb.cc
-  COMMAND Python::Interpreter -m cython
-          "${CMAKE_CURRENT_SOURCE_DIR}/libtiledb.pyx" --output-file libtiledb.cc
-  DEPENDS libtiledb.pyx
-  VERBATIM
-)
-
-python_add_library(
-    libtiledb
-    MODULE
-    libtiledb.cc
-    WITH_SOABI
-)
-
-target_link_libraries(
-    libtiledb
-    PUBLIC
-    Python::NumPy
-    TileDB::tiledb_shared
-)
-
-target_compile_features(
-    libtiledb
-    PUBLIC
-    cxx_std_20
-)
-
 # Pybind11
 
 pybind11_add_module(
@@ -64,7 +34,7 @@ if (TILEDB_SERIALIZATION)
     )
 endif()
 
-install(TARGETS main libtiledb DESTINATION tiledb)
+install(TARGETS main DESTINATION tiledb)
 
 if(TILEDB_DOWNLOADED)
     message(STATUS "Adding \"libtiledb\" into install group")
@@ -73,18 +43,15 @@ if(TILEDB_DOWNLOADED)
 
     if (APPLE)
         set_target_properties(main PROPERTIES INSTALL_RPATH "@loader_path")
-        set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "@loader_path")
     elseif(UNIX)
         set_target_properties(main PROPERTIES INSTALL_RPATH "\$ORIGIN")
-        set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "\$ORIGIN")
     endif()
 else()
     # If using external TileDB core library force it to be linked at runtime using RPATH
     get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION)
     get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY)
     message(STATUS "Setting RPATH for targets \"main\" and \"libtiledb\" to ${TILEDB_LOCATION}")
     set_target_properties(main PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION})
-    set_target_properties(libtiledb PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION})
 endif()
 
-add_subdirectory(cc)
+add_subdirectory(libtiledb)

tiledb/__init__.py

@@ -2,15 +2,6 @@
 import os
 import sys
 
-# un-comment this section to fix Cython backtrace line-numbers in
-# IPython/Jupyter. see https://bugs.python.org/issue32797#msg323167
-# ---
-# try:
-#    from importlib.machinery import ExtensionFileLoader
-# else:
-#    del ExtensionFileLoader.get_source
-# ---
-
 if os.name == "posix":
     if sys.platform == "darwin":
         lib_name = "libtiledb.dylib"
@@ -19,17 +10,25 @@
 else:
     lib_name = "tiledb"
 
+import numpy as np
+
+# TODO: get rid of this - It is currently used for unified numpy printing accross numpy versions
+np.set_printoptions(
+    legacy="1.21" if np.lib.NumpyVersion(np.__version__) >= "1.22.0" else False
+)
+del np
+
 from tiledb.libtiledb import version as libtiledb_version
 
-if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 25:
+if libtiledb_version()[0] == 2 and libtiledb_version()[1] >= 26:
     from .current_domain import CurrentDomain
     from .ndrectangle import NDRectangle
 
 del libtiledb_version  # no longer needed
 
+from .array import Array
 from .array_schema import ArraySchema
 from .attribute import Attr
-from .cc import TileDBError
 from .consolidation_plan import ConsolidationPlan
 from .ctx import Config, Ctx, default_ctx, scope_ctx
 from .dataframe_ import from_csv, from_pandas, open_dataframe
@@ -87,10 +86,7 @@
     vacuum,
     walk,
 )
-from .libtiledb import (
-    Array,
-    Ctx,
-)
+from .libtiledb import TileDBError
 from .multirange_indexing import EmptyRange
 from .object import Object
 from .parquet_ import from_parquet

tiledb/aggregation.py

@@ -1,4 +1,4 @@
-import tiledb.cc as lt
+import tiledb.libtiledb as lt
 
 
 class Aggregation:
@@ -39,9 +39,9 @@ def __getitem__(self, selection):
                 "or 'U' (TILEDB_UNORDERED)"
             )
 
-        q = PyAgg(array._ctx_(), array, layout, self.attr_to_aggs)
+        q = PyAgg(array.ctx, array, layout, self.attr_to_aggs)
 
-        from .libtiledb import (
+        from .array import (
             index_as_tuple,
             index_domain_subarray,
             replace_ellipsis,
@@ -54,7 +54,7 @@ def __getitem__(self, selection):
         idx, drop_axes = replace_scalars_slice(dom, idx)
         dim_ranges = index_domain_subarray(array, dom, idx)
 
-        subarray = Subarray(array, array._ctx_())
+        subarray = Subarray(array, array.ctx)
         subarray.add_ranges([list([x]) for x in dim_ranges])
         q.set_subarray(subarray)