Merge branch 'main' into kerl/wheels-macos-14

single-cell-data · May 21, 2024 · defa29e · defa29e
2 parents 54cda83 + d837dd2
commit defa29e
Show file tree

Hide file tree

Showing 109 changed files with 26,003 additions and 1,383 deletions.
diff --git a/.github/workflows/python-ci-minimal.yml b/.github/workflows/python-ci-minimal.yml
@@ -12,8 +12,14 @@ on:
     branches:
       - main
       - 'release-*'
-    paths-ignore:
-      - 'apis/r/**'
+    paths:
+      - '**'
+      - '!**.md'
+      - '!apis/r/**'
+      - '!docs/**'
+      - '!.github/**'
+      - '.github/workflows/python-ci-minimal.yml'
+      - '.github/workflows/python-ci-single.yml'
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/python-so-copying.yml → .github/workflows/python-ci-packaging.yml b/.github/workflows/python-so-copying.yml → .github/workflows/python-ci-packaging.yml
@@ -1,13 +1,16 @@
-# Confirm shared object copying when building the Python package
-# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
+# This CI confirms the Python package continues to install properly in the many
+# different required contexts
 
-name: Python SO copying
+name: TileDB-SOMA Python CI (Packaging)
 
 on:
   push:
     paths:
-      - '.github/workflows/python-so-copying.yml'
+      - '.github/workflows/python-ci-packaging.yml'
+      - 'apis/python/MANIFEST.in'
+      - 'apis/python/pyproject.toml'
       - 'apis/python/setup.py'
+      - 'apis/python/version.py'
       - 'apis/python/src/tiledbsoma/__init__.py'
       - 'libtiledbsoma/cmake/inputs/Config.cmake.in'
       - 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
@@ -17,8 +20,11 @@ on:
       - 'libtiledbsoma/CMakeLists.txt'
   pull_request:
     paths:
-      - '.github/workflows/python-so-copying.yml'
+      - '.github/workflows/python-ci-packaging.yml'
+      - 'apis/python/MANIFEST.in'
+      - 'apis/python/pyproject.toml'
       - 'apis/python/setup.py'
+      - 'apis/python/version.py'
       - 'apis/python/src/tiledbsoma/__init__.py'
       - 'libtiledbsoma/cmake/inputs/Config.cmake.in'
       - 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
@@ -30,9 +36,11 @@ on:
 
 jobs:
 
+  # Confirm shared object copying when building the Python package
+  # https://github.com/single-cell-data/TileDB-SOMA/pull/1937
   docker:
     runs-on: ubuntu-latest
-    name: "docker TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
+    name: "SO copying (docker) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
     strategy:
       fail-fast: false
       matrix:
@@ -68,7 +76,7 @@ jobs:
         run: |
           mkdir -p external
           # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-linux-x86_64-2.22.0-52e981e.tar.gz
+          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz
           tar -C external -xzf tiledb-linux-x86_64-*.tar.gz
           ls external/lib/
           echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
@@ -138,9 +146,12 @@ jobs:
           find . -name '*tile*.so*'
           ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
 
+  # Confirm shared object copying when building the Python package
+  # https://github.com/single-cell-data/TileDB-SOMA/pull/1937
+  # Same as job above, but running on macOS instead of in a Docker container
   macos:
     runs-on: macos-13
-    name: "macos TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
+    name: "SO copying (macos) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
     strategy:
       fail-fast: false
       matrix:
@@ -160,7 +171,7 @@ jobs:
         run: |
           mkdir -p external
           # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-macos-x86_64-2.22.0-52e981e.tar.gz
+          wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz
           tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
           ls external/lib/
           echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
@@ -236,7 +247,7 @@ jobs:
   # Tests that the --libtiledbsoma flag to setup.py continues working
   setuptools:
     runs-on: ${{ matrix.os }}
-    name: "${{ matrix.os }} setuptools"
+    name: "setuptools (${{ matrix.os }})"
     strategy:
       fail-fast: false
       matrix:
@@ -251,10 +262,10 @@ jobs:
           if [ `uname -s` == "Darwin" ];
           then
             # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-macos-x86_64-2.22.0-52e981e.tar.gz
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-macos-x86_64-2.23.0-152093b.tar.gz
           else
             # Please do not edit manually -- let scripts/update-tiledb-version.py update this
-            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.22.0/tiledb-linux-x86_64-2.22.0-52e981e.tar.gz
+            wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.23.0/tiledb-linux-x86_64-2.23.0-152093b.tar.gz
           fi
           tar -C external -xzf tiledb-*.tar.gz
           ls external/lib/
@@ -312,3 +323,57 @@ jobs:
         run: ./venv-soma/bin/python -m pip install --prefer-binary `grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt`
       - name: Runtime test
         run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
+
+  # Build a wheel from a source tarball (confirms all required files are
+  # distributed in the sdist). This is in preparation for the official building of
+  # the PyPI wheels in python-packaging.yml.
+  # https://github.com/single-cell-data/TileDB-SOMA/pull/2506
+  #
+  #
+  #  Differences to python-packaging.yml:
+  #
+  #  * Uses transparent python commands instead of the opaque action
+  #    `pypa/cibuildwheel`, and therefore easier to debug
+  #  * Only builds two wheels total (python 3.11 on Ubuntu and macOS), so it is
+  #    much faster. It's not creating the wheels for distribution, but instead just
+  #    providing a quick check that all the required files are distributed in the
+  #    source tarball
+  #  * Runs whenever an installation-related file is modified instead of only
+  #    after a release
+  #
+  # In summary, the goal is to identify any potential problems with building
+  # the PyPI wheels when a PR is submitted, and not at release time.
+  sdist:
+    runs-on: ${{ matrix.os }}
+    name: "Wheel from sdist (${{ matrix.os }})"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-22.04", "macos-12"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: TileDB-SOMA
+          fetch-depth: 0 # for setuptools-scm
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: python -m pip install --prefer-binary pybind11 wheel
+      - name: Build source tarball (sdist)
+        run: |
+          cd TileDB-SOMA/apis/python
+          python setup.py sdist
+      - name: Extract source tarball
+        run: |
+          tar --list -f TileDB-SOMA/apis/python/dist/tiledbsoma-*.tar.gz
+          tar -xzf TileDB-SOMA/apis/python/dist/tiledbsoma-*.tar.gz
+      - name: Build wheel
+        run: |
+          cd tiledbsoma-*/
+          python setup.py bdist_wheel
+      - name: Install wheel
+        run: |
+          python -m pip install --prefer-binary tiledbsoma-*/dist/tiledbsoma-*.whl
+          python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml
@@ -118,7 +118,7 @@ jobs:
           # * unzip whatever.zip
           # * pip install tiledbsoma-i.j.k-cp310-cp310-macosx_11_0_arm64.whl
           #
-          #- os: macos-11
+          #- os: macos-12
           #  platform: macosx
           #  arch: arm64
           #  cc: clang

diff --git a/.github/workflows/r-ci.yml b/.github/workflows/r-ci.yml
@@ -2,9 +2,13 @@ name: TileDB-SOMA R CI
 
 on:
   pull_request:
-    paths-ignore:
-      - "apis/python/**"
-      - ".pre-commit-config.yaml"
+    paths:
+      - '**'
+      - '!**.md'
+      - '!apis/python/**'
+      - '!docs/**'
+      - '!.github/**'
+      - '.github/workflows/r-ci.yml'
   push:
     branches:
       - main
@@ -16,6 +20,7 @@ env:
   COVERAGE_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   _R_CHECK_TESTS_NLINES_: 0
   CATCHSEGV: "TRUE"
+  R_REMOTES_UPGRADE: "never"
 
 jobs:
   ci:
@@ -52,10 +57,26 @@ jobs:
       - name: Bootstrap
         run: cd apis/r && tools/r-ci.sh bootstrap
 
+      - name: Set additional repositories (macOS)
+        if: ${{ matrix.os != 'ubuntu-latest' }}
+        run: echo 'options(repos = c("https://tiledb-inc.r-universe.dev", getOption("repos")))' | tee -a ~/.Rprofile
+
+      - name: Set additional repositories (Linux)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          rversion <- paste(strsplit(as.character(getRversion()), split = '\\.')[[1L]][1:2], collapse = '.')
+          codename <-  system('. /etc/os-release; echo ${VERSION_CODENAME}', intern = TRUE)
+          repo <- "https://tiledb-inc.r-universe.dev"
+          (opt <- sprintf('options(repos = c("%s/bin/linux/%s/%s", "%s", getOption("repos")))', repo, codename, rversion, repo))
+          cat(opt, "\n", file = "~/.Rprofile", append = TRUE)
+        shell: Rscript {0}
+
+      - name: Install tiledb-r
+        run: cd apis/r && Rscript tools/install-tiledb-r.R
+
       - name: Install BioConductor package SingleCellExperiment
         run: cd apis/r && tools/r-ci.sh install_bioc SingleCellExperiment
 
-
       # Uncomment these next two stanzas as needed whenever we've just released a new tiledb-r for
       # which source is available but CRAN releases (and hence update r2u binaries) are not yet:
       #
@@ -82,7 +103,7 @@ jobs:
       #  run: cd apis/r && Rscript -e "options(bspm.version.check=TRUE); install.packages('tiledb', repos = c('https://eddelbuettel.r-universe.dev/bin/linux/jammy/4.3/', 'https://cloud.r-project.org'))"
 
       - name: Dependencies
-        run: cd apis/r && tools/r-ci.sh install_all
+        run: cd apis/r && Rscript -e "remotes::install_deps(dependencies = TRUE, upgrade = FALSE)"
 
       # - name: Install dataset packages from source (macOS)
       #   if: ${{ matrix.os == 'macOS-latest' }}

diff --git a/.github/workflows/r-python-interop-testing.yml b/.github/workflows/r-python-interop-testing.yml
@@ -32,8 +32,21 @@ jobs:
       - name: Bootstrap
         run: cd apis/r && tools/r-ci.sh bootstrap
 
+      - name: Set additional repositories (Linux)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          rversion <- paste(strsplit(as.character(getRversion()), split = '\\.')[[1L]][1:2], collapse = '.')
+          codename <-  system('. /etc/os-release; echo ${VERSION_CODENAME}', intern = TRUE)
+          repo <- "https://tiledb-inc.r-universe.dev"
+          (opt <- sprintf('options(repos = c("%s/bin/linux/%s/%s", "%s", getOption("repos")))', repo, codename, rversion, repo))
+          cat(opt, "\n", file = "~/.Rprofile", append = TRUE)
+        shell: Rscript {0}
+
+      - name: Install tiledb-r
+        run: cd apis/r && Rscript tools/install-tiledb-r.R
+
       - name: Dependencies
-        run: cd apis/r && tools/r-ci.sh install_all
+        run: cd apis/r && Rscript -e "remotes::install_deps(dependencies = TRUE, upgrade = FALSE)"
 
       - name: CMake
         uses: lukka/get-cmake@latest

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,45 +1,12 @@
-# ================================================================
-# The main branch is not published to readthedocs at present.
-# We only publish from main-old for now (2022-09-26).
-# ================================================================
-
-# Note: builds at
-# https://readthedocs.com/projects/tiledb-inc-tiledb-soma/builds/
-# (permissioned)
-
-# Don't build any extra formats
-formats: []
-
-# Use RTD config version 2
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
+# Build TileDB-SOMA readthedocs site: https://tiledbsoma.readthedocs.io
+# Build history can be found at https://readthedocs.org/projects/tiledbsoma/builds
 version: 2
-
 sphinx:
   configuration: doc/source/conf.py
-
-# python:
-#   version: 3.8
-#   install:
-#     # this order is important: we need to get cmake
-#     - requirements: doc/requirements_doc.txt
-#     - method: setuptools
-#       path: .
-
 build:
-  os: "ubuntu-22.04"
+  os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.11"
   commands:
-    # Sandbox alert: `pip install -e .` will _not_ let python find the tiledbsoma package
-    # within sphinx build
-    - python -m pip install --upgrade pip
-
-    - python -m pip install -r doc/requirements_doc.txt
-
-    - python -m pip install -v apis/python
-
-    - python scripts/show-versions.py
-
-    - sphinx-build --version
-
-    - sphinx-build -T -E -b html -d _readthedocs/doctrees -D language=en doc/source _readthedocs/html
+    - pip install --upgrade pip
+    - doc/build.sh -r -V  # install deps, don't make a venv
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -18,3 +18,12 @@ Thanks for your interest in TileDB-SOMA. The notes below give some pointers for
 - Make changes locally, then rebuild as appropriate for the level of changes (e.g.: `make` for `libtilebsc` or `python setup.py develop` for `apis/python`).
 - Make sure to run `make check`, or `pytest` to verify changes against tests (add new tests where applicable).
 - Please submit [pull requests](https://help.github.com/en/desktop/contributing-to-projects/creating-a-pull-request) against the default [`main` branch of TileDB-SOMA](https://github.com/TileDB-Inc/TileDB-SOMA/tree/master).
+- If you edit the Python files, please run the pre-commit hooks
+
+  ```sh
+  python -m venv ./pre-commit
+  source ./pre-commit/bin/activate
+  python -m pip -v install pre-commit
+  pre-commit run -a -v
+  deactivate
+  ```