Pyproject toml (#324)

Major overhaul of build system: - Migration to pyproject.toml with meson. - Migrated away from versioneer to custom version.py script. - Migrated away from miniconda/tox-conda to "regular" python install for unit tests - Migrated CI workflows to use cibuildwheel for all platforms - Move utilities related to build into <root>/build_utils. - Updated testing workflows to add appropriate compilers. - Enable testing on Intel Mac as well as ARM - Update docs on building with (and without) C extension --------- Co-authored-by: Adam J. Jackson <[email protected]>
pace-neutrons · Nov 22, 2024 · c829321 · c829321
1 parent d54d4ff
commit c829321
Show file tree

Hide file tree

Showing 19 changed files with 318 additions and 3,199 deletions.
diff --git a/.github/workflows/build_upload_pypi_wheels.yml b/.github/workflows/build_upload_pypi_wheels.yml
@@ -14,64 +14,88 @@ jobs:
         include:
           - os: windows-latest
             wheelname: win
+            cibw_archs: "AMD64"
           - os: macos-13
-            wheelname: macos-intel
+            wheelname: macosx
+            cibw_archs: "x86_64"
           - os: macos-latest
-            wheelname: macos-arm
+            wheelname: macosx
+            cibw_archs: "arm64"
           - os: ubuntu-latest
             wheelname: manylinux
+            cibw_archs: "x86_64"
           - python-version: '3.10'
-            manylinux-version-tag: cp310
+            version-tag: cp310
           - python-version: '3.11'
-            manylinux-version-tag: cp311
-          - python-version: '3.11'
-            manylinux-version-tag: cp311
+            version-tag: cp311
+          - python-version: '3.12'
+            version-tag: cp312
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # Ensure tags are fetched for versioning
-      - name: Setup Python ${{ matrix.python-version }} with Conda
-        uses: conda-incubator/setup-miniconda@v3
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          channels: conda-forge,defaults
-          channel-priority: true
-      - name: Update Python pip, wheel, and twine
-        shell: bash -l {0}
-        run: |
-          python -m pip install --upgrade pip wheel twine
       - name: Install llvm on Macos
         if: startsWith(matrix.os, 'macos')
-        run: brew install llvm
-
-      - name: Build Python wheel
+        shell: bash -l {0}
+        env:
+          # Homebrew location is different on Intel Mac
+          LLVM_DIR: ${{ (matrix.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }}
+        run: |
+          brew install llvm
+          echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV
+          echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV
+          echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV
 
-        if: matrix.os != 'ubuntu-latest'
+      - name: Windows - find MSVC and set environment variables
+        if: startsWith(matrix.os, 'windows')
         shell: bash -l {0}
         env:
-          NUMPY_VERSION: 2.0
+          MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC'
         run: |
-          # Build against lowest required Numpy version
-          python -m pip install numpy==${NUMPY_VERSION}
-          python -m pip wheel . -w wheelhouse --no-deps
+          echo "Available MSVC installations:"
+          ls "$MSVC_PREFIX"
+
+          MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64
+          CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe"
+          echo "CC: $CC"
+          echo "CC=$CC" >> $GITHUB_ENV
 
-      - name: Build manylinux Python wheel
-        if: matrix.os == 'ubuntu-latest'
-        uses: pypa/[email protected]
+          CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe"
+          echo "CC_LD: $CC_LD"
+          echo "CC_LD=$CC_LD" >> $GITHUB_ENV
+
+      - name: Update Python pip, build, wheel, and twine
+        shell: bash -l {0}
+        run: |
+          python -m pip install --upgrade pip build wheel twine
+
+      - name: Build wheels
+        uses: pypa/[email protected]
         env:
-          CIBW_BUILD: ${{ matrix.manylinux-version-tag}}-manylinux*
-          CIBW_BEFORE_BUILD: python -mpip install numpy==2.0
-          CIBW_ARCHS: x86_64
+          CIBW_BUILD_FRONTEND: build
+          CIBW_BUILD: ${{ matrix.version-tag }}-*
+          CIBW_ARCHS: ${{ matrix.cibw_archs }}
+          CIBW_SKIP: "*-musllinux*"
+
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
+
+          CIBW_TEST_EXTRAS: "test,brille,phonopy_reader,matplotlib"
+          CIBW_TEST_COMMAND: python {package}/tests_and_analysis/test/run_tests.py
+
         with:
           output-dir: wheelhouse
 
       - name: Create source distribution
         if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
         shell: bash -l {0}
         run: |
-          python setup.py sdist
+          python -m build --sdist .
 
       - name: Upload source dist as build artifact
         if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' && github.event_name == 'release'
@@ -83,8 +107,9 @@ jobs:
       - name: Upload wheels as build artifacts
         uses: actions/upload-artifact@v4
         with:
-            name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}
-            path: wheelhouse/*-${{ matrix.wheelname }}*.whl
+            name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }}
+            path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl
+            if-no-files-found: error
 
   publish:
     if: github.event_name == 'release'

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -17,25 +17,57 @@ jobs:
        (github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'no_ci'))
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+
       fail-fast: false
+
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: '3.10'
-          channels: conda-forge,defaults
-          channel-priority: true
-      - name: Install llvm on Macos
+          fetch-depth: 0 # Ensure tags are fetched for versioning
+      - uses: actions/setup-python@v5
+        with:
+          python-version: |
+              3.10
+              3.11
+              3.12
+      - name: Install llvm on MacOS
         if: startsWith(matrix.os, 'macos')
+        shell: bash -l {0}
+        env:
+          # Homebrew location is different on Intel Mac
+          LLVM_DIR: ${{ (matrix.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }}
         run: |
           brew install llvm
+          echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV
+          echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV
+          echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV
+
+      - name: Windows - find MSVC and set environment variables
+        if: startsWith(matrix.os, 'windows')
+        shell: bash -l {0}
+        env:
+          MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC'
+        run: |
+          echo "Available MSVC installations:"
+          ls "$MSVC_PREFIX"
+
+          MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64
+          CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe"
+          echo "CC: $CC"
+          echo "CC=$CC" >> $GITHUB_ENV
+
+          CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe"
+          echo "CC_LD: $CC_LD"
+          echo "CC_LD=$CC_LD" >> $GITHUB_ENV
+
       - name: Update pip and install dependencies
         shell: bash -l {0}
         run: |
           python -m pip install --upgrade pip
           python -m pip install -r tests_and_analysis/ci_requirements.txt
+
       - name: Run tests, skip Python 3.11 unless workflow dispatch
         if: github.event_name != 'workflow_dispatch'
         env:
@@ -46,6 +78,7 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
         shell: bash -l {0}
         run: python -m tox
+
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -68,11 +101,9 @@ jobs:
     if: success() || failure()
     steps:
       - uses: actions/checkout@v4
-      - uses: conda-incubator/setup-miniconda@v3
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-          channels: conda-forge,defaults
-          channel-priority: true
+          python-version: "3.10"
       - name: Update pip and install dependencies
         shell: bash -l {0}
         run: |

diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml
@@ -33,7 +33,7 @@ jobs:
         shell: bash -l {0}
         env:
           EUPHONIC_VERSION: ${{ github.event.inputs.version }}
-        run: python -m tox -c release_tox.ini
+        run: python -m tox -c build_utils/release_tox.ini
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4
@@ -54,4 +54,3 @@ jobs:
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:
           junit_files: artifacts/**/junit_report*.xml
-
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -83,6 +83,9 @@
     support addition of Spectrum2DCollection and improve
     maintainability.
 
+  - Entire build system rework, migrating to ``pyproject.toml`` form
+    with ``meson-python``, ``cibuildwheel`` and removing
+    ``versioneer`` to simplify future development and maintenance.
 
 `v1.3.2 <https://github.com/pace-neutrons/Euphonic/compare/v1.3.1...v1.3.2>`_
 -----------------------------------------------------------------------------
@@ -619,7 +622,7 @@
   - Default Monkhorst-Pack meshes (i.e. [6, 6, 6] in ``euphonic-dos``
     and [20, 20, 20] in ``sample_sphere_structure_factor()``) have
     been replaced by default grid-spacing values.
-  
+
   - The scaling of density of states has changed, due to a change
     in implementation
 

diff --git a/release.py → build_utils/release.py b/release.py → build_utils/release.py
diff --git a/release_tox.ini → build_utils/release_tox.ini b/release_tox.ini → build_utils/release_tox.ini
diff --git a/build_utils/version.py b/build_utils/version.py
@@ -0,0 +1,70 @@
+"""Script to compute version from git tags.
+
+Provides 3 means of updating:
+
+- python version.py
+
+  Print computed version to stdout
+
+- python version.py --dump
+
+  Update version in `euphonic/version.py`
+
+- python version.py --dist
+
+  Update versions in both `euphonic/version.py` and in meson sdist build directory.
+"""
+
+from pathlib import Path
+import subprocess
+import sys
+import os
+
+if new_dir := os.getenv("MESON_SOURCE_ROOT"):
+    os.chdir(new_dir)
+
+gits = ["git"]
+if sys.platform == "win32":
+    gits += ["git.cmd", "git.exe"]
+
+match sys.argv:
+    case [_, "--dist"]:
+        COMMAND = "dist"
+    case [_, "--dump"]:
+        COMMAND = "dump"
+    case [_]:
+        COMMAND = "print"
+
+version_file = Path(__file__).parent.parent / "euphonic" / "version.py"
+
+for gitcmd in gits:
+    try:
+        proc = subprocess.run([gitcmd, "describe", "--tags", "--dirty"],
+                              capture_output=True, check=True, text=True)
+    except subprocess.CalledProcessError as err:
+        print(f"Tried {gitcmd}, returned: {err}", file=sys.stderr)
+        print(f"Stdout: '{err.stdout.strip()}'", file=sys.stderr)
+        print(f"Stdout: '{err.stderr.strip()}'", file=sys.stderr)
+        continue
+
+    version, *dirty = proc.stdout.strip().split("-")
+    if dirty:
+        version += f"+{dirty[0]}.{dirty[1]}{'.dirty' if len(dirty) > 2 and COMMAND != 'dump' else ''}"
+    break
+
+else:  # Can't use git
+    version = version_file.read_text().split("=")[1].strip('"\n ')
+
+match COMMAND:
+    case "dist":
+        version_file.write_text(f'__version__ = "{version}"', encoding="utf-8")
+        dist_path = os.getenv("MESON_DIST_ROOT")
+        dist_version_file = Path(dist_path) / "euphonic" / "version.py"
+        dist_version_file.parent.mkdir(parents=True, exist_ok=True)
+        dist_version_file.write_text(f'__version__ = "{version}"', encoding="utf-8")
+
+    case "dump":
+        version_file.write_text(f'__version__ = "{version}"', encoding="utf-8")
+
+    case "print":
+        print(version)
diff --git a/build_utils/version.py.in b/build_utils/version.py.in
@@ -0,0 +1 @@
+__version__ = "@VERSION@"
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
@@ -106,8 +106,8 @@ the same pip commands as above.
 
 **Linux**
 
-You should have a version of ``gcc`` on your path (currently tested with
-``4.8.5``). If ``gcc`` can be found the Euphonic extension will be
+You should have a C-compatible compiler on your path (currently tested with
+``gcc``). If a C compiler with OpenMP can be found, the Euphonic extension will be
 automatically installed when using the same pip commands as above.
 
 **Mac OSX**
@@ -119,6 +119,12 @@ before running pip install run:
 
   brew install llvm
 
+.. note::
+
+  You may need to add the llvm-clang compiler to your path or export
+  the variables recommended by ``brew`` to ensure that the llvm-clang compiler
+  is chosen ahead of the native Apple compiler (which does not support OpenMP).
+
 Installing Euphonic without the C extension
 ===========================================
 
@@ -127,10 +133,4 @@ install the Python parts only with:
 
 .. code-block:: bash
 
-  pip install --install-option="--python-only" euphonic
-
-Note that using this option disables the use of wheels which, if they haven't
-been installed already, actually makes installing other packages such as Numpy
-more difficult. The easiest way around this is running the usual install
-command first (which will install all the dependencies), then running again
-with the ``--install-option="--python-only"`` option.
+  pip install -Csetup-args="-Dpython_only=true" euphonic
diff --git a/euphonic/__init__.py b/euphonic/__init__.py
@@ -1,8 +1,7 @@
 from importlib.resources import files
 
 from . import data
-from . import _version
-__version__ = _version.get_versions()['version']
+from .version import __version__
 
 import pint
 from pint import UnitRegistry