From 0214a9219dafe033fe9630a7d031fe5b9715312c Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 25 Nov 2024 17:01:44 +0100 Subject: [PATCH 01/12] prototype of so-only wheel --- .github/workflows/build-wheel-linux.yml | 88 +++++++++++++++++++ python_wrapper/Dockerfile | 23 +++++ python_wrapper/buildconfig | 15 ++++ python_wrapper/buildscripts/compile.sh | 22 +++++ .../buildscripts/setup_utils/__init__.py | 52 +++++++++++ python_wrapper/buildscripts/test-wheel.sh | 26 ++++++ python_wrapper/buildscripts/wheel-linux.sh | 35 ++++++++ python_wrapper/dockerbuild.sh | 12 +++ python_wrapper/setup.cfg | 4 + python_wrapper/setup.py | 2 + 10 files changed, 279 insertions(+) create mode 100644 .github/workflows/build-wheel-linux.yml create mode 100644 python_wrapper/Dockerfile create mode 100644 python_wrapper/buildconfig create mode 100755 python_wrapper/buildscripts/compile.sh create mode 100644 python_wrapper/buildscripts/setup_utils/__init__.py create mode 100644 python_wrapper/buildscripts/test-wheel.sh create mode 100755 python_wrapper/buildscripts/wheel-linux.sh create mode 100755 python_wrapper/dockerbuild.sh create mode 100644 python_wrapper/setup.cfg create mode 100644 python_wrapper/setup.py diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 000000000..289f3e27f --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,88 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + + build: + + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: wheelmaker_2_28:0.1 + + name: Build manylinux_2_28 + + steps: + - uses: actions/checkout@v2 + - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig + + ################################################################ + - run: /buildscripts/wheel-linux.sh ./eckit/python_wrapper/buildconfig 3.11 + - uses: actions/upload-artifact@v4 + name: Upload wheel 3.11 + with: + name: wheel-manylinux2_28-3.11 + path: /build/wheel/*.whl + + # TODO other python versions, once the above is correct. + # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps + + test: + + needs: build + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable + + name: Test with ${{ matrix.python-version }} + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + container: wheelmaker_2_28:0.1 + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v4 + with: + name: wheel-manylinux2_28-${{ matrix.python-version }} + - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} + +# TODO enable and test +# deploy: +# +# if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} +# needs: [test, build] +# strategy: +# fail-fast: false +# matrix: +# python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable +# +# name: Deploy wheel ${{ matrix.python-version }} +# runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] +# container: wheelmaker_2_28:0.1 +# steps: +# - run: mkdir artifact-${{ matrix.python-version }} +# - uses: actions/checkout@v2 +# - uses: actions/download-artifact@v4 +# with: +# name: wheel-manylinux2_28-${{ matrix.python-version }} +# path: artifact-${{ matrix.python-version }} +# - run: | +# /buildsripts/upload-twine.sh ${{ matrix.python-version }} +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/python_wrapper/Dockerfile b/python_wrapper/Dockerfile new file mode 100644 index 000000000..df68acad5 --- /dev/null +++ b/python_wrapper/Dockerfile @@ -0,0 +1,23 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# this docker image contains utilities for building shared libraries and wrap them in python wheel, test and publish + +# TODO parametrize this via build arg to support other manylinuxes +from quay.io/pypa/manylinux_2_28_x86_64:2024.11.24-1 + +run \ + yum install -y bison flex \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && mkdir /src /target \ + && git clone --branch master --depth=1 https://github.com/ecmwf/ecbuild.git /src/ecbuild +copy buildscripts /buildscripts + +# we workdir in /src so that the action checkout result ends up where expected +workdir /src + diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 000000000..52b529198 --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,15 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent + +NAME="eckit" +CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1" +PYPROJECT_DIR="python_wrapper" diff --git a/python_wrapper/buildscripts/compile.sh b/python_wrapper/buildscripts/compile.sh new file mode 100755 index 000000000..ed2a782c7 --- /dev/null +++ b/python_wrapper/buildscripts/compile.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +set -euo pipefail + +# this script is configured by sourcing its first param. Namely, we expect the following: +# $NAME -- name of the project being compiled. Used for /src/$NAME and /target/$NAME, should thus correspond to git repo name +# $CMAKE_PARAMS -- passed to ecbuild after the `--`. Eg "-DENABLE_THIS=1 -DENABLE_THAT=0" +# $PYPROJECT_DIR -- ignored + +source $1 + +mkdir /build && cd /build +/src/ecbuild/bin/ecbuild --prefix=/target/$NAME -- $CMAKE_PARAMS /src/$NAME +make -j10 +make install diff --git a/python_wrapper/buildscripts/setup_utils/__init__.py b/python_wrapper/buildscripts/setup_utils/__init__.py new file mode 100644 index 000000000..0bc9fd39f --- /dev/null +++ b/python_wrapper/buildscripts/setup_utils/__init__.py @@ -0,0 +1,52 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +""" +Utilities for setup invocation for shared-libs-only python wheels + +Customization should mostly happen in setup.cfg + +Resulting package's name & version come from env vars named NAME and VERSION +""" + +from setuptools import setup, find_packages +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel +import pathlib +import os + +class BinaryDistribution(Distribution): + # we do this for making the packager aware of presence of binary modules, to include + # the python version in the classifier + def has_ext_modules(foo): + return True + +class bdist_wheel_ext(bdist_wheel): + # This forces the platform tag from linux_x64_64 to manylinux_2_28 + # There is a chance that this is wrong, ie, the wheels are actually not compatible + # More reliable would be to auditwheel this, eg, + # LD_LIBRARY_PATH=/target/$NAME/lib64/ auditwheel repair dist/*whl --plat manylinux_2_28_x86_64 -L "libs" + # However, the problem is that this messes up with the whole idea of creating a chain of wheels + # Thus we would need to rip out the added .so files out and fix the rpaths, only making sure we preserve + # the right .so files in case auditwheel decided to make changes (this manifests as eg `libeckit-xxyyzz.so` + # appearing in the auditwheel-outputted wheel, alongside the original). Alternatively, we could replicate + # what auditwheel is doing -- in our original compilation, thus address the incompatibility at the root + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, abi, "manylinux_2_28" + +def plain_setup(): + setup( + name=os.environ["NAME"], + version=os.environ["VERSION"], + package_dir={"": "src"}, + packages=find_packages(where="src"), + package_data={"": ["*.so"]}, + distclass=BinaryDistribution, + cmdclass={"bdist_wheel": bdist_wheel_ext}, + ) diff --git a/python_wrapper/buildscripts/test-wheel.sh b/python_wrapper/buildscripts/test-wheel.sh new file mode 100644 index 000000000..792f9b3a3 --- /dev/null +++ b/python_wrapper/buildscripts/test-wheel.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# **** + +# just tests whether the package can be installed, imported, and __version__ matches what is expected + +set -euo pipefail + +source $1 +PYTHON="python$2" + +uv venv --python $PYTHON /tmp/venv +source /tmp/venv/bin/activate +uv pip install ./wheel-manylinux2_28-$2 + +INSTALLED_VERSION=$(python -c "import ${NAME}libs as l; print(l.__version__)") +EXPECTED_VERSION=$(cat /src/$NAME/VERSION) + +test "$INSTALLED_VERSION" == "$EXPECTED_VERSION" diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh new file mode 100755 index 000000000..2a077c432 --- /dev/null +++ b/python_wrapper/buildscripts/wheel-linux.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +set -euo pipefail + +# this script is configured by sourcing its first param. Namely, we expect the following: +# - $NAME -- name of the project being compiled. Used for /target/$NAME/lib64 where the shared libraries are expected, and for naming the python module `$NAME-libs` +# - $CMAKE_PARAMS -- ignored +# - $PYPROJECT_DIR -- where pyproject.toml/setup.py are expected, prefixed with src/$NAME. Param to `python -m build` +# additionally: +# - second param is the target python version, in the x.y format +# - version, license, authors and readme are expected to be root-located in src/$NAME + +source $1 +PYTHON="python$2" +PYPROJECT_DIR=/src/$NAME/$PYPROJECT_DIR + +mkdir -p $PYPROJECT_DIR/src +ln -s /target/$NAME/lib64 $PYPROJECT_DIR/src/${NAME}libs +ln -s ../LICENSE $PYPROJECT_DIR +ln -s ../README.md $PYPROJECT_DIR +ln -s ../AUTHORS $PYPROJECT_DIR +VERSION=$(cat /src/$NAME/VERSION) +echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py + +PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv $PYPROJECT_DIR + +mkdir /build/wheel +mv dist/*whl /build/wheel diff --git a/python_wrapper/dockerbuild.sh b/python_wrapper/dockerbuild.sh new file mode 100755 index 000000000..38c0a3b00 --- /dev/null +++ b/python_wrapper/dockerbuild.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR +docker build -t wheelmaker_2_28:0.1 . diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 000000000..2c746b9b2 --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,4 @@ +[metadata] +description = "eckit" +long_description = file: README.md +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 000000000..c64ad6d78 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup() From 5035a7ce50e8c2c2daf3aa83e0062316b4134ba7 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 26 Nov 2024 13:36:44 +0100 Subject: [PATCH 02/12] fix tag, update rpath --- python_wrapper/buildscripts/compile.sh | 2 +- python_wrapper/buildscripts/setup_utils/__init__.py | 2 +- python_wrapper/buildscripts/wheel-linux.sh | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python_wrapper/buildscripts/compile.sh b/python_wrapper/buildscripts/compile.sh index ed2a782c7..ef58b8efa 100755 --- a/python_wrapper/buildscripts/compile.sh +++ b/python_wrapper/buildscripts/compile.sh @@ -16,7 +16,7 @@ set -euo pipefail source $1 -mkdir /build && cd /build +rm -rf /build && mkdir /build && cd /build /src/ecbuild/bin/ecbuild --prefix=/target/$NAME -- $CMAKE_PARAMS /src/$NAME make -j10 make install diff --git a/python_wrapper/buildscripts/setup_utils/__init__.py b/python_wrapper/buildscripts/setup_utils/__init__.py index 0bc9fd39f..82e5c6760 100644 --- a/python_wrapper/buildscripts/setup_utils/__init__.py +++ b/python_wrapper/buildscripts/setup_utils/__init__.py @@ -38,7 +38,7 @@ class bdist_wheel_ext(bdist_wheel): # what auditwheel is doing -- in our original compilation, thus address the incompatibility at the root def get_tag(self): python, abi, plat = bdist_wheel.get_tag(self) - return python, abi, "manylinux_2_28" + return python, abi, "manylinux_2_28_x86_64" def plain_setup(): setup( diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh index 2a077c432..92dd78db1 100755 --- a/python_wrapper/buildscripts/wheel-linux.sh +++ b/python_wrapper/buildscripts/wheel-linux.sh @@ -29,7 +29,13 @@ ln -s ../AUTHORS $PYPROJECT_DIR VERSION=$(cat /src/$NAME/VERSION) echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py -PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv $PYPROJECT_DIR +# if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime +for e in $(find /target/$NAME/lib64 -name '*.so'); do + RPATH_MODIF=$(readelf -d $e | grep RPATH | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g') + patchelf --set-rpath "$RPATH_MODIF" $e +done + +PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv --wheel $PYPROJECT_DIR mkdir /build/wheel -mv dist/*whl /build/wheel +mv $PYPROJECT_DIR/dist/*whl /build/wheel From c6d4f456ecfab9c93fb9cedb28d07e5638300388 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Wed, 27 Nov 2024 12:21:37 +0100 Subject: [PATCH 03/12] more updates to rpath --- python_wrapper/buildscripts/wheel-linux.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh index 92dd78db1..077e3c25c 100755 --- a/python_wrapper/buildscripts/wheel-linux.sh +++ b/python_wrapper/buildscripts/wheel-linux.sh @@ -29,9 +29,10 @@ ln -s ../AUTHORS $PYPROJECT_DIR VERSION=$(cat /src/$NAME/VERSION) echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py -# if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime for e in $(find /target/$NAME/lib64 -name '*.so'); do - RPATH_MODIF=$(readelf -d $e | grep RPATH | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g') + # 1/ if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime + # 2/ we change $ORIGIN/../lib64 to just $ORIGIN, for the self-reference within the package + RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' | sed 's#$ORIGIN/../lib64#$ORIGIN#g') patchelf --set-rpath "$RPATH_MODIF" $e done From 279abc84224c0a15e34b4607fae3cd758930cec3 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 6 Dec 2024 17:10:37 +0100 Subject: [PATCH 04/12] support findlibs dependencies --- python_wrapper/buildconfig | 1 + python_wrapper/buildscripts/wheel-linux.sh | 7 ++++++- python_wrapper/dockerbuild.sh | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index 52b529198..95c47ea5a 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -13,3 +13,4 @@ NAME="eckit" CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1" PYPROJECT_DIR="python_wrapper" +FINDLIBS_DEPENDENCIES='[]' diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh index 077e3c25c..c0f39b3a7 100755 --- a/python_wrapper/buildscripts/wheel-linux.sh +++ b/python_wrapper/buildscripts/wheel-linux.sh @@ -13,6 +13,7 @@ set -euo pipefail # - $NAME -- name of the project being compiled. Used for /target/$NAME/lib64 where the shared libraries are expected, and for naming the python module `$NAME-libs` # - $CMAKE_PARAMS -- ignored # - $PYPROJECT_DIR -- where pyproject.toml/setup.py are expected, prefixed with src/$NAME. Param to `python -m build` +# - $FINDLIBS_DEPENDENCIES -- names of python modules (presumably with dylibs) that this one depends on, to be utilized by `findlibs` # additionally: # - second param is the target python version, in the x.y format # - version, license, authors and readme are expected to be root-located in src/$NAME @@ -28,11 +29,15 @@ ln -s ../README.md $PYPROJECT_DIR ln -s ../AUTHORS $PYPROJECT_DIR VERSION=$(cat /src/$NAME/VERSION) echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py +echo "findlibs_dependencies = $FINDLIBS_DEPENDENCIES" >> $PYPROJECT_DIR/src/${NAME}libs/__init__.py for e in $(find /target/$NAME/lib64 -name '*.so'); do # 1/ if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime + # 1a/ we actually dont, we rely on findlibs_dependencies instead + # | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' # 2/ we change $ORIGIN/../lib64 to just $ORIGIN, for the self-reference within the package - RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' | sed 's#$ORIGIN/../lib64#$ORIGIN#g') + # TODO just force the $ORIGIN here universally? + RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#$ORIGIN/../lib64#$ORIGIN#g') patchelf --set-rpath "$RPATH_MODIF" $e done diff --git a/python_wrapper/dockerbuild.sh b/python_wrapper/dockerbuild.sh index 38c0a3b00..bb4cdec7f 100755 --- a/python_wrapper/dockerbuild.sh +++ b/python_wrapper/dockerbuild.sh @@ -9,4 +9,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR -docker build -t wheelmaker_2_28:0.1 . +docker build -t wheelmaker_2_28:0.2 . From fed30f620f4d53b09c17c98534debe1c55650c7c Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 9 Dec 2024 10:38:41 +0100 Subject: [PATCH 05/12] Factor wheelmaker docker to a different repo --- .github/workflows/build-wheel-linux.yml | 55 +++++++++---------- python_wrapper/Dockerfile | 23 -------- python_wrapper/buildscripts/compile.sh | 22 -------- .../buildscripts/setup_utils/__init__.py | 52 ------------------ python_wrapper/buildscripts/test-wheel.sh | 26 --------- python_wrapper/buildscripts/wheel-linux.sh | 47 ---------------- python_wrapper/dockerbuild.sh | 12 ---- 7 files changed, 27 insertions(+), 210 deletions(-) delete mode 100644 python_wrapper/Dockerfile delete mode 100755 python_wrapper/buildscripts/compile.sh delete mode 100644 python_wrapper/buildscripts/setup_utils/__init__.py delete mode 100644 python_wrapper/buildscripts/test-wheel.sh delete mode 100755 python_wrapper/buildscripts/wheel-linux.sh delete mode 100755 python_wrapper/dockerbuild.sh diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 289f3e27f..899c86971 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -24,7 +24,7 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? - container: wheelmaker_2_28:0.1 + container: eccr.ecmwf.int/wheelmaker/2_28:latest name: Build manylinux_2_28 @@ -41,7 +41,7 @@ jobs: path: /build/wheel/*.whl # TODO other python versions, once the above is correct. - # NOTE if Matrix, then break into (compile & upload) ; (wheel & upload)[matix] steps + # NOTE if Matrix, then break into (compile) && (wheel & upload)@[matix] steps test: @@ -53,7 +53,7 @@ jobs: name: Test with ${{ matrix.python-version }} runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: wheelmaker_2_28:0.1 + container: eccr.ecmwf.int/wheelmaker/2_28:latest steps: - uses: actions/checkout@v2 - uses: actions/download-artifact@v4 @@ -61,28 +61,27 @@ jobs: name: wheel-manylinux2_28-${{ matrix.python-version }} - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} -# TODO enable and test -# deploy: -# -# if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} -# needs: [test, build] -# strategy: -# fail-fast: false -# matrix: -# python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable -# -# name: Deploy wheel ${{ matrix.python-version }} -# runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] -# container: wheelmaker_2_28:0.1 -# steps: -# - run: mkdir artifact-${{ matrix.python-version }} -# - uses: actions/checkout@v2 -# - uses: actions/download-artifact@v4 -# with: -# name: wheel-manylinux2_28-${{ matrix.python-version }} -# path: artifact-${{ matrix.python-version }} -# - run: | -# /buildsripts/upload-twine.sh ${{ matrix.python-version }} -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + deploy: + + if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} + needs: [test, build] + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable + + name: Deploy wheel ${{ matrix.python-version }} + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + container: eccr.ecmwf.int/wheelmaker/2_28:latest + steps: + - run: mkdir artifact-${{ matrix.python-version }} + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v4 + with: + name: wheel-manylinux2_28-${{ matrix.python-version }} + path: artifact-${{ matrix.python-version }} + - run: | + /buildsripts/upload-twine.sh ${{ matrix.python-version }} + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/python_wrapper/Dockerfile b/python_wrapper/Dockerfile deleted file mode 100644 index df68acad5..000000000 --- a/python_wrapper/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -# this docker image contains utilities for building shared libraries and wrap them in python wheel, test and publish - -# TODO parametrize this via build arg to support other manylinuxes -from quay.io/pypa/manylinux_2_28_x86_64:2024.11.24-1 - -run \ - yum install -y bison flex \ - && curl -LsSf https://astral.sh/uv/install.sh | sh \ - && mkdir /src /target \ - && git clone --branch master --depth=1 https://github.com/ecmwf/ecbuild.git /src/ecbuild -copy buildscripts /buildscripts - -# we workdir in /src so that the action checkout result ends up where expected -workdir /src - diff --git a/python_wrapper/buildscripts/compile.sh b/python_wrapper/buildscripts/compile.sh deleted file mode 100755 index ef58b8efa..000000000 --- a/python_wrapper/buildscripts/compile.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -set -euo pipefail - -# this script is configured by sourcing its first param. Namely, we expect the following: -# $NAME -- name of the project being compiled. Used for /src/$NAME and /target/$NAME, should thus correspond to git repo name -# $CMAKE_PARAMS -- passed to ecbuild after the `--`. Eg "-DENABLE_THIS=1 -DENABLE_THAT=0" -# $PYPROJECT_DIR -- ignored - -source $1 - -rm -rf /build && mkdir /build && cd /build -/src/ecbuild/bin/ecbuild --prefix=/target/$NAME -- $CMAKE_PARAMS /src/$NAME -make -j10 -make install diff --git a/python_wrapper/buildscripts/setup_utils/__init__.py b/python_wrapper/buildscripts/setup_utils/__init__.py deleted file mode 100644 index 82e5c6760..000000000 --- a/python_wrapper/buildscripts/setup_utils/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -""" -Utilities for setup invocation for shared-libs-only python wheels - -Customization should mostly happen in setup.cfg - -Resulting package's name & version come from env vars named NAME and VERSION -""" - -from setuptools import setup, find_packages -from setuptools.dist import Distribution -from wheel.bdist_wheel import bdist_wheel -import pathlib -import os - -class BinaryDistribution(Distribution): - # we do this for making the packager aware of presence of binary modules, to include - # the python version in the classifier - def has_ext_modules(foo): - return True - -class bdist_wheel_ext(bdist_wheel): - # This forces the platform tag from linux_x64_64 to manylinux_2_28 - # There is a chance that this is wrong, ie, the wheels are actually not compatible - # More reliable would be to auditwheel this, eg, - # LD_LIBRARY_PATH=/target/$NAME/lib64/ auditwheel repair dist/*whl --plat manylinux_2_28_x86_64 -L "libs" - # However, the problem is that this messes up with the whole idea of creating a chain of wheels - # Thus we would need to rip out the added .so files out and fix the rpaths, only making sure we preserve - # the right .so files in case auditwheel decided to make changes (this manifests as eg `libeckit-xxyyzz.so` - # appearing in the auditwheel-outputted wheel, alongside the original). Alternatively, we could replicate - # what auditwheel is doing -- in our original compilation, thus address the incompatibility at the root - def get_tag(self): - python, abi, plat = bdist_wheel.get_tag(self) - return python, abi, "manylinux_2_28_x86_64" - -def plain_setup(): - setup( - name=os.environ["NAME"], - version=os.environ["VERSION"], - package_dir={"": "src"}, - packages=find_packages(where="src"), - package_data={"": ["*.so"]}, - distclass=BinaryDistribution, - cmdclass={"bdist_wheel": bdist_wheel_ext}, - ) diff --git a/python_wrapper/buildscripts/test-wheel.sh b/python_wrapper/buildscripts/test-wheel.sh deleted file mode 100644 index 792f9b3a3..000000000 --- a/python_wrapper/buildscripts/test-wheel.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -# **** - -# just tests whether the package can be installed, imported, and __version__ matches what is expected - -set -euo pipefail - -source $1 -PYTHON="python$2" - -uv venv --python $PYTHON /tmp/venv -source /tmp/venv/bin/activate -uv pip install ./wheel-manylinux2_28-$2 - -INSTALLED_VERSION=$(python -c "import ${NAME}libs as l; print(l.__version__)") -EXPECTED_VERSION=$(cat /src/$NAME/VERSION) - -test "$INSTALLED_VERSION" == "$EXPECTED_VERSION" diff --git a/python_wrapper/buildscripts/wheel-linux.sh b/python_wrapper/buildscripts/wheel-linux.sh deleted file mode 100755 index c0f39b3a7..000000000 --- a/python_wrapper/buildscripts/wheel-linux.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -set -euo pipefail - -# this script is configured by sourcing its first param. Namely, we expect the following: -# - $NAME -- name of the project being compiled. Used for /target/$NAME/lib64 where the shared libraries are expected, and for naming the python module `$NAME-libs` -# - $CMAKE_PARAMS -- ignored -# - $PYPROJECT_DIR -- where pyproject.toml/setup.py are expected, prefixed with src/$NAME. Param to `python -m build` -# - $FINDLIBS_DEPENDENCIES -- names of python modules (presumably with dylibs) that this one depends on, to be utilized by `findlibs` -# additionally: -# - second param is the target python version, in the x.y format -# - version, license, authors and readme are expected to be root-located in src/$NAME - -source $1 -PYTHON="python$2" -PYPROJECT_DIR=/src/$NAME/$PYPROJECT_DIR - -mkdir -p $PYPROJECT_DIR/src -ln -s /target/$NAME/lib64 $PYPROJECT_DIR/src/${NAME}libs -ln -s ../LICENSE $PYPROJECT_DIR -ln -s ../README.md $PYPROJECT_DIR -ln -s ../AUTHORS $PYPROJECT_DIR -VERSION=$(cat /src/$NAME/VERSION) -echo "__version__ = '$VERSION'" > $PYPROJECT_DIR/src/${NAME}libs/__init__.py -echo "findlibs_dependencies = $FINDLIBS_DEPENDENCIES" >> $PYPROJECT_DIR/src/${NAME}libs/__init__.py - -for e in $(find /target/$NAME/lib64 -name '*.so'); do - # 1/ if there were some dependencies on other libraries from ecmwf stack, we patch the rpath to locate them at runtime - # 1a/ we actually dont, we rely on findlibs_dependencies instead - # | sed 's#/target/\([^/]*\)/lib64#$ORIGIN/../\1libs#g' - # 2/ we change $ORIGIN/../lib64 to just $ORIGIN, for the self-reference within the package - # TODO just force the $ORIGIN here universally? - RPATH_MODIF=$(readelf -d $e | grep "RPATH\|RUNPATH" | sed 's/.*\[\(.*\)\]/\1/' | sed 's#$ORIGIN/../lib64#$ORIGIN#g') - patchelf --set-rpath "$RPATH_MODIF" $e -done - -PYTHONPATH=/buildscripts NAME=$NAME VERSION=$VERSION uv run --python $PYTHON python -m build --installer uv --wheel $PYPROJECT_DIR - -mkdir /build/wheel -mv $PYPROJECT_DIR/dist/*whl /build/wheel diff --git a/python_wrapper/dockerbuild.sh b/python_wrapper/dockerbuild.sh deleted file mode 100755 index bb4cdec7f..000000000 --- a/python_wrapper/dockerbuild.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR -docker build -t wheelmaker_2_28:0.2 . From 1578c0317e86ccfe0b8a7b847384bfaf737d2f6c Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 9 Dec 2024 15:44:30 +0100 Subject: [PATCH 06/12] support dependency generation --- .github/workflows/build-wheel-linux.yml | 77 ++++++------------------- README.md | 4 +- python_wrapper/buildconfig | 4 +- python_wrapper/setup.cfg | 1 + 4 files changed, 23 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 899c86971..6120dbdda 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -19,69 +19,28 @@ on: # TODO automation trigger jobs: - build: - - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? - container: eccr.ecmwf.int/wheelmaker/2_28:latest - name: Build manylinux_2_28 - - steps: - - uses: actions/checkout@v2 - - run: /buildscripts/compile.sh ./eckit/python_wrapper/buildconfig - - ################################################################ - - run: /buildscripts/wheel-linux.sh ./eckit/python_wrapper/buildconfig 3.11 - - uses: actions/upload-artifact@v4 - name: Upload wheel 3.11 - with: - name: wheel-manylinux2_28-3.11 - path: /build/wheel/*.whl - - # TODO other python versions, once the above is correct. - # NOTE if Matrix, then break into (compile) && (wheel & upload)@[matix] steps - - test: - - needs: build - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable - - name: Test with ${{ matrix.python-version }} runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: eccr.ecmwf.int/wheelmaker/2_28:latest - steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2_28-${{ matrix.python-version }} - - run: /buildscripts/test-wheel.sh ${{ matrix.python-version }} - - deploy: - - if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }} - needs: [test, build] - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] # TODO enable - - name: Deploy wheel ${{ matrix.python-version }} - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - container: eccr.ecmwf.int/wheelmaker/2_28:latest + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:0.9 + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} steps: - - run: mkdir artifact-${{ matrix.python-version }} - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v4 - with: - name: wheel-manylinux2_28-${{ matrix.python-version }} - path: artifact-${{ matrix.python-version }} - - run: | - /buildsripts/upload-twine.sh ${{ matrix.python-version }} + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig + - run: cd /proj && /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/README.md b/README.md index 424338622..65447cfe9 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ It features facilities to easily handle, in a cross-platform way: * MPI object-oriented wrapper * linear algebra abstraction with multiple backends (BLAS, MKL, Armadillo, Eigen3) * advanced container classes - * space partition trees - * file-mapped arrays +* space partition trees +* file-mapped arrays Requirements diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index 95c47ea5a..2b0e4436d 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -11,6 +11,6 @@ # TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent NAME="eckit" -CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1" +CMAKE_PARAMS="-DENABLE_MPI=0 -DENABLE_ECKIT_GEO=1 -DENABLE_BUILD_TOOLS=OFF" PYPROJECT_DIR="python_wrapper" -FINDLIBS_DEPENDENCIES='[]' +DEPENDENCIES='[]' diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg index 2c746b9b2..fda226b6e 100644 --- a/python_wrapper/setup.cfg +++ b/python_wrapper/setup.cfg @@ -1,4 +1,5 @@ [metadata] description = "eckit" long_description = file: README.md +long_description_content_type = text/markdown author = file: AUTHORS From cc2f17fe3d0e2e03f14d43cdb5a3f63ff56f4955 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 12 Dec 2024 10:58:26 +0100 Subject: [PATCH 07/12] Macos wheel action --- .github/workflows/build-wheel-linux.yml | 8 +-- .github/workflows/build-wheel-macos.yml | 68 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build-wheel-macos.yml diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 6120dbdda..c146e1f7e 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -24,7 +24,7 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? container: - image: eccr.ecmwf.int/wheelmaker/2_28:0.9 + image: eccr.ecmwf.int/wheelmaker/2_28:0.10 credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} @@ -36,9 +36,9 @@ jobs: - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig - - run: cd /proj && /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /build/wheel/*whl - - run: cd /proj && /buildscripts/upload-pypi.sh /build/wheel/*whl + - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/build-wheel-macos.yml b/.github/workflows/build-wheel-macos.yml new file mode 100644 index 000000000..a11d75aec --- /dev/null +++ b/.github/workflows/build-wheel-macos.yml @@ -0,0 +1,68 @@ +# 1) +# install bison, flex, uv, python setuptools/build +# 2) +# run compile.sh + +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for MacOS + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + build: + name: Build macos wheel + # TODO enable after all tested + # strategy: + # matrix: + # arch_type: [ARM64, X64] + # runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] + runs-on: [self-hosted, macOS, ARM64 ] + steps: + # TODO convert this to be matrix-friendly for python versions. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: | + if [ -z "$(which uv)" ] ; then curl -LsSf https://astral.sh/uv/install.sh | sh ; fi + rm -rf ecbuild wheelmaker + git clone --depth=1 https://github.com/ecmwf/ecbuild ecbuild + # git clone --depth=1 --branch="wheelmaker" https://github.com/ecmwf/ci-utils wheelmaker # TODO use token here to get rid of the checkout action below + - uses: actions/checkout@v4 + with: + repository: ecmwf/ci-utils + ref: wheelmaker + path: ci-utils + token: ${{ secrets.GH_REPO_READ_TOKEN }} + - run: rm -rf proj && git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY proj + - run: | + rm -rf /tmp/prereq* + cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: | + rm -rf /tmp/build + cd proj && PATH="$PATH:$GITHUB_WORKSPACE/ecbuild/bin/" $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/compile.sh ./python_wrapper/buildconfig + - run: | + uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build + rm -rf /tmp/build/wheel + cd proj && PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl + - run: cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github From f98793f9f9287c792aa216ce9fddfadd230890ac Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 16 Dec 2024 12:57:42 +0100 Subject: [PATCH 08/12] Python package name consolidation --- .github/workflows/build-wheel-linux.yml | 2 +- python_wrapper/setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index c146e1f7e..92fce3000 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -24,7 +24,7 @@ jobs: runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? container: - image: eccr.ecmwf.int/wheelmaker/2_28:0.10 + image: eccr.ecmwf.int/wheelmaker/2_28:latest credentials: username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg index fda226b6e..7af3c6f0f 100644 --- a/python_wrapper/setup.cfg +++ b/python_wrapper/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description = "eckit" +description = "eckitlib" long_description = file: README.md long_description_content_type = text/markdown author = file: AUTHORS From 12e34c3007116f795a7c50db3eb993ebecfdce80 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 10 Jan 2025 09:00:08 +0100 Subject: [PATCH 09/12] Switch to test pypi --- .github/workflows/build-wheel-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 92fce3000..d60437410 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -41,6 +41,6 @@ jobs: - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_PASSWORD: ${{ secrets.PYPI_TEST_API_TOKEN }} # NOTE temporary thing until all the mess gets cleared - run: rm -rf ./* ./.git ./.github From 52f5ec2f3bb08cac74b0f5261f19db6da98a7725 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 21 Jan 2025 15:34:11 +0100 Subject: [PATCH 10/12] Fix action --- .github/workflows/build-wheel-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index d60437410..158220b14 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -37,8 +37,8 @@ jobs: - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl - - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TEST_API_TOKEN }} From 750a12639235bd5a67121422cba4f5710d41af01 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 3 Feb 2025 12:01:50 +0100 Subject: [PATCH 11/12] Fix macos action --- .github/workflows/build-wheel-macos.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheel-macos.yml b/.github/workflows/build-wheel-macos.yml index a11d75aec..f35558d62 100644 --- a/.github/workflows/build-wheel-macos.yml +++ b/.github/workflows/build-wheel-macos.yml @@ -50,17 +50,15 @@ jobs: token: ${{ secrets.GH_REPO_READ_TOKEN }} - run: rm -rf proj && git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY proj - run: | - rm -rf /tmp/prereq* cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 - run: | - rm -rf /tmp/build cd proj && PATH="$PATH:$GITHUB_WORKSPACE/ecbuild/bin/" $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/compile.sh ./python_wrapper/buildconfig - run: | - uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build - rm -rf /tmp/build/wheel - cd proj && PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - - run: cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl - - run: cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + cd proj + rm -rf /tmp/buildvenv && uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build twine + PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh ./python_wrapper/buildconfig env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} From 2e384fabc11871ed7bcaf03e4f03224cf744e558 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 3 Feb 2025 16:25:54 +0100 Subject: [PATCH 12/12] Switch to prod pypi --- .github/workflows/build-wheel-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 158220b14..24a7add98 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -41,6 +41,6 @@ jobs: - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TEST_API_TOKEN }} + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} # NOTE temporary thing until all the mess gets cleared - run: rm -rf ./* ./.git ./.github