CI: Re-enable Emscripten/Pyodide CI job for NumPy

This commit performs the following actions: 1. Adds WASM builds to the CPU family for Meson configurations, but without SSE or SIMD instructions. 2. Removes CPU feature detection message for an unsupported architecture. 3. Enables `IEEE_QUAD_LE` longdouble format for the wasm32 target (cross-builds). 4. Enables run for Emscripten/Pyodide wheels by setting the `if:` condition to `true`. 5. Uses recursive submodules to ensure that vendored-meson is received. 6. Moves the Meson cross file to `tools/ci/emscripten/` (i.e., creates a separate Emscripten folder to store relevant files) 7. Adds a patch for vendored-meson detection for Pyodide and applies this Pyodide-meson patch in the Emscripten CI jobs 8. Builds wasm32 wheels without BLAS and LAPACK support (see numpy#24750 (comment)) 9. Forces coloured and prettified outputs for test runs Some of these changes have been copied with updates and suggestions received from numpy#24603 on 23/02/2024 and authorship is preserved with this commit. [skip cirrus] [skip circle] [skip azp] Co-Authored-By: Ralf Gommers <[email protected]>
agriyakhetarpal · Feb 27, 2024 · 891c3cc · 891c3cc
1 parent 43b61d7
commit 891c3cc
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 32 deletions.
diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml
@@ -1,6 +1,6 @@
 # To enable this workflow on a fork, comment out:
 #
-# if: github.repository == 'numpy/numpy'
+if: github.repository == 'numpy/numpy'
 name: Test Emscripten/Pyodide build
 
 on:
@@ -9,6 +9,9 @@ on:
       - main
       - maintenance/**
 
+env:
+  FORCE_COLOR: 3
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
@@ -20,50 +23,64 @@ jobs:
   build-wasm-emscripten:
     name: Build NumPy distribution for Pyodide
     runs-on: ubuntu-22.04
-    if: false  # NOTE: job disabled, it needs to be moved to Meson (see gh-24603)
-    # if: "github.repository == 'numpy/numpy'"
+    if: github.repository == 'numpy/numpy'
     env:
-      PYODIDE_VERSION: 0.23.1
+      PYODIDE_VERSION: 0.25.0
       # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
       # The appropriate versions can be found in the Pyodide repodata.json
       # "info" field, or in Makefile.envs:
       # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
-      PYTHON_VERSION: 3.11.2
-      EMSCRIPTEN_VERSION: 3.1.32
+      PYTHON_VERSION: 3.11.3
+      EMSCRIPTEN_VERSION: 3.1.46
       NODE_VERSION: 18
     steps:
-      - name: Checkout numpy
+      - name: Checkout NumPy
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          submodules: true
-          # versioneer.py requires the latest tag to be reachable. Here we
-          # fetch the complete history to get access to the tags.
-          # A shallow clone can work when the following issue is resolved:
-          # https://github.com/actions/checkout/issues/338
-          fetch-depth: 0
-
-      - name: set up python
+          submodules: recursive
+          # The fetch-tags input shall fetch tags for versioneer.py
+          # without the need to fetch the entire history, see
+          # https://github.com/actions/checkout#usage
+          fetch-tags: true
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         id: setup-python
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
+      - name: Set up Emscripten toolchain
+        uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
         with:
           version: ${{ env.EMSCRIPTEN_VERSION }}
           actions-cache-folder: emsdk-cache
 
       - name: Install pyodide-build
-        run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION
+        run: pip install "pydantic<2" pyodide-build==${{ env.PYODIDE_VERSION }}
+
+      - name: Find installation for pyodide-build
+        shell: python
+        run: |
+          import os
+          import pyodide_build
+
+          pyodide_build_path = pyodide_build.__file__[:-12]
+
+          env_file = os.getenv('GITHUB_ENV')
+
+          with open(env_file, "a") as myfile:
+              myfile.write(f"PYODIDE_BUILD_PATH={pyodide_build_path}\n")
+
+      - name: Apply patch(es) for pyodide-build installation
+        run: |
+          ls -a ${{ env.PYODIDE_BUILD_PATH }}
+          patch -d "${{ env.PYODIDE_BUILD_PATH }}" -p1 < tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch
 
-      - name: Build
+      - name: Build NumPy for Pyodide
         run: |
-          # Pyodide is still in the process of adding better/easier support for
-          # non-setup.py based builds.
-          cp pyproject.toml.setuppy pyproject.toml
-          CFLAGS=-g2 LDFLAGS=-g2 pyodide build
+          pyodide build -Cbuild-dir=build -Csetup-args="--cross-file=$PWD/tools/ci/emscripten/emscripten.meson.cross" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none"
 
-      - name: set up node
+      - name: Set up Node.js
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
           node-version: ${{ env.NODE_VERSION }}
@@ -73,11 +90,10 @@ jobs:
           pyodide venv .venv-pyodide
           source .venv-pyodide/bin/activate
           pip install dist/*.whl
-          python -c "import sys; print(sys.platform)"
           pip install -r requirements/emscripten_test_requirements.txt
 
       - name: Test NumPy for Pyodide
         run: |
           source .venv-pyodide/bin/activate
           cd ..
-          python numpy/runtests.py -n -vv
+          pytest --pyargs numpy -m "not slow"
diff --git a/.gitignore b/.gitignore
@@ -97,6 +97,10 @@ pip-wheel-metadata
 *.patch
 *.diff
 
+# Do not ignore the following patches: #
+########################################
+!tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch
+
 # OS generated files #
 ######################
 .DS_Store*

diff --git a/meson_cpu/meson.build b/meson_cpu/meson.build
@@ -92,30 +92,27 @@ min_features = {
   'ppc64': [],
   's390x': [],
   'arm': [],
-  'aarch64': [ASIMD]
+  'aarch64': [ASIMD],
+  'wasm32': [],
 }.get(cpu_family, [])
 if host_machine.endian() == 'little' and cpu_family == 'ppc64'
   min_features = [VSX2]
 endif
 
-# Used by build option 'max/native/detect'
+# Used by build option 'max'
 max_features_dict = {
   'x86': X86_FEATURES,
   'x86_64': X86_FEATURES,
   'ppc64': PPC64_FEATURES,
   's390x': S390X_FEATURES,
   'arm': ARM_FEATURES,
   'aarch64': ARM_FEATURES,
+  'wasm32': {},
 }.get(cpu_family, {})
 max_features = []
 foreach fet_name, fet_obj : max_features_dict
   max_features += [fet_obj]
 endforeach
-if max_features.length() == 0
-  message('Disabling CPU feature detection due to unsupported architecture: "' + cpu_family + '"')
-  CPU_CONF_BASELINE = 'none'
-  CPU_CONF_DISPATCH = 'none'
-endif
 
 parse_options = {
   'cpu-baseline': CPU_CONF_BASELINE,

diff --git a/tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch b/tools/ci/emscripten/0001-do-not-set-meson-environment-variable-pyodide-gh-4502.patch
@@ -0,0 +1,55 @@
+From e08ebf0e90f632547c8ff5b396ec0c4ddd65aad4 Mon Sep 17 00:00:00 2001
+From: Gyeongjae Choi <[email protected]>
+Date: Sat, 10 Feb 2024 03:28:01 +0900
+Subject: [PATCH] Update numpy to 1.26.4 and don't set MESON env variable
+ (#4502)
+
+From meson-python 0.15, $MESON env variable is used to overwrite the meson binary
+path. We don't want that behavior.
+---
+ pypabuild.py | 22 +++++++++++++++-------
+ 1 file changed, 15 insertions(+), 7 deletions(-)
+
+diff --git a/pypabuild.py b/pypabuild.py
+index 9d0107a8..6961b14e 100644
+--- a/pypabuild.py
++++ b/pypabuild.py
+@@ -40,6 +40,19 @@ AVOIDED_REQUIREMENTS = [
+     "patchelf",
+ ]
+
++# corresponding env variables for symlinks
++SYMLINK_ENV_VARS = {
++    "cc": "CC",
++    "c++": "CXX",
++    "ld": "LD",
++    "lld": "LLD",
++    "ar": "AR",
++    "gcc": "GCC",
++    "ranlib": "RANLIB",
++    "strip": "STRIP",
++    "gfortran": "FC",  # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables
++}
++
+
+ def _gen_runner(
+     cross_build_env: Mapping[str, str],
+@@ -207,13 +220,8 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]:
+             symlink_path.unlink()
+
+         symlink_path.symlink_to(pywasmcross_exe)
+-        if symlink == "c++":
+-            var = "CXX"
+-        elif symlink == "gfortran":
+-            var = "FC"  # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables
+-        else:
+-            var = symlink.upper()
+-        env[var] = str(symlink_path)
++        if symlink in SYMLINK_ENV_VARS:
++            env[SYMLINK_ENV_VARS[symlink]] = str(symlink_path)
+
+     return env
+
+-- 
+2.39.3 (Apple Git-145)
+
diff --git a/tools/ci/emscripten/emscripten.meson.cross b/tools/ci/emscripten/emscripten.meson.cross
@@ -0,0 +1,15 @@
+# compiler paths are omitted intentionally so we can override the compiler using environment variables
+[binaries]
+exe_wrapper = 'node'
+pkgconfig = 'pkg-config'
+
+[properties]
+needs_exe_wrapper = true
+skip_sanity_check = true
+longdouble_format = 'IEEE_QUAD_LE' # for numpy
+
+[host_machine]
+system = 'emscripten'
+cpu_family = 'wasm32'
+cpu = 'wasm'
+endian = 'little'