Build and test namespace_package wheel for emscripten-wasm32

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - emscripten-ci
   pull_request:
 
 concurrency:
@@ -332,6 +333,44 @@ jobs:
             python3 -c "from namespace_package import rust; assert rust.rust_func() == 14"
             python3 -c "from namespace_package import python; assert python.python_func() == 15"
 
+  emscripten:
+    name: emscripten
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: 3.11-dev
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          target: wasm32-unknown-emscripten
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 14
+      - run: pip install nox
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: |
+            .nox/emscripten
+          key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
+      - uses: Swatinem/rust-cache@v1
+        with:
+          key: cargo-emscripten-wasm32
+      - name: Build libpython
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: nox -s build-emscripten-libpython
+      - name: Build interpreter
+        run: nox -s build-emscripten-interpreter
+      - name: Build namespace wheel
+        run: nox -s build-emscripten-namespace-package-wheel
+      - name: Test
+        run: nox -s test-emscripten-namespace-package-wheel
+
+
   test-cibuildwheel:
     runs-on: macos-latest
     steps:

emscripten/.gitignore

@@ -0,0 +1,4 @@
+builddir
+main.*
+!main.c
+pybuilddir.txt

emscripten/Makefile

@@ -0,0 +1,148 @@
+CURDIR=$(abspath .)
+
+# These three are passed in from nox.
+BUILDROOT ?= $(CURDIR)/builddir
+PYMAJORMINORMICRO ?= 3.11.0
+PYPRERELEASE ?= b1# I'm not sure how to split 3.11.0b1 in Make.
+
+
+EMSCRIPTEN_VERSION ?= 3.1.13
+EMSCRIPTEN_VERSION_UNDERSCORE := $(subst .,_,$(EMSCRIPTEN_VERSION:v%=%))
+
+export PLATFORM_TRIPLET=wasm32-emscripten
+export SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET)
+export PLATFORM=emscripten_$(EMSCRIPTEN_VERSION_UNDERSCORE)_wasm32
+
+
+export EMSDKDIR = $(BUILDROOT)/emsdk
+
+# BASH_ENV tells bash to source emsdk_env.sh on startup.
+export BASH_ENV := $(CURDIR)/env.sh
+# Use bash to run each command so that env.sh will be used.
+SHELL := /bin/bash
+
+
+# Set version variables.
+version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
+PYMAJOR=$(word 1,$(version_tuple))
+PYMINOR=$(word 2,$(version_tuple))
+PYMICRO=$(word 3,$(version_tuple))
+PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE)
+PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
+
+
+PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
+PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
+PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
+
+PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib
+PYTHONINCLUDEDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/include
+
+NAMESPACE_PACKAGE_DIST_DIR=../examples/namespace_package/dist
+NAMESPACE_PACKAGE_WHEEL_NAME=namespace_package-0.1.0-cp$(PYMAJOR)$(PYMINOR)-cp$(PYMAJOR)$(PYMINOR)-$(PLATFORM).whl
+NAMESPACE_PACKAGE_WHEEL_PATH=$(NAMESPACE_PACKAGE_DIST_DIR)/$(NAMESPACE_PACKAGE_WHEEL_NAME)
+
+export CARGO_HOME ?= $(HOME)/.cargo
+export CARGO_BUILD_TARGET=wasm32-unknown-emscripten
+export CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=$(CURDIR)/emcc_wrapper.py
+export PYO3_CONFIG_FILE=$(CURDIR)/pyo3_config.ini
+export RUSTFLAGS=\
+	-C relocation-model=pic \
+	-C target-feature=+mutable-globals \
+	-C link-arg=-sSIDE_MODULE=1 \
+	-C link-arg=-sWASM_BIGINT 
+
+
+all: libpython
+
+namespace_package_wheel: $(NAMESPACE_PACKAGE_WHEEL_PATH)
+
+python-interpreter: interpreter/main.js
+
+libpython: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
+
+$(BUILDROOT)/.exists: 
+	mkdir -p $(BUILDROOT)
+	touch $@
+
+
+# Install emscripten
+$(EMSDKDIR)/.exists : $(BUILDROOT)/.exists
+	git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR)
+	$(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION)
+	cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1
+	$(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION)
+	touch $(EMSDKDIR)/.exists
+
+
+$(PYTHONTARBALL):
+	[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
+	wget -q -O $@ $(PYTHONURL)
+
+$(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
+	[ -d $(PYTHONBUILD) ] || ( \
+		mkdir -p $(dir $(PYTHONBUILD));\
+		tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
+	)
+	touch $@
+
+$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(EMSDKDIR)/.exists
+	cd $(PYTHONBUILD) && \
+	CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+	PLATFORM_TRIPLET="$(PLATFORM_TRIPLET)" \
+  	emconfigure ./configure -C \
+		--host=wasm32-unknown-emscripten \
+		--build=$(shell $(PYTHONBUILD)/config.guess) \
+		--with-emscripten-target=browser \
+		--enable-wasm-dynamic-linking \
+		--with-build-python=python3.11
+
+$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile
+	cd $(PYTHONBUILD) && \
+		emmake make -j3 libpython$(PYMAJORMINOR).a
+
+	# Generate sysconfigdata
+	_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars
+
+	mkdir -p $(PYTHONINCLUDEDIR)
+	mkdir -p $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
+	mkdir -p $(PYTHONLIBDIR)/sysconfigdata/
+	# Copy libexpat.a, libmpdec.a, and libpython3.11.a
+	# In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS
+	find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
+	# Install Python stdlib
+	cp -r $(PYTHONBUILD)/Lib/* $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
+	cp -r $(PYTHONBUILD)/Include/* $(PYTHONINCLUDEDIR)
+	cp -r $(PYTHONBUILD)/pyconfig.h $(PYTHONINCLUDEDIR)
+	cp `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
+	cp `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(PYTHONLIBDIR)/sysconfigdata/
+
+
+interpreter/main.js: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a interpreter/main.c interpreter/pre.js
+	cd interpreter && emcc -c main.c -o main.o -I$(PYTHONINCLUDEDIR) -fPIC
+	cd interpreter &&  \
+	emcc main.o -o main.js \
+		-L$(PYTHONLIBDIR) \
+		-lpython$(PYMAJORMINOR) \
+		-lmpdec \
+		-lexpat \
+		-s MAIN_MODULE=1 \
+		-s WASM_BIGINT \
+		--preload-file $(PYTHONLIBDIR)/python$(PYMAJORMINOR)@/lib/python$(PYMAJORMINOR) \
+		--pre-js pre.js \
+		-lnodefs.js
+
+
+
+$(NAMESPACE_PACKAGE_WHEEL_PATH) : $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
+	cd ../examples/namespace_package && \
+		RUSTUP_TOOLCHAIN=nightly \
+		_SETUPTOOLSRUST_BUILD_STD=1 \
+		PYTHONPATH=$(PYTHONLIBDIR)/sysconfigdata/ \
+		_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) \
+		_PYTHON_HOST_PLATFORM=$(PLATFORM) \
+		python setup.py bdist_wheel
+
+
+clean:
+	rm -rf $(BUILDROOT)

emscripten/emcc_wrapper.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+
+
+def update_args(args):
+    # https://github.com/emscripten-core/emscripten/issues/17109
+    args.insert(0, "-Wl,--no-whole-archive")
+
+    # Remove -s ASSERTIONS=1
+    # See https://github.com/rust-lang/rust/pull/97928
+    for i in range(len(args)):
+        if "ASSERTIONS" in args[i]:
+            del args[i - 1 : i + 1]
+            break
+
+    # remove -lc. Not sure if it makes a difference but -lc doesn't belong here.
+    # https://github.com/emscripten-core/emscripten/issues/17191
+    for i in reversed(range(len(args))):
+        if args[i] == "c" and args[i - 1] == "-l":
+            del args[i - 1 : i + 1]
+
+    # Prevent a bunch of errors caused by buggy behavior in
+    # `esmcripten/tools/building.py:lld_flags_for_executable` REQUIRED_EXPORTS
+    # contains symbols that should come from the main module.
+    # https://github.com/emscripten-core/emscripten/issues/17202
+    args.append("-sERROR_ON_UNDEFINED_SYMBOLS=0")
+    # Seems like --no-entry should be implied by SIDE_MODULE but apparently it
+    # isn't?
+    args.append("-Wl,--no-entry")
+
+    return args
+
+
+def main(args):
+    args = update_args(args)
+    return subprocess.call(["emcc"] + args)
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    sys.exit(main(args))

emscripten/env.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Activate emsdk environment. emsdk_env.sh writes a lot to stderr so we suppress
+# the output. This also prevents it from complaining when emscripten isn't yet
+# installed.
+source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true

emscripten/interpreter/main.c

@@ -0,0 +1,45 @@
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include <assert.h>
+#include <emscripten.h>
+
+
+#define FAIL_IF_STATUS_EXCEPTION(status)                                       \
+  if (PyStatus_Exception(status)) {                                            \
+    goto finally;                                                              \
+  }
+
+
+// Initialize python. exit() and print message to stderr on failure.
+static void
+initialize_python()
+{
+  int success = 0;
+  PyStatus status;
+  PyConfig config;
+  PyConfig_InitPythonConfig(&config);
+  status = PyConfig_SetBytesString(&config, &config.home, "/");
+  FAIL_IF_STATUS_EXCEPTION(status);
+  config.write_bytecode = 0;
+  status = Py_InitializeFromConfig(&config);
+  FAIL_IF_STATUS_EXCEPTION(status);
+
+  success = 1;
+finally:
+  PyConfig_Clear(&config);
+  if (!success) {
+    // This will exit().
+    Py_ExitStatusException(status);
+  }
+}
+
+int
+main(int argc, char** argv)
+{
+  initialize_python();
+  emscripten_exit_with_live_runtime();
+  // More convenient to construct a multiline string from Javascript than in C,
+  // so leave the actual action in pre.js
+  return 0;
+}
+

emscripten/interpreter/pre.js

@@ -0,0 +1,16 @@
+const nodefs = require("fs");
+
+function runPython(string) {
+  let ptr = Module.stringToNewUTF8(string);
+  let result = Module._PyRun_SimpleString(ptr);
+  Module._free(ptr);
+  return result;
+}
+
+Module.postRun = function () {
+  const test_code = nodefs.readFileSync("test.py", { encoding: "utf8" });
+  FS.mkdir("/package_dir");
+  FS.mount(NODEFS, { root: process.argv[2] }, "/package_dir");
+  let errcode = runPython(test_code);
+  process.exit(errcode);
+};

emscripten/interpreter/test.py

@@ -0,0 +1,13 @@
+import sys
+
+sys.path.append("/package_dir")
+
+from namespace_package import python
+
+print("python.python_func()", python.python_func())
+assert python.python_func() == 15
+
+from namespace_package import rust
+
+print("rust.rust_func()", rust.rust_func())
+assert rust.rust_func() == 14

emscripten/pyo3_config.ini

@@ -0,0 +1,7 @@
+implementation=CPython
+version=3.10
+shared=true
+abi3=false
+lib_name=python3.10
+pointer_width=32
+suppress_build_script_link_lines=false