diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d104046..47a4eb6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - emscripten-ci pull_request: concurrency: @@ -332,6 +333,44 @@ jobs: python3 -c "from namespace_package import rust; assert rust.rust_func() == 14" python3 -c "from namespace_package import python; assert python.python_func() == 15" + emscripten: + name: emscripten + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + id: setup-python + with: + python-version: 3.11-dev + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + target: wasm32-unknown-emscripten + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: pip install nox + - uses: actions/cache@v3 + id: cache + with: + path: | + .nox/emscripten + key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }} + - uses: Swatinem/rust-cache@v1 + with: + key: cargo-emscripten-wasm32 + - name: Build libpython + if: steps.cache.outputs.cache-hit != 'true' + run: nox -s build-emscripten-libpython + - name: Build interpreter + run: nox -s build-emscripten-interpreter + - name: Build namespace wheel + run: nox -s build-emscripten-namespace-package-wheel + - name: Test + run: nox -s test-emscripten-namespace-package-wheel + + test-cibuildwheel: runs-on: macos-latest steps: diff --git a/emscripten/.gitignore b/emscripten/.gitignore new file mode 100644 index 00000000..9b0247b0 --- /dev/null +++ b/emscripten/.gitignore @@ -0,0 +1,4 @@ +builddir +main.* +!main.c +pybuilddir.txt diff --git a/emscripten/Makefile b/emscripten/Makefile new file mode 100644 index 00000000..118aa69e --- /dev/null +++ b/emscripten/Makefile @@ -0,0 +1,148 @@ +CURDIR=$(abspath .) + +# These three are passed in from nox. +BUILDROOT ?= $(CURDIR)/builddir +PYMAJORMINORMICRO ?= 3.11.0 +PYPRERELEASE ?= b1# I'm not sure how to split 3.11.0b1 in Make. + + +EMSCRIPTEN_VERSION ?= 3.1.13 +EMSCRIPTEN_VERSION_UNDERSCORE := $(subst .,_,$(EMSCRIPTEN_VERSION:v%=%)) + +export PLATFORM_TRIPLET=wasm32-emscripten +export SYSCONFIG_NAME=_sysconfigdata__emscripten_$(PLATFORM_TRIPLET) +export PLATFORM=emscripten_$(EMSCRIPTEN_VERSION_UNDERSCORE)_wasm32 + + +export EMSDKDIR = $(BUILDROOT)/emsdk + +# BASH_ENV tells bash to source emsdk_env.sh on startup. +export BASH_ENV := $(CURDIR)/env.sh +# Use bash to run each command so that env.sh will be used. +SHELL := /bin/bash + + +# Set version variables. +version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%)) +PYMAJOR=$(word 1,$(version_tuple)) +PYMINOR=$(word 2,$(version_tuple)) +PYMICRO=$(word 3,$(version_tuple)) +PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE) +PYMAJORMINOR=$(PYMAJOR).$(PYMINOR) + + +PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz +PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz +PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION) + +PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib +PYTHONINCLUDEDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/include + +NAMESPACE_PACKAGE_DIST_DIR=../examples/namespace_package/dist +NAMESPACE_PACKAGE_WHEEL_NAME=namespace_package-0.1.0-cp$(PYMAJOR)$(PYMINOR)-cp$(PYMAJOR)$(PYMINOR)-$(PLATFORM).whl +NAMESPACE_PACKAGE_WHEEL_PATH=$(NAMESPACE_PACKAGE_DIST_DIR)/$(NAMESPACE_PACKAGE_WHEEL_NAME) + +export CARGO_HOME ?= $(HOME)/.cargo +export CARGO_BUILD_TARGET=wasm32-unknown-emscripten +export CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=$(CURDIR)/emcc_wrapper.py +export PYO3_CONFIG_FILE=$(CURDIR)/pyo3_config.ini +export RUSTFLAGS=\ + -C relocation-model=pic \ + -C target-feature=+mutable-globals \ + -C link-arg=-sSIDE_MODULE=1 \ + -C link-arg=-sWASM_BIGINT + + +all: libpython + +namespace_package_wheel: $(NAMESPACE_PACKAGE_WHEEL_PATH) + +python-interpreter: interpreter/main.js + +libpython: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a + +$(BUILDROOT)/.exists: + mkdir -p $(BUILDROOT) + touch $@ + + +# Install emscripten +$(EMSDKDIR)/.exists : $(BUILDROOT)/.exists + git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR) + $(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION) + cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1 + $(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION) + touch $(EMSDKDIR)/.exists + + +$(PYTHONTARBALL): + [ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads + wget -q -O $@ $(PYTHONURL) + +$(PYTHONBUILD)/.patched: $(PYTHONTARBALL) + [ -d $(PYTHONBUILD) ] || ( \ + mkdir -p $(dir $(PYTHONBUILD));\ + tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \ + ) + touch $@ + +$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(EMSDKDIR)/.exists + cd $(PYTHONBUILD) && \ + CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ + PLATFORM_TRIPLET="$(PLATFORM_TRIPLET)" \ + emconfigure ./configure -C \ + --host=wasm32-unknown-emscripten \ + --build=$(shell $(PYTHONBUILD)/config.guess) \ + --with-emscripten-target=browser \ + --enable-wasm-dynamic-linking \ + --with-build-python=python3.11 + +$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile + cd $(PYTHONBUILD) && \ + emmake make -j3 libpython$(PYMAJORMINOR).a + + # Generate sysconfigdata + _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars + + mkdir -p $(PYTHONINCLUDEDIR) + mkdir -p $(PYTHONLIBDIR)/python$(PYMAJORMINOR) + mkdir -p $(PYTHONLIBDIR)/sysconfigdata/ + # Copy libexpat.a, libmpdec.a, and libpython3.11.a + # In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS + find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \; + # Install Python stdlib + cp -r $(PYTHONBUILD)/Lib/* $(PYTHONLIBDIR)/python$(PYMAJORMINOR) + cp -r $(PYTHONBUILD)/Include/* $(PYTHONINCLUDEDIR) + cp -r $(PYTHONBUILD)/pyconfig.h $(PYTHONINCLUDEDIR) + cp `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(PYTHONLIBDIR)/python$(PYMAJORMINOR) + cp `cat pybuilddir.txt`/$(SYSCONFIG_NAME).py $(PYTHONLIBDIR)/sysconfigdata/ + + +interpreter/main.js: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a interpreter/main.c interpreter/pre.js + cd interpreter && emcc -c main.c -o main.o -I$(PYTHONINCLUDEDIR) -fPIC + cd interpreter && \ + emcc main.o -o main.js \ + -L$(PYTHONLIBDIR) \ + -lpython$(PYMAJORMINOR) \ + -lmpdec \ + -lexpat \ + -s MAIN_MODULE=1 \ + -s WASM_BIGINT \ + --preload-file $(PYTHONLIBDIR)/python$(PYMAJORMINOR)@/lib/python$(PYMAJORMINOR) \ + --pre-js pre.js \ + -lnodefs.js + + + +$(NAMESPACE_PACKAGE_WHEEL_PATH) : $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a + cd ../examples/namespace_package && \ + RUSTUP_TOOLCHAIN=nightly \ + _SETUPTOOLSRUST_BUILD_STD=1 \ + PYTHONPATH=$(PYTHONLIBDIR)/sysconfigdata/ \ + _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIG_NAME) \ + _PYTHON_HOST_PLATFORM=$(PLATFORM) \ + python setup.py bdist_wheel + + +clean: + rm -rf $(BUILDROOT) diff --git a/emscripten/emcc_wrapper.py b/emscripten/emcc_wrapper.py new file mode 100755 index 00000000..2bf73fa1 --- /dev/null +++ b/emscripten/emcc_wrapper.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import subprocess +import sys + + +def update_args(args): + # https://github.com/emscripten-core/emscripten/issues/17109 + args.insert(0, "-Wl,--no-whole-archive") + + # Remove -s ASSERTIONS=1 + # See https://github.com/rust-lang/rust/pull/97928 + for i in range(len(args)): + if "ASSERTIONS" in args[i]: + del args[i - 1 : i + 1] + break + + # remove -lc. Not sure if it makes a difference but -lc doesn't belong here. + # https://github.com/emscripten-core/emscripten/issues/17191 + for i in reversed(range(len(args))): + if args[i] == "c" and args[i - 1] == "-l": + del args[i - 1 : i + 1] + + # Prevent a bunch of errors caused by buggy behavior in + # `esmcripten/tools/building.py:lld_flags_for_executable` REQUIRED_EXPORTS + # contains symbols that should come from the main module. + # https://github.com/emscripten-core/emscripten/issues/17202 + args.append("-sERROR_ON_UNDEFINED_SYMBOLS=0") + # Seems like --no-entry should be implied by SIDE_MODULE but apparently it + # isn't? + args.append("-Wl,--no-entry") + + return args + + +def main(args): + args = update_args(args) + return subprocess.call(["emcc"] + args) + + +if __name__ == "__main__": + args = sys.argv[1:] + sys.exit(main(args)) diff --git a/emscripten/env.sh b/emscripten/env.sh new file mode 100644 index 00000000..87b7b551 --- /dev/null +++ b/emscripten/env.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Activate emsdk environment. emsdk_env.sh writes a lot to stderr so we suppress +# the output. This also prevents it from complaining when emscripten isn't yet +# installed. +source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true diff --git a/emscripten/interpreter/main.c b/emscripten/interpreter/main.c new file mode 100644 index 00000000..0bcb3721 --- /dev/null +++ b/emscripten/interpreter/main.c @@ -0,0 +1,45 @@ +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include +#include + + +#define FAIL_IF_STATUS_EXCEPTION(status) \ + if (PyStatus_Exception(status)) { \ + goto finally; \ + } + + +// Initialize python. exit() and print message to stderr on failure. +static void +initialize_python() +{ + int success = 0; + PyStatus status; + PyConfig config; + PyConfig_InitPythonConfig(&config); + status = PyConfig_SetBytesString(&config, &config.home, "/"); + FAIL_IF_STATUS_EXCEPTION(status); + config.write_bytecode = 0; + status = Py_InitializeFromConfig(&config); + FAIL_IF_STATUS_EXCEPTION(status); + + success = 1; +finally: + PyConfig_Clear(&config); + if (!success) { + // This will exit(). + Py_ExitStatusException(status); + } +} + +int +main(int argc, char** argv) +{ + initialize_python(); + emscripten_exit_with_live_runtime(); + // More convenient to construct a multiline string from Javascript than in C, + // so leave the actual action in pre.js + return 0; +} + diff --git a/emscripten/interpreter/pre.js b/emscripten/interpreter/pre.js new file mode 100644 index 00000000..c5d51bc7 --- /dev/null +++ b/emscripten/interpreter/pre.js @@ -0,0 +1,16 @@ +const nodefs = require("fs"); + +function runPython(string) { + let ptr = Module.stringToNewUTF8(string); + let result = Module._PyRun_SimpleString(ptr); + Module._free(ptr); + return result; +} + +Module.postRun = function () { + const test_code = nodefs.readFileSync("test.py", { encoding: "utf8" }); + FS.mkdir("/package_dir"); + FS.mount(NODEFS, { root: process.argv[2] }, "/package_dir"); + let errcode = runPython(test_code); + process.exit(errcode); +}; diff --git a/emscripten/interpreter/test.py b/emscripten/interpreter/test.py new file mode 100644 index 00000000..31d409d2 --- /dev/null +++ b/emscripten/interpreter/test.py @@ -0,0 +1,13 @@ +import sys + +sys.path.append("/package_dir") + +from namespace_package import python + +print("python.python_func()", python.python_func()) +assert python.python_func() == 15 + +from namespace_package import rust + +print("rust.rust_func()", rust.rust_func()) +assert rust.rust_func() == 14 diff --git a/emscripten/pyo3_config.ini b/emscripten/pyo3_config.ini new file mode 100644 index 00000000..a91b4f9c --- /dev/null +++ b/emscripten/pyo3_config.ini @@ -0,0 +1,7 @@ +implementation=CPython +version=3.10 +shared=true +abi3=false +lib_name=python3.10 +pointer_width=32 +suppress_build_script_link_lines=false diff --git a/noxfile.py b/noxfile.py index f12faec4..25588f47 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,8 +1,11 @@ import os +import re +import sys import tarfile from glob import glob from pathlib import Path + import nox @@ -35,3 +38,104 @@ def mypy(session: nox.Session): def test(session: nox.Session): session.install("pytest", ".") session.run("pytest", "setuptools_rust", *session.posargs) + + +class EmscriptenInfo: + def __init__(self): + rootdir = Path(__file__).parent + self.emscripten_dir = rootdir / "emscripten" + self.builddir = rootdir / ".nox/emscripten" + self.builddir.mkdir(exist_ok=True, parents=True) + + self.pyversion = sys.version.split()[0] + self.pymajor, self.pyminor, self.pymicro = self.pyversion.split(".") + self.pymicro, self.pydev = re.match( + "([0-9]*)([^0-9].*)?", self.pymicro + ).groups() + if self.pydev is None: + self.pydev = "" + + self.pymajorminor = f"{self.pymajor}.{self.pyminor}" + self.pymajorminormicro = f"{self.pymajorminor}.{self.pymicro}" + self.emscripten_version = "3.1.13" + + underscore_emscripten_version = self.emscripten_version.replace(".", "_") + cp = f"cp{self.pymajor}{self.pyminor}" + self.wheel_suffix = ( + f"{cp}-{cp}-emscripten_{underscore_emscripten_version}_wasm32.whl" + ) + + def build(self, session, target): + session.run( + "make", + "-C", + str(self.emscripten_dir), + target, + f"BUILDROOT={self.builddir}", + f"PYMAJORMINORMICRO={self.pymajorminormicro}", + f"PYPRERELEASE={self.pydev}", + f"EMSCRIPTEN_VERSION={self.emscripten_version}", + external=True, + ) + + +@nox.session(name="build-emscripten-libpython", venv_backend="none") +def build_emscripten_libpython(session: nox.Session): + info = EmscriptenInfo() + info.build(session, "libpython") + + +@nox.session(name="build-emscripten-namespace-package-wheel") +def build_emscripten_namespace_package_wheel(session: nox.Session): + session.install(".") + info = EmscriptenInfo() + session.run( + "rustup", + "target", + "add", + "wasm32-unknown-emscripten", + "--toolchain", + "nightly", + external=True, + ) + session.run( + "rustup", + "component", + "add", + "rust-src", + "--toolchain", + "nightly", + external=True, + ) + info.build(session, "namespace_package_wheel") + + +@nox.session(name="build-emscripten-interpreter", venv_backend="none") +def build_emscripten_interpreter(session: nox.Session): + info = EmscriptenInfo() + info.build(session, "python-interpreter") + + +@nox.session(name="test-emscripten-namespace-package-wheel") +def test_emscripten_namespace_package_wheel(session: nox.Session): + session.install("wheel") + + info = EmscriptenInfo() + dist_dir = Path("examples/namespace_package/dist/").resolve() + pkg = "namespace_package-0.1.0" + with session.chdir(dist_dir): + session.run( + "wheel", + "unpack", + f"{pkg}-{info.wheel_suffix}", + external=True, + ) + + with session.chdir("emscripten/interpreter"): + session.run( + "node", + "--experimental-wasm-bigint", + "main.js", + str(dist_dir / pkg), + external=True, + ) diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index d3bd8c0d..930e94e3 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -258,10 +258,18 @@ def build_extension( f"unable to find executable '{name}' in '{artifacts_dir}'" ) else: - if sys.platform == "win32" or sys.platform == "cygwin": + print( + "\n\n=====================\nsysconfig.get_platform():", + sysconfig.get_platform(), + "\n===================\n", + ) + platform = sysconfig.get_platform() + if "win" in platform: dylib_ext = "dll" - elif sys.platform == "darwin": + elif platform.startswith("macosx"): dylib_ext = "dylib" + elif "wasm32" in platform: + dylib_ext = "wasm" else: dylib_ext = "so" @@ -502,6 +510,9 @@ def _cargo_args( if ext.args is not None: args.extend(ext.args) + if "_SETUPTOOLSRUST_BUILD_STD" in os.environ: + args.append("-Zbuild-std") + return args @@ -622,7 +633,7 @@ def _detect_unix_cross_compile_info() -> Optional["_CrossCompileInfo"]: linker = None linker_args = None else: - [linker, linker_args] = bldshared.split(maxsplit=1) + [linker, _, linker_args] = bldshared.partition(" ") return _CrossCompileInfo(host_type, cross_lib, linker, linker_args)