diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6df0a6f4f..0eb20dd40 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,39 @@ jobs: export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt cargo run -- build -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-unknown-linux-gnu --zig + test-emscripten: + name: Test Emscripten + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + components: rust-src + target: wasm32-unknown-emscripten + override: true + - uses: mymindstorm/setup-emsdk@v11 + with: + version: 3.1.13 + actions-cache-folder: emsdk-cache + - uses: actions/setup-python@v2 + id: setup-python + with: + python-version: "3.10" + - uses: actions/setup-node@v3 + with: + node-version: 18 + - run: pip install nox + - uses: actions/cache@v3 + with: + path: | + tests/pyodide + key: ${{ hashFiles('tests/*.js') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }} + - uses: Swatinem/rust-cache@v1 + - name: Run tests + run: nox -s test-emscripten + test-alpine: name: Test Alpine Linux runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 02af83eb5..0ef13b4de 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ tags test-crates/wheels/ test-crates/targets/ test-crates/venvs/ +tests/pyodide/ diff --git a/Changelog.md b/Changelog.md index 4be276e81..266d2cf62 100644 --- a/Changelog.md +++ b/Changelog.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add support for cross compiling PyPy wheels when abi3 feature is enabled in [#963](https://github.com/PyO3/maturin/pull/963) * Add `--find-interpreter` option to `build` and `publish` commands to search for python interpreters in [#964](https://github.com/PyO3/maturin/pull/964) * Infer target triple from `ARCHFLAGS` for macOS to be compatible with `cibuildwheel` in [#967](https://github.com/PyO3/maturin/pull/967) +* Add support for `wasm32-unknown-emscripten` target in [#974](https://github.com/PyO3/maturin/pull/974) ## [0.12.20] - 2022-06-15 diff --git a/MANIFEST.in b/MANIFEST.in index 7be329e68..c6b09d281 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include Cargo.toml Cargo.lock include Readme.md include license-apache license-mit -recursive-include src *.rs +recursive-include src *.rs *.py recursive-include src/auditwheel *.json recursive-include src/python_interpreter *.py *.json recursive-include src/templates *.j2 diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 000000000..f26776dad --- /dev/null +++ b/noxfile.py @@ -0,0 +1,54 @@ +import sys +from pathlib import Path + +import nox + + +def download_pyodide(session: nox.Session, pyodide_dir: Path) -> None: + pyodide_dir.mkdir() + + PYODIDE_DEV = "https://pyodide-cdn2.iodide.io/dev/full/" + pyodide_files = [ + "pyodide.js", + "packages.json", + "pyodide.asm.js", + "pyodide.asm.data", + "pyodide.asm.wasm", + "pyodide_py.tar", + ] + with session.chdir(pyodide_dir): + for file in pyodide_files: + session.run("wget", "-q", PYODIDE_DEV + file, external=True) + session.run("npm", "i", "node-fetch", external=True) + + +@nox.session(name="test-emscripten") +def test_emscripten(session: nox.Session): + emscripten_dir = Path("./tests").resolve() + pyodide_dir = emscripten_dir / "pyodide" + if not pyodide_dir.exists(): + download_pyodide(session, pyodide_dir) + + test_crates = [ + "test-crates/pyo3-mixed", + ] + for crate in test_crates: + crate = Path(crate).resolve() + + ver = sys.version_info + session.run( + "cargo", + "+nightly", + "run", + "build", + "-m", + str(crate / "Cargo.toml"), + "--target", + "wasm32-unknown-emscripten", + "-i", + f"python{ver.major}.{ver.minor}", + external=True, + ) + + with session.chdir(emscripten_dir): + session.run("node", "emscripten_runner.js", str(crate), external=True) diff --git a/src/auditwheel/policy.rs b/src/auditwheel/policy.rs index 3d72b2aa1..d28e63acb 100644 --- a/src/auditwheel/policy.rs +++ b/src/auditwheel/policy.rs @@ -104,6 +104,7 @@ impl Policy { Arch::X86 => "libc.musl-x86.so.1", Arch::X86_64 => "libc.musl-x86_64.so.1", Arch::S390X => "libc.musl-s390x.so.1", + _ => "", }; if !new_soname.is_empty() { self.lib_whitelist.insert(new_soname.to_string()); diff --git a/src/compile.rs b/src/compile.rs index e15c0f055..170bfb5da 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -5,7 +5,9 @@ use fat_macho::FatWriter; use fs_err::{self as fs, File}; use std::collections::HashMap; use std::env; -use std::io::{BufReader, Read}; +use std::io::{BufReader, Read, Write}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::str; @@ -14,6 +16,8 @@ use std::str; /// without `PYO3_NO_PYTHON` environment variable const PYO3_ABI3_NO_PYTHON_VERSION: (u64, u64, u64) = (0, 16, 4); +const EMCC_WRAPPER: &str = include_str!("emcc_wrapper.py"); + /// Builds the rust crate into a native module (i.e. an .so or .dll) for a /// specific python version. Returns a mapping from crate type (e.g. cdylib) /// to artifact location. @@ -214,6 +218,18 @@ fn compile_target( ]; rustc_args.extend(mac_args); } + } else if target.is_emscripten() { + shared_args.push("-Zbuild-std"); + rust_flags + .get_or_insert_with(Default::default) + .push(" -C relocation-model=pic"); + let emscripten_args = &[ + "-C", + "link-arg=-sSIDE_MODULE=2", + "-C", + "link-arg=-sWASM_BIGINT", + ]; + rustc_args.extend(emscripten_args); } if context.strip { @@ -270,6 +286,28 @@ fn compile_target( } } + if target.is_emscripten() { + // Workaround https://github.com/emscripten-core/emscripten/issues/17191 + let cache_dir = dirs::cache_dir() + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")); + fs::create_dir_all(&cache_dir)?; + let emcc_wrapper = cache_dir.join("emcc_wrapper.py"); + let mut emcc_wrapper_file = fs::File::create(&emcc_wrapper)?; + emcc_wrapper_file.write_all(EMCC_WRAPPER.as_bytes())?; + #[cfg(unix)] + { + let metadata = emcc_wrapper_file.metadata()?; + let mut permissions = metadata.permissions(); + permissions.set_mode(0o755); + emcc_wrapper_file.set_permissions(permissions)?; + } + build_command.env( + "CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER", + emcc_wrapper, + ); + } + build_command .args(&build_args) // We need to capture the json messages diff --git a/src/emcc_wrapper.py b/src/emcc_wrapper.py new file mode 100755 index 000000000..86969af51 --- /dev/null +++ b/src/emcc_wrapper.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import subprocess +import sys + + +def update_args(args): + # remove -lc. Not sure if it makes a difference but -lc doesn't belong here. + # https://github.com/emscripten-core/emscripten/issues/17191 + for i in reversed(range(len(args))): + if args[i] == "c" and args[i - 1] == "-l": + del args[i - 1 : i + 1] + + return args + + +def main(args): + args = update_args(args) + return subprocess.call(["emcc"] + args) + + +if __name__ == "__main__": + args = sys.argv[1:] + sys.exit(main(args)) diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs index 3c21aea1a..53786f862 100644 --- a/src/python_interpreter/config.rs +++ b/src/python_interpreter/config.rs @@ -37,6 +37,11 @@ static WELLKNOWN_SYSCONFIG: Lazy write!(f, "DragonFly"), Os::Illumos => write!(f, "Illumos"), Os::Haiku => write!(f, "Haiku"), + Os::Emscripten => write!(f, "Emscripten"), } } } @@ -57,6 +59,7 @@ pub enum Arch { X86, X86_64, S390X, + Wasm32, } impl fmt::Display for Arch { @@ -70,6 +73,7 @@ impl fmt::Display for Arch { Arch::X86 => write!(f, "i686"), Arch::X86_64 => write!(f, "x86_64"), Arch::S390X => write!(f, "s390x"), + Arch::Wasm32 => write!(f, "wasm32"), } } } @@ -100,6 +104,7 @@ fn get_supported_architectures(os: &Os) -> Vec { Os::Dragonfly => vec![Arch::X86_64], Os::Illumos => vec![Arch::X86_64], Os::Haiku => vec![Arch::X86_64], + Os::Emscripten => vec![Arch::Wasm32], } } @@ -123,7 +128,7 @@ impl Target { /// /// Fails if the target triple isn't supported pub fn from_target_triple(target_triple: Option) -> Result { - use target_lexicon::ArmArchitecture; + use target_lexicon::{Architecture, ArmArchitecture, OperatingSystem}; let host_triple = get_host_target()?; let (platform, triple) = if let Some(ref target_triple) = target_triple { @@ -139,30 +144,31 @@ impl Target { }; let os = match platform.operating_system { - target_lexicon::OperatingSystem::Linux => Os::Linux, - target_lexicon::OperatingSystem::Windows => Os::Windows, - target_lexicon::OperatingSystem::MacOSX { .. } - | target_lexicon::OperatingSystem::Darwin => Os::Macos, - target_lexicon::OperatingSystem::Netbsd => Os::NetBsd, - target_lexicon::OperatingSystem::Freebsd => Os::FreeBsd, - target_lexicon::OperatingSystem::Openbsd => Os::OpenBsd, - target_lexicon::OperatingSystem::Dragonfly => Os::Dragonfly, - target_lexicon::OperatingSystem::Illumos => Os::Illumos, - target_lexicon::OperatingSystem::Haiku => Os::Haiku, + OperatingSystem::Linux => Os::Linux, + OperatingSystem::Windows => Os::Windows, + OperatingSystem::MacOSX { .. } | OperatingSystem::Darwin => Os::Macos, + OperatingSystem::Netbsd => Os::NetBsd, + OperatingSystem::Freebsd => Os::FreeBsd, + OperatingSystem::Openbsd => Os::OpenBsd, + OperatingSystem::Dragonfly => Os::Dragonfly, + OperatingSystem::Illumos => Os::Illumos, + OperatingSystem::Haiku => Os::Haiku, + OperatingSystem::Emscripten => Os::Emscripten, unsupported => bail!("The operating system {:?} is not supported", unsupported), }; let arch = match platform.architecture { - target_lexicon::Architecture::X86_64 => Arch::X86_64, - target_lexicon::Architecture::X86_32(_) => Arch::X86, - target_lexicon::Architecture::Arm(arm_arch) => match arm_arch { + Architecture::X86_64 => Arch::X86_64, + Architecture::X86_32(_) => Arch::X86, + Architecture::Arm(arm_arch) => match arm_arch { ArmArchitecture::Arm | ArmArchitecture::Armv6 => Arch::Armv6L, _ => Arch::Armv7L, }, - target_lexicon::Architecture::Aarch64(_) => Arch::Aarch64, - target_lexicon::Architecture::Powerpc64 => Arch::Powerpc64, - target_lexicon::Architecture::Powerpc64le => Arch::Powerpc64Le, - target_lexicon::Architecture::S390x => Arch::S390X, + Architecture::Aarch64(_) => Arch::Aarch64, + Architecture::Powerpc64 => Arch::Powerpc64, + Architecture::Powerpc64le => Arch::Powerpc64Le, + Architecture::S390x => Arch::S390X, + Architecture::Wasm32 => Arch::Wasm32, unsupported => bail!("The architecture {} is not supported", unsupported), }; @@ -235,6 +241,7 @@ impl Target { "x86_64" ) } + // Illumos (Os::Illumos, Arch::X86_64) => { let info = PlatformInfo::new()?; let mut release = info.release().replace('.', "_").replace('-', "_"); @@ -258,6 +265,7 @@ impl Target { arch ) } + // Linux (Os::Linux, _) => { let arch = if self.cross_compiling { self.arch.to_string() @@ -277,6 +285,7 @@ impl Target { } tags.join(".") } + // macOS (Os::Macos, Arch::X86_64) => { let ((x86_64_major, x86_64_minor), (arm64_major, arm64_minor)) = macosx_deployment_target(env::var("MACOSX_DEPLOYMENT_TARGET").ok().as_deref(), universal2)?; if universal2 { @@ -305,9 +314,15 @@ impl Target { format!("macosx_{}_{}_arm64", arm64_major, arm64_minor) } } + // Windows (Os::Windows, Arch::X86) => "win32".to_string(), (Os::Windows, Arch::X86_64) => "win_amd64".to_string(), (Os::Windows, Arch::Aarch64) => "win_arm64".to_string(), + // Emscripten + (Os::Emscripten, Arch::Wasm32) => { + let version = emcc_version()?; + format!("emscripten_{}_wasm32", version.replace('.', "_")) + } (_, _) => panic!("unsupported target should not have reached get_platform_tag()"), }; Ok(tag) @@ -324,6 +339,7 @@ impl Target { Arch::X86 => "i386", Arch::X86_64 => "x86_64", Arch::S390X => "s390x", + Arch::Wasm32 => "wasm32", } } @@ -339,6 +355,7 @@ impl Target { Os::Dragonfly => "dragonfly", Os::Illumos => "sunos", Os::Haiku => "haiku", + Os::Emscripten => "emscripten", } } @@ -349,7 +366,7 @@ impl Target { PlatformTag::manylinux2014() } Arch::X86 | Arch::X86_64 => PlatformTag::manylinux2010(), - Arch::Armv6L => PlatformTag::Linux, + Arch::Armv6L | Arch::Wasm32 => PlatformTag::Linux, } } @@ -357,7 +374,7 @@ impl Target { pub fn pointer_width(&self) -> usize { match self.arch { Arch::Aarch64 | Arch::Powerpc64 | Arch::Powerpc64Le | Arch::X86_64 | Arch::S390X => 64, - Arch::Armv6L | Arch::Armv7L | Arch::X86 => 32, + Arch::Armv6L | Arch::Armv7L | Arch::X86 | Arch::Wasm32 => 32, } } @@ -377,7 +394,8 @@ impl Target { | Os::OpenBsd | Os::Dragonfly | Os::Illumos - | Os::Haiku => true, + | Os::Haiku + | Os::Emscripten => true, } } @@ -431,6 +449,11 @@ impl Target { self.os == Os::Haiku } + /// Returns true if the current platform is Emscripten + pub fn is_emscripten(&self) -> bool { + self.os == Os::Emscripten + } + /// Returns true if the current platform's target env is Musl pub fn is_musl_target(&self) -> bool { matches!( @@ -580,6 +603,22 @@ fn macosx_deployment_target( Ok((x86_64_ver, arm64_ver)) } +fn emcc_version() -> Result { + use regex::bytes::Regex; + use std::process::Command; + + let emcc = Command::new("emcc") + .arg("--version") + .output() + .context("Failed to run emcc to get the version")?; + let pattern = Regex::new(r"^emcc .+? (\d+\.\d+\.\d+).*").unwrap(); + let caps = pattern + .captures(&emcc.stdout) + .context("Failed to parse emcc version")?; + let version = caps.get(1).context("Failed to parse emcc version")?; + Ok(String::from_utf8(version.as_bytes().to_vec())?) +} + #[cfg(test)] mod test { use super::macosx_deployment_target; diff --git a/test-crates/pyo3-mixed/test_pyo3_mixed.py b/test-crates/pyo3-mixed/tests/test_pyo3_mixed.py similarity index 100% rename from test-crates/pyo3-mixed/test_pyo3_mixed.py rename to test-crates/pyo3-mixed/tests/test_pyo3_mixed.py diff --git a/test-crates/pyo3-mixed/tox.ini b/test-crates/pyo3-mixed/tox.ini index fd19ca2b2..421774193 100644 --- a/test-crates/pyo3-mixed/tox.ini +++ b/test-crates/pyo3-mixed/tox.ini @@ -4,4 +4,4 @@ isolated_build = True [testenv] deps = pytest -commands = pytest \ No newline at end of file +commands = pytest tests/ diff --git a/tests/emscripten_runner.js b/tests/emscripten_runner.js new file mode 100644 index 000000000..afa3a8455 --- /dev/null +++ b/tests/emscripten_runner.js @@ -0,0 +1,75 @@ +const http = require("http"); +const fs = require("fs"); +const { opendir } = require("node:fs/promises"); + +const { loadPyodide } = require("./pyodide/pyodide.js"); + +const PORT = 8124; + +const server = http + .createServer(function (request, response) { + const filePath = "." + request.url; + const contentType = "application/octet-stream"; + fs.readFile( + process.argv[2] + "/target/wheels/" + filePath, + function (error, content) { + if (error) { + if (error.code == "ENOENT") { + response.writeHead(404); + response.end("Not found"); + response.end(); + } else { + response.writeHead(500); + response.end("error: " + error.code); + response.end(); + } + } else { + response.writeHead(200, { "Content-Type": contentType }); + response.end(content, "utf-8"); + } + } + ); + }) + .listen(PORT); + +async function findWheel(distDir) { + const dir = await opendir(distDir); + for await (const dirent of dir) { + if (dirent.name.endsWith("whl")) { + return dirent.name; + } + } +} + +const localhost = `http://0.0.0.0:${PORT}`; +const pkgDir = process.argv[2]; +const distDir = pkgDir + "/target/wheels"; +const testDir = pkgDir + "/tests"; + +async function main() { + const wheelName = await findWheel(distDir); + const wheelURL = `${localhost}/${wheelName}`; + + let errcode = 1; + try { + pyodide = await loadPyodide({ indexURL: "./pyodide", fullStdLib: false }); + pyodide._api.setCdnUrl("https://pyodide-cdn2.iodide.io/dev/full/"); + const FS = pyodide.FS; + const NODEFS = FS.filesystems.NODEFS; + FS.mkdir("/test_dir"); + FS.mount(NODEFS, { root: testDir }, "/test_dir"); + await pyodide.loadPackage(["micropip", "pytest", "tomli"]); + const micropip = pyodide.pyimport("micropip"); + await micropip.install(wheelURL); + const pytest = pyodide.pyimport("pytest"); + errcode = pytest.main(pyodide.toPy(["/test_dir", "-vv"])); + } catch (e) { + console.error(e); + errcode = 1; + } finally { + server.close(); + } + process.exit(errcode); +} + +main();