Build platform independent rust binary wheel through wasi (#1107)

* Build platform independent rust binary wheel through wasi

We build the rust binaries to wasi and put them in a wheel with a wasmtime launcher. The resulting wheels are platform independent (py3-none-any) and run on all wasmtime platforms. Those are currently less than maturin natively, but you still only have to build one wheel and i hope for some nice things in the future (isolation capabilities, wasm runtimes on less-supported platforms)

* Test fixes

* fix typo

* Add wasm32-wasi target

* Add wasm32-wasi target in cirrus ci

* windows ci

* bin name validation and wasmtime 1.0

* changelog

.cirrus.yml

@@ -5,6 +5,7 @@ build_and_test: &BUILD_AND_TEST
   setup_script:
     - curl https://sh.rustup.rs -sSf --output rustup.sh
     - sh rustup.sh -y --default-toolchain stable
+    - rustup target add wasm32-wasi
     - python3 -m pip install --upgrade cffi virtualenv
   build_script:
     - cargo build

.github/workflows/test.yml

@@ -47,6 +47,7 @@ jobs:
           profile: minimal
           toolchain: stable
           override: true
+          target: wasm32-wasi # Additional target
       - name: Install cargo-nextest
         uses: taiki-e/install-action@nextest
       - name: Install aarch64-apple-darwin Rust target
@@ -351,7 +352,7 @@ jobs:
     strategy:
       fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     steps:
       - uses: actions/checkout@v3
       - uses: dorny/paths-filter@v2

Changelog.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+ * Initial support for shipping bin targets as wasm32-wasi binaries that are run through wasmtime in [#1107](https://github.com/PyO3/maturin/pull/1107). Note that wasmtime currently only support the five most popular platforms and that wasi binaries have restrictions when interacting with the host. Usage is by setting `--target wasm32-wasi`.
+
 ## [0.13.6] - 2022-10-08
 
 * Fix `maturin develop` in Windows conda virtual environment in [#1146](https://github.com/PyO3/maturin/pull/1146)

src/build_context.rs

@@ -3,7 +3,8 @@ use crate::auditwheel::{PlatformTag, Policy};
 use crate::build_options::CargoOptions;
 use crate::compile::warn_missing_py_init;
 use crate::module_writer::{
-    add_data, write_bin, write_bindings_module, write_cffi_module, write_python_part, WheelWriter,
+    add_data, write_bin, write_bindings_module, write_cffi_module, write_python_part,
+    write_wasm_launcher, WheelWriter,
 };
 use crate::project_layout::ProjectLayout;
 use crate::source_distribution::source_distribution;
@@ -14,6 +15,7 @@ use anyhow::{anyhow, bail, Context, Result};
 use cargo_metadata::Metadata;
 use fs_err as fs;
 use lddtree::Library;
+use regex::Regex;
 use sha2::{Digest, Sha256};
 use std::collections::{HashMap, HashSet};
 use std::fmt::{Display, Formatter};
@@ -74,6 +76,59 @@ impl Display for BridgeModel {
     }
 }
 
+/// Insert wasm launcher scripts as entrypoints and the wasmtime dependency
+fn bin_wasi_helper(
+    artifacts_and_files: &[(&BuildArtifact, String)],
+    mut metadata21: Metadata21,
+) -> Result<Metadata21> {
+    eprintln!("⚠️  Warning: wasi support is experimental");
+    // escaped can contain [\w\d.], but i don't know how we'd handle dots correctly here
+    if metadata21.get_distribution_escaped().contains('.') {
+        bail!(
+            "Can't build wasm wheel if there is a dot in the name ('{}')",
+            metadata21.get_distribution_escaped()
+        )
+    }
+    if !metadata21.entry_points.is_empty() {
+        bail!("You can't define entrypoints yourself for a binary project");
+    }
+
+    let mut console_scripts = HashMap::new();
+    for (_, bin_name) in artifacts_and_files {
+        let base_name = bin_name
+            .strip_suffix(".wasm")
+            .context("No .wasm suffix in wasi binary")?;
+        console_scripts.insert(
+            base_name.to_string(),
+            format!(
+                "{}.{}:main",
+                metadata21.get_distribution_escaped(),
+                base_name.replace('-', "_")
+            ),
+        );
+    }
+
+    metadata21
+        .entry_points
+        .insert("console_scripts".to_string(), console_scripts);
+
+    // A real pip version specification parser would be better, but bearing this we use this regex
+    // which tries to find the name wasmtime and then any specification
+    let wasmtime_re = Regex::new("^wasmtime[^a-zA-Z.-_]").unwrap();
+    if !metadata21
+        .requires_dist
+        .iter()
+        .any(|requirement| wasmtime_re.is_match(requirement))
+    {
+        // Having the wasmtime version hardcoded is not ideal, it's easy enough to overwrite
+        metadata21
+            .requires_dist
+            .push("wasmtime>=1.0.1,<2.0.0".to_string());
+    }
+
+    Ok(metadata21)
+}
+
 /// Contains all the metadata required to build the crate
 #[derive(Clone)]
 pub struct BuildContext {
@@ -653,28 +708,59 @@ impl BuildContext {
         };
 
         if !self.metadata21.scripts.is_empty() {
-            bail!("Defining entrypoints and working with a binary doesn't mix well");
+            bail!("Defining scripts and working with a binary doesn't mix well");
         }
 
-        let mut writer = WheelWriter::new(&tag, &self.out, &self.metadata21, &tags)?;
+        let mut artifacts_and_files = Vec::new();
+        for artifact in artifacts {
+            // I wouldn't know of any case where this would be the wrong (and neither do
+            // I know a better alternative)
+            let bin_name = artifact
+                .path
+                .file_name()
+                .context("Couldn't get the filename from the binary produced by cargo")?
+                .to_str()
+                .context("binary produced by cargo has non-utf8 filename")?
+                .to_string();
+
+            // From https://packaging.python.org/en/latest/specifications/entry-points/
+            // > The name may contain any characters except =, but it cannot start or end with any
+            // > whitespace character, or start with [. For new entry points, it is recommended to
+            // > use only letters, numbers, underscores, dots and dashes (regex [\w.-]+).
+            // All of these rules are already enforced by cargo:
+            // https://github.com/rust-lang/cargo/blob/58a961314437258065e23cb6316dfc121d96fb71/src/cargo/util/restricted_names.rs#L39-L84
+            // i.e. we don't need to do any bin name validation here anymore
+
+            artifacts_and_files.push((artifact, bin_name))
+        }
+
+        let metadata21 = if self.target.is_wasi() {
+            bin_wasi_helper(&artifacts_and_files, self.metadata21.clone())?
+        } else {
+            self.metadata21.clone()
+        };
+
+        let mut writer = WheelWriter::new(&tag, &self.out, &metadata21, &tags)?;
 
         if let Some(python_module) = &self.project_layout.python_module {
+            if self.target.is_wasi() {
+                // TODO: Can we have python code and the wasm launchers coexisting
+                // without clashes?
+                bail!("Sorry, adding python code to a wasm binary is currently not supported")
+            }
             if !self.editable {
                 write_python_part(&mut writer, python_module)
                     .context("Failed to add the python module to the package")?;
             }
         }
 
         let mut artifacts_ref = Vec::with_capacity(artifacts.len());
-        for artifact in artifacts {
-            artifacts_ref.push(artifact);
-            // I wouldn't know of any case where this would be the wrong (and neither do
-            // I know a better alternative)
-            let bin_name = artifact
-                .path
-                .file_name()
-                .expect("Couldn't get the filename from the binary produced by cargo");
+        for (artifact, bin_name) in &artifacts_and_files {
+            artifacts_ref.push(*artifact);
             write_bin(&mut writer, &artifact.path, &self.metadata21, bin_name)?;
+            if self.target.is_wasi() {
+                write_wasm_launcher(&mut writer, &self.metadata21, bin_name)?;
+            }
         }
         self.add_external_libs(&mut writer, &artifacts_ref, ext_libs)?;
 

src/module_writer.rs

@@ -761,7 +761,7 @@ pub fn write_bin(
     writer: &mut impl ModuleWriter,
     artifact: &Path,
     metadata: &Metadata21,
-    bin_name: &OsStr,
+    bin_name: &str,
 ) -> Result<()> {
     let data_dir = PathBuf::from(format!(
         "{}-{}.data",
@@ -777,6 +777,58 @@ pub fn write_bin(
     Ok(())
 }
 
+/// Adds a wrapper script that start the wasm binary through wasmtime.
+///
+/// Note that the wasm binary needs to be written separately by [write_bin]
+pub fn write_wasm_launcher(
+    writer: &mut impl ModuleWriter,
+    metadata: &Metadata21,
+    bin_name: &str,
+) -> Result<()> {
+    let entrypoint_script = format!(
+        r#"from pathlib import Path
+
+from wasmtime import Store, Module, Engine, WasiConfig, Linker
+
+import sysconfig
+
+def main():
+    # The actual executable
+    program_location = Path(sysconfig.get_path("scripts")).joinpath("{}")
+    # wasmtime-py boilerplate
+    engine = Engine()
+    store = Store(engine)
+    # TODO: is there an option to just get the default of the wasmtime cli here?
+    wasi = WasiConfig()
+    wasi.inherit_argv()
+    wasi.inherit_env()
+    wasi.inherit_stdout()
+    wasi.inherit_stderr()
+    wasi.inherit_stdin()
+    store.set_wasi(wasi)
+    linker = Linker(engine)
+    linker.define_wasi()
+    module = Module.from_file(store.engine, str(program_location))
+    linking1 = linker.instantiate(store, module)
+    # TODO: this is taken from https://docs.wasmtime.dev/api/wasmtime/struct.Linker.html#method.get_default
+    #       is this always correct?
+    start = linking1.exports(store).get("") or linking1.exports(store)["_start"]
+    start(store)
+
+if __name__ == '__main__':
+    main()
+    "#,
+        bin_name
+    );
+
+    // We can't use add_file since we want to mark the file as executable
+    let launcher_path = Path::new(&metadata.get_distribution_escaped())
+        .join(bin_name.replace('-', "_"))
+        .with_extension("py");
+    writer.add_bytes_with_permissions(&launcher_path, entrypoint_script.as_bytes(), 0o755)?;
+    Ok(())
+}
+
 /// Adds the python part of a mixed project to the writer,
 pub fn write_python_part(
     writer: &mut impl ModuleWriter,

src/target.rs

@@ -28,6 +28,7 @@ pub enum Os {
     Illumos,
     Haiku,
     Emscripten,
+    Wasi,
 }
 
 impl fmt::Display for Os {
@@ -43,6 +44,7 @@ impl fmt::Display for Os {
             Os::Illumos => write!(f, "Illumos"),
             Os::Haiku => write!(f, "Haiku"),
             Os::Emscripten => write!(f, "Emscripten"),
+            Os::Wasi => write!(f, "Wasi"),
         }
     }
 }
@@ -123,7 +125,7 @@ fn get_supported_architectures(os: &Os) -> Vec<Arch> {
         Os::Dragonfly => vec![Arch::X86_64],
         Os::Illumos => vec![Arch::X86_64],
         Os::Haiku => vec![Arch::X86_64],
-        Os::Emscripten => vec![Arch::Wasm32],
+        Os::Emscripten | Os::Wasi => vec![Arch::Wasm32],
     }
 }
 
@@ -176,6 +178,7 @@ impl Target {
             OperatingSystem::Illumos => Os::Illumos,
             OperatingSystem::Haiku => Os::Haiku,
             OperatingSystem::Emscripten => Os::Emscripten,
+            OperatingSystem::Wasi => Os::Wasi,
             unsupported => bail!("The operating system {:?} is not supported", unsupported),
         };
 
@@ -353,6 +356,9 @@ impl Target {
                 let release = release.replace('.', "_").replace('-', "_");
                 format!("emscripten_{}_wasm32", release)
             }
+            (Os::Wasi, Arch::Wasm32) => {
+                "any".to_string()
+            }
             (_, _) => panic!("unsupported target should not have reached get_platform_tag()"),
         };
         Ok(tag)
@@ -406,6 +412,8 @@ impl Target {
             Os::Illumos => "sunos",
             Os::Haiku => "haiku",
             Os::Emscripten => "emscripten",
+            // This isn't real, there's no sys.platform here
+            Os::Wasi => "wasi",
         }
     }
 
@@ -476,7 +484,8 @@ impl Target {
             | Os::Dragonfly
             | Os::Illumos
             | Os::Haiku
-            | Os::Emscripten => true,
+            | Os::Emscripten
+            | Os::Wasi => true,
         }
     }
 
@@ -535,6 +544,11 @@ impl Target {
         self.os == Os::Emscripten
     }
 
+    /// Returns true if we're building a binary for wasm32-wasi
+    pub fn is_wasi(&self) -> bool {
+        self.os == Os::Wasi
+    }
+
     /// Returns true if the current platform's target env is Musl
     pub fn is_musl_target(&self) -> bool {
         matches!(

test-crates/cargo-mock/src/main.rs

@@ -33,7 +33,8 @@ fn run() -> Result<()> {
         .replace("--quiet", "")
         .replace(&cwd, "")
         .replace(" ", "-")
-        .replace("/", "-");
+        .replace("/", "-")
+        .replace("-----C-link-arg=-s", "");
 
     let cache_path = base_cache_path.join(&env_key).join(&cargo_key);
     let stdout_path = cache_path.join("cargo.stdout");

test-crates/hello-world/check_installed/check_installed.py

@@ -7,7 +7,7 @@ def main():
         raise Exception(output)
 
     output = check_output(["foo"]).decode("utf-8").strip()
-    if not output == "Hello, world!":
+    if not output == "🦀 Hello, world! 🦀":
         raise Exception(output)
     print("SUCCESS")
 

test-crates/hello-world/src/bin/foo.rs

@@ -1,3 +1,3 @@
 fn main() {
-    println!("Hello, world!");
+    println!("🦀 Hello, world! 🦀");
 }

tests/common/integration.rs

@@ -15,6 +15,7 @@ pub fn test_integration(
     bindings: Option<String>,
     unique_name: &str,
     zig: bool,
+    target: Option<&str>,
 ) -> Result<()> {
     maybe_mock_cargo();
 
@@ -45,6 +46,11 @@ pub fn test_integration(
         cli.push(bindings);
     }
 
+    if let Some(target) = target {
+        cli.push("--target");
+        cli.push(target)
+    }
+
     let test_zig = if zig && (env::var("GITHUB_ACTIONS").is_ok() || Zig::find_zig().is_ok()) {
         cli.push("--zig");
         true
@@ -84,11 +90,14 @@ pub fn test_integration(
             };
             assert!(filename.to_string_lossy().ends_with(file_suffix))
         }
-        let venv_suffix = if supported_version == "py3" {
+        let mut venv_suffix = if supported_version == "py3" {
             "py3".to_string()
         } else {
             format!("{}.{}", python_interpreter.major, python_interpreter.minor,)
         };
+        if let Some(target) = target {
+            venv_suffix = format!("{}-{}", venv_suffix, target);
+        }
         let (venv_dir, python) = create_virtualenv(
             &package,
             &venv_suffix,

tests/common/mod.rs

@@ -76,7 +76,7 @@ pub fn handle_result<T>(result: Result<T>) -> T {
     match result {
         Err(e) => {
             for cause in e.chain().collect::<Vec<_>>().iter().rev() {
-                eprintln!("{}", cause);
+                eprintln!("Cause: {}", cause);
             }
             panic!("{}", e);
         }