Merge pull request #3944 from wasmerio/create-exe-webc-compat

Port create-exe and the wasmer C API over to the webc compatibility layer

lib/c-api/src/wasm_c_api/wasi/mod.rs

@@ -226,12 +226,12 @@ unsafe fn wasi_env_with_filesystem_inner(
         config,
         &mut store.store_mut(),
         module,
-        std::mem::transmute(fs.ptr), // cast wasi_filesystem_t.ptr as &'static [u8]
+        &*(fs.ptr as *const u8), // cast wasi_filesystem_t.ptr as &'static [u8]
         fs.size,
         package,
     )?;
 
-    imports_set_buffer(&store, module, import_object, imports)?;
+    imports_set_buffer(store, module, import_object, imports)?;
 
     Some(Box::new(wasi_env_t {
         inner: wasi_env,
@@ -268,7 +268,7 @@ fn prepare_webc_env(
         })
         .collect::<Vec<_>>();
 
-    let filesystem = Box::new(StaticFileSystem::init(slice, &package_name)?);
+    let filesystem = Box::new(StaticFileSystem::init(slice, package_name)?);
     let mut builder = config.builder;
 
     if !config.inherit_stdout {
@@ -288,7 +288,7 @@ fn prepare_webc_env(
     }
     let env = builder.finalize(store).ok()?;
 
-    let import_object = env.import_object(store, &module).ok()?;
+    let import_object = env.import_object(store, module).ok()?;
     Some((env, import_object))
 }
 

lib/cli/src/cli.rs

@@ -9,7 +9,7 @@ use crate::commands::CreateExe;
 #[cfg(feature = "wast")]
 use crate::commands::Wast;
 use crate::commands::{
-    Add, Cache, Config, Init, Inspect, List, Login, Publish, Run, SelfUpdate, Validate, Whoami,
+    Add, Cache, Config, Init, Inspect, Login, Publish, Run, SelfUpdate, Validate, Whoami,
 };
 #[cfg(feature = "static-artifact-create")]
 use crate::commands::{CreateObj, GenCHeader};
@@ -37,9 +37,6 @@ use clap::{error::ErrorKind, CommandFactory, Parser};
 )]
 /// The options for the wasmer Command Line Interface
 enum WasmerCLIOptions {
-    /// List all locally installed packages
-    List(List),
-
     /// Login into a wapm.io-like registry
     Login(Login),
 
@@ -188,7 +185,6 @@ impl WasmerCLIOptions {
             Self::Config(config) => config.execute(),
             Self::Inspect(inspect) => inspect.execute(),
             Self::Init(init) => init.execute(),
-            Self::List(list) => list.execute(),
             Self::Login(login) => login.execute(),
             Self::Publish(publish) => publish.execute(),
             #[cfg(feature = "static-artifact-create")]
@@ -257,8 +253,9 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> {
         match command.unwrap_or(&String::new()).as_ref() {
             "add" | "cache" | "compile" | "config" | "create-obj" | "create-exe" | "help"
             | "gen-c-header" | "inspect" | "init" | "run" | "run-unstable" | "self-update"
-            | "validate" | "wast" | "binfmt" | "list" | "login" | "publish" | "app"
-            | "namespace" | "" => WasmerCLIOptions::parse(),
+            | "validate" | "wast" | "binfmt" | "login" | "publish" | "app" | "namespace" | "" => {
+                WasmerCLIOptions::parse()
+            }
             _ => {
                 WasmerCLIOptions::try_parse_from(args.iter()).unwrap_or_else(|e| {
                     match e.kind() {

lib/cli/src/commands.rs

@@ -14,7 +14,6 @@ mod create_obj;
 mod gen_c_header;
 mod init;
 mod inspect;
-mod list;
 mod login;
 mod publish;
 mod run;
@@ -33,7 +32,7 @@ pub use create_exe::*;
 #[cfg(feature = "wast")]
 pub use wast::*;
 pub use {
-    add::*, cache::*, config::*, init::*, inspect::*, list::*, login::*, publish::*, run::Run,
+    add::*, cache::*, config::*, init::*, inspect::*, login::*, publish::*, run::Run,
     self_update::*, validate::*, whoami::*,
 };
 #[cfg(feature = "static-artifact-create")]

lib/cli/src/commands/create_exe.rs

@@ -16,7 +16,10 @@ use tar::Archive;
 use wasmer::*;
 use wasmer_object::{emit_serialized, get_object_for_target};
 use wasmer_types::{compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo};
-use webc::v1::{ParseOptions, WebCMmap};
+use webc::{
+    compat::{Container, Volume as WebcVolume},
+    PathSegments,
+};
 
 const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"];
 
@@ -229,31 +232,30 @@ impl CreateExe {
         };
         std::fs::create_dir_all(&tempdir)?;
 
-        let atoms =
-            if let Ok(pirita) = WebCMmap::parse(input_path.clone(), &ParseOptions::default()) {
-                // pirita file
-                compile_pirita_into_directory(
-                    &pirita,
-                    &tempdir,
-                    &self.compiler,
-                    &self.cpu_features,
-                    &cross_compilation.target,
-                    &self.precompiled_atom,
-                    AllowMultiWasm::Allow,
-                    self.debug_dir.is_some(),
-                )
-            } else {
-                // wasm file
-                prepare_directory_from_single_wasm_file(
-                    &input_path,
-                    &tempdir,
-                    &self.compiler,
-                    &cross_compilation.target,
-                    &self.cpu_features,
-                    &self.precompiled_atom,
-                    self.debug_dir.is_some(),
-                )
-            }?;
+        let atoms = if let Ok(pirita) = Container::from_disk(&input_path) {
+            // pirita file
+            compile_pirita_into_directory(
+                &pirita,
+                &tempdir,
+                &self.compiler,
+                &self.cpu_features,
+                &cross_compilation.target,
+                &self.precompiled_atom,
+                AllowMultiWasm::Allow,
+                self.debug_dir.is_some(),
+            )
+        } else {
+            // wasm file
+            prepare_directory_from_single_wasm_file(
+                &input_path,
+                &tempdir,
+                &self.compiler,
+                &cross_compilation.target,
+                &self.cpu_features,
+                &self.precompiled_atom,
+                self.debug_dir.is_some(),
+            )
+        }?;
 
         get_module_infos(&store, &tempdir, &atoms)?;
         let mut entrypoint = get_entrypoint(&tempdir)?;
@@ -334,7 +336,7 @@ pub enum AllowMultiWasm {
 /// Given a pirita file, compiles the .wasm files into the target directory
 #[allow(clippy::too_many_arguments)]
 pub(super) fn compile_pirita_into_directory(
-    pirita: &WebCMmap,
+    pirita: &Container,
     target_dir: &Path,
     compiler: &CompilerOptions,
     cpu_features: &[CpuFeature],
@@ -345,39 +347,17 @@ pub(super) fn compile_pirita_into_directory(
 ) -> anyhow::Result<Vec<(String, Vec<u8>)>> {
     let all_atoms = match &allow_multi_wasm {
         AllowMultiWasm::Allow | AllowMultiWasm::Reject(None) => {
-            pirita.get_all_atoms().into_iter().collect::<Vec<_>>()
+            pirita.atoms().into_iter().collect::<Vec<_>>()
         }
         AllowMultiWasm::Reject(Some(s)) => {
-            vec![(
-                s.to_string(),
-                pirita
-                    .get_atom(&pirita.get_package_name(), s)
-                    .with_context(|| {
-                        anyhow::anyhow!(
-                            "could not find atom {s} in package {}",
-                            pirita.get_package_name()
-                        )
-                    })?,
-            )]
+            let atom = pirita
+                .get_atom(s)
+                .with_context(|| format!("could not find atom \"{s}\"",))?;
+            vec![(s.to_string(), atom)]
         }
     };
 
-    if allow_multi_wasm == AllowMultiWasm::Reject(None) && all_atoms.len() > 1 {
-        let keys = all_atoms
-            .iter()
-            .map(|(name, _)| name.clone())
-            .collect::<Vec<_>>();
-        return Err(anyhow::anyhow!(
-            "where <ATOM> is one of: {}",
-            keys.join(", ")
-        ))
-        .context(anyhow::anyhow!(
-            "note: use --atom <ATOM> to specify which atom to compile"
-        ))
-        .context(anyhow::anyhow!(
-            "cannot compile more than one atom at a time"
-        ));
-    }
+    allow_multi_wasm.validate(&all_atoms)?;
 
     std::fs::create_dir_all(target_dir)
         .map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
@@ -392,7 +372,8 @@ pub(super) fn compile_pirita_into_directory(
         )
     })?;
 
-    let volume_bytes = pirita.get_volumes_as_fileblock();
+    let volumes = pirita.volumes();
+    let volume_bytes = volume_file_block(&volumes);
     let volume_name = "VOLUMES";
     let volume_path = target_dir.join("volumes").join("volume.o");
     write_volume_obj(&volume_bytes, volume_name, &volume_path, target)?;
@@ -479,6 +460,105 @@ pub(super) fn compile_pirita_into_directory(
     Ok(atoms_from_file)
 }
 
+/// Serialize a set of volumes so they can be read by the C API.
+///
+/// This is really backwards, but the only way to create a v1 volume "fileblock"
+/// is by first reading every file in a [`webc::compat::Volume`], serializing it
+/// to webc v1's binary form, parsing those bytes into a [`webc::v1::Volume`],
+/// then constructing a dummy [`webc::v1::WebC`] object just so we can make a
+/// copy of its file block.
+fn volume_file_block(volumes: &BTreeMap<String, WebcVolume>) -> Vec<u8> {
+    let serialized_volumes: Vec<(&String, Vec<u8>)> = volumes
+        .iter()
+        .map(|(name, volume)| (name, serialize_volume_to_webc_v1(volume)))
+        .collect();
+
+    let parsed_volumes: indexmap::IndexMap<String, webc::v1::Volume<'_>> = serialized_volumes
+        .iter()
+        .filter_map(|(name, serialized_volume)| {
+            let volume = webc::v1::Volume::parse(serialized_volume).ok()?;
+            Some((name.to_string(), volume))
+        })
+        .collect();
+
+    let webc = webc::v1::WebC {
+        version: 0,
+        checksum: None,
+        signature: None,
+        manifest: webc::metadata::Manifest::default(),
+        atoms: webc::v1::Volume::default(),
+        volumes: parsed_volumes,
+    };
+
+    webc.get_volumes_as_fileblock()
+}
+
+fn serialize_volume_to_webc_v1(volume: &WebcVolume) -> Vec<u8> {
+    fn read_dir(
+        volume: &WebcVolume,
+        path: &mut PathSegments,
+        files: &mut BTreeMap<webc::v1::DirOrFile, Vec<u8>>,
+    ) {
+        for (segment, meta) in volume.read_dir(&*path).unwrap_or_default() {
+            path.push(segment);
+
+            match meta {
+                webc::compat::Metadata::Dir => {
+                    files.insert(
+                        webc::v1::DirOrFile::Dir(path.to_string().into()),
+                        Vec::new(),
+                    );
+                    read_dir(volume, path, files);
+                }
+                webc::compat::Metadata::File { .. } => {
+                    if let Some(contents) = volume.read_file(&*path) {
+                        files.insert(
+                            webc::v1::DirOrFile::File(path.to_string().into()),
+                            contents.into(),
+                        );
+                    }
+                }
+            }
+
+            path.pop();
+        }
+    }
+
+    let mut path = PathSegments::ROOT;
+    let mut files = BTreeMap::new();
+
+    read_dir(volume, &mut path, &mut files);
+
+    webc::v1::Volume::serialize_files(files)
+}
+
+impl AllowMultiWasm {
+    fn validate(
+        &self,
+        all_atoms: &Vec<(String, webc::compat::SharedBytes)>,
+    ) -> Result<(), anyhow::Error> {
+        if matches!(self, AllowMultiWasm::Reject(None)) && all_atoms.len() > 1 {
+            let keys = all_atoms
+                .iter()
+                .map(|(name, _)| name.clone())
+                .collect::<Vec<_>>();
+
+            return Err(anyhow::anyhow!(
+                "where <ATOM> is one of: {}",
+                keys.join(", ")
+            ))
+            .context(anyhow::anyhow!(
+                "note: use --atom <ATOM> to specify which atom to compile"
+            ))
+            .context(anyhow::anyhow!(
+                "cannot compile more than one atom at a time"
+            ));
+        }
+
+        Ok(())
+    }
+}
+
 /// Prefix map used during compilation of object files
 #[derive(Debug, Default, PartialEq)]
 pub(crate) struct PrefixMapCompilation {

lib/cli/src/commands/create_obj.rs

@@ -84,11 +84,9 @@ impl CreateObj {
         println!("Compiler: {}", compiler_type.to_string());
         println!("Target: {}", target.triple());
 
-        let atoms = if let Ok(pirita) =
-            webc::v1::WebCMmap::parse(input_path.clone(), &webc::v1::ParseOptions::default())
-        {
+        let atoms = if let Ok(webc) = webc::compat::Container::from_disk(&input_path) {
             crate::commands::create_exe::compile_pirita_into_directory(
-                &pirita,
+                &webc,
                 &output_directory_path,
                 &self.compiler,
                 &self.cpu_features,