feat: allow mounting folders in workers

crates/data-kv/src/lib.rs

@@ -8,7 +8,7 @@ use std::collections::HashMap;
 use store::KVStore;
 
 /// The Key/Value store configuration. This information is read from workers TOML files.
-#[derive(Deserialize, Clone)]
+#[derive(Deserialize, Clone, Default)]
 pub struct KVConfigData {
     /// The namespace the worker will access in the global Key / Value store
     pub namespace: String,

crates/router/src/files.rs

@@ -8,6 +8,8 @@ use wws_config::Config;
 use wws_runtimes_manager::check_runtime;
 use wws_store::STORE_FOLDER;
 
+pub const IGNORE_PATH_PREFIX: &str = "_";
+
 /// Manages the files associated to a Wasm Workers Run.
 /// It uses glob patterns to detect the workers and
 /// provide utilities to work with public folders and
@@ -53,35 +55,22 @@ impl Files {
 
         glob.walk(&self.root)
             .filter_map(|el| match el {
-                Ok(entry)
-                    if !self.is_in_public_folder(entry.path())
-                        && !self.is_in_wws_folder(entry.path()) =>
-                {
-                    Some(entry)
-                }
+                Ok(entry) if !self.should_ignore(entry.path()) => Some(entry),
                 _ => None,
             })
             .collect()
     }
 
-    /// Checks if the given filepath is inside the "public" folder.
-    /// It will return an early false if the project doesn't have
-    /// a public folder.
-    fn is_in_public_folder(&self, path: &Path) -> bool {
-        if !self.has_public {
-            return false;
-        }
-
-        path.components().any(|c| match c {
-            Component::Normal(os_str) => os_str == OsStr::new("public"),
-            _ => false,
-        })
-    }
-
-    /// Checks if the given filepath is inside the ".wws" special folder.
-    fn is_in_wws_folder(&self, path: &Path) -> bool {
+    /// Perform multiple checks to confirm if the given file should be ignored.
+    /// The current checks are: file is not inside the public or .wws folder, and
+    /// any component starts with _.
+    fn should_ignore(&self, path: &Path) -> bool {
         path.components().any(|c| match c {
-            Component::Normal(os_str) => os_str == OsStr::new(STORE_FOLDER),
+            Component::Normal(os_str) => {
+                (self.has_public && os_str == OsStr::new("public"))
+                    || os_str == OsStr::new(STORE_FOLDER)
+                    || os_str.to_string_lossy().starts_with(IGNORE_PATH_PREFIX)
+            }
             _ => false,
         })
     }
@@ -107,8 +96,7 @@ mod tests {
         for t in tests {
             let config = Config::default();
             assert_eq!(
-                Files::new(Path::new("../../tests/data"), &config)
-                    .is_in_public_folder(Path::new(t.0)),
+                Files::new(Path::new("../../tests/data"), &config).should_ignore(Path::new(t.0)),
                 t.1
             )
         }
@@ -130,8 +118,7 @@ mod tests {
         for t in tests {
             let config = Config::default();
             assert_eq!(
-                Files::new(Path::new("..\\..\\tests\\data"), &config)
-                    .is_in_public_folder(Path::new(t.0)),
+                Files::new(Path::new("..\\..\\tests\\data"), &config).should_ignore(Path::new(t.0)),
                 t.1
             )
         }

crates/router/src/route.rs

@@ -5,11 +5,10 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use std::{
     ffi::OsStr,
-    fs,
     path::{Component, Path, PathBuf},
 };
 use wws_config::Config as ProjectConfig;
-use wws_worker::{config::Config, Worker};
+use wws_worker::Worker;
 
 lazy_static! {
     static ref PARAMETER_REGEX: Regex = Regex::new(r"\[\w+\]").unwrap();
@@ -45,37 +44,25 @@ pub struct Route {
     pub path: String,
     /// The associated worker
     pub worker: Worker,
-    /// The associated configuration if available
-    pub config: Option<Config>,
 }
 
 impl Route {
     /// Initialize a new route from the given folder and filepath. It will calculate the
     /// proper URL path based on the filename.
     ///
     /// This method also initializes the Runner and loads the Config if available.
-    pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str, config: &ProjectConfig) -> Self {
-        let worker = Worker::new(base_path, &filepath, config).unwrap();
-
-        // Load configuration
-        let mut config_path = filepath.clone();
-        config_path.set_extension("toml");
-        let mut config = None::<Config>;
-
-        if fs::metadata(&config_path).is_ok() {
-            match Config::try_from_file(config_path) {
-                Ok(c) => config = Some(c),
-                Err(err) => {
-                    eprintln!("{err}");
-                }
-            }
-        }
+    pub fn new(
+        base_path: &Path,
+        filepath: PathBuf,
+        prefix: &str,
+        project_config: &ProjectConfig,
+    ) -> Self {
+        let worker = Worker::new(base_path, &filepath, project_config).unwrap();
 
         Self {
             path: Self::retrieve_route(base_path, &filepath, prefix),
             handler: filepath,
             worker,
-            config,
         }
     }
 

crates/server/src/handlers/worker.rs

@@ -8,7 +8,7 @@ use actix_web::{
     web::{Bytes, Data},
     HttpRequest, HttpResponse,
 };
-use std::{collections::HashMap, sync::RwLock};
+use std::sync::RwLock;
 use wws_router::Routes;
 use wws_worker::io::WasmOutput;
 
@@ -55,17 +55,8 @@ pub async fn handle_worker(req: HttpRequest, body: Bytes) -> HttpResponse {
         let body_str = String::from_utf8(body.to_vec()).unwrap_or_else(|_| String::from(""));
 
         // Init from configuration
-        let empty_hash = HashMap::new();
-        let mut vars = &empty_hash;
-        let mut kv_namespace = None;
-
-        match &route.config {
-            Some(config) => {
-                kv_namespace = config.data_kv_namespace();
-                vars = &config.vars;
-            }
-            None => {}
-        };
+        let vars = &route.worker.config.vars;
+        let kv_namespace = route.worker.config.data_kv_namespace();
 
         let store = match &kv_namespace {
             Some(namespace) => {

crates/server/src/lib.rs

@@ -52,7 +52,7 @@ pub async fn serve(
             app = app.service(web::resource(route.actix_path()).to(handle_worker));
 
             // Configure KV
-            if let Some(namespace) = route.config.as_ref().and_then(|c| c.data_kv_namespace()) {
+            if let Some(namespace) = route.worker.config.data_kv_namespace() {
                 data.write().unwrap().kv.create_store(&namespace);
             }
         }

crates/worker/src/config.rs

@@ -10,21 +10,23 @@ use toml::from_str;
 use wws_data_kv::KVConfigData;
 
 /// Workers configuration. These files are optional when no configuration change is required.
-#[derive(Deserialize, Clone)]
+#[derive(Deserialize, Clone, Default)]
 pub struct Config {
     /// Worker name. For logging purposes
     pub name: Option<String>,
     /// Mandatory version of the file
     pub version: String,
     /// Optional data configuration
     pub data: Option<ConfigData>,
+    /// Optional folders
+    pub folders: Option<Vec<Folder>>,
     /// Optional environment configuration
     #[serde(deserialize_with = "read_environment_variables", default)]
     pub vars: HashMap<String, String>,
 }
 
 /// Configure a data plugin for the worker
-#[derive(Deserialize, Clone)]
+#[derive(Deserialize, Clone, Default)]
 pub struct ConfigData {
     /// Creates a Key/Value store associated to the given worker
     pub kv: Option<KVConfigData>,
@@ -107,3 +109,41 @@ where
         Err(err) => Err(err),
     }
 }
+
+/// A folder to mount in the worker
+#[derive(Deserialize, Clone, Default)]
+pub struct Folder {
+    /// Local folder
+    #[serde(deserialize_with = "deserialize_path", default)]
+    pub from: PathBuf,
+    /// Mount point
+    pub to: String,
+}
+
+/// Deserialize a valid path for the given platform. This method checks and
+/// split the path by the different separators and join the final path
+/// using the current OS requirements.
+fn deserialize_path<'de, D>(deserializer: D) -> Result<PathBuf, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let result: Result<String, D::Error> = Deserialize::deserialize(deserializer);
+
+    match result {
+        Ok(value) => {
+            let split = if value.contains('/') {
+                // Unix separator
+                value.split('/')
+            } else {
+                // Windows separator
+                value.split('\\')
+            };
+
+            Ok(split.fold(PathBuf::new(), |mut acc, el| {
+                acc.push(el);
+                acc
+            }))
+        }
+        Err(err) => Err(err),
+    }
+}

crates/worker/src/lib.rs

@@ -6,12 +6,15 @@ pub mod io;
 
 use actix_web::HttpRequest;
 use anyhow::Result;
+use config::Config;
 use io::{WasmInput, WasmOutput};
+use std::fs;
+use std::path::PathBuf;
 use std::{collections::HashMap, path::Path};
 use wasi_common::pipe::{ReadPipe, WritePipe};
 use wasmtime::{Engine, Linker, Module, Store};
-use wasmtime_wasi::WasiCtxBuilder;
-use wws_config::Config;
+use wasmtime_wasi::{Dir, WasiCtxBuilder};
+use wws_config::Config as ProjectConfig;
 use wws_runtimes::{init_runtime, Runtime};
 
 /// A worker contains the engine and the associated runtime.
@@ -24,13 +27,30 @@ pub struct Worker {
     module: Module,
     /// Worker runtime
     runtime: Box<dyn Runtime + Sync + Send>,
+    /// Current config
+    pub config: Config,
+    /// Project base path
+    project_root: PathBuf,
 }
 
 impl Worker {
     /// Creates a new Worker
-    pub fn new(project_root: &Path, path: &Path, config: &Config) -> Result<Self> {
+    pub fn new(project_root: &Path, path: &Path, project_config: &ProjectConfig) -> Result<Self> {
+        // Load configuration
+        let mut config_path = path.to_path_buf();
+        config_path.set_extension("toml");
+        let mut config = Config::default();
+
+        if fs::metadata(&config_path).is_ok() {
+            if let Ok(c) = Config::try_from_file(config_path) {
+                config = c;
+            } else {
+                println!("Error loading the config!");
+            }
+        }
+
         let engine = Engine::default();
-        let runtime = init_runtime(project_root, path, config)?;
+        let runtime = init_runtime(project_root, path, project_config)?;
         let bytes = runtime.module_bytes()?;
         let module = Module::from_binary(&engine, &bytes)?;
 
@@ -41,6 +61,8 @@ impl Worker {
             engine,
             module,
             runtime,
+            config,
+            project_root: project_root.to_path_buf(),
         })
     }
 
@@ -72,6 +94,15 @@ impl Worker {
             .stderr(Box::new(stderr.clone()))
             .envs(&tuple_vars)?;
 
+        // Mount folders from the configuration
+        if let Some(folders) = self.config.folders.as_ref() {
+            for folder in folders {
+                let source = fs::File::open(&self.project_root.join(&folder.from))?;
+                wasi_builder =
+                    wasi_builder.preopened_dir(Dir::from_std_file(source), &folder.to)?;
+            }
+        }
+
         // Pass to the runtime to add any WASI specific requirement
         wasi_builder = self.runtime.prepare_wasi_ctx(wasi_builder)?;
 

examples/python-mount/.wws.toml

@@ -0,0 +1,44 @@
+version = 1
+
+[[repositories]]
+name = "wasmlabs"
+url = "https://workers.wasmlabs.dev/repository/v1/index.toml"
+
+[[repositories.runtimes]]
+name = "python"
+version = "3.11.1+20230217"
+tags = [
+    "latest",
+    "3.11",
+    "3.11.1",
+]
+status = "active"
+extensions = ["py"]
+args = [
+    "--",
+    "/src/index.py",
+]
+
+[repositories.runtimes.binary]
+url = "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/python%2F3.11.1%2B20230217-15dfbed/python-3.11.1.wasm"
+filename = "python.wasm"
+
+[repositories.runtimes.binary.checksum]
+type = "sha256"
+value = "66589b289f76bd716120f76f234e4dd663064ed5b6256c92d441d84e51d7585d"
+
+[repositories.runtimes.polyfill]
+url = "https://workers.wasmlabs.dev/repository/v1/files/python/3/poly.py"
+filename = "poly.py"
+
+[repositories.runtimes.polyfill.checksum]
+type = "sha256"
+value = "2027b73556ca02155f026cee751ab736985917d2f28bbcad5aac928c719e1112"
+
+[repositories.runtimes.wrapper]
+url = "https://workers.wasmlabs.dev/repository/v1/files/python/3/wrapper.txt"
+filename = "wrapper.txt"
+
+[repositories.runtimes.wrapper.checksum]
+type = "sha256"
+value = "cf1edc5b1427180ec09d18f4d169580379f1b12001f30e330759f9a0f8745357"

examples/python-mount/_assets/index.html

@@ -0,0 +1 @@
+<h1>This page was loaded from a mounted file!</h1>

examples/python-mount/index.py

@@ -0,0 +1,7 @@
+# Read a mounted file and return it
+def worker(request):
+    s = ""
+    with open("/src/assets/index.html") as f:
+        s = f.read()
+
+    return Response(s)

examples/python-mount/index.toml

@@ -0,0 +1,5 @@
+version = "1"
+
+[[folders]]
+from = "./_assets"
+to = "/src/assets"