feat: run workers in installed language runtimes

examples/python-basic/worker.py

@@ -0,0 +1,2 @@
+def worker(req):
+  return Response("Hello from Python!")

examples/ruby-basic/worker.rb

@@ -0,0 +1,3 @@
+def worker(_req)
+  Response.new("Hello from Ruby!")
+end

src/config.rs

@@ -44,16 +44,7 @@ impl Config {
                 anyhow!("Error opening the .wws.toml file. The file format is not correct")
             })
         } else {
-            let new_repo = ConfigRepository {
-                name: DEFAULT_REPO_NAME.to_string(),
-                url: DEFAULT_REPO_URL.to_string(),
-                runtimes: Vec::new(),
-            };
-
-            Ok(Self {
-                version: 1,
-                repositories: vec![new_repo],
-            })
+            Ok(Self::default())
         }
     }
 
@@ -115,6 +106,22 @@ impl Config {
     }
 }
 
+impl Default for Config {
+    // Initialize an empty repository by default
+    fn default() -> Self {
+        let new_repo = ConfigRepository {
+            name: DEFAULT_REPO_NAME.to_string(),
+            url: DEFAULT_REPO_URL.to_string(),
+            runtimes: Vec::new(),
+        };
+
+        Self {
+            version: 1,
+            repositories: vec![new_repo],
+        }
+    }
+}
+
 #[derive(Deserialize, Serialize)]
 pub struct ConfigRepository {
     /// Local name to identify the repository. It avoids collisions when installing

src/main.rs

@@ -13,6 +13,7 @@ mod runtimes;
 mod store;
 mod workers;
 
+use crate::config::Config;
 use actix_files::{Files, NamedFile};
 use actix_web::dev::{fn_service, ServiceRequest, ServiceResponse};
 use actix_web::{
@@ -257,8 +258,20 @@ async fn main() -> std::io::Result<()> {
     } else {
         // TODO(Angelmmiguel): refactor this into a separate command!
         // Initialize the routes
+
+        // Loading the local configuration if available.
+        let config = match Config::load(&args.path) {
+            Ok(c) => c,
+            Err(err) => {
+                println!("⚠️  There was an error reading the .wws.toml file. It will be ignored");
+                println!("⚠️  Error: {err}");
+
+                Config::default()
+            }
+        };
+
         println!("⚙️  Loading routes from: {}", &args.path.display());
-        let routes = Data::new(Routes::new(&args.path, &args.prefix));
+        let routes = Data::new(Routes::new(&args.path, &args.prefix, &config));
 
         let data = Data::new(RwLock::new(DataConnectors { kv: KV::new() }));
 

src/router/files.rs

@@ -1,6 +1,7 @@
 // Copyright 2022 VMware, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
+use crate::config::Config;
 use crate::store::STORE_FOLDER;
 use std::ffi::OsStr;
 use std::path::{Component, Path, PathBuf};
@@ -11,25 +12,43 @@ use wax::{Glob, WalkEntry};
 /// provide utilities to work with public folders and
 /// other related resources.
 pub struct Files {
+    /// Root path
     root: PathBuf,
+    /// Available extensions based on the config
+    extensions: Vec<String>,
+    /// Check if the public folder exists
     has_public: bool,
 }
 
 impl Files {
     /// Initializes a new files instance. It will detect
     /// relevant resources for WWS like the public folder.
-    pub fn new(root: &Path) -> Self {
+    pub fn new(root: &Path, config: &Config) -> Self {
+        let mut extensions = vec![String::from("js"), String::from("wasm")];
+
+        for repo in &config.repositories {
+            for runtime in &repo.runtimes {
+                for ext in &runtime.extensions {
+                    if !extensions.contains(ext) {
+                        extensions.push(ext.clone());
+                    }
+                }
+            }
+        }
+
         Self {
             root: root.to_path_buf(),
+            extensions,
             has_public: root.join(Path::new("public")).exists(),
         }
     }
 
     /// Walk through all the different files associated to this
     /// project using a Glob pattern
     pub fn walk(&self) -> Vec<WalkEntry> {
+        let glob_pattern = format!("**/*.{{{}}}", self.extensions.join(","));
         let glob =
-            Glob::new("**/*.{wasm,js}").expect("Failed to read the files in the current directory");
+            Glob::new(&glob_pattern).expect("Failed to read the files in the current directory");
 
         glob.walk(&self.root)
             .filter_map(|el| match el {
@@ -85,8 +104,9 @@ mod tests {
         ];
 
         for t in tests {
+            let config = Config::default();
             assert_eq!(
-                Files::new(Path::new("./tests/data")).is_in_public_folder(Path::new(t.0)),
+                Files::new(Path::new("./tests/data"), &config).is_in_public_folder(Path::new(t.0)),
                 t.1
             )
         }
@@ -106,8 +126,10 @@ mod tests {
         ];
 
         for t in tests {
+            let config = Config::default();
             assert_eq!(
-                Files::new(Path::new(".\\tests\\data")).is_in_public_folder(Path::new(t.0)),
+                Files::new(Path::new(".\\tests\\data"), &config)
+                    .is_in_public_folder(Path::new(t.0)),
                 t.1
             )
         }

src/router/route.rs

@@ -1,7 +1,7 @@
 // Copyright 2022 VMware, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::{workers::config::Config, workers::Worker};
+use crate::{config::Config as ProjectConfig, workers::config::Config, workers::Worker};
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::{
@@ -53,8 +53,8 @@ impl Route {
     /// proper URL path based on the filename.
     ///
     /// This method also initializes the Runner and loads the Config if available.
-    pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str) -> Self {
-        let worker = Worker::new(base_path, &filepath).unwrap();
+    pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str, config: &ProjectConfig) -> Self {
+        let worker = Worker::new(base_path, &filepath, config).unwrap();
 
         // Load configuration
         let mut config_path = filepath.clone();

src/router/routes.rs

@@ -9,6 +9,7 @@ use std::path::Path;
 
 use super::files::Files;
 use super::route::{Route, RouteAffinity};
+use crate::config::Config;
 
 /// Contains all registered routes
 pub struct Routes {
@@ -20,13 +21,13 @@ impl Routes {
     /// Initialize the list of routes from the given folder. This method will look for
     /// different files and will create the associated routes. This routing approach
     /// is pretty popular in web development and static sites.
-    pub fn new(path: &Path, base_prefix: &str) -> Self {
+    pub fn new(path: &Path, base_prefix: &str, config: &Config) -> Self {
         let mut routes = Vec::new();
         let prefix = Self::format_prefix(base_prefix);
-        let files = Files::new(path);
+        let files = Files::new(path, config);
 
         for entry in files.walk() {
-            routes.push(Route::new(path, entry.into_path(), &prefix));
+            routes.push(Route::new(path, entry.into_path(), &prefix, config));
         }
 
         Self { routes, prefix }
@@ -91,10 +92,12 @@ mod tests {
     #[test]
     fn route_path_affinity() {
         let build_route = |file: &str| -> Route {
+            let project_config = Config::default();
             Route::new(
                 Path::new("./tests/data/params"),
                 PathBuf::from(format!("./tests/data/params{file}")),
                 "",
+                &project_config,
             )
         };
 
@@ -123,10 +126,12 @@ mod tests {
     #[test]
     fn best_route_by_affinity() {
         let build_route = |file: &str| -> Route {
+            let project_config = Config::default();
             Route::new(
                 Path::new("./tests/data/params"),
                 PathBuf::from(format!("./tests/data/params{file}")),
                 "",
+                &project_config,
             )
         };
 

src/runtimes/manager.rs

@@ -3,10 +3,10 @@
 
 use super::{
     metadata::{RemoteFile, Runtime as RuntimeMetadata},
-    modules::{javascript::JavaScriptRuntime, native::NativeRuntime},
+    modules::{external::ExternalRuntime, javascript::JavaScriptRuntime, native::NativeRuntime},
     runtime::Runtime,
 };
-use crate::{fetch::fetch_and_validate, store::Store};
+use crate::{config::Config, fetch::fetch_and_validate, store::Store};
 use anyhow::{anyhow, Result};
 use std::path::Path;
 
@@ -15,7 +15,11 @@ use std::path::Path;
 /// Initializes a runtime based on the file extension. In the future,
 /// This will contain a more complete struct that will identify local
 /// runtimes.
-pub fn init_runtime(project_root: &Path, path: &Path) -> Result<Box<dyn Runtime + Sync + Send>> {
+pub fn init_runtime(
+    project_root: &Path,
+    path: &Path,
+    config: &Config,
+) -> Result<Box<dyn Runtime + Sync + Send>> {
     if let Some(ext) = path.extension() {
         let ext_as_str = ext.to_str().unwrap();
 
@@ -25,9 +29,34 @@ pub fn init_runtime(project_root: &Path, path: &Path) -> Result<Box<dyn Runtime
                 path.to_path_buf(),
             )?)),
             "wasm" => Ok(Box::new(NativeRuntime::new(path.to_path_buf()))),
-            _ => Err(anyhow!(format!(
-                "The '{ext_as_str}' extension does not have an associated runtime"
-            ))),
+            other => {
+                let mut runtime_config = None;
+                let mut repo_name = "";
+                let other_string = other.to_string();
+
+                'outer: for repo in &config.repositories {
+                    for r in &repo.runtimes {
+                        if r.extensions.contains(&other_string) {
+                            runtime_config = Some(r);
+                            repo_name = &repo.name;
+                            break 'outer;
+                        }
+                    }
+                }
+
+                if let Some(runtime_config) = runtime_config {
+                    Ok(Box::new(ExternalRuntime::new(
+                        project_root,
+                        path.to_path_buf(),
+                        repo_name,
+                        runtime_config.clone(),
+                    )?))
+                } else {
+                    Err(anyhow!(format!(
+                        "The '{ext_as_str}' extension does not have an associated runtime"
+                    )))
+                }
+            }
         }
     } else {
         Err(anyhow!("The given file does not have a valid extension"))

src/runtimes/modules/external.rs

@@ -0,0 +1,107 @@
+// Copyright 2022 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{
+    runtimes::{metadata::Runtime as RuntimeMetadata, runtime::Runtime},
+    store::Store,
+};
+use anyhow::Result;
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+use wasmtime_wasi::{Dir, WasiCtxBuilder};
+
+/// Run language runtimes that were downloaded externally. This
+/// runtime prepare the worker and configure the WASI context
+/// based on the given metadata.
+pub struct ExternalRuntime {
+    /// Path of the given module
+    path: PathBuf,
+    /// Utils to store temporary files for this runtime
+    store: Store,
+    /// Associated runtime metadata
+    metadata: RuntimeMetadata,
+    /// Runtime store to load different files
+    runtime_store: Store,
+}
+
+impl ExternalRuntime {
+    /// Initializes the External runtime. This runtime will use
+    /// the associated metadata to properly prepare the worker
+    /// and the WASI environment
+    pub fn new(
+        project_root: &Path,
+        path: PathBuf,
+        repository: &str,
+        metadata: RuntimeMetadata,
+    ) -> Result<Self> {
+        let hash = Store::file_hash(&path)?;
+        // TODO: May move to a different folder strucuture when having multiple extensions?
+        let worker_folder = metadata.extensions.first().unwrap_or(&metadata.name);
+        let store = Store::create(project_root, &["workers", worker_folder, &hash])?;
+        let runtime_store = Store::new(
+            project_root,
+            &["runtimes", repository, &metadata.name, &metadata.version],
+        );
+
+        Ok(Self {
+            path,
+            store,
+            metadata,
+            runtime_store,
+        })
+    }
+}
+
+impl Runtime for ExternalRuntime {
+    /// Prepare the environment to run this specific worker. Since
+    /// the current folder received by argument may include multiple
+    /// files (workers), we use the Data struct to write the JS source
+    /// file into an isolated and separate folder. Then, we will mount
+    /// it during the [prepare_wasi_ctx] call.
+    fn prepare(&self) -> Result<()> {
+        let filename = format!("index.{}", self.metadata.extensions.first().unwrap());
+
+        // If wrapper, modify the worker and write the data
+        if let Some(wrapper) = &self.metadata.wrapper {
+            let wrapper_data = String::from_utf8(self.runtime_store.read(&[&wrapper.filename])?)?;
+            let source_data = fs::read_to_string(&self.path)?;
+
+            self.store.write(
+                &[&filename],
+                wrapper_data.replace("{source}", &source_data).as_bytes(),
+            )?;
+        } else {
+            // If not, copy the worker
+            self.store.copy(&self.path, &[&filename])?;
+        }
+
+        // Copy polyfills file if available
+        if let Some(polyfill) = &self.metadata.polyfill {
+            self.store.copy(
+                &self.runtime_store.build_folder_path(&[&polyfill.filename]),
+                &[&polyfill.filename],
+            )?;
+        }
+
+        Ok(())
+    }
+
+    /// Mount the source code in the WASI context so it can be
+    /// processed by the engine
+    fn prepare_wasi_ctx(&self, builder: WasiCtxBuilder) -> Result<WasiCtxBuilder> {
+        let source = fs::File::open(&self.store.folder)?;
+
+        Ok(builder
+            .preopened_dir(Dir::from_std_file(source), "/src")?
+            .args(&self.metadata.args)?)
+    }
+
+    /// Returns a reference to the Wasm module that should
+    /// run this worker. It can be a custom (native) or a
+    /// shared module (others).
+    fn module_bytes(&self) -> Result<Vec<u8>> {
+        self.runtime_store.read(&[&self.metadata.binary.filename])
+    }
+}