diff --git a/examples/python-basic/worker.py b/examples/python-basic/worker.py new file mode 100644 index 00000000..6aa4009c --- /dev/null +++ b/examples/python-basic/worker.py @@ -0,0 +1,2 @@ +def worker(req): + return Response("Hello from Python!") \ No newline at end of file diff --git a/examples/ruby-basic/worker.rb b/examples/ruby-basic/worker.rb new file mode 100644 index 00000000..23cd9086 --- /dev/null +++ b/examples/ruby-basic/worker.rb @@ -0,0 +1,3 @@ +def worker(_req) + Response.new("Hello from Ruby!") +end \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index d2f464e8..556b769a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,16 +44,7 @@ impl Config { anyhow!("Error opening the .wws.toml file. The file format is not correct") }) } else { - let new_repo = ConfigRepository { - name: DEFAULT_REPO_NAME.to_string(), - url: DEFAULT_REPO_URL.to_string(), - runtimes: Vec::new(), - }; - - Ok(Self { - version: 1, - repositories: vec![new_repo], - }) + Ok(Self::default()) } } @@ -115,6 +106,22 @@ impl Config { } } +impl Default for Config { + // Initialize an empty repository by default + fn default() -> Self { + let new_repo = ConfigRepository { + name: DEFAULT_REPO_NAME.to_string(), + url: DEFAULT_REPO_URL.to_string(), + runtimes: Vec::new(), + }; + + Self { + version: 1, + repositories: vec![new_repo], + } + } +} + #[derive(Deserialize, Serialize)] pub struct ConfigRepository { /// Local name to identify the repository. It avoids collisions when installing diff --git a/src/main.rs b/src/main.rs index 3aa87df3..0f57697f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ mod runtimes; mod store; mod workers; +use crate::config::Config; use actix_files::{Files, NamedFile}; use actix_web::dev::{fn_service, ServiceRequest, ServiceResponse}; use actix_web::{ @@ -257,8 +258,20 @@ async fn main() -> std::io::Result<()> { } else { // TODO(Angelmmiguel): refactor this into a separate command! // Initialize the routes + + // Loading the local configuration if available. + let config = match Config::load(&args.path) { + Ok(c) => c, + Err(err) => { + println!("⚠️ There was an error reading the .wws.toml file. It will be ignored"); + println!("⚠️ Error: {err}"); + + Config::default() + } + }; + println!("⚙️ Loading routes from: {}", &args.path.display()); - let routes = Data::new(Routes::new(&args.path, &args.prefix)); + let routes = Data::new(Routes::new(&args.path, &args.prefix, &config)); let data = Data::new(RwLock::new(DataConnectors { kv: KV::new() })); diff --git a/src/router/files.rs b/src/router/files.rs index b5d9458f..1fea82a3 100644 --- a/src/router/files.rs +++ b/src/router/files.rs @@ -1,6 +1,7 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 +use crate::config::Config; use crate::store::STORE_FOLDER; use std::ffi::OsStr; use std::path::{Component, Path, PathBuf}; @@ -11,16 +12,33 @@ use wax::{Glob, WalkEntry}; /// provide utilities to work with public folders and /// other related resources. pub struct Files { + /// Root path root: PathBuf, + /// Available extensions based on the config + extensions: Vec, + /// Check if the public folder exists has_public: bool, } impl Files { /// Initializes a new files instance. It will detect /// relevant resources for WWS like the public folder. - pub fn new(root: &Path) -> Self { + pub fn new(root: &Path, config: &Config) -> Self { + let mut extensions = vec![String::from("js"), String::from("wasm")]; + + for repo in &config.repositories { + for runtime in &repo.runtimes { + for ext in &runtime.extensions { + if !extensions.contains(ext) { + extensions.push(ext.clone()); + } + } + } + } + Self { root: root.to_path_buf(), + extensions, has_public: root.join(Path::new("public")).exists(), } } @@ -28,8 +46,9 @@ impl Files { /// Walk through all the different files associated to this /// project using a Glob pattern pub fn walk(&self) -> Vec { + let glob_pattern = format!("**/*.{{{}}}", self.extensions.join(",")); let glob = - Glob::new("**/*.{wasm,js}").expect("Failed to read the files in the current directory"); + Glob::new(&glob_pattern).expect("Failed to read the files in the current directory"); glob.walk(&self.root) .filter_map(|el| match el { @@ -85,8 +104,9 @@ mod tests { ]; for t in tests { + let config = Config::default(); assert_eq!( - Files::new(Path::new("./tests/data")).is_in_public_folder(Path::new(t.0)), + Files::new(Path::new("./tests/data"), &config).is_in_public_folder(Path::new(t.0)), t.1 ) } @@ -106,8 +126,10 @@ mod tests { ]; for t in tests { + let config = Config::default(); assert_eq!( - Files::new(Path::new(".\\tests\\data")).is_in_public_folder(Path::new(t.0)), + Files::new(Path::new(".\\tests\\data"), &config) + .is_in_public_folder(Path::new(t.0)), t.1 ) } diff --git a/src/router/route.rs b/src/router/route.rs index bacebb6f..e7ca5f2d 100644 --- a/src/router/route.rs +++ b/src/router/route.rs @@ -1,7 +1,7 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -use crate::{workers::config::Config, workers::Worker}; +use crate::{config::Config as ProjectConfig, workers::config::Config, workers::Worker}; use lazy_static::lazy_static; use regex::Regex; use std::{ @@ -53,8 +53,8 @@ impl Route { /// proper URL path based on the filename. /// /// This method also initializes the Runner and loads the Config if available. - pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str) -> Self { - let worker = Worker::new(base_path, &filepath).unwrap(); + pub fn new(base_path: &Path, filepath: PathBuf, prefix: &str, config: &ProjectConfig) -> Self { + let worker = Worker::new(base_path, &filepath, config).unwrap(); // Load configuration let mut config_path = filepath.clone(); diff --git a/src/router/routes.rs b/src/router/routes.rs index 7e4964ac..f76058e9 100644 --- a/src/router/routes.rs +++ b/src/router/routes.rs @@ -9,6 +9,7 @@ use std::path::Path; use super::files::Files; use super::route::{Route, RouteAffinity}; +use crate::config::Config; /// Contains all registered routes pub struct Routes { @@ -20,13 +21,13 @@ impl Routes { /// Initialize the list of routes from the given folder. This method will look for /// different files and will create the associated routes. This routing approach /// is pretty popular in web development and static sites. - pub fn new(path: &Path, base_prefix: &str) -> Self { + pub fn new(path: &Path, base_prefix: &str, config: &Config) -> Self { let mut routes = Vec::new(); let prefix = Self::format_prefix(base_prefix); - let files = Files::new(path); + let files = Files::new(path, config); for entry in files.walk() { - routes.push(Route::new(path, entry.into_path(), &prefix)); + routes.push(Route::new(path, entry.into_path(), &prefix, config)); } Self { routes, prefix } @@ -91,10 +92,12 @@ mod tests { #[test] fn route_path_affinity() { let build_route = |file: &str| -> Route { + let project_config = Config::default(); Route::new( Path::new("./tests/data/params"), PathBuf::from(format!("./tests/data/params{file}")), "", + &project_config, ) }; @@ -123,10 +126,12 @@ mod tests { #[test] fn best_route_by_affinity() { let build_route = |file: &str| -> Route { + let project_config = Config::default(); Route::new( Path::new("./tests/data/params"), PathBuf::from(format!("./tests/data/params{file}")), "", + &project_config, ) }; diff --git a/src/runtimes/manager.rs b/src/runtimes/manager.rs index 20e15647..6b40a403 100644 --- a/src/runtimes/manager.rs +++ b/src/runtimes/manager.rs @@ -3,10 +3,10 @@ use super::{ metadata::{RemoteFile, Runtime as RuntimeMetadata}, - modules::{javascript::JavaScriptRuntime, native::NativeRuntime}, + modules::{external::ExternalRuntime, javascript::JavaScriptRuntime, native::NativeRuntime}, runtime::Runtime, }; -use crate::{fetch::fetch_and_validate, store::Store}; +use crate::{config::Config, fetch::fetch_and_validate, store::Store}; use anyhow::{anyhow, Result}; use std::path::Path; @@ -15,7 +15,11 @@ use std::path::Path; /// Initializes a runtime based on the file extension. In the future, /// This will contain a more complete struct that will identify local /// runtimes. -pub fn init_runtime(project_root: &Path, path: &Path) -> Result> { +pub fn init_runtime( + project_root: &Path, + path: &Path, + config: &Config, +) -> Result> { if let Some(ext) = path.extension() { let ext_as_str = ext.to_str().unwrap(); @@ -25,15 +29,49 @@ pub fn init_runtime(project_root: &Path, path: &Path) -> Result Ok(Box::new(NativeRuntime::new(path.to_path_buf()))), - _ => Err(anyhow!(format!( - "The '{ext_as_str}' extension does not have an associated runtime" - ))), + other => init_external_runtime(project_root, config, path, other), } } else { Err(anyhow!("The given file does not have a valid extension")) } } +/// Initialize an external runtime. It looks for the right runtime in the configuration +/// metadata. Then, it will init the runtime with it. +fn init_external_runtime( + project_root: &Path, + config: &Config, + path: &Path, + extension: &str, +) -> Result> { + let mut runtime_config = None; + let mut repo_name = ""; + let other_string = extension.to_string(); + + 'outer: for repo in &config.repositories { + for r in &repo.runtimes { + if r.extensions.contains(&other_string) { + runtime_config = Some(r); + repo_name = &repo.name; + break 'outer; + } + } + } + + if let Some(runtime_config) = runtime_config { + Ok(Box::new(ExternalRuntime::new( + project_root, + path.to_path_buf(), + repo_name, + runtime_config.clone(), + )?)) + } else { + Err(anyhow!(format!( + "The '{extension}' extension does not have an associated runtime" + ))) + } +} + // Install a given runtime based on its metadata pub async fn install_runtime( project_root: &Path, diff --git a/src/runtimes/modules/external.rs b/src/runtimes/modules/external.rs new file mode 100644 index 00000000..33f3018d --- /dev/null +++ b/src/runtimes/modules/external.rs @@ -0,0 +1,107 @@ +// Copyright 2022 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + runtimes::{metadata::Runtime as RuntimeMetadata, runtime::Runtime}, + store::Store, +}; +use anyhow::Result; +use std::{ + fs, + path::{Path, PathBuf}, +}; +use wasmtime_wasi::{Dir, WasiCtxBuilder}; + +/// Run language runtimes that were downloaded externally. This +/// runtime prepare the worker and configure the WASI context +/// based on the given metadata. +pub struct ExternalRuntime { + /// Path of the given module + path: PathBuf, + /// Utils to store temporary files for this runtime + store: Store, + /// Associated runtime metadata + metadata: RuntimeMetadata, + /// Runtime store to load different files + runtime_store: Store, +} + +impl ExternalRuntime { + /// Initializes the External runtime. This runtime will use + /// the associated metadata to properly prepare the worker + /// and the WASI environment + pub fn new( + project_root: &Path, + path: PathBuf, + repository: &str, + metadata: RuntimeMetadata, + ) -> Result { + let hash = Store::file_hash(&path)?; + // TODO: May move to a different folder strucuture when having multiple extensions? + let worker_folder = metadata.extensions.first().unwrap_or(&metadata.name); + let store = Store::create(project_root, &["workers", worker_folder, &hash])?; + let runtime_store = Store::new( + project_root, + &["runtimes", repository, &metadata.name, &metadata.version], + ); + + Ok(Self { + path, + store, + metadata, + runtime_store, + }) + } +} + +impl Runtime for ExternalRuntime { + /// Prepare the environment to run this specific worker. Since + /// the current folder received by argument may include multiple + /// files (workers), we use the Data struct to write the JS source + /// file into an isolated and separate folder. Then, we will mount + /// it during the [prepare_wasi_ctx] call. + fn prepare(&self) -> Result<()> { + let filename = format!("index.{}", self.metadata.extensions.first().unwrap()); + + // If wrapper, modify the worker and write the data + if let Some(wrapper) = &self.metadata.wrapper { + let wrapper_data = String::from_utf8(self.runtime_store.read(&[&wrapper.filename])?)?; + let source_data = fs::read_to_string(&self.path)?; + + self.store.write( + &[&filename], + wrapper_data.replace("{source}", &source_data).as_bytes(), + )?; + } else { + // If not, copy the worker + self.store.copy(&self.path, &[&filename])?; + } + + // Copy polyfills file if available + if let Some(polyfill) = &self.metadata.polyfill { + self.store.copy( + &self.runtime_store.build_folder_path(&[&polyfill.filename]), + &[&polyfill.filename], + )?; + } + + Ok(()) + } + + /// Mount the source code in the WASI context so it can be + /// processed by the engine + fn prepare_wasi_ctx(&self, builder: WasiCtxBuilder) -> Result { + let source = fs::File::open(&self.store.folder)?; + + Ok(builder + .preopened_dir(Dir::from_std_file(source), "/src")? + .args(&self.metadata.args)?) + } + + /// Returns a reference to the Wasm module that should + /// run this worker. It can be a custom (native) or a + /// shared module (others). + fn module_bytes(&self) -> Result> { + self.runtime_store.read(&[&self.metadata.binary.filename]) + } +} diff --git a/src/runtimes/modules/mod.rs b/src/runtimes/modules/mod.rs index 9764954c..5bf5c51d 100644 --- a/src/runtimes/modules/mod.rs +++ b/src/runtimes/modules/mod.rs @@ -1,5 +1,6 @@ // Copyright 2022 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 +pub(super) mod external; pub(super) mod javascript; pub(super) mod native; diff --git a/src/store.rs b/src/store.rs index 6e182ddb..6360a8b1 100644 --- a/src/store.rs +++ b/src/store.rs @@ -82,6 +82,18 @@ impl Store { Ok(()) } + /// Read the file content in the given store + pub fn read(&self, path: &[&str]) -> Result> { + let file_path = self.build_folder_path(path); + fs::read(&file_path).map_err(|err| { + anyhow!( + "There was an error reading the {} file: {}", + &file_path.display(), + err + ) + }) + } + /// Copy file inside the configured root folder pub fn copy(&self, source: &Path, dest: &[&str]) -> Result<()> { let file_path = self.build_folder_path(dest); @@ -91,7 +103,7 @@ impl Store { } /// This method builds a path in the context of the instance folder - fn build_folder_path(&self, source: &[&str]) -> PathBuf { + pub fn build_folder_path(&self, source: &[&str]) -> PathBuf { source .iter() .fold(self.folder.clone(), |acc, comp| acc.join(comp)) diff --git a/src/workers/worker.rs b/src/workers/worker.rs index d864a680..e1189a96 100644 --- a/src/workers/worker.rs +++ b/src/workers/worker.rs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use super::wasm_io::{WasmInput, WasmOutput}; -use crate::runtimes::{manager::init_runtime, runtime::Runtime}; +use crate::{ + config::Config, + runtimes::{manager::init_runtime, runtime::Runtime}, +}; use actix_web::HttpRequest; use anyhow::Result; use std::{collections::HashMap, path::Path}; @@ -24,9 +27,9 @@ pub struct Worker { impl Worker { /// Creates a new Worker - pub fn new(project_root: &Path, path: &Path) -> Result { + pub fn new(project_root: &Path, path: &Path, config: &Config) -> Result { let engine = Engine::default(); - let runtime = init_runtime(project_root, path)?; + let runtime = init_runtime(project_root, path, config)?; let bytes = runtime.module_bytes()?; let module = Module::from_binary(&engine, &bytes)?;