From 1fef939ed944c3d92a43f51118b157dc90195991 Mon Sep 17 00:00:00 2001 From: Angel M De Miguel <dangel@vmware.com> Date: Fri, 2 Jun 2023 11:21:06 +0200 Subject: [PATCH 1/4] feat: use remote git repositories (https) as valid project location Signed-off-by: Angel M De Miguel <dangel@vmware.com> --- Cargo.lock | 57 ++++++++++++++ Cargo.toml | 1 + crates/project/Cargo.toml | 4 + crates/project/src/lib.rs | 118 ++++++++++++++++++++++++++++- crates/project/src/options.rs | 27 +++++++ crates/project/src/types/git.rs | 61 +++++++++++++++ crates/project/src/types/mod.rs | 4 + crates/project/tests/data/index.js | 0 crates/router/Cargo.toml | 2 +- src/commands/runtimes.rs | 109 ++------------------------ src/main.rs | 96 +++++++++++++++++++---- src/utils/mod.rs | 5 ++ src/utils/options.rs | 36 +++++++++ src/utils/runtimes.rs | 102 +++++++++++++++++++++++++ 14 files changed, 503 insertions(+), 119 deletions(-) create mode 100644 crates/project/src/options.rs create mode 100644 crates/project/src/types/git.rs create mode 100644 crates/project/src/types/mod.rs create mode 100644 crates/project/tests/data/index.js create mode 100644 src/utils/mod.rs create mode 100644 src/utils/options.rs create mode 100644 src/utils/runtimes.rs diff --git a/Cargo.lock b/Cargo.lock index a397a3b1..ebf2c04c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1224,6 +1224,21 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "git2" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "glob" version = "0.3.1" @@ -1564,6 +1579,20 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "libgit2-sys" +version = "0.15.2+1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.7.4" @@ -1574,6 +1603,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "libssh2-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -3628,6 +3683,8 @@ name = "wws-project" version = "1.2.0" dependencies = [ "anyhow", + "git2", + "path-slash", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 36822550..c30a2eb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,3 +82,4 @@ wws-project = { path = "./crates/project" } wasmtime = "6.0.2" wasmtime-wasi = "6.0.2" wasi-common = "6.0.2" +path-slash = "0.2.1" diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml index cba4f772..d46c0b40 100644 --- a/crates/project/Cargo.toml +++ b/crates/project/Cargo.toml @@ -15,3 +15,7 @@ wws-store = { workspace = true } url = "2.3.1" sha256 = "1.1.1" reqwest = "0.11" +git2 = "0.17.2" + +[dev-dependencies] +path-slash = { workspace = true } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 01188c2c..b16485d2 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1,15 +1,72 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 mod fetch; pub mod metadata; +pub mod options; +pub mod types; -use anyhow::Result; +use anyhow::{bail, Result}; use fetch::fetch_and_validate; use metadata::{RemoteFile, Runtime}; -use std::path::Path; +use options::Options; +use std::path::{Path, PathBuf}; +use types::git::prepare_git_project; use wws_store::Store; +pub enum ProjectType { + Local, + Git, +} + +/// Prepare a project from the given String. This argument could represent +/// different things: +/// +/// - A local path +/// - A git repository +/// - Etc. +/// +/// Depending on the type, the project preparation requires different steps. +/// For example, a git repository requires to clone it. +/// +/// However, the result of any type is the same: a local folder to point to. +/// This is the value we return from this function. +pub async fn prepare_project( + location: &str, + force_type: Option<ProjectType>, + options: Option<Options>, +) -> Result<PathBuf> { + let project_type = if force_type.is_some() { + force_type.unwrap() + } else { + identify_type(location)? + }; + + match project_type { + ProjectType::Local => Ok(PathBuf::from(location)), + ProjectType::Git => prepare_git_project(location, options), + } +} + +/// Identify the type of the project based on different rules related to the location. +/// For example, an URL that ends in .git is considered a git repository. For any +/// unknown pattern, it will default to "Local" +pub fn identify_type(location: &str) -> Result<ProjectType> { + if (location.starts_with("https://") || location.starts_with("http://")) + && location.ends_with(".git") + { + Ok(ProjectType::Git) + } else { + let path = Path::new(location); + + if path.exists() { + Ok(ProjectType::Local) + } else { + bail!("The given path does not exist in the local filesystem.") + } + } +} + /// Install a runtime locally. It reads the provided configuration and /// dowload the files. All files are saved in a store that references /// the repository, the runtime name and version @@ -89,3 +146,58 @@ async fn download_file(file: &RemoteFile, store: &Store) -> Result<()> { let contents = fetch_and_validate(&file.url, &file.checksum).await?; store.write(&[&file.filename], &contents) } + +#[cfg(test)] +mod tests { + use super::*; + use path_slash::PathBufExt as _; + + #[test] + fn identify_local_locations() { + let tests = ["tests", "tests/data", "./tests", "./tests/data"]; + + for test in tests { + let file_route = PathBuf::from_slash(test); + + match identify_type(file_route.to_str().unwrap()) { + Ok(project_type) => { + assert!(matches!(project_type, ProjectType::Local)); + } + Err(err) => panic!("Error identifying a the project type: {err}"), + } + } + } + + #[test] + fn identify_local_error_when_missing() { + let tests = [ + "missing", + "missing/missing", + "./missing/missing", + "./missing/missing", + ]; + + for test in tests { + let file_route = PathBuf::from_slash(test); + + match identify_type(file_route.to_str().unwrap()) { + Ok(_) => { + panic!("The folder doesn't exist, so identifying it should fail."); + } + Err(err) => assert!(err.to_string().contains("does not exist")), + } + } + } + + #[test] + fn identify_git_repository_locations() { + let location = "https://github.com/vmware-labs/wasm-workers-server.git"; + + match identify_type(location) { + Ok(project_type) => { + assert!(matches!(project_type, ProjectType::Git)); + } + Err(err) => panic!("Error identifying a the project type: {err}"), + } + } +} diff --git a/crates/project/src/options.rs b/crates/project/src/options.rs new file mode 100644 index 00000000..ce0dd2ae --- /dev/null +++ b/crates/project/src/options.rs @@ -0,0 +1,27 @@ +/// Defines the different options to configure the project. +/// Every type has their own options. +#[derive(Default)] +pub struct Options { + /// Options for Git repositories + pub git: Option<GitOptions>, + /// Options for local repositories + pub local: Option<LocalOptions>, +} + +/// For now, we don't have any particular option for this type. +/// I'm keeping it as a placeholder +#[derive(Default)] +pub struct LocalOptions {} + +/// The different git options you can configure. +#[derive(Default)] +pub struct GitOptions { + /// Use a specific commit + pub commit: Option<String>, + /// Use a specific tag + pub tag: Option<String>, + /// Use a specific git branch + pub branch: Option<String>, + /// Change the directory to run the workers + pub folder: Option<String>, +} diff --git a/crates/project/src/types/git.rs b/crates/project/src/types/git.rs new file mode 100644 index 00000000..dcc4f28e --- /dev/null +++ b/crates/project/src/types/git.rs @@ -0,0 +1,61 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::options::Options; +use anyhow::{bail, Result}; +use git2::{Oid, Repository}; +use sha256::digest as sha256_digest; +use std::{env::temp_dir, fs::remove_dir_all, path::PathBuf}; + +// Default remote for git repos +static DEFAULT_REMOTE: &str = "origin"; + +/// Prepare a project based on a git repository. This method +/// clones the repo locally and return the path in which it's located. +pub fn prepare_git_project(location: &str, options: Option<Options>) -> Result<PathBuf> { + // By default, we use temporary dirs + let mut dir = temp_dir().join(sha256_digest(location)); + + if dir.exists() { + // Clean up a previous download + remove_dir_all(&dir)?; + } + + let repo = match Repository::clone(location, &dir) { + Ok(repo) => repo, + Err(e) => bail!("There was an error cloning the repository: {e}"), + }; + + if let Some(options) = options { + if let Some(git) = options.git { + // These options are prioritized + if let Some(commit) = git.commit { + let oid = Oid::from_str(&commit)?; + let commit = repo.find_commit(oid)?; + repo.checkout_tree(commit.as_object(), None)?; + } else if let Some(tag) = git.tag { + let mut remote = repo.find_remote(DEFAULT_REMOTE)?; + let tag_remote = format!("refs/tags/{tag}:refs/tags/{tag}"); + remote.fetch(&[&tag_remote], None, None)?; + + let oid = Oid::from_str(&tag)?; + let tag = repo.find_tag(oid)?; + repo.checkout_tree(tag.as_object(), None)?; + } else if let Some(branch) = git.branch { + let mut remote = repo.find_remote(DEFAULT_REMOTE)?; + let head_remote = format!("refs/heads/{branch}:refs/heads/{branch}"); + remote.fetch(&[&head_remote], None, None)?; + + let branch = repo.find_branch(&branch, git2::BranchType::Local)?; + let reference = branch.into_reference(); + repo.checkout_tree(&reference.peel(git2::ObjectType::Tree)?, None)?; + } + + if let Some(folder) = git.folder { + dir = dir.join(folder); + } + } + } + + Ok(dir) +} diff --git a/crates/project/src/types/mod.rs b/crates/project/src/types/mod.rs new file mode 100644 index 00000000..86bcb990 --- /dev/null +++ b/crates/project/src/types/mod.rs @@ -0,0 +1,4 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +pub mod git; diff --git a/crates/project/tests/data/index.js b/crates/project/tests/data/index.js new file mode 100644 index 00000000..e69de29b diff --git a/crates/router/Cargo.toml b/crates/router/Cargo.toml index ea176eb6..fdf7b050 100644 --- a/crates/router/Cargo.toml +++ b/crates/router/Cargo.toml @@ -21,4 +21,4 @@ regex = "1" wax = { git = "https://github.com/olson-sean-k/wax.git", rev = "6d66a10" } [dev-dependencies] -path-slash = "0.2.1" +path-slash = { workspace = true } diff --git a/src/commands/runtimes.rs b/src/commands/runtimes.rs index fdd098c0..bef89efb 100644 --- a/src/commands/runtimes.rs +++ b/src/commands/runtimes.rs @@ -3,31 +3,24 @@ use std::path::Path; +use crate::utils::runtimes::{ + get_repo_name, get_repo_url, install_from_repository, install_missing_runtimes, +}; use anyhow::{anyhow, Result}; use clap::{Args, Parser, Subcommand}; use prettytable::{format, Cell, Row, Table}; -use std::env; use wws_config::Config; -use wws_project::{check_runtime, install_runtime, metadata::Repository, uninstall_runtime}; - -/// Default repository name -pub const DEFAULT_REPO_NAME: &str = "wasmlabs"; -/// Default repository URL -pub const DEFAULT_REPO_URL: &str = "https://workers.wasmlabs.dev/repository/v1/index.toml"; - -/// Environment variable to set the repository name -pub const WWS_REPO_NAME: &str = "WWS_REPO_NAME"; -pub const WWS_REPO_URL: &str = "WWS_REPO_URL"; +use wws_project::{check_runtime, metadata::Repository, uninstall_runtime}; /// Manage the language runtimes in your project #[derive(Parser, Debug)] pub struct Runtimes { /// Set a different repository URL #[arg(long)] - repo_url: Option<String>, + pub repo_url: Option<String>, /// Gives a name to the given repository URL #[arg(long)] - repo_name: Option<String>, + pub repo_name: Option<String>, #[command(subcommand)] pub runtime_commands: RuntimesCommands, @@ -56,82 +49,13 @@ impl Install { pub async fn run(&self, project_root: &Path, args: &Runtimes) -> Result<()> { match (&self.name, &self.version) { (Some(name), Some(version)) => { - self.install_from_repository(project_root, args, name, version) - .await + install_from_repository(project_root, args, name, version).await } (Some(_), None) | (None, Some(_)) => Err(anyhow!( "The name and version are mandatory when installing a runtime from a repository" )), - (None, None) => self.install_missing_runtimes(project_root).await, - } - } - - /// Retrieves the remote repository and install the desired runtime. - /// It will return an error if the desired runtime is not present in - /// the repo. - async fn install_from_repository( - &self, - project_root: &Path, - args: &Runtimes, - name: &str, - version: &str, - ) -> Result<()> { - let repo_name = get_repo_name(args); - let repo_url = get_repo_url(args); - - println!("βοΈ Fetching data from the repository..."); - let repo = Repository::from_remote_file(&repo_url).await?; - let runtime = repo.find_runtime(name, version); - - if let Some(runtime) = runtime { - if check_runtime(project_root, &repo_name, runtime) { - println!("β The runtime is already installed"); - Ok(()) - } else { - println!("π Installing the runtime..."); - install_runtime(project_root, &repo_name, runtime).await?; - - // Update the configuration - let mut config = Config::load(project_root)?; - config.save_runtime(&repo_name, &repo_url, runtime); - config.save(project_root)?; - - println!("β Done"); - Ok(()) - } - } else { - Err(anyhow!( - "The runtime with name = '{}' and version = '{}' is not present in the repository", - name, - version - )) - } - } - - /// Loads the local configuration and install any missing runtime from it. - /// It will check all the different repositories and install missing - /// runtimes inside them. - async fn install_missing_runtimes(&self, project_root: &Path) -> Result<()> { - println!("βοΈ Checking local configuration..."); - // Retrieve the configuration - let config = Config::load(project_root)?; - - for repo in &config.repositories { - for runtime in &repo.runtimes { - let is_installed = check_runtime(project_root, &repo.name, runtime); - - if !is_installed { - println!( - "π Installing: {} - {} / {}", - &repo.name, &runtime.name, &runtime.version - ); - install_runtime(project_root, &repo.name, runtime).await?; - } - } + (None, None) => install_missing_runtimes(project_root).await, } - - println!("β Done"); - Ok(()) } } @@ -286,20 +210,3 @@ impl Uninstall { Ok(()) } } - -/// Utility to retrieve the repository name for the given command. -/// It will look first for the flag and fallback to the default value. -fn get_repo_name(args: &Runtimes) -> String { - let default_value = env::var(WWS_REPO_NAME).unwrap_or_else(|_| DEFAULT_REPO_NAME.into()); - args.repo_name - .as_ref() - .unwrap_or(&default_value) - .to_string() -} - -/// Utility to retrieve the repository url for the given command. -/// It will look first for the flag and fallback to the default value. -fn get_repo_url(args: &Runtimes) -> String { - let default_value = env::var(WWS_REPO_URL).unwrap_or_else(|_| DEFAULT_REPO_URL.into()); - args.repo_url.as_ref().unwrap_or(&default_value).to_string() -} diff --git a/src/main.rs b/src/main.rs index c29a57e6..3b4cad88 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,19 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 mod commands; +mod utils; +use crate::utils::options; +use crate::utils::runtimes::install_missing_runtimes; use clap::Parser; use commands::main::Main; use commands::runtimes::RuntimesCommands; use std::io::{Error, ErrorKind}; use std::path::PathBuf; +use std::process::exit; use wws_config::Config; +use wws_project::{identify_type, prepare_project, ProjectType}; use wws_router::Routes; use wws_server::serve; @@ -24,9 +29,9 @@ pub struct Args { #[arg(short, long, default_value_t = 8080)] port: u16, - /// Folder to read WebAssembly modules from + /// Location of the wws project. It could be a local folder or a git repository. #[arg(value_parser, default_value = ".")] - path: PathBuf, + location: String, /// Prepend the given path to all URLs #[arg(long, default_value = "")] @@ -36,6 +41,26 @@ pub struct Args { #[arg(long, default_value = "")] ignore: Vec<String>, + /// Install missing runtimes automatically. + #[arg(long, short)] + install_runtimes: bool, + + /// Set the commit when using a git repository as project + #[arg(long)] + git_commit: Option<String>, + + /// Set the tag when using a git repository as project + #[arg(long)] + git_tag: Option<String>, + + /// Set the branch when using a git repository as project + #[arg(long)] + git_branch: Option<String>, + + /// Change the directory when using a git repository as project + #[arg(long)] + git_folder: Option<String>, + /// Manage language runtimes in your project #[command(subcommand)] commands: Option<Main>, @@ -52,6 +77,24 @@ async fn main() -> std::io::Result<()> { if let Some(Main::Runtimes(sub)) = &args.commands { let mut run_result = Ok(()); + match identify_type(&args.location) { + Ok(project_type) => match project_type { + ProjectType::Local => {} + _ => { + eprintln!("β You an only manage runtimes in local projects"); + exit(1); + } + }, + Err(err) => { + eprintln!("β There was an error preparing the project."); + eprintln!("β Here you can find more information: {err}."); + + exit(1); + } + } + + let project_path = PathBuf::from(&args.location); + match &sub.runtime_commands { RuntimesCommands::List(list) => { if let Err(err) = list.run(sub).await { @@ -61,21 +104,21 @@ async fn main() -> std::io::Result<()> { } } RuntimesCommands::Install(install) => { - if let Err(err) = install.run(&args.path, sub).await { + if let Err(err) = install.run(&project_path, sub).await { println!("β There was an error installing the runtime from the repository"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); } } RuntimesCommands::Uninstall(uninstall) => { - if let Err(err) = uninstall.run(&args.path, sub) { + if let Err(err) = uninstall.run(&project_path, sub) { println!("β There was an error uninstalling the runtime"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); } } RuntimesCommands::Check(check) => { - if let Err(err) = check.run(&args.path) { + if let Err(err) = check.run(&project_path) { println!("β There was an error checking the local runtimes"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); @@ -86,10 +129,23 @@ async fn main() -> std::io::Result<()> { run_result } else { // TODO(Angelmmiguel): refactor this into a separate command! - // Initialize the routes + + // Set the final options + let project_opts = options::build_project_options(&args); + + println!("βοΈ Preparing the project from: {}", &args.location); + let project_path = match prepare_project(&args.location, None, Some(project_opts)).await { + Ok(p) => p, + Err(err) => { + eprintln!("β There was an error preparing the project."); + eprintln!("β Here you can find more information: {err}."); + + exit(1); + } + }; // Loading the local configuration if available. - let config = match Config::load(&args.path) { + let config = match Config::load(&project_path) { Ok(c) => c, Err(err) => { println!("β οΈ There was an error reading the .wws.toml file. It will be ignored"); @@ -100,13 +156,25 @@ async fn main() -> std::io::Result<()> { }; // Check if there're missing runtimes - if config.is_missing_any_runtime(&args.path) { - println!("β οΈ Required language runtimes are not installed. Some files may not be considered workers"); - println!("β οΈ You can install the missing runtimes with: wws runtimes install"); + if config.is_missing_any_runtime(&project_path) { + if args.install_runtimes { + match install_missing_runtimes(&project_path).await { + Ok(_) => {} + Err(err) => { + eprintln!("β There was an error installing the missing runtimes."); + eprintln!("β Here you can find more information: {err}."); + + exit(1); + } + } + } else { + println!("β οΈ Required language runtimes are not installed. Some files may not be considered workers"); + println!("β οΈ You can install the missing runtimes adding the --install-runtimes / -i flag"); + } } - println!("βοΈ Loading routes from: {}", &args.path.display()); - let routes = Routes::new(&args.path, &args.prefix, args.ignore, &config); + println!("βοΈ Loading routes from: {}", &project_path.display()); + let routes = Routes::new(&project_path, &args.prefix, args.ignore, &config); for route in routes.routes.iter() { println!( " - http://{}:{}{}\n => {}", @@ -117,7 +185,7 @@ async fn main() -> std::io::Result<()> { ); } - let server = serve(&args.path, routes, &args.hostname, args.port, None) + let server = serve(&project_path, routes, &args.hostname, args.port, None) .await .map_err(|err| Error::new(ErrorKind::AddrInUse, err))?; diff --git a/src/utils/mod.rs b/src/utils/mod.rs new file mode 100644 index 00000000..7c6a9749 --- /dev/null +++ b/src/utils/mod.rs @@ -0,0 +1,5 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +pub mod options; +pub mod runtimes; diff --git a/src/utils/options.rs b/src/utils/options.rs new file mode 100644 index 00000000..dfef80d3 --- /dev/null +++ b/src/utils/options.rs @@ -0,0 +1,36 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::Args; +use wws_project::options::{GitOptions, Options}; + +/// Create the project options from the CLI arguments +pub fn build_project_options(args: &Args) -> Options { + Options { + local: None, + git: Some(build_git_options(args)), + } +} + +/// Create the Git options from the CLI arguments +pub fn build_git_options(args: &Args) -> GitOptions { + let mut git_opts = GitOptions::default(); + + if let Some(commit) = args.git_commit.as_ref() { + git_opts.commit = Some(commit.clone()); + } + + if let Some(tag) = args.git_tag.as_ref() { + git_opts.tag = Some(tag.clone()); + } + + if let Some(branch) = args.git_branch.as_ref() { + git_opts.branch = Some(branch.clone()); + } + + if let Some(folder) = args.git_folder.as_ref() { + git_opts.folder = Some(folder.clone()); + } + + git_opts +} diff --git a/src/utils/runtimes.rs b/src/utils/runtimes.rs new file mode 100644 index 00000000..a3a6ad9d --- /dev/null +++ b/src/utils/runtimes.rs @@ -0,0 +1,102 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{anyhow, Result}; +use std::{env, path::Path}; +use wws_config::Config; +use wws_project::{check_runtime, install_runtime, metadata::Repository}; + +use crate::commands::runtimes::Runtimes; + +/// Default repository name +pub const DEFAULT_REPO_NAME: &str = "wasmlabs"; +/// Default repository URL +pub const DEFAULT_REPO_URL: &str = "https://workers.wasmlabs.dev/repository/v1/index.toml"; + +/// Environment variable to set the repository name +pub const WWS_REPO_NAME: &str = "WWS_REPO_NAME"; +pub const WWS_REPO_URL: &str = "WWS_REPO_URL"; + +/// Loads the local configuration and install any missing runtime from it. +/// It will check all the different repositories and install missing +/// runtimes inside them. +pub async fn install_missing_runtimes(project_root: &Path) -> Result<()> { + println!("βοΈ Checking local configuration..."); + // Retrieve the configuration + let config = Config::load(project_root)?; + + for repo in &config.repositories { + for runtime in &repo.runtimes { + let is_installed = check_runtime(project_root, &repo.name, runtime); + + if !is_installed { + println!( + "π Installing: {} - {} / {}", + &repo.name, &runtime.name, &runtime.version + ); + install_runtime(project_root, &repo.name, runtime).await?; + } + } + } + + println!("β Done"); + Ok(()) +} + +/// Retrieves the remote repository and install the desired runtime. +/// It will return an error if the desired runtime is not present in +/// the repo. +pub async fn install_from_repository( + project_root: &Path, + args: &Runtimes, + name: &str, + version: &str, +) -> Result<()> { + let repo_name = get_repo_name(args); + let repo_url = get_repo_url(args); + + println!("βοΈ Fetching data from the repository..."); + let repo = Repository::from_remote_file(&repo_url).await?; + let runtime = repo.find_runtime(name, version); + + if let Some(runtime) = runtime { + if check_runtime(project_root, &repo_name, runtime) { + println!("β The runtime is already installed"); + Ok(()) + } else { + println!("π Installing the runtime..."); + install_runtime(project_root, &repo_name, runtime).await?; + + // Update the configuration + let mut config = Config::load(project_root)?; + config.save_runtime(&repo_name, &repo_url, runtime); + config.save(project_root)?; + + println!("β Done"); + Ok(()) + } + } else { + Err(anyhow!( + "The runtime with name = '{}' and version = '{}' is not present in the repository", + name, + version + )) + } +} + +/// Utility to retrieve the repository name for the given command. +/// It will look first for the flag and fallback to the default value. +pub fn get_repo_name(args: &Runtimes) -> String { + let default_value = env::var(WWS_REPO_NAME).unwrap_or_else(|_| DEFAULT_REPO_NAME.into()); + args.repo_name + .as_ref() + .unwrap_or(&default_value) + .to_string() +} + +/// Utility to retrieve the repository url for the given command. +/// It will look first for the flag and fallback to the default value. +pub fn get_repo_url(args: &Runtimes) -> String { + let default_value = env::var(WWS_REPO_URL).unwrap_or_else(|_| DEFAULT_REPO_URL.into()); + args.repo_url.as_ref().unwrap_or(&default_value).to_string() +} From 3a200c3bdbc284563875e9adbad600bdb25bac96 Mon Sep 17 00:00:00 2001 From: Angel M De Miguel <dangel@vmware.com> Date: Fri, 2 Jun 2023 11:22:05 +0200 Subject: [PATCH 2/4] fix: add missing file notices --- crates/project/src/options.rs | 3 +++ src/commands/runtimes.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/project/src/options.rs b/crates/project/src/options.rs index ce0dd2ae..d3022eb5 100644 --- a/crates/project/src/options.rs +++ b/crates/project/src/options.rs @@ -1,3 +1,6 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + /// Defines the different options to configure the project. /// Every type has their own options. #[derive(Default)] diff --git a/src/commands/runtimes.rs b/src/commands/runtimes.rs index bef89efb..4c5676a0 100644 --- a/src/commands/runtimes.rs +++ b/src/commands/runtimes.rs @@ -1,4 +1,4 @@ -// Copyright 2022 VMware, Inc. +// Copyright 2022-2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 use std::path::Path; From d9aad6a5e456985ad890bc877487c665658d0871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81ngel=20M?= <dangel@vmware.com> Date: Mon, 5 Jun 2023 09:25:07 +0200 Subject: [PATCH 3/4] fix: improve CLI output and fix typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafael FernΓ‘ndez LΓ³pez <rfernandezl@vmware.com> --- crates/project/src/types/git.rs | 2 +- src/main.rs | 11 ++++------- src/utils/runtimes.rs | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/crates/project/src/types/git.rs b/crates/project/src/types/git.rs index dcc4f28e..8c1eba25 100644 --- a/crates/project/src/types/git.rs +++ b/crates/project/src/types/git.rs @@ -11,7 +11,7 @@ use std::{env::temp_dir, fs::remove_dir_all, path::PathBuf}; static DEFAULT_REMOTE: &str = "origin"; /// Prepare a project based on a git repository. This method -/// clones the repo locally and return the path in which it's located. +/// clones the repo locally and returns the path in which it's located. pub fn prepare_git_project(location: &str, options: Option<Options>) -> Result<PathBuf> { // By default, we use temporary dirs let mut dir = temp_dir().join(sha256_digest(location)); diff --git a/src/main.rs b/src/main.rs index 3b4cad88..49666825 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,13 +81,12 @@ async fn main() -> std::io::Result<()> { Ok(project_type) => match project_type { ProjectType::Local => {} _ => { - eprintln!("β You an only manage runtimes in local projects"); + eprintln!("β You can only manage runtimes in local projects"); exit(1); } }, Err(err) => { - eprintln!("β There was an error preparing the project."); - eprintln!("β Here you can find more information: {err}."); + eprintln!("β There was an error preparing the project: {err}"); exit(1); } @@ -137,8 +136,7 @@ async fn main() -> std::io::Result<()> { let project_path = match prepare_project(&args.location, None, Some(project_opts)).await { Ok(p) => p, Err(err) => { - eprintln!("β There was an error preparing the project."); - eprintln!("β Here you can find more information: {err}."); + eprintln!("β There was an error preparing the project: {err}"); exit(1); } @@ -161,8 +159,7 @@ async fn main() -> std::io::Result<()> { match install_missing_runtimes(&project_path).await { Ok(_) => {} Err(err) => { - eprintln!("β There was an error installing the missing runtimes."); - eprintln!("β Here you can find more information: {err}."); + eprintln!("β There was an error installing the missing runtimes: {err}"); exit(1); } diff --git a/src/utils/runtimes.rs b/src/utils/runtimes.rs index a3a6ad9d..d753bd66 100644 --- a/src/utils/runtimes.rs +++ b/src/utils/runtimes.rs @@ -17,7 +17,7 @@ pub const DEFAULT_REPO_URL: &str = "https://workers.wasmlabs.dev/repository/v1/i pub const WWS_REPO_NAME: &str = "WWS_REPO_NAME"; pub const WWS_REPO_URL: &str = "WWS_REPO_URL"; -/// Loads the local configuration and install any missing runtime from it. +/// Loads the local configuration and installs any missing runtime from it. /// It will check all the different repositories and install missing /// runtimes inside them. pub async fn install_missing_runtimes(project_root: &Path) -> Result<()> { @@ -43,7 +43,7 @@ pub async fn install_missing_runtimes(project_root: &Path) -> Result<()> { Ok(()) } -/// Retrieves the remote repository and install the desired runtime. +/// Retrieves the remote repository and installs the desired runtime. /// It will return an error if the desired runtime is not present in /// the repo. pub async fn install_from_repository( From 3eff29c879fcdcc6d7018b68f3a357e6877fb4ef Mon Sep 17 00:00:00 2001 From: Angel M De Miguel <dangel@vmware.com> Date: Mon, 5 Jun 2023 10:45:21 +0200 Subject: [PATCH 4/4] feat: identify git references with an enum and revert Path -> String --- crates/project/src/lib.rs | 20 ++++++---- crates/project/src/options.rs | 17 ++++++--- crates/project/src/types/git.rs | 66 +++++++++++++++++++-------------- src/main.rs | 16 ++++---- src/utils/options.rs | 17 ++++----- 5 files changed, 77 insertions(+), 59 deletions(-) diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index b16485d2..3f0f42ee 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -10,7 +10,10 @@ use anyhow::{bail, Result}; use fetch::fetch_and_validate; use metadata::{RemoteFile, Runtime}; use options::Options; -use std::path::{Path, PathBuf}; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, +}; use types::git::prepare_git_project; use wws_store::Store; @@ -32,7 +35,7 @@ pub enum ProjectType { /// However, the result of any type is the same: a local folder to point to. /// This is the value we return from this function. pub async fn prepare_project( - location: &str, + location: &Path, force_type: Option<ProjectType>, options: Option<Options>, ) -> Result<PathBuf> { @@ -51,9 +54,12 @@ pub async fn prepare_project( /// Identify the type of the project based on different rules related to the location. /// For example, an URL that ends in .git is considered a git repository. For any /// unknown pattern, it will default to "Local" -pub fn identify_type(location: &str) -> Result<ProjectType> { +pub fn identify_type(location: &Path) -> Result<ProjectType> { if (location.starts_with("https://") || location.starts_with("http://")) - && location.ends_with(".git") + && location + .extension() + .filter(|e| *e == OsStr::new("git")) + .is_some() { Ok(ProjectType::Git) } else { @@ -159,7 +165,7 @@ mod tests { for test in tests { let file_route = PathBuf::from_slash(test); - match identify_type(file_route.to_str().unwrap()) { + match identify_type(&file_route) { Ok(project_type) => { assert!(matches!(project_type, ProjectType::Local)); } @@ -180,7 +186,7 @@ mod tests { for test in tests { let file_route = PathBuf::from_slash(test); - match identify_type(file_route.to_str().unwrap()) { + match identify_type(&file_route) { Ok(_) => { panic!("The folder doesn't exist, so identifying it should fail."); } @@ -191,7 +197,7 @@ mod tests { #[test] fn identify_git_repository_locations() { - let location = "https://github.com/vmware-labs/wasm-workers-server.git"; + let location = Path::new("https://github.com/vmware-labs/wasm-workers-server.git"); match identify_type(location) { Ok(project_type) => { diff --git a/crates/project/src/options.rs b/crates/project/src/options.rs index d3022eb5..c6cc261a 100644 --- a/crates/project/src/options.rs +++ b/crates/project/src/options.rs @@ -16,15 +16,20 @@ pub struct Options { #[derive(Default)] pub struct LocalOptions {} -/// The different git options you can configure. -#[derive(Default)] -pub struct GitOptions { +/// Defines a different reference when cloning the repository +pub enum GitReference { /// Use a specific commit - pub commit: Option<String>, + Commit(String), /// Use a specific tag - pub tag: Option<String>, + Tag(String), /// Use a specific git branch - pub branch: Option<String>, + Branch(String), +} + +/// The different git options you can configure. +#[derive(Default)] +pub struct GitOptions { + pub git_ref: Option<GitReference>, /// Change the directory to run the workers pub folder: Option<String>, } diff --git a/crates/project/src/types/git.rs b/crates/project/src/types/git.rs index 8c1eba25..17ab0458 100644 --- a/crates/project/src/types/git.rs +++ b/crates/project/src/types/git.rs @@ -1,54 +1,66 @@ // Copyright 2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 -use crate::options::Options; -use anyhow::{bail, Result}; +use crate::options::{GitReference, Options}; +use anyhow::{anyhow, bail, Result}; use git2::{Oid, Repository}; use sha256::digest as sha256_digest; -use std::{env::temp_dir, fs::remove_dir_all, path::PathBuf}; +use std::{ + env::temp_dir, + fs::remove_dir_all, + path::{Path, PathBuf}, +}; // Default remote for git repos static DEFAULT_REMOTE: &str = "origin"; /// Prepare a project based on a git repository. This method /// clones the repo locally and returns the path in which it's located. -pub fn prepare_git_project(location: &str, options: Option<Options>) -> Result<PathBuf> { +pub fn prepare_git_project(location: &Path, options: Option<Options>) -> Result<PathBuf> { + let project_url = location + .to_str() + .ok_or(anyhow!("The project URL cannot be retrieved"))?; // By default, we use temporary dirs - let mut dir = temp_dir().join(sha256_digest(location)); + let mut dir = temp_dir().join(sha256_digest(project_url)); if dir.exists() { // Clean up a previous download remove_dir_all(&dir)?; } - let repo = match Repository::clone(location, &dir) { + let repo = match Repository::clone(project_url, &dir) { Ok(repo) => repo, Err(e) => bail!("There was an error cloning the repository: {e}"), }; if let Some(options) = options { if let Some(git) = options.git { - // These options are prioritized - if let Some(commit) = git.commit { - let oid = Oid::from_str(&commit)?; - let commit = repo.find_commit(oid)?; - repo.checkout_tree(commit.as_object(), None)?; - } else if let Some(tag) = git.tag { - let mut remote = repo.find_remote(DEFAULT_REMOTE)?; - let tag_remote = format!("refs/tags/{tag}:refs/tags/{tag}"); - remote.fetch(&[&tag_remote], None, None)?; - - let oid = Oid::from_str(&tag)?; - let tag = repo.find_tag(oid)?; - repo.checkout_tree(tag.as_object(), None)?; - } else if let Some(branch) = git.branch { - let mut remote = repo.find_remote(DEFAULT_REMOTE)?; - let head_remote = format!("refs/heads/{branch}:refs/heads/{branch}"); - remote.fetch(&[&head_remote], None, None)?; - - let branch = repo.find_branch(&branch, git2::BranchType::Local)?; - let reference = branch.into_reference(); - repo.checkout_tree(&reference.peel(git2::ObjectType::Tree)?, None)?; + if let Some(git_ref) = git.git_ref { + match git_ref { + GitReference::Commit(commit) => { + let oid = Oid::from_str(&commit)?; + let commit = repo.find_commit(oid)?; + repo.checkout_tree(commit.as_object(), None)?; + } + GitReference::Tag(tag) => { + let mut remote = repo.find_remote(DEFAULT_REMOTE)?; + let tag_remote = format!("refs/tags/{tag}:refs/tags/{tag}"); + remote.fetch(&[&tag_remote], None, None)?; + + let oid = Oid::from_str(&tag)?; + let tag = repo.find_tag(oid)?; + repo.checkout_tree(tag.as_object(), None)?; + } + GitReference::Branch(branch) => { + let mut remote = repo.find_remote(DEFAULT_REMOTE)?; + let head_remote = format!("refs/heads/{branch}:refs/heads/{branch}"); + remote.fetch(&[&head_remote], None, None)?; + + let branch = repo.find_branch(&branch, git2::BranchType::Local)?; + let reference = branch.into_reference(); + repo.checkout_tree(&reference.peel(git2::ObjectType::Tree)?, None)?; + } + } } if let Some(folder) = git.folder { diff --git a/src/main.rs b/src/main.rs index 49666825..acd0243e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,7 +31,7 @@ pub struct Args { /// Location of the wws project. It could be a local folder or a git repository. #[arg(value_parser, default_value = ".")] - location: String, + path: PathBuf, /// Prepend the given path to all URLs #[arg(long, default_value = "")] @@ -77,7 +77,7 @@ async fn main() -> std::io::Result<()> { if let Some(Main::Runtimes(sub)) = &args.commands { let mut run_result = Ok(()); - match identify_type(&args.location) { + match identify_type(&args.path) { Ok(project_type) => match project_type { ProjectType::Local => {} _ => { @@ -92,8 +92,6 @@ async fn main() -> std::io::Result<()> { } } - let project_path = PathBuf::from(&args.location); - match &sub.runtime_commands { RuntimesCommands::List(list) => { if let Err(err) = list.run(sub).await { @@ -103,21 +101,21 @@ async fn main() -> std::io::Result<()> { } } RuntimesCommands::Install(install) => { - if let Err(err) = install.run(&project_path, sub).await { + if let Err(err) = install.run(&args.path, sub).await { println!("β There was an error installing the runtime from the repository"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); } } RuntimesCommands::Uninstall(uninstall) => { - if let Err(err) = uninstall.run(&project_path, sub) { + if let Err(err) = uninstall.run(&args.path, sub) { println!("β There was an error uninstalling the runtime"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); } } RuntimesCommands::Check(check) => { - if let Err(err) = check.run(&project_path) { + if let Err(err) = check.run(&args.path) { println!("β There was an error checking the local runtimes"); println!("π {err}"); run_result = Err(Error::new(ErrorKind::InvalidData, "")); @@ -132,8 +130,8 @@ async fn main() -> std::io::Result<()> { // Set the final options let project_opts = options::build_project_options(&args); - println!("βοΈ Preparing the project from: {}", &args.location); - let project_path = match prepare_project(&args.location, None, Some(project_opts)).await { + println!("βοΈ Preparing the project from: {}", &args.path.display()); + let project_path = match prepare_project(&args.path, None, Some(project_opts)).await { Ok(p) => p, Err(err) => { eprintln!("β There was an error preparing the project: {err}"); diff --git a/src/utils/options.rs b/src/utils/options.rs index dfef80d3..078520ab 100644 --- a/src/utils/options.rs +++ b/src/utils/options.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::Args; -use wws_project::options::{GitOptions, Options}; +use wws_project::options::{GitOptions, GitReference, Options}; /// Create the project options from the CLI arguments pub fn build_project_options(args: &Args) -> Options { @@ -16,16 +16,13 @@ pub fn build_project_options(args: &Args) -> Options { pub fn build_git_options(args: &Args) -> GitOptions { let mut git_opts = GitOptions::default(); + // This conditional is prioritized: commit > tag > branch if let Some(commit) = args.git_commit.as_ref() { - git_opts.commit = Some(commit.clone()); - } - - if let Some(tag) = args.git_tag.as_ref() { - git_opts.tag = Some(tag.clone()); - } - - if let Some(branch) = args.git_branch.as_ref() { - git_opts.branch = Some(branch.clone()); + git_opts.git_ref = Some(GitReference::Commit(commit.clone())); + } else if let Some(tag) = args.git_tag.as_ref() { + git_opts.git_ref = Some(GitReference::Tag(tag.clone())); + } else if let Some(branch) = args.git_branch.as_ref() { + git_opts.git_ref = Some(GitReference::Branch(branch.clone())); } if let Some(folder) = args.git_folder.as_ref() {