diff --git a/Cargo.lock b/Cargo.lock index 4926328a..e68236dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", @@ -4397,6 +4397,7 @@ dependencies = [ "wasm-compose", "wasm-encoder 0.36.1", "wasmparser 0.116.0", + "wasmprinter", "wasmtime", "wasmtime-wasi", "wat 1.0.78", @@ -4409,19 +4410,23 @@ name = "warg-client" version = "0.3.0-dev" dependencies = [ "anyhow", + "async-recursion", "async-trait", "bytes", "clap", "dirs 5.0.1", "futures-util", + "indexmap 2.0.0", "itertools 0.11.0", "libc", "normpath", "once_cell", "pathdiff", "reqwest", + "semver", "serde 1.0.171", "serde_json", + "sha256", "tempfile", "thiserror", "tokio", @@ -4433,6 +4438,10 @@ dependencies = [ "warg-crypto", "warg-protocol", "warg-transparency", + "wasm-compose", + "wasm-encoder 0.36.1", + "wasmparser 0.116.0", + "wasmprinter", "windows-sys 0.48.0", ] @@ -4785,14 +4794,24 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.118.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" +dependencies = [ + "indexmap 2.0.0", + "semver", +] + [[package]] name = "wasmprinter" -version = "0.2.60" +version = "0.2.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b76cb909fe3d9b0de58cee1f4072247e680ff5cc1558ccad2790a9de14a23993" +checksum = "3d027eb8294904fc715ac0870cebe6b0271e96b90605ee21511e7565c4ce568c" dependencies = [ "anyhow", - "wasmparser 0.108.0", + "wasmparser 0.118.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b836b0f1..b20b4582 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ async-recursion = "1.0.4" indexmap.workspace = true semver.workspace = true sha256 = "1.4.0" +wat = "1.0.67" +wasmprinter = "0.2.75" [dev-dependencies] reqwest = { workspace = true } diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index 12f98018..cbdec7f9 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -14,7 +14,7 @@ warg-crypto = { workspace = true } warg-protocol = { workspace = true } warg-api = { workspace = true } warg-transparency = { workspace = true } -anyhow = { workspace = true } +anyhow.workspace = true thiserror = { workspace = true } clap = { workspace = true } serde = { workspace = true } @@ -30,11 +30,19 @@ url = { workspace = true } libc = { workspace = true } tracing = { workspace = true } itertools = { workspace = true } +wasmparser = { workspace = true } +wasm-compose = { workspace = true } dirs = { workspace = true } once_cell = { workspace = true } walkdir = { workspace = true } normpath = { workspace = true } pathdiff = { workspace = true } +indexmap.workspace = true +async-recursion = "1.0.5" +semver.workspace = true +wasm-encoder.workspace = true +wasmprinter = "0.2.75" +sha256 = "1.4.0" [target.'cfg(windows)'.dependencies.windows-sys] version = "0.48" diff --git a/crates/client/src/depsolve.rs b/crates/client/src/depsolve.rs new file mode 100644 index 00000000..5cbb6d23 --- /dev/null +++ b/crates/client/src/depsolve.rs @@ -0,0 +1,542 @@ +use anyhow::{bail, Result}; +use async_recursion::async_recursion; +use indexmap::IndexSet; +use semver::{Comparator, Prerelease, Version, VersionReq}; +use std::fs; +use warg_protocol::package::ReleaseState; +use warg_protocol::registry::PackageName; +use wasm_encoder::{ + Component, ComponentImportSection, ComponentSectionId, ComponentTypeRef, RawSection, +}; +use wasmparser::names::KebabStr; +use wasmparser::{Chunk, ComponentImportSectionReader, Parser, Payload}; + +use super::Client; +use crate::storage::{ContentStorage, PackageInfo, RegistryStorage}; +/// Import Kinds found in components +#[derive(Debug, Eq, PartialEq, Hash)] +pub enum ImportKind { + /// Locked Version + Locked(Option), + /// Unlocked Version Range + Unlocked, + /// Interface + Interface(Option), +} + +/// Dependency in dep solve +#[derive(Debug, Eq, PartialEq, Hash)] +pub struct Import { + /// Import name + pub name: String, + /// Version Requirements + pub req: VersionReq, + /// Import kind + pub kind: ImportKind, +} + +/// Parser for dep solve deps +pub struct DependencyImportParser<'a> { + /// string to be parsed + pub next: &'a str, + /// index of parser + pub offset: usize, +} + +impl<'a> DependencyImportParser<'a> { + /// Parses import + pub fn parse(&mut self) -> Result { + if self.eat_str("unlocked-dep=") { + self.expect_str("<")?; + let imp = self.pkgidset_up_to('>')?; + self.expect_str(">")?; + return Ok(imp); + } + + if self.eat_str("locked-dep=") { + self.expect_str("<")?; + let imp = self.pkgver()?; + return Ok(imp); + } + + let name = self.eat_until('@'); + let v = self.semver(self.next)?; + let comp = Comparator { + op: semver::Op::Exact, + major: v.major, + minor: Some(v.minor), + patch: Some(v.patch), + pre: v.pre, + }; + let req = VersionReq { + comparators: vec![comp], + }; + Ok(Import { + name: name.unwrap().to_string(), + req, + kind: ImportKind::Interface(Some(self.next.to_string())), + }) + } + + fn eat_str(&mut self, prefix: &str) -> bool { + match self.next.strip_prefix(prefix) { + Some(rest) => { + self.next = rest; + true + } + None => false, + } + } + + fn expect_str(&mut self, prefix: &str) -> Result<()> { + if self.eat_str(prefix) { + Ok(()) + } else { + bail!(format!( + "expected `{prefix}` at `{}` at {}", + self.next, self.offset + )); + } + } + + fn eat_up_to(&mut self, c: char) -> Option<&'a str> { + let i = self.next.find(c)?; + let (a, b) = self.next.split_at(i); + self.next = b; + Some(a) + } + + fn eat_until(&mut self, c: char) -> Option<&'a str> { + let ret = self.eat_up_to(c); + if ret.is_some() { + self.next = &self.next[c.len_utf8()..]; + } + ret + } + + fn kebab(&self, s: &'a str) -> Result<&'a KebabStr> { + match KebabStr::new(s) { + Some(name) => Ok(name), + None => bail!(format!("`{s}` is not in kebab case at {}", self.offset)), + } + } + + fn take_until(&mut self, c: char) -> Result<&'a str> { + match self.eat_until(c) { + Some(s) => Ok(s), + None => bail!(format!("failed to find `{c}` character at {}", self.offset)), + } + } + + fn take_up_to(&mut self, c: char) -> Result<&'a str> { + match self.eat_up_to(c) { + Some(s) => Ok(s), + None => bail!(format!("failed to find `{c}` character at {}", self.offset)), + } + } + + fn semver(&self, s: &str) -> Result { + match Version::parse(s) { + Ok(v) => Ok(v), + Err(e) => bail!(format!( + "`{s}` is not a valid semver: {e} at {}", + self.offset + )), + } + } + + fn pkgver(&mut self) -> Result { + let namespace = self.take_until(':')?; + self.kebab(namespace)?; + let name = match self.eat_until('@') { + Some(name) => name, + // a:b + None => { + let name = self.take_up_to(',')?; + self.kebab(name)?; + return Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq::STAR, + kind: ImportKind::Locked(None), + }); + } + }; + let version = self.eat_until('>'); + let req = if let Some(v) = version { + let v = self.semver(v)?; + let comp = Comparator { + op: semver::Op::Exact, + major: v.major, + minor: Some(v.minor), + patch: Some(v.patch), + pre: Prerelease::default(), + }; + VersionReq { + comparators: vec![comp], + } + } else { + VersionReq::STAR + }; + let digest = if self.eat_str(",") { + self.eat_until('<'); + self.eat_until('>').map(|d| d.to_string()) + } else { + None + }; + Ok(Import { + name: format!("{namespace}:{name}"), + req, + kind: ImportKind::Locked(digest), + }) + } + fn pkgidset_up_to(&mut self, end: char) -> Result { + let namespace = self.take_until(':')?; + self.kebab(namespace)?; + let name = match self.eat_until('@') { + Some(name) => name, + // a:b + None => { + let name = self.take_up_to(end)?; + self.kebab(name)?; + return Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq::STAR, + kind: ImportKind::Unlocked, + }); + } + }; + self.kebab(name)?; + // a:b@* + if self.eat_str("*") { + return Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq::STAR, + kind: ImportKind::Unlocked, + }); + } + self.expect_str("{")?; + if self.eat_str(">=") { + match self.eat_until(' ') { + Some(lower) => { + let lower = self.semver(lower)?; + self.expect_str("<")?; + let upper = self.take_until('}')?; + let upper = self.semver(upper)?; + let lc = Comparator { + op: semver::Op::GreaterEq, + major: lower.major, + minor: Some(lower.minor), + patch: Some(lower.patch), + pre: Prerelease::default(), + }; + let uc = Comparator { + op: semver::Op::Less, + major: upper.major, + minor: Some(upper.minor), + patch: Some(upper.patch), + pre: Prerelease::default(), + }; + let comparators = vec![lc, uc]; + return Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq { comparators }, + kind: ImportKind::Unlocked, + }); + } + // a:b@{>=1.2.3} + None => { + let lower = self.take_until('}')?; + let lower = self.semver(lower)?; + let comparator = Comparator { + op: semver::Op::GreaterEq, + major: lower.major, + minor: Some(lower.minor), + patch: Some(lower.patch), + pre: Prerelease::default(), + }; + let comparators = vec![comparator]; + return Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq { comparators }, + kind: ImportKind::Unlocked, + }); + } + } + } + + // a:b@{<1.2.3} + // .. or + // a:b@{<1.2.3 >=1.2.3} + self.expect_str("<")?; + let upper = self.take_until('}')?; + let upper = self.semver(upper)?; + let uc = Comparator { + op: semver::Op::Less, + major: upper.major, + minor: Some(upper.minor), + patch: Some(upper.patch), + pre: Prerelease::default(), + }; + let mut comparators: Vec = Vec::new(); + comparators.push(uc); + Ok(Import { + name: format!("{namespace}:{name}"), + req: VersionReq { comparators }, + kind: ImportKind::Unlocked, + }) + } +} + +/// Creates list of dependenies for locking components +pub struct LockListBuilder { + /// List of deps to include in locked component + pub lock_list: IndexSet, +} + +impl LockListBuilder { + /// New LockListBuilder + pub fn new() -> Self { + Self { + lock_list: IndexSet::new(), + } + } + + fn parse_import( + &mut self, + parser: &ComponentImportSectionReader, + imports: &mut Vec, + ) -> Result<()> { + let clone = parser.clone(); + for import in clone.into_iter_with_offsets() { + let (_, imp) = import?; + imports.push(imp.name.0.to_string()); + } + Ok(()) + } + + #[async_recursion] + async fn parse_package( + &mut self, + client: &Client, + mut bytes: &[u8], + ) -> Result<()> { + let mut parser = Parser::new(0); + let mut imports: Vec = Vec::new(); + loop { + let payload = match parser.parse(bytes, true)? { + Chunk::NeedMoreData(_) => unreachable!(), + Chunk::Parsed { payload, consumed } => { + bytes = &bytes[consumed..]; + payload + } + }; + match payload { + Payload::ComponentImportSection(s) => { + self.parse_import(&s, &mut imports)?; + } + Payload::CodeSectionStart { + count: _, + range: _, + size: _, + } => { + parser.skip_section(); + } + Payload::ModuleSection { range, .. } => { + let offset = range.end - range.start; + if offset > bytes.len() { + bail!("invalid module or component section range"); + } + bytes = &bytes[offset..]; + } + Payload::ComponentSection { range, .. } => { + let offset = range.end - range.start; + if offset > bytes.len() { + bail!("invalid module or component section range"); + } + bytes = &bytes[offset..]; + } + Payload::End(_) => { + break; + } + _ => {} + } + } + for import in imports { + let mut resolver = DependencyImportParser { + next: &import, + offset: 0, + }; + + let import = resolver.parse()?; + match import.kind { + ImportKind::Locked(_) | ImportKind::Unlocked => { + let id = PackageName::new(import.name.clone())?; + if let Some(info) = client.registry().load_package(&id).await? { + let release = info.state.releases().last(); + if let Some(r) = release { + let state = &r.state; + if let ReleaseState::Released { content } = state { + let path = client.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(p)?; + self.parse_package(client, &bytes).await?; + } + } + } + self.lock_list.insert(import); + } else { + client.download(&id, &VersionReq::STAR).await?; + if let Some(info) = client.registry().load_package(&id).await? { + let release = info.state.releases().last(); + if let Some(r) = release { + let state = &r.state; + if let ReleaseState::Released { content } = state { + let path = client.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(p)?; + self.parse_package(client, &bytes).await?; + } + } + } + self.lock_list.insert(import); + } + } + } + ImportKind::Interface(_) => {} + } + } + Ok(()) + } + + /// List of deps for building + #[async_recursion] + pub async fn build_list( + &mut self, + client: &Client, + info: &PackageInfo, + ) -> Result<()> { + let release = info.state.releases().last(); + if let Some(r) = release { + let state = &r.state; + if let ReleaseState::Released { content } = state { + let path = client.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(p)?; + self.parse_package(client, &bytes).await?; + } + } + } + Ok(()) + } +} + +/// Bundles Dependencies +pub struct Bundler<'a, R, C> +where + R: RegistryStorage, + C: ContentStorage, +{ + /// Warg client used for bundling + client: &'a Client, +} + +impl<'a, R, C> Bundler<'a, R, C> +where + R: RegistryStorage, + C: ContentStorage, +{ + /// New Bundler + pub fn new(client: &'a Client) -> Self { + Self { client } + } + + async fn parse_imports( + &mut self, + parser: ComponentImportSectionReader<'a>, + component: &mut Component, + ) -> Result> { + let mut imports = ComponentImportSection::new(); + for import in parser.into_iter_with_offsets() { + let (_, imp) = import?; + let mut dep_parser = DependencyImportParser { + next: imp.name.0, + offset: 0, + }; + let parsed_imp = dep_parser.parse()?; + if !parsed_imp.name.contains('/') { + let pkg_id = PackageName::new(parsed_imp.name)?; + if let Some(info) = self.client.registry().load_package(&pkg_id).await? { + let release = if parsed_imp.req != VersionReq::STAR { + info.state + .releases() + .filter(|r| parsed_imp.req.matches(&r.version)) + .last() + } else { + info.state.releases().last() + }; + if let Some(r) = release { + let release_state = &r.state; + if let ReleaseState::Released { content } = release_state { + let path = self.client.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(p)?; + component.section(&RawSection { + id: ComponentSectionId::Component.into(), + data: &bytes, + }); + } + } + } + } + } else if let wasmparser::ComponentTypeRef::Instance(i) = imp.ty { + imports.import(imp.name.0, ComponentTypeRef::Instance(i)); + } + } + component.section(&imports); + Ok(Vec::new()) + } + + /// Parse bytes for bundling + pub async fn parse(&mut self, mut bytes: &'a [u8]) -> Result { + let constant = bytes.clone(); + let mut parser = Parser::new(0); + let mut component = Component::new(); + loop { + let payload = match parser.parse(bytes, true)? { + Chunk::NeedMoreData(_) => unreachable!(), + Chunk::Parsed { payload, consumed } => { + bytes = &bytes[consumed..]; + payload + } + }; + match payload { + Payload::ComponentImportSection(s) => { + self.parse_imports(s, &mut component).await?; + } + Payload::ModuleSection { range, .. } => { + let offset = range.end - range.start; + component.section(&RawSection { + id: 1, + data: &constant[range], + }); + if offset > bytes.len() { + panic!(); + } + bytes = &bytes[offset..]; + } + Payload::End(_) => { + break; + } + _ => { + if let Some((id, range)) = payload.as_section() { + component.section(&RawSection { + id, + data: &constant[range], + }); + } + } + } + } + Ok(component) + } +} diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 4de3673d..dd73a4f9 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -1,17 +1,18 @@ //! A client library for Warg component registries. #![deny(missing_docs)] - -use crate::storage::{PackageInfo, PublishEntry}; +use crate::storage::PackageInfo; use anyhow::{anyhow, Context, Result}; use reqwest::{Body, IntoUrl}; +use semver::{Version, VersionReq}; +use std::fs; +use std::str::FromStr; use std::{borrow::Cow, collections::HashMap, path::PathBuf, time::Duration}; use storage::{ ContentStorage, FileSystemContentStorage, FileSystemRegistryStorage, PublishInfo, RegistryStorage, }; use thiserror::Error; -use warg_api::v1::package::ContentSource; use warg_api::v1::{ fetch::{FetchError, FetchLogsRequest, FetchLogsResponse}, package::{ @@ -20,18 +21,21 @@ use warg_api::v1::{ }, proof::{ConsistencyRequest, InclusionRequest}, }; -use warg_crypto::{ - hash::{AnyHash, Sha256}, - signing, Encode, Signable, -}; +use warg_crypto::hash::Sha256; +use warg_crypto::{hash::AnyHash, signing, Encode, Signable}; +use warg_protocol::package::ReleaseState; use warg_protocol::{ operator, package, registry::{LogId, LogLeaf, PackageName, RecordId, TimestampedCheckpoint}, - PublishedProtoEnvelope, SerdeEnvelope, Version, VersionReq, + PublishedProtoEnvelope, SerdeEnvelope, }; +use wasm_compose::graph::{CompositionGraph, EncodeOptions, ExportIndex, InstanceId}; pub mod api; mod config; +/// Tools for locking and bundling components +pub mod depsolve; +use depsolve::{Bundler, Import, ImportKind, LockListBuilder}; pub mod lock; mod registry_url; pub mod storage; @@ -89,6 +93,135 @@ impl Client { .or(Err(ClientError::ClearContentCacheFailed)) } + /// Locks component + pub async fn lock_component(&self, info: &PackageInfo) -> ClientResult> { + let mut builder = LockListBuilder::new(); + builder.build_list(self, info).await?; + let top = Import { + name: format!("{}:{}", info.name.namespace(), info.name.name()), + req: VersionReq::STAR, + kind: ImportKind::Unlocked, + }; + builder.lock_list.insert(top); + let mut composer = CompositionGraph::new(); + let mut handled = HashMap::::new(); + for package in builder.lock_list { + let name = package.name.clone(); + let version = package.req; + let id = PackageName::new(name)?; + let info = self.registry().load_package(&id).await?; + if let Some(inf) = info { + let release = if version != VersionReq::STAR { + inf.state + .releases() + .filter(|r| version.matches(&r.version)) + .last() + } else { + inf.state.releases().last() + }; + + if let Some(r) = release { + let state = &r.state; + if let ReleaseState::Released { content } = state { + let mut locked_package = package.name.clone(); + locked_package = format!( + "locked-dep=<{}@{}>,integrity=<{}>", + locked_package, + &r.version.to_string(), + content.to_string().replace(':', "-") + ); + let path = self.content().content_location(content); + if let Some(p) = path { + let bytes = fs::read(&p).map_err(|_| ClientError::ContentNotFound { + digest: content.clone(), + })?; + + let read_digest = + AnyHash::from_str(&format!("sha256:{}", sha256::digest(bytes))) + .unwrap(); + if content != &read_digest { + return Err(ClientError::IncorrectContent { + digest: read_digest, + expected: content.clone(), + }); + } + let component = + wasm_compose::graph::Component::from_file(&locked_package, p)?; + let component_index = + if let Some(c) = composer.get_component_by_name(&locked_package) { + c.0 + } else { + composer.add_component(component)? + }; + let instance_id = composer.instantiate(component_index)?; + let added = composer.get_component(component_index); + let ver = version.clone().to_string(); + let range = if ver == "*" { + "".to_string() + } else { + format!("@{{{}}}", ver.replace(',', "")) + }; + handled.insert(format!("{}{range}", package.name), instance_id); + let mut args = Vec::new(); + if let Some(added) = added { + for (index, name, _) in added.imports() { + let kindless_name = name.splitn(2, '=').last(); + if let Some(name) = kindless_name { + let iid = handled.get(&name[1..name.len() - 1]); + if let Some(arg) = iid { + args.push((arg, index)); + } + } + } + } + for arg in args { + composer.connect( + *arg.0, + None::, + instance_id, + arg.1, + )?; + } + } + } + } + } + } + let final_name = &format!("{}:{}", info.name.namespace(), &info.name.name()); + let id = handled.get(final_name); + let options = if let Some(id) = id { + EncodeOptions { + define_components: false, + export: Some(*id), + validate: false, + } + } else { + EncodeOptions { + define_components: false, + export: None, + validate: false, + } + }; + let locked = composer.encode(options)?; + fs::write("./locked.wasm", locked.as_slice()).map_err(|e| ClientError::Other(e.into()))?; + Ok(locked) + } + + /// Bundles component + pub async fn bundle_component(&self, info: &PackageInfo) -> ClientResult> { + let mut bundler = Bundler::new(self); + let path = PathBuf::from("./locked.wasm"); + let locked = if !path.is_file() { + self.lock_component(info).await? + } else { + fs::read("./locked.wasm").map_err(|e| ClientError::Other(e.into()))? + }; + let bundled = bundler.parse(&locked).await?; + fs::write("./bundled.wasm", bundled.as_slice()) + .map_err(|e| ClientError::Other(e.into()))?; + Ok(bundled.as_slice().to_vec()) + } + /// Submits the publish information in client storage. /// /// If there's no publishing information in client storage, an error is returned. @@ -799,6 +932,15 @@ pub enum ClientError { digest: AnyHash, }, + /// Content digest was different than expected. + #[error("content with digest `{digest}` was not found expected `{expected}`")] + IncorrectContent { + /// The digest of the missing content. + digest: AnyHash, + /// The expected + expected: AnyHash, + }, + /// The package log is empty and cannot be validated. #[error("package log is empty and cannot be validated")] PackageLogEmpty { diff --git a/src/commands/bundle.rs b/src/commands/bundle.rs index 4e90b2c5..4aabfb9f 100644 --- a/src/commands/bundle.rs +++ b/src/commands/bundle.rs @@ -1,20 +1,9 @@ -use super::{CommonOptions, DependencyImportParser}; +use super::CommonOptions; use anyhow::Result; use clap::Args; use semver::VersionReq; -use std::fs; -use warg_client::{ - storage::{ - ContentStorage, FileSystemContentStorage, FileSystemRegistryStorage, RegistryStorage, - }, - Client, -}; -use warg_protocol::{package::ReleaseState, registry::PackageName}; -use wasm_encoder::{ - Component, ComponentImportSection, ComponentSectionId, ComponentTypeRef, RawSection, -}; -use wasmparser::{Chunk, ComponentImportSectionReader, Parser, Payload}; - +use warg_client::storage::RegistryStorage; +use warg_protocol::registry::PackageName; /// Bundle With Registry Dependencies #[derive(Args)] pub struct BundleCommand { @@ -32,112 +21,17 @@ impl BundleCommand { pub async fn exec(self) -> Result<()> { let config = self.common.read_config()?; let client = self.common.create_client(&config)?; - println!("registry: {url}", url = client.url()); - let mut bundler = Bundler::new(&client); - let locked = fs::read("./locked.wasm")?; - let bundled = bundler.parse(&locked).await?; - fs::write("./bundled.wasm", bundled.as_slice())?; - Ok(()) - } -} - -/// Bundles Dependencies -pub struct Bundler<'a> { - client: &'a Client, -} - -impl<'a> Bundler<'a> { - fn new(client: &'a Client) -> Self { - Self { client } - } - - async fn parse_imports( - &mut self, - parser: ComponentImportSectionReader<'a>, - component: &mut Component, - ) -> Result> { - let mut imports = ComponentImportSection::new(); - for import in parser.into_iter_with_offsets() { - let (_, imp) = import?; - let mut dep_parser = DependencyImportParser { - next: imp.name.0, - offset: 0, - }; - let parsed_imp = dep_parser.parse()?; - if !parsed_imp.name.contains('/') { - let pkg_id = PackageName::new(parsed_imp.name)?; - if let Some(info) = self.client.registry().load_package(&pkg_id).await? { - let release = if parsed_imp.req != VersionReq::STAR { - info.state - .releases() - .filter(|r| parsed_imp.req.matches(&r.version)) - .last() - } else { - info.state.releases().last() - }; - if let Some(r) = release { - let release_state = &r.state; - if let ReleaseState::Released { content } = release_state { - let path = self.client.content().content_location(&content); - if let Some(p) = path { - let bytes = fs::read(p)?; - component.section(&RawSection { - id: ComponentSectionId::Component.into(), - data: &bytes, - }); - } - } - } + if let Some(package) = self.package { + if let Some(info) = client.registry().load_package(&package).await? { + client.bundle_component(&info).await?; + } else { + client.download(&package, &VersionReq::STAR).await?; + if let Some(info) = client.registry().load_package(&package).await? { + client.bundle_component(&info).await?; } - } else if let wasmparser::ComponentTypeRef::Instance(i) = imp.ty { - imports.import(imp.name.0, ComponentTypeRef::Instance(i)); } } - component.section(&imports); - Ok(Vec::new()) - } - - async fn parse(&mut self, mut bytes: &'a [u8]) -> Result { - let constant = bytes.clone(); - let mut parser = Parser::new(0); - let mut component = Component::new(); - loop { - let payload = match parser.parse(bytes, true)? { - Chunk::NeedMoreData(_) => unreachable!(), - Chunk::Parsed { payload, consumed } => { - bytes = &bytes[consumed..]; - payload - } - }; - match payload { - Payload::ComponentImportSection(s) => { - self.parse_imports(s, &mut component).await?; - } - Payload::ModuleSection { range, .. } => { - let offset = range.end - range.start; - component.section(&RawSection { - id: 1, - data: &constant[range], - }); - if offset > bytes.len() { - panic!(); - } - bytes = &bytes[offset..]; - } - Payload::End(_) => { - break; - } - _ => { - if let Some((id, range)) = payload.as_section() { - component.section(&RawSection { - id, - data: &constant[range], - }); - } - } - } - } - Ok(component) + Ok(()) } } diff --git a/src/commands/dependencies.rs b/src/commands/dependencies.rs index 4c424bba..0e623d19 100644 --- a/src/commands/dependencies.rs +++ b/src/commands/dependencies.rs @@ -1,5 +1,4 @@ use super::CommonOptions; -use crate::commands::lock::DependencyImportParser; use anyhow::{bail, Result}; use async_recursion::async_recursion; use clap::Args; @@ -7,6 +6,7 @@ use ptree::{output::print_tree, TreeBuilder}; use semver::Op; use std::fs; use warg_client::{ + depsolve::{DependencyImportParser, ImportKind}, storage::{ContentStorage, PackageInfo, RegistryStorage}, FileSystemClient, }; @@ -78,13 +78,12 @@ impl DependenciesCommand { let grand_child = node.begin_child(format!("{}@{}", dep.name.to_string(), v)); match dep.kind { - crate::commands::ImportKind::Locked(_) - | crate::commands::ImportKind::Unlocked => { + ImportKind::Locked(_) | ImportKind::Unlocked => { let id = PackageName::new(dep.name)?; Self::parse_deps(&id, dep.req, client, grand_child, parser) .await?; } - crate::commands::ImportKind::Interface(_) => {} + ImportKind::Interface(_) => {} } grand_child.end_child(); } @@ -127,8 +126,7 @@ impl DependenciesCommand { }; let child = tree.begin_child(format!("{}@{}", dep.name.to_string(), v)); match dep.kind { - crate::commands::ImportKind::Locked(_) - | crate::commands::ImportKind::Unlocked => { + ImportKind::Locked(_) | ImportKind::Unlocked => { Self::parse_deps( &PackageName::new(dep.name)?, dep.req, @@ -138,7 +136,7 @@ impl DependenciesCommand { ) .await?; } - crate::commands::ImportKind::Interface(_) => {} + ImportKind::Interface(_) => {} } child.end_child(); } diff --git a/src/commands/lock.rs b/src/commands/lock.rs index c8179623..b716647c 100644 --- a/src/commands/lock.rs +++ b/src/commands/lock.rs @@ -1,425 +1,12 @@ use super::CommonOptions; -use anyhow::{bail, Result}; -use async_recursion::async_recursion; +use anyhow::Result; use clap::Args; -use indexmap::IndexSet; -use semver::{Comparator, Prerelease, Version, VersionReq}; -use sha256; -use std::{collections::HashMap, fs}; +use semver::VersionReq; use warg_client::{ - storage::{ContentStorage, PackageInfo, RegistryStorage}, + storage::{PackageInfo, RegistryStorage}, FileSystemClient, }; -use warg_protocol::{package::ReleaseState, registry::PackageName}; -use wasm_compose::graph::{CompositionGraph, EncodeOptions, ExportIndex, InstanceId}; -use wasmparser::{names::KebabStr, Chunk, ComponentImportSectionReader, Parser, Payload}; - -/// Parser for dep solve deps -pub struct DependencyImportParser<'a> { - /// string to be parsed - pub next: &'a str, - /// index of parser - pub offset: usize, -} - -/// Import Kinds found in components -#[derive(Debug, Eq, PartialEq, Hash)] -pub enum ImportKind { - /// Locked Version - Locked(Option), - /// Unlocked Version Range - Unlocked, - /// Interface - Interface(Option), -} - -/// Dependency in dep solve -#[derive(Debug, Eq, PartialEq, Hash)] -pub struct Import { - /// Import name - pub name: String, - /// Version Requirements - pub req: VersionReq, - /// Import kind - pub kind: ImportKind, -} - -impl<'a> DependencyImportParser<'a> { - /// Parses import - pub fn parse(&mut self) -> Result { - if self.eat_str("unlocked-dep=") { - self.expect_str("<")?; - let imp = self.pkgidset_up_to('>')?; - self.expect_str(">")?; - return Ok(imp); - } - - if self.eat_str("locked-dep=") { - self.expect_str("<")?; - let imp = self.pkgver()?; - return Ok(imp); - } - - let name = self.eat_until('@'); - let v = self.semver(self.next)?; - let comp = Comparator { - op: semver::Op::Exact, - major: v.major, - minor: Some(v.minor), - patch: Some(v.patch), - pre: v.pre, - }; - let req = VersionReq { - comparators: vec![comp], - }; - Ok(Import { - name: name.unwrap().to_string(), - req, - kind: ImportKind::Interface(Some(self.next.to_string())), - }) - } - - fn eat_str(&mut self, prefix: &str) -> bool { - match self.next.strip_prefix(prefix) { - Some(rest) => { - self.next = rest; - true - } - None => false, - } - } - - fn expect_str(&mut self, prefix: &str) -> Result<()> { - if self.eat_str(prefix) { - Ok(()) - } else { - bail!(format!( - "expected `{prefix}` at `{}` at {}", - self.next, self.offset - )); - } - } - - fn eat_up_to(&mut self, c: char) -> Option<&'a str> { - let i = self.next.find(c)?; - let (a, b) = self.next.split_at(i); - self.next = b; - Some(a) - } - - fn eat_until(&mut self, c: char) -> Option<&'a str> { - let ret = self.eat_up_to(c); - if ret.is_some() { - self.next = &self.next[c.len_utf8()..]; - } - ret - } - - fn kebab(&self, s: &'a str) -> Result<&'a KebabStr> { - match KebabStr::new(s) { - Some(name) => Ok(name), - None => bail!(format!("`{s}` is not in kebab case at {}", self.offset)), - } - } - - fn take_until(&mut self, c: char) -> Result<&'a str> { - match self.eat_until(c) { - Some(s) => Ok(s), - None => bail!(format!("failed to find `{c}` character at {}", self.offset)), - } - } - - fn take_up_to(&mut self, c: char) -> Result<&'a str> { - match self.eat_up_to(c) { - Some(s) => Ok(s), - None => bail!(format!("failed to find `{c}` character at {}", self.offset)), - } - } - - fn semver(&self, s: &str) -> Result { - match Version::parse(s) { - Ok(v) => Ok(v), - Err(e) => bail!(format!( - "`{s}` is not a valid semver: {e} at {}", - self.offset - )), - } - } - - fn pkgver(&mut self) -> Result { - let namespace = self.take_until(':')?; - self.kebab(namespace)?; - let name = match self.eat_until('@') { - Some(name) => name, - // a:b - None => { - let name = self.take_up_to(',')?; - self.kebab(name)?; - return Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq::STAR, - kind: ImportKind::Locked(None), - }); - } - }; - let version = self.eat_until('>'); - let req = if let Some(v) = version { - let v = self.semver(v)?; - let comp = Comparator { - op: semver::Op::Exact, - major: v.major, - minor: Some(v.minor), - patch: Some(v.patch), - pre: Prerelease::default(), - }; - VersionReq { - comparators: vec![comp], - } - } else { - VersionReq::STAR - }; - let digest = if self.eat_str(",") { - self.eat_until('<'); - self.eat_until('>').map(|d| d.to_string()) - } else { - None - }; - Ok(Import { - name: format!("{namespace}:{name}"), - req, - kind: ImportKind::Locked(digest), - }) - } - fn pkgidset_up_to(&mut self, end: char) -> Result { - let namespace = self.take_until(':')?; - self.kebab(namespace)?; - let name = match self.eat_until('@') { - Some(name) => name, - // a:b - None => { - let name = self.take_up_to(end)?; - self.kebab(name)?; - return Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq::STAR, - kind: ImportKind::Unlocked, - }); - } - }; - self.kebab(name)?; - // a:b@* - if self.eat_str("*") { - return Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq::STAR, - kind: ImportKind::Unlocked, - }); - } - self.expect_str("{")?; - if self.eat_str(">=") { - match self.eat_until(' ') { - Some(lower) => { - let lower = self.semver(lower)?; - self.expect_str("<")?; - let upper = self.take_until('}')?; - let upper = self.semver(upper)?; - let lc = Comparator { - op: semver::Op::GreaterEq, - major: lower.major, - minor: Some(lower.minor), - patch: Some(lower.patch), - pre: Prerelease::default(), - }; - let uc = Comparator { - op: semver::Op::Less, - major: upper.major, - minor: Some(upper.minor), - patch: Some(upper.patch), - pre: Prerelease::default(), - }; - let comparators = vec![lc, uc]; - return Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq { comparators }, - kind: ImportKind::Unlocked, - }); - } - // a:b@{>=1.2.3} - None => { - let lower = self.take_until('}')?; - let lower = self.semver(lower)?; - let comparator = Comparator { - op: semver::Op::GreaterEq, - major: lower.major, - minor: Some(lower.minor), - patch: Some(lower.patch), - pre: Prerelease::default(), - }; - let comparators = vec![comparator]; - return Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq { comparators }, - kind: ImportKind::Unlocked, - }); - } - } - } - - // a:b@{<1.2.3} - // .. or - // a:b@{<1.2.3 >=1.2.3} - self.expect_str("<")?; - let upper = self.take_until('}')?; - let upper = self.semver(upper)?; - let uc = Comparator { - op: semver::Op::Less, - major: upper.major, - minor: Some(upper.minor), - patch: Some(upper.patch), - pre: Prerelease::default(), - }; - let mut comparators: Vec = Vec::new(); - comparators.push(uc); - Ok(Import { - name: format!("{namespace}:{name}"), - req: VersionReq { comparators }, - kind: ImportKind::Unlocked, - }) - } -} -/// Builds list of packages in order that they should be added to locked component -pub struct LockListBuilder { - // lock_list: IndexSet, - lock_list: IndexSet, -} - -impl LockListBuilder { - /// New LockListBuilder - fn new() -> Self { - Self { - lock_list: IndexSet::new(), - } - } - - fn parse_import( - &mut self, - parser: &ComponentImportSectionReader, - imports: &mut Vec, - ) -> Result<()> { - let clone = parser.clone(); - for import in clone.into_iter_with_offsets() { - let (_, imp) = import?; - imports.push(imp.name.0.to_string()); - } - Ok(()) - } - - #[async_recursion] - async fn parse_package(&mut self, client: &FileSystemClient, mut bytes: &[u8]) -> Result<()> { - let mut parser = Parser::new(0); - let mut imports: Vec = Vec::new(); - loop { - let payload = match parser.parse(bytes, true)? { - Chunk::NeedMoreData(_) => unreachable!(), - Chunk::Parsed { payload, consumed } => { - bytes = &bytes[consumed..]; - payload - } - }; - match payload { - Payload::ComponentImportSection(s) => { - self.parse_import(&s, &mut imports)?; - } - Payload::CodeSectionStart { - count: _, - range: _, - size: _, - } => { - parser.skip_section(); - } - Payload::ModuleSection { range, .. } => { - let offset = range.end - range.start; - if offset > bytes.len() { - bail!("invalid module or component section range"); - } - bytes = &bytes[offset..]; - } - Payload::ComponentSection { range, .. } => { - let offset = range.end - range.start; - if offset > bytes.len() { - bail!("invalid module or component section range"); - } - bytes = &bytes[offset..]; - } - Payload::End(_) => { - break; - } - _ => {} - } - } - for import in imports { - let mut resolver = DependencyImportParser { - next: &import, - offset: 0, - }; - - let import = resolver.parse()?; - match import.kind { - ImportKind::Locked(_) | ImportKind::Unlocked => { - let id = PackageName::new(import.name.clone())?; - if let Some(info) = client.registry().load_package(&id).await? { - let release = info.state.releases().last(); - if let Some(r) = release { - let state = &r.state; - if let ReleaseState::Released { content } = state { - let path = client.content().content_location(content); - if let Some(p) = path { - let bytes = fs::read(p)?; - self.parse_package(client, &bytes).await?; - } - } - } - self.lock_list.insert(import); - } else { - client.download(&id, &VersionReq::STAR).await?; - if let Some(info) = client.registry().load_package(&id).await? { - let release = info.state.releases().last(); - if let Some(r) = release { - let state = &r.state; - if let ReleaseState::Released { content } = state { - let path = client.content().content_location(content); - if let Some(p) = path { - let bytes = fs::read(p)?; - self.parse_package(client, &bytes).await?; - } - } - } - self.lock_list.insert(import); - } - } - } - ImportKind::Interface(_) => {} - } - } - Ok(()) - } - - #[async_recursion] - async fn build_list(&mut self, client: &FileSystemClient, info: &PackageInfo) -> Result<()> { - let release = info.state.releases().last(); - if let Some(r) = release { - let state = &r.state; - if let ReleaseState::Released { content } = state { - let path = client.content().content_location(content); - if let Some(p) = path { - let bytes = fs::read(p)?; - self.parse_package(client, &bytes).await?; - } - } - } - Ok(()) - } -} +use warg_protocol::registry::PackageName; /// Print Dependency Tree #[derive(Args)] @@ -431,10 +18,6 @@ pub struct LockCommand { /// Only show information for the specified package. #[clap(value_name = "PACKAGE")] pub package: Option, - - /// Only show information for the specified package. - #[clap(long = "exec", value_name = "EXEC", required = false)] - pub executable: bool, } impl LockCommand { @@ -445,118 +28,19 @@ impl LockCommand { println!("registry: {url}", url = client.url()); if let Some(package) = self.package { if let Some(info) = client.registry().load_package(&package).await? { - Self::lock(client, &info, self.executable).await?; + Self::lock(client, &info).await?; } else { client.download(&package, &VersionReq::STAR).await?; if let Some(info) = client.registry().load_package(&package).await? { - Self::lock(client, &info, self.executable).await?; + Self::lock(client, &info).await?; } } } Ok(()) } - async fn lock(client: FileSystemClient, info: &PackageInfo, should_bundle: bool) -> Result<()> { - let mut builder = LockListBuilder::new(); - builder.build_list(&client, info).await?; - let top = Import { - name: format!("{}:{}", info.name.namespace(), info.name.name()), - req: VersionReq::STAR, - kind: ImportKind::Unlocked, - }; - builder.lock_list.insert(top); - let mut composer = CompositionGraph::new(); - let mut handled = HashMap::::new(); - for package in builder.lock_list { - let name = package.name.clone(); - let version = package.req; - let id = PackageName::new(name)?; - let info = client.registry().load_package(&id).await?; - if let Some(inf) = info { - let release = if version != VersionReq::STAR { - inf.state - .releases() - .filter(|r| version.matches(&r.version)) - .last() - } else { - inf.state.releases().last() - }; - - if let Some(r) = release { - let state = &r.state; - if let ReleaseState::Released { content } = state { - let mut locked_package = package.name.clone(); - locked_package = format!( - "locked-dep=<{}@{}>,integrity=<{}>", - locked_package, - &r.version.to_string(), - content.to_string().replace(':', "-") - ); - let path = client.content().content_location(content); - if let Some(p) = path { - let read_digest = sha256::try_digest(&p)?; - if content.to_string().split(':').last().unwrap() != read_digest { - bail!("Expected content digest to be `{content}`, instead found `{read_digest}`"); - } - let component = - wasm_compose::graph::Component::from_file(&locked_package, p)?; - let component_index = - if let Some(c) = composer.get_component_by_name(&locked_package) { - c.0 - } else { - composer.add_component(component)? - }; - let instance_id = composer.instantiate(component_index)?; - let added = composer.get_component(component_index); - let ver = version.clone().to_string(); - let range = if ver == "*" { - "".to_string() - } else { - format!("@{{{}}}", ver.replace(',', "")) - }; - handled.insert(format!("{}{range}", package.name), instance_id); - let mut args = Vec::new(); - if let Some(added) = added { - for (index, name, _) in added.imports() { - let kindless_name = name.splitn(2, '=').last(); - if let Some(name) = kindless_name { - let iid = handled.get(&name[1..name.len() - 1]); - if let Some(arg) = iid { - args.push((arg, index)); - } - } - } - } - for arg in args { - composer.connect( - *arg.0, - None::, - instance_id, - arg.1, - )?; - } - } - } - } - } - } - let final_name = &format!("{}:{}", info.name.namespace(), &info.name.name()); - let id = handled.get(final_name); - let options = if let Some(id) = id { - EncodeOptions { - define_components: should_bundle, - export: Some(*id), - validate: false, - } - } else { - EncodeOptions { - define_components: false, - export: None, - validate: false, - } - }; - let locked = composer.encode(options)?; - fs::write("./locked.wasm", locked.as_slice())?; + async fn lock(client: FileSystemClient, info: &PackageInfo) -> Result<()> { + client.lock_component(info).await?; Ok(()) } } diff --git a/tests/components/add.wat b/tests/components/add.wat new file mode 100644 index 00000000..52f09081 --- /dev/null +++ b/tests/components/add.wat @@ -0,0 +1,14 @@ +(component + (core module $numbers + (func (export "add") (param i32 i32) (result i32) + local.get 0 + local.get 1 + i32.add + ) + ) + (core instance $firstInstance (instantiate $numbers)) + (alias core export 0 "add" (core func)) + (type (func (param "left" u32) (param "right" u32) (result u32))) + (func (type 0) (canon lift (core func 0))) + (export "add" (func 0)) +) \ No newline at end of file diff --git a/tests/components/five.wat b/tests/components/five.wat new file mode 100644 index 00000000..3693d219 --- /dev/null +++ b/tests/components/five.wat @@ -0,0 +1,33 @@ +(component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (alias export 0 "add" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (core module $numbers (;0;) + (type (;0;) (func (param i32 i32) (result i32))) + (type (;1;) (func (param i32) (result i32))) + (import "adder" "add" (func (;0;) (type 0))) + (func (;1;) (type 1) (param i32) (result i32) + local.get 0 + i32.const 5 + call 0 + ) + (export "both" (func 1)) + ) + (core instance (;0;) + (export "add" (func 0)) + ) + (core instance $firstInstance (;1;) (instantiate $numbers + (with "adder" (instance 0)) + ) + ) + (alias core export $firstInstance "both" (core func (;1;))) + (type (;1;) (func (param "input" u32) (result u32))) + (func (;1;) (type 1) (canon lift (core func 1))) + (export (;2;) "second" (func 1)) +) \ No newline at end of file diff --git a/tests/components/inc.wat b/tests/components/inc.wat new file mode 100644 index 00000000..91a81977 --- /dev/null +++ b/tests/components/inc.wat @@ -0,0 +1,33 @@ +(component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (alias export 0 "add" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (core module $numbers (;0;) + (type (;0;) (func (param i32 i32) (result i32))) + (type (;1;) (func (param i32) (result i32))) + (import "adder" "add" (func (;0;) (type 0))) + (func (;1;) (type 1) (param i32) (result i32) + local.get 0 + i32.const 2 + call 0 + ) + (export "both" (func 1)) + ) + (core instance (;0;) + (export "add" (func 0)) + ) + (core instance $firstInstance (;1;) (instantiate $numbers + (with "adder" (instance 0)) + ) + ) + (alias core export $firstInstance "both" (core func (;1;))) + (type (;1;) (func (param "input" u32) (result u32))) + (func (;1;) (type 1) (canon lift (core func 1))) + (export (;2;) "first" (func 1)) +) \ No newline at end of file diff --git a/tests/components/meet.wat b/tests/components/meet.wat new file mode 100644 index 00000000..596a2e63 --- /dev/null +++ b/tests/components/meet.wat @@ -0,0 +1,49 @@ +(component + (type (;0;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 0))) + ) + ) + (type (;1;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 1))) + (alias export 0 "first" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (alias export 1 "second" (func (;1;))) + (core func (;1;) (canon lower (func 1))) + (core module $meet + (type (;0;) (func (param i32 ) (result i32))) + (type (;1;) (func (param i32 i32) (result i32))) + (import "firsty" "first" (func (;0;) (type 0))) + (import "secondy" "second" (func (;1;) (type 0))) + (func (;2;) (type 1) (param i32 i32) (result i32) + local.get 0 + call 0 + local.get 1 + call 1 + i32.add + ) + (export "full" (func 2)) + ) + (core instance (;0;) + (export "first" (func 0)) + ) + (core instance (;1;) + (export "second" (func 1)) + ) + (core instance $total (;1;) (instantiate $meet + (with "firsty" (instance 0)) + (with "secondy" (instance 1)) + ) + ) + (alias core export $total "full" (core func (;2;))) + (type (;2;) (func (param "left" u32) (param "right" u32) (result u32))) + (func (;1;) (type 2) (canon lift (core func 2))) + (export (;2;) "full" (func 2)) +) \ No newline at end of file diff --git a/tests/components/meet_bundled.wat b/tests/components/meet_bundled.wat new file mode 100644 index 00000000..ae36356f --- /dev/null +++ b/tests/components/meet_bundled.wat @@ -0,0 +1,201 @@ +(component + (type (;0;) + (component + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (component (;0;) + (core module $numbers (;0;) + (type (;0;) (func (param i32 i32) (result i32))) + (func (;0;) (type 0) (param i32 i32) (result i32) + local.get 0 + local.get 1 + i32.add + ) + (export "add" (func 0)) + ) + (core instance $firstInstance (;0;) (instantiate $numbers)) + (alias core export $firstInstance "add" (core func (;0;))) + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (func (;0;) (type 0) (canon lift (core func 0))) + (export (;1;) "add" (func 0)) + ) + (type (;1;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 1))) + ) + ) + (component (;1;) + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (alias export 0 "add" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (core module $numbers (;0;) + (type (;0;) (func (param i32 i32) (result i32))) + (type (;1;) (func (param i32) (result i32))) + (import "adder" "add" (func (;0;) (type 0))) + (func (;1;) (type 1) (param i32) (result i32) + local.get 0 + i32.const 2 + call 0 + ) + (export "both" (func 1)) + ) + (core instance (;0;) + (export "add" (func 0)) + ) + (core instance $firstInstance (;1;) (instantiate $numbers + (with "adder" (instance 0)) + ) + ) + (alias core export $firstInstance "both" (core func (;1;))) + (type (;1;) (func (param "input" u32) (result u32))) + (func (;1;) (type 1) (canon lift (core func 1))) + (export (;2;) "first" (func 1)) + ) + (type (;2;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 1))) + ) + ) + (component (;2;) + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (alias export 0 "add" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (core module $numbers (;0;) + (type (;0;) (func (param i32 i32) (result i32))) + (type (;1;) (func (param i32) (result i32))) + (import "adder" "add" (func (;0;) (type 0))) + (func (;1;) (type 1) (param i32) (result i32) + local.get 0 + i32.const 5 + call 0 + ) + (export "both" (func 1)) + ) + (core instance (;0;) + (export "add" (func 0)) + ) + (core instance $firstInstance (;1;) (instantiate $numbers + (with "adder" (instance 0)) + ) + ) + (alias core export $firstInstance "both" (core func (;1;))) + (type (;1;) (func (param "input" u32) (result u32))) + (func (;1;) (type 1) (canon lift (core func 1))) + (export (;2;) "second" (func 1)) + ) + (type (;3;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;1;) (type 1))) + (type (;2;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "full" (func (type 2))) + ) + ) + (component (;3;) + (type (;0;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 0))) + ) + ) + (type (;1;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (import "unlocked-dep==1.0.0}>" (instance (;1;) (type 1))) + (alias export 0 "first" (func (;0;))) + (core func (;0;) (canon lower (func 0))) + (alias export 1 "second" (func (;1;))) + (core func (;1;) (canon lower (func 1))) + (core module $meet (;0;) + (type (;0;) (func (param i32) (result i32))) + (type (;1;) (func (param i32 i32) (result i32))) + (import "firsty" "first" (func (;0;) (type 0))) + (import "secondy" "second" (func (;1;) (type 0))) + (func (;2;) (type 1) (param i32 i32) (result i32) + local.get 0 + call 0 + local.get 1 + call 1 + i32.add + ) + (export "full" (func 2)) + ) + (core instance (;0;) + (export "first" (func 0)) + ) + (core instance (;1;) + (export "second" (func 1)) + ) + (core instance $total (;2;) (instantiate $meet + (with "firsty" (instance 0)) + (with "secondy" (instance 1)) + ) + ) + (alias core export $total "full" (core func (;2;))) + (type (;2;) (func (param "left" u32) (param "right" u32) (result u32))) + (func (;2;) (type 2) (canon lift (core func 2))) + (export (;3;) "full" (func 2)) + ) + (instance (;0;) (instantiate 0)) + (instance (;1;) (instantiate 1 + (with "unlocked-dep==1.0.0}>" (instance 0)) + ) + ) + (instance (;2;) (instantiate 2 + (with "unlocked-dep==1.0.0}>" (instance 0)) + ) + ) + (instance (;3;) (instantiate 3 + (with "unlocked-dep==1.0.0}>" (instance 1)) + (with "unlocked-dep==1.0.0}>" (instance 2)) + ) + ) + (alias export 3 "full" (func (;0;))) + (export (;1;) "full" (func 0)) +) \ No newline at end of file diff --git a/tests/components/meet_locked.wat b/tests/components/meet_locked.wat new file mode 100644 index 00000000..aca4251a --- /dev/null +++ b/tests/components/meet_locked.wat @@ -0,0 +1,74 @@ +(component + (type (;0;) + (component + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "locked-dep=,integrity=" (component (;0;) (type 0))) + (type (;1;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 1))) + ) + ) + (import "locked-dep=,integrity=" (component (;1;) (type 1))) + (type (;2;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "add" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 1))) + ) + ) + (import "locked-dep=,integrity=" (component (;2;) (type 2))) + (type (;3;) + (component + (type (;0;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "first" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;0;) (type 0))) + (type (;1;) + (instance + (type (;0;) (func (param "input" u32) (result u32))) + (export (;0;) "second" (func (type 0))) + ) + ) + (import "unlocked-dep==1.0.0}>" (instance (;1;) (type 1))) + (type (;2;) (func (param "left" u32) (param "right" u32) (result u32))) + (export (;0;) "full" (func (type 2))) + ) + ) + (import "locked-dep=,integrity=" (component (;3;) (type 3))) + (instance (;0;) (instantiate 0)) + (instance (;1;) (instantiate 1 + (with "unlocked-dep==1.0.0}>" (instance 0)) + ) + ) + (instance (;2;) (instantiate 2 + (with "unlocked-dep==1.0.0}>" (instance 0)) + ) + ) + (instance (;3;) (instantiate 3 + (with "unlocked-dep==1.0.0}>" (instance 1)) + (with "unlocked-dep==1.0.0}>" (instance 2)) + ) + ) + (alias export 3 "full" (func (;0;))) + (export (;1;) "full" (func 0)) +) \ No newline at end of file diff --git a/tests/depsolve.rs b/tests/depsolve.rs new file mode 100644 index 00000000..0628a2d7 --- /dev/null +++ b/tests/depsolve.rs @@ -0,0 +1,153 @@ +use self::support::*; +use anyhow::{Context, Result}; +use std::time::Duration; +use warg_client::{ + storage::{ + ContentStorage, FileSystemContentStorage, FileSystemRegistryStorage, PublishEntry, + PublishInfo, RegistryStorage, + }, + Client, +}; +use warg_crypto::signing::PrivateKey; +use warg_protocol::registry::{PackageName, RecordId}; + +pub mod support; + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn depsolve() -> Result<()> { + let (_server, config) = spawn_server(&root().await?, None, None, None).await?; + + let client = create_client(&config)?; + let signing_key = support::test_signing_key(); + + let mut head = publish_package( + &client, + &signing_key, + "test:add", + "tests/components/add.wat", + ) + .await?; + client + .wait_for_publish( + &PackageName::new("test:add")?, + &head, + Duration::from_millis(100), + ) + .await?; + head = publish_package( + &client, + &signing_key, + "test:five", + "tests/components/five.wat", + ) + .await?; + client + .wait_for_publish( + &PackageName::new("test:five")?, + &head, + Duration::from_millis(100), + ) + .await?; + head = publish_package( + &client, + &signing_key, + "test:inc", + "tests/components/inc.wat", + ) + .await?; + client + .wait_for_publish( + &PackageName::new("test:inc")?, + &head, + Duration::from_millis(100), + ) + .await?; + head = publish_package( + &client, + &signing_key, + "test:meet", + "tests/components/meet.wat", + ) + .await?; + client + .wait_for_publish( + &PackageName::new("test:meet")?, + &head, + Duration::from_millis(100), + ) + .await?; + + client.update().await?; + client + .upsert([ + &PackageName::new("test:add")?, + &PackageName::new("test:inc")?, + &PackageName::new("test:five")?, + &PackageName::new("test:meet")?, + ]) + .await?; + + let info = client + .registry() + .load_package(&PackageName::new("test:meet")?) + .await? + .context("package does not exist in client storage")?; + + let locked_bytes = client.lock_component(&info).await?; + let expected_locked = wat::parse_file("tests/components/meet_locked.wat")?; + assert_eq!( + wasmprinter::print_bytes(&locked_bytes)?, + wasmprinter::print_bytes(expected_locked)? + ); + let bundled_bytes = client.bundle_component(&info).await?; + let expected_bundled = wat::parse_file("tests/components/meet_bundled.wat")?; + assert_eq!( + wasmprinter::print_bytes(bundled_bytes)?, + wasmprinter::print_bytes(expected_bundled)? + ); + Ok(()) +} + +async fn publish_package( + client: &Client, + signing_key: &PrivateKey, + name: &str, + path: &str, +) -> Result { + let comp = wat::parse_file(path)?; + let name = PackageName::new(name)?; + let add_digest = client + .content() + .store_content( + Box::pin(futures::stream::once(async move { Ok(comp.into()) })), + None, + ) + .await?; + let mut head = client + .publish_with_info( + signing_key, + PublishInfo { + name: name.clone(), + head: None, + entries: vec![PublishEntry::Init], + }, + ) + .await?; + client + .wait_for_publish(&name.clone(), &head, Duration::from_millis(100)) + .await?; + head = client + .publish_with_info( + signing_key, + PublishInfo { + name: name.clone(), + head: Some(head), + entries: vec![PublishEntry::Release { + version: format!("1.0.0").parse().unwrap(), + content: add_digest.clone(), + }], + }, + ) + .await?; + Ok(head) +}