Skip to content

Commit

Permalink
feat(*)!: Uses wasm-pkg-client for loading dependencies
Browse files Browse the repository at this point in the history
This is a fairly large PR because this dependency is used everywhere.
cargo component now uses the new wasm-pkg-tools toolchain to load deps,
which means that both OCI and Warg are supported. This tries to stay as
close to the original code style as possible, but I did have to make
large changes to how dependencies were being resolved to account for the
new library. There are a couple major breaking changes to be aware of and
one question to answer. This PR is already quite large, so I figured it
would be better to probably merge this and then deal with follow ups to
any of the outstanding questions below:

- Bindings are now generated every time. Where the actual dependency is
  being loaded from is now abstracted away by the client, so we can't
  check if those files have changed on disk. I personally don't think this
  is the worst thing, but if people really want that functionality, we
  can add support for that into the package tools.
- Offline deps currently does not work, but that is something that will
  be added into wasm-pkg-client instead of doing checks within cargo
  component itself.
- Currently the registries specified within Cargo.toml are not used in
  favor of using the config file for wasm-pkg-tools. I am not sure if we
  wanted to use that to override the wasm-pkg-tools config or just drop
  it entirely. I would like to address this one before merge.
- Until wasm-pkg-client has support for publishing, I have left support
  in to use warg directly. Once we have that added to wasm-pkg-tools, we
  can change the behavior here

Signed-off-by: Taylor Thomas <[email protected]>
  • Loading branch information
thomastaylor312 committed Jul 16, 2024
1 parent 7cf7368 commit f3840de
Show file tree
Hide file tree
Showing 47 changed files with 2,596 additions and 1,935 deletions.
1,495 changes: 1,017 additions & 478 deletions Cargo.lock

Large diffs are not rendered by default.

121 changes: 64 additions & 57 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,95 +18,102 @@ keywords = ["webassembly", "wasm", "components", "component-model"]
repository = "https://github.com/bytecodealliance/cargo-component"

[dependencies]
cargo-component-core = { workspace = true }
anyhow = { workspace = true }
bytes = { workspace = true }
cargo_metadata = { workspace = true }
cargo-component-core = { workspace = true }
cargo-config2 = { workspace = true }
clap = { workspace = true }
toml_edit = { workspace = true }
pretty_env_logger = { workspace = true }
log = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
futures = { workspace = true }
heck = { workspace = true }
semver = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
indexmap = { workspace = true }
url = { workspace = true }
wit-bindgen-rust = { workspace = true }
wit-bindgen-core = { workspace = true }
wit-parser = { workspace = true }
wit-component = { workspace = true }
wasm-metadata = { workspace = true }
wasmparser = { workspace = true }
parse_arg = { workspace = true }
cargo_metadata = { workspace = true }
cargo-config2 = { workspace = true }
libc = { workspace = true }
warg-protocol = { workspace = true }
warg-crypto = { workspace = true }
warg-client = { workspace = true }
log = { workspace = true }
p256 = { workspace = true }
parse_arg = { workspace = true }
pretty_env_logger = { workspace = true }
rand_core = { workspace = true }
rpassword = { workspace = true }
futures = { workspace = true }
bytes = { workspace = true }
which = { workspace = true }
semver = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
shell-escape = "0.1.5"
tempfile = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
toml_edit = { workspace = true }
url = { workspace = true }
warg-client = { workspace = true }
warg-crypto = { workspace = true }
warg-protocol = { workspace = true }
wasm-metadata = { workspace = true }
wasm-pkg-client = { workspace = true }
wasmparser = { workspace = true }
which = { workspace = true }
wit-bindgen-core = { workspace = true }
wit-bindgen-rust = { workspace = true }
wit-component = { workspace = true }
wit-parser = { workspace = true }

[dev-dependencies]
assert_cmd = { workspace = true }
predicates = { workspace = true }
wat = { workspace = true }
warg-server = { workspace = true }
tempfile = { workspace = true }
warg-server = { workspace = true }
wasmprinter = { workspace = true }
wat = { workspace = true }

[workspace]
members = ["crates/core", "crates/wit"]

[workspace.dependencies]
cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
warg-protocol = "0.7.0"
warg-crypto = "0.7.0"
warg-client = "0.7.0"
warg-server = "0.7.0"
anyhow = "1.0.82"
clap = { version = "4.5.4", features = ["derive"] }
toml_edit = { version = "0.22.9", features = ["serde"] }
pretty_env_logger = "0.5.0"
log = "0.4.21"
tokio = { version = "1.37.0", default-features = false, features = ["macros", "rt-multi-thread"] }
tokio-util = "0.7.10"
heck = "0.5.0"
semver = "1.0.22"
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.115"
indexmap = "2.2.6"
url = { version = "2.5.0", features = ["serde"] }
wit-parser = "0.208.1"
wit-component = "0.208.1"
wasm-metadata = "0.208.1"
parse_arg = "0.1.4"
assert_cmd = "2.0.14"
bytes = "1.6.0"
cargo_metadata = "0.18.1"
cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
cargo-config2 = "0.1.24"
clap = { version = "4.5.4", features = ["derive", "env"] }
dirs = "5"
futures = "0.3.30"
heck = "0.5.0"
indexmap = "2.2.6"
libc = "0.2.153"
log = "0.4.21"
oci-distribution = "0.11"
owo-colors = "4.0.0"
unicode-width = "0.1.11"
p256 = "0.13.2"
parse_arg = "0.1.4"
predicates = "3.1.0"
pretty_env_logger = "0.5.0"
rand_core = "0.6.4"
rpassword = "7.3.1"
futures = "0.3.30"
bytes = "1.6.0"
which = "6.0.1"
wit-bindgen-rust = "0.25.0"
wit-bindgen-core = "0.25.0"
semver = "1"
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.115"
tempfile = "3.10.1"
assert_cmd = "2.0.14"
predicates = "3.1.0"
tokio = { version = "1.37.0", default-features = false, features = [
"macros",
"rt-multi-thread",
] }
tokio-util = "0.7.10"
toml_edit = { version = "0.22.9", features = ["serde"] }
unicode-width = "0.1.11"
url = { version = "2.5.0", features = ["serde"] }
warg-client = "0.7.0"
warg-crypto = "0.7.0"
warg-protocol = "0.7.0"
warg-server = "0.7.0"
wasm-metadata = "0.208.1"
wasm-pkg-client = { git = "https://github.com/thomastaylor312/wasm-pkg-tools.git", branch = "fix/cached_versions" }
wasmparser = "0.208.1"
wat = "1.208.1"
wasmprinter = "0.208.1"
wat = "1.208.1"
which = "6.0.1"
wit-bindgen-core = "0.25.0"
wit-bindgen-rust = "0.25.0"
wit-component = "0.208.1"
wit-parser = "0.208.1"

[profile.release]
panic = "abort"
Expand Down
23 changes: 13 additions & 10 deletions crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,26 @@ repository = { workspace = true }

[dependencies]
anyhow = { workspace = true }
clap = { workspace = true }
dirs = { workspace = true }
futures = { workspace = true }
indexmap = { workspace = true }
libc = { workspace = true }
log = { workspace = true }
owo-colors = { workspace = true }
unicode-width = { workspace = true }
warg-crypto = { workspace = true }
warg-protocol = { workspace = true }
warg-client = { workspace = true }
toml_edit = { workspace = true }
semver = { workspace = true }
serde = { workspace = true }
indexmap = { workspace = true }
futures = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true, features = ["io"] }
toml_edit = { workspace = true }
unicode-width = { workspace = true }
url = { workspace = true }
warg-client = { workspace = true }
warg-crypto = { workspace = true }
warg-protocol = { workspace = true }
wasm-pkg-client = { workspace = true }
wit-component = { workspace = true }
wit-parser = { workspace = true }
log = { workspace = true }
tokio = { workspace = true }
clap = { workspace = true }

[target.'cfg(windows)'.dependencies.windows-sys]
version = "0.52"
Expand Down
17 changes: 16 additions & 1 deletion crates/core/src/command.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
//! Module for common command implementation.
use std::path::PathBuf;

use crate::terminal::{Color, Terminal, Verbosity};
use clap::{ArgAction, Args};

use crate::terminal::{Color, Terminal, Verbosity};

/// The environment variable name for setting a cache directory location
pub const CACHE_DIR_ENV_VAR: &str = "CARGO_COMPONENT_CACHE_DIR";
/// The environment variable name for setting a path to a config file
pub const CONFIG_FILE_ENV_VAR: &str = "CARGO_COMPONENT_CONFIG_FILE";

/// Common options for commands.
#[derive(Args)]
#[command(
Expand All @@ -24,6 +31,14 @@ pub struct CommonOptions {
/// Coloring: auto, always, never
#[clap(long = "color", value_name = "WHEN")]
pub color: Option<Color>,

/// The path to the cache directory to store component dependencies.
#[clap(long = "cache-dir", env = CACHE_DIR_ENV_VAR)]
pub cache_dir: Option<PathBuf>,

/// The path to the pkg-tools config file
#[clap(long = "config", env = CONFIG_FILE_ENV_VAR)]
pub config: Option<PathBuf>,
}

impl CommonOptions {
Expand Down
28 changes: 25 additions & 3 deletions crates/core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,44 @@
#![deny(missing_docs)]

use std::path::PathBuf;
use std::str::FromStr;

use anyhow::Context;
use semver::VersionReq;
use std::str::FromStr;
use warg_protocol::registry::PackageName;
use wasm_pkg_client::PackageRef;

pub mod command;
pub mod lock;
pub mod progress;
pub mod registry;
pub mod terminal;

/// The root directory name used for default cargo component directories
pub const CARGO_COMPONENT_DIR: &str = "cargo-component";
/// The cache directory name used by default
pub const CACHE_DIR: &str = "cache";

/// Returns the path to the default cache directory, returning an error if a cache directory cannot be found.
pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
dirs::cache_dir()
.map(|p| p.join(CARGO_COMPONENT_DIR).join(CACHE_DIR))
.ok_or_else(|| anyhow::anyhow!("failed to find cache directory"))
}

/// A helper that fetches the default directory if the given directory is `None`.
pub fn cache_dir(dir: Option<PathBuf>) -> anyhow::Result<PathBuf> {
match dir {
Some(dir) => Ok(dir),
None => default_cache_dir(),
}
}

/// Represents a versioned component package name.
#[derive(Clone)]
pub struct VersionedPackageName {
/// The package name.
pub name: PackageName,
pub name: PackageRef,
/// The optional package version.
pub version: Option<VersionReq>,
}
Expand Down
27 changes: 17 additions & 10 deletions crates/core/src/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ use std::{
path::{Path, PathBuf},
};
use toml_edit::{DocumentMut, Item, Value};
use warg_crypto::hash::AnyHash;
use warg_protocol::registry::PackageName;
use wasm_pkg_client::{ContentDigest, PackageRef};

/// The file format version of the lock file.
const LOCK_FILE_VERSION: i64 = 1;
Expand All @@ -21,7 +20,7 @@ const LOCK_FILE_VERSION: i64 = 1;
#[serde(rename_all = "kebab-case")]
pub struct LockedPackage {
/// The name of the locked package.
pub name: PackageName,
pub name: PackageRef,
/// The registry the package was resolved from.
///
/// Defaults to the default registry if not specified.
Expand All @@ -37,9 +36,10 @@ pub struct LockedPackage {

impl LockedPackage {
/// Gets the key used in sorting and searching the package list.
pub fn key(&self) -> (&PackageName, &str) {
pub fn key(&self) -> (&str, &str, &str) {
(
&self.name,
self.name.namespace().as_ref(),
self.name.name().as_ref(),
self.registry.as_deref().unwrap_or(DEFAULT_REGISTRY_NAME),
)
}
Expand All @@ -53,7 +53,7 @@ pub struct LockedPackageVersion {
/// The version the package is locked to.
pub version: Version,
/// The digest of the package contents.
pub digest: AnyHash,
pub digest: ContentDigest,
}

impl LockedPackageVersion {
Expand Down Expand Up @@ -81,13 +81,20 @@ impl<'a> LockFileResolver<'a> {
pub fn resolve(
&'a self,
registry: &str,
name: &PackageName,
package_ref: &PackageRef,
requirement: &VersionReq,
) -> Result<Option<&'a LockedPackageVersion>> {
if let Some(pkg) = self
.0
.packages
.binary_search_by_key(&(name, registry), LockedPackage::key)
.binary_search_by_key(
&(
package_ref.namespace().as_ref(),
package_ref.name().as_ref(),
registry,
),
LockedPackage::key,
)
.ok()
.map(|i| &self.0.packages[i])
{
Expand All @@ -96,12 +103,12 @@ impl<'a> LockFileResolver<'a> {
.binary_search_by_key(&requirement.to_string().as_str(), LockedPackageVersion::key)
{
let locked = &pkg.versions[index];
log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
return Ok(Some(locked));
}
}

log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
Ok(None)
}
}
Expand Down
Loading

0 comments on commit f3840de

Please sign in to comment.