locking and bundling

Cargo.toml

@@ -6,11 +6,15 @@ authors = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
+wasm-bundle = { workspace = true }
+wasm-deps = { workspace = true }
+wasmparser = { workspace = true }
+wasm-compose = { workspace = true }
 warg-crypto = { workspace = true }
 warg-protocol = { workspace = true }
 warg-client = { workspace = true }
 clap = { workspace = true }
-anyhow = { workspace = true }
+anyhow.workspace = true
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { workspace = true }
@@ -30,6 +34,11 @@ itertools = "0.11.0"
 # TODO: remove these demo-related dependencies
 wasmtime = "10.0"
 wasmtime-wasi = "10.0"
+reqwest.workspace = true
+warg-api.workspace = true
+ptree = "0.4.0"
+async-recursion = "1.0.4"
+indexmap.workspace = true
 
 [dev-dependencies]
 reqwest = { workspace = true }
@@ -118,6 +127,9 @@ diesel_migrations = "2.1.0"
 diesel-derive-enum = "2.1.0"
 chrono = "0.4.26"
 regex = "1"
-wasmparser = "0.108.0"
 protox = "0.4.1"
 toml = "0.7.6"
+wasm-bundle = { path = "../mine/wasm-tools/crates/wasm-bundle" }
+wasm-deps = { path = "../mine/wasm-tools/crates/wasm-deps" }
+wasmparser = { path = "../mine/wasm-tools/crates/wasmparser" }
+wasm-compose = { path = "../mine/wasm-tools/crates/wasm-compose" }

crates/client/src/lib.rs

@@ -2,7 +2,7 @@
 
 #![deny(missing_docs)]
 
-use crate::storage::PackageInfo;
+use crate::storage::{PackageInfo, PublishEntry};
 use anyhow::{anyhow, Context, Result};
 use reqwest::{Body, IntoUrl};
 use std::{borrow::Cow, collections::HashMap, path::PathBuf, time::Duration};
@@ -11,6 +11,7 @@ use storage::{
     RegistryStorage,
 };
 use thiserror::Error;
+use warg_api::v1::package::ContentSource;
 use warg_api::v1::{
     fetch::{FetchError, FetchLogsRequest, FetchLogsResponse},
     package::{
@@ -137,6 +138,30 @@ impl<R: RegistryStorage, C: ContentStorage> Client<R, C> {
             _ => (),
         }
 
+        let mut content_source_list: Vec<ContentSource> = Vec::new();
+        for entry in &info.entries {
+            if let PublishEntry::Release {
+                version: _,
+                content,
+            } = entry
+            {
+                let mut url = self.url().to_string();
+                url.push_str("content/sha256-");
+                let full_hash = content.to_string();
+                let mut split = full_hash.split(':');
+                split.next();
+                let hash = split.next().unwrap();
+                url.push_str(hash);
+
+                content_source_list.push(ContentSource::Http { url });
+            }
+        }
+
+        let mut content_sources = HashMap::new();
+        if let Some(h) = info.head.clone() {
+            content_sources.insert(h.0, content_source_list);
+        }
+
         let record = info.finalize(signing_key)?;
         let log_id = LogId::package_log::<Sha256>(&package.id);
         let record = self
@@ -146,7 +171,7 @@ impl<R: RegistryStorage, C: ContentStorage> Client<R, C> {
                 PublishRecordRequest {
                     id: Cow::Borrowed(&package.id),
                     record: Cow::Owned(record.into()),
-                    content_sources: Default::default(),
+                    content_sources,
                 },
             )
             .await
@@ -230,14 +255,15 @@ impl<R: RegistryStorage, C: ContentStorage> Client<R, C> {
     }
 
     /// Updates every package log in client storage to the latest registry checkpoint.
-    pub async fn update(&self) -> ClientResult<()> {
+    pub async fn update(&self) -> ClientResult<SerdeEnvelope<TimestampedCheckpoint>> {
         tracing::info!("updating all packages to latest checkpoint");
 
         let mut updating = self.registry.load_packages().await?;
+        let checkpoint = &self.api.latest_checkpoint().await?;
         self.update_checkpoint(&self.api.latest_checkpoint().await?, &mut updating)
             .await?;
 
-        Ok(())
+        Ok(checkpoint.to_owned())
     }
 
     /// Inserts or updates the logs of the specified packages in client storage to
@@ -349,7 +375,8 @@ impl<R: RegistryStorage, C: ContentStorage> Client<R, C> {
         })
     }
 
-    async fn update_checkpoint<'a>(
+    /// Update checkpoint for list of packages
+    pub async fn update_checkpoint<'a>(
         &self,
         ts_checkpoint: &SerdeEnvelope<TimestampedCheckpoint>,
         packages: impl IntoIterator<Item = &mut PackageInfo>,

crates/protocol/src/registry.rs

@@ -131,7 +131,7 @@ impl VisitBytes for LogLeaf {
 /// where both parts are also valid WIT identifiers (i.e. kebab-cased).
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub struct PackageId {
-    id: String,
+    pub id: String,
     colon: usize,
 }
 
@@ -259,7 +259,7 @@ impl AsRef<[u8]> for LogId {
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct RecordId(AnyHash);
+pub struct RecordId(pub AnyHash);
 
 impl RecordId {
     pub fn algorithm(&self) -> HashAlgorithm {

crates/server/Cargo.toml

@@ -36,6 +36,8 @@ diesel_migrations = { workspace = true, optional = true }
 diesel-derive-enum = { workspace = true, optional = true, features = ["postgres"] }
 serde_json = { workspace = true, optional = true }
 chrono = { workspace = true, optional = true }
+reqwest.workspace = true
+wasm-deps = { path = "../../../mine/wasm-tools/crates/wasm-deps" }
 
 [features]
 default = []

crates/server/src/api/v1/fetch.rs

@@ -10,6 +10,7 @@ use axum::{
     Router,
 };
 use std::collections::HashMap;
+use std::sync::Arc;
 use warg_api::v1::fetch::{FetchError, FetchLogsRequest, FetchLogsResponse};
 use warg_crypto::hash::Sha256;
 use warg_protocol::registry::{LogId, TimestampedCheckpoint};

crates/server/src/api/v1/package.rs

@@ -31,6 +31,8 @@ use warg_protocol::{
     registry::{LogId, RecordId},
     ProtoEnvelope, Record as _,
 };
+use wasm_deps::DepsParser;
+use wasmparser::ComponentImportName;
 
 #[derive(Clone)]
 pub struct Config {
@@ -207,11 +209,11 @@ async fn publish_record(
         .map_err(PackageApiError::bad_request)?;
 
     // Specifying content sources is not allowed in this implementation
-    if !body.content_sources.is_empty() {
-        return Err(PackageApiError::unsupported(
-            "specifying content sources is not supported",
-        ));
-    }
+    // if !body.content_sources.is_empty() {
+    //     return Err(PackageApiError::unsupported(
+    //         "specifying content sources is not supported",
+    //     ));
+    // }
 
     // Preemptively perform the policy check on the record before storing it
     // This is performed here so that we never store an unauthorized record
@@ -384,7 +386,7 @@ async fn upload_content(
     {
         config
             .core_service
-            .submit_package_record(log_id, record_id)
+            .submit_package_record(log_id, record_id.clone())
             .await;
     }
 

src/bin/warg.rs

@@ -3,8 +3,8 @@ use clap::Parser;
 use std::process::exit;
 use tracing_subscriber::EnvFilter;
 use warg_cli::commands::{
-    ConfigCommand, DownloadCommand, InfoCommand, KeyCommand, PublishCommand, RunCommand,
-    UpdateCommand,
+    BundleCommand, ConfigCommand, DependenciesCommand, DownloadCommand, InfoCommand, KeyCommand,
+    LockCommand, PublishCommand, RunCommand, UpdateCommand,
 };
 use warg_client::ClientError;
 
@@ -25,6 +25,9 @@ enum WargCli {
     Config(ConfigCommand),
     Info(InfoCommand),
     Key(KeyCommand),
+    Lock(LockCommand),
+    Bundle(BundleCommand),
+    Dependencies(DependenciesCommand),
     Download(DownloadCommand),
     Update(UpdateCommand),
     #[clap(subcommand)]
@@ -42,6 +45,9 @@ async fn main() -> Result<()> {
         WargCli::Config(cmd) => cmd.exec().await,
         WargCli::Info(cmd) => cmd.exec().await,
         WargCli::Key(cmd) => cmd.exec().await,
+        WargCli::Lock(cmd) => cmd.exec().await,
+        WargCli::Bundle(cmd) => cmd.exec().await,
+        WargCli::Dependencies(cmd) => cmd.exec().await,
         WargCli::Download(cmd) => cmd.exec().await,
         WargCli::Update(cmd) => cmd.exec().await,
         WargCli::Publish(cmd) => cmd.exec().await,

src/commands.rs

@@ -8,20 +8,26 @@ use warg_client::RegistryUrl;
 use warg_client::{ClientError, Config, FileSystemClient, StorageLockResult};
 use warg_crypto::signing::PrivateKey;
 
+mod bundle;
 mod config;
+mod dependencies;
 mod download;
 mod info;
 mod key;
+mod lock;
 mod publish;
 mod run;
 mod update;
 
 use crate::keyring::get_signing_key;
 
+pub use self::bundle::*;
 pub use self::config::*;
+pub use self::dependencies::*;
 pub use self::download::*;
 pub use self::info::*;
 pub use self::key::*;
+pub use self::lock::*;
 pub use self::publish::*;
 pub use self::run::*;
 pub use self::update::*;

src/commands/bundle.rs

@@ -0,0 +1,34 @@
+use super::CommonOptions;
+use anyhow::Result;
+use clap::Args;
+use std::fs;
+use warg_protocol::registry::PackageId;
+use wasm_bundle::Bundler;
+
+/// Bundle With Registry Dependencies
+#[derive(Args)]
+pub struct BundleCommand {
+    /// The common command options.
+    #[clap(flatten)]
+    pub common: CommonOptions,
+
+    /// Only show information for the specified package.
+    #[clap(value_name = "PACKAGE")]
+    pub package: Option<PackageId>,
+}
+
+impl BundleCommand {
+    /// Executes the command.
+    pub async fn exec(self) -> Result<()> {
+        let config = self.common.read_config()?;
+        let client = self.common.create_client(&config)?;
+
+        println!("registry: {url}", url = client.url());
+        println!("\npackages in client storage:");
+        let mut bundler = Bundler::new(&client);
+        let locked = fs::read("./locked.wasm")?;
+        let bundled = bundler.parse(&locked).await?;
+        fs::write("./bundled.wasm", bundled.as_slice())?;
+        Ok(())
+    }
+}