Added ability for wick reg pull to pull to the local dir (vs module hierarchy)

Cargo.toml

@@ -203,8 +203,8 @@ itertools = { version = "0.11", default-features = false, features = [
 jaq-core = { version = "0.10", default-features = false }
 json_dotpath = { version = "1.1.0", default-features = false }
 lazy_static = { version = "1.4", default-features = false }
-liquid = { version = "0.26", default-features = false }
-liquid-json = { version = "0.5", default-features = false }
+liquid = { package = "loose-liquid", version = "0.27", default-features = false }
+liquid-json = { version = "0.6", default-features = false }
 markup-converter = { version = "0.2", default-features = false }
 mslnk = { version = "0.1.8", default-features = false }
 nkeys = { version = "0.3", default-features = false }

crates/wick/wick-oci-utils/src/options.rs

@@ -24,7 +24,11 @@ pub struct OciOptions {
   #[getset(get = "pub", set = "pub")]
   pub(crate) cache_dir: PathBuf,
   #[getset(get = "pub", set = "pub")]
+  pub(crate) flatten: bool,
+  #[getset(get = "pub", set = "pub")]
   pub(crate) on_existing: OnExisting,
+  #[getset(get = "pub", set = "pub")]
+  pub(crate) ignore_manifest: bool,
 }
 
 impl Default for OciOptions {
@@ -35,8 +39,10 @@ impl Default for OciOptions {
       allow_insecure: vec![],
       username: None,
       password: None,
+      flatten: false,
       cache_dir: xdg.global().cache().clone(),
       on_existing: OnExisting::Ignore,
+      ignore_manifest: false,
     }
   }
 }

crates/wick/wick-oci-utils/src/package.rs

@@ -13,16 +13,16 @@ pub mod media_types;
 /// Represents a single file in a Wick package.
 #[derive(Debug, Clone)]
 pub struct PackageFile {
-  path: PathBuf,
+  package_path: PathBuf,
   hash: String,
   media_type: String,
   contents: bytes::Bytes,
 }
 
 impl PackageFile {
-  pub fn new(path: PathBuf, hash: String, media_type: String, contents: bytes::Bytes) -> Self {
+  pub fn new(package_path: PathBuf, hash: String, media_type: String, contents: bytes::Bytes) -> Self {
     Self {
-      path,
+      package_path,
       hash,
       media_type,
       contents,
@@ -31,8 +31,8 @@ impl PackageFile {
 
   /// Get path for the file.
   #[must_use]
-  pub const fn path(&self) -> &PathBuf {
-    &self.path
+  pub const fn package_path(&self) -> &PathBuf {
+    &self.package_path
   }
 
   /// Get hash for the file.

crates/wick/wick-oci-utils/src/package/pull.rs

@@ -27,11 +27,15 @@ pub struct PullResult {
 pub async fn pull(reference: &str, options: &OciOptions) -> Result<PullResult, Error> {
   let (image_ref, protocol) = crate::utils::parse_reference_and_protocol(reference, &options.allow_insecure)?;
 
-  let cache_dir = get_cache_directory(reference, &options.cache_dir)?;
+  let cache_dir = if options.flatten {
+    options.cache_dir.clone()
+  } else {
+    get_cache_directory(reference, &options.cache_dir)?
+  };
 
   let manifest_file = cache_dir.join(AssetManifest::FILENAME);
 
-  if manifest_file.exists() {
+  if !options.ignore_manifest && manifest_file.exists() {
     debug!(cache_hit = true, "remote asset");
     let json = tokio::fs::read_to_string(&manifest_file).await?;
 
@@ -194,9 +198,12 @@ pub async fn pull(reference: &str, options: &OciOptions) -> Result<PullResult, E
   };
 
   let root_file = root_file.ok_or_else(|| Error::PackageReadFailed("No root file found".to_owned()))?;
-  let manifest = AssetManifest::new(PathBuf::from(&root_file), version);
-  let contents = serde_json::to_string(&manifest).unwrap();
-  tokio::fs::write(cache_dir.join(AssetManifest::FILENAME), contents).await?;
+
+  if !options.ignore_manifest {
+    let manifest = AssetManifest::new(PathBuf::from(&root_file), version);
+    let contents = serde_json::to_string(&manifest).unwrap();
+    tokio::fs::write(cache_dir.join(AssetManifest::FILENAME), contents).await?;
+  }
 
   debug!(path = root_file, "Root file");
   Ok(PullResult {

crates/wick/wick-oci-utils/src/package/push.rs

@@ -27,7 +27,7 @@ pub async fn push(
   for file in files {
     let mut annotations_map: HashMap<String, String> = HashMap::new();
 
-    annotations_map.insert(annotations::TITLE.to_owned(), file.path().display().to_string());
+    annotations_map.insert(annotations::TITLE.to_owned(), file.package_path().display().to_string());
     let media_type = file.media_type().to_owned();
     let digest = file.hash().to_owned();
     let data = file.into_contents();

crates/wick/wick-package/src/package.rs

@@ -122,6 +122,7 @@ pub struct WickPackage {
   absolute_path: PathBuf,
   registry: Option<RegistryConfig>,
   root: String,
+  basedir: Option<PathBuf>,
 }
 
 impl WickPackage {
@@ -130,14 +131,17 @@ impl WickPackage {
   /// The provided path can be a file or directory. If it is a directory, the WickPackage will be created
   /// based on the files within the directory.
   #[allow(clippy::too_many_lines)]
-  pub async fn from_path(path: &Path) -> Result<Self, Error> {
+  pub async fn from_path(basedir: Option<PathBuf>, path: &Path) -> Result<Self, Error> {
+    let path = basedir
+      .as_ref()
+      .map_or_else(|| path.to_path_buf(), |basedir| basedir.join(path));
     //add check to see if its a path or directory and call appropriate api to find files based on that.
     if path.is_dir() {
-      return Err(Error::Directory(path.to_path_buf()));
+      return Err(Error::Directory(path));
     }
 
     let options = wick_config::FetchOptions::default();
-    let config = WickConfiguration::fetch(path, options).await?.into_inner();
+    let config = WickConfiguration::fetch(&path, options).await?.into_inner();
     if !matches!(
       config,
       WickConfiguration::App(_) | WickConfiguration::Component(_) | WickConfiguration::Types(_)
@@ -146,7 +150,7 @@ impl WickPackage {
     }
     let full_path = path
       .normalize()
-      .map_err(|e| Error::ReadFile(path.to_path_buf(), e))?
+      .map_err(|e| Error::ReadFile(path.clone(), e))?
       .into_path_buf();
 
     let parent_dir = full_path
@@ -197,9 +201,7 @@ impl WickPackage {
     let assets = config.assets();
     let mut wick_files: Vec<PackageFile> = Vec::new();
 
-    let root_bytes = fs::read(path)
-      .await
-      .map_err(|e| Error::ReadFile(path.to_path_buf(), e))?;
+    let root_bytes = fs::read(&path).await.map_err(|e| Error::ReadFile(path.clone(), e))?;
     let root_hash = format!("sha256:{}", digest(root_bytes.as_slice()));
 
     let root_file = PackageFile::new(
@@ -231,7 +233,7 @@ impl WickPackage {
     let (_, return_assets) = process_assets(Default::default(), assets, parent_dir.clone(), parent_dir.clone()).await?;
     //merge return assets  vector to wick_files
     wick_files.extend(return_assets);
-    trace!(files = ?wick_files.iter().map(|f| f.path()).collect::<Vec<_>>(),
+    trace!(files = ?wick_files.iter().map(|f| f.package_path()).collect::<Vec<_>>(),
       "package files"
     );
 
@@ -244,6 +246,7 @@ impl WickPackage {
       absolute_path: full_path,
       registry,
       root: root_file_path,
+      basedir,
     })
   }
 
@@ -253,6 +256,12 @@ impl WickPackage {
     self.files.iter().collect()
   }
 
+  #[must_use]
+  /// Return the base directory of the WickPackage if it came from the filesystem.
+  pub const fn basedir(&self) -> Option<&PathBuf> {
+    self.basedir.as_ref()
+  }
+
   #[must_use]
   /// Returns a list of the files contained within the WickPackage.
   pub const fn path(&self) -> &PathBuf {
@@ -321,7 +330,7 @@ impl WickPackage {
   pub async fn pull(reference: &str, options: &OciOptions) -> Result<Self, Error> {
     let result = wick_oci_utils::package::pull(reference, options).await?;
 
-    let package = Self::from_path(&result.base_dir.join(Path::new(&result.root_path))).await;
+    let package = Self::from_path(Some(result.base_dir), &result.root_path).await;
 
     match package {
       Ok(package) => Ok(package),

crates/wick/wick-package/tests/wick_package_integration.rs

@@ -28,7 +28,7 @@ mod integration_test {
     // Run the push operation
     let package_path = Path::new("./tests/files/jinja.wick");
     println!("Package path: {:?}", package_path);
-    let mut package = WickPackage::from_path(package_path).await.unwrap();
+    let mut package = WickPackage::from_path(None, package_path).await.unwrap();
     // Set the host to the whatever docker registry the tests are using.
     package.registry_mut().map(|r| r.set_host(host));
 
@@ -71,22 +71,26 @@ mod integration_test {
     // Sort both the pushed_files and pulled_files by path
     let mut pushed_files_sorted = pushed_files.clone();
     let mut pulled_files_sorted = pulled_files.clone();
-    pushed_files_sorted.sort_by_key(|file| file.path());
-    pulled_files_sorted.sort_by_key(|file| file.path());
+    pushed_files_sorted.sort_by_key(|file| file.package_path());
+    pulled_files_sorted.sort_by_key(|file| file.package_path());
 
     for (pushed_file, pulled_file) in pushed_files_sorted.iter().zip(pulled_files_sorted.iter()) {
-      let pushed_file_path = pushed_file.path().to_str().unwrap().trim_start_matches(&crate_dir);
+      let pushed_file_path = pushed_file
+        .package_path()
+        .to_str()
+        .unwrap()
+        .trim_start_matches(&crate_dir);
       let pulled_file_path = pulled_file
-        .path()
+        .package_path()
         .to_str()
         .unwrap()
         .trim_start_matches(tempdir.to_str().unwrap());
       assert_eq!(pushed_file_path, pulled_file_path, "Mismatch in file paths");
       //if pushed_file.path() ends with .tar.gz, don't compare hashes
-      if pushed_file.path().to_str().unwrap().ends_with(".tar.gz") {
+      if pushed_file.package_path().to_str().unwrap().ends_with(".tar.gz") {
         continue;
       }
-      println!("Comparing hashes for file: {:?}", pushed_file.path());
+      println!("Comparing hashes for file: {:?}", pushed_file.package_path());
       assert_eq!(pushed_file.hash(), pulled_file.hash(), "Mismatch in file hashes");
       assert_eq!(
         pushed_file.media_type(),