diff --git a/crates/wick/wick-oci-utils/src/options.rs b/crates/wick/wick-oci-utils/src/options.rs index 576b176f..bd250c6e 100644 --- a/crates/wick/wick-oci-utils/src/options.rs +++ b/crates/wick/wick-oci-utils/src/options.rs @@ -24,7 +24,11 @@ pub struct OciOptions { #[getset(get = "pub", set = "pub")] pub(crate) cache_dir: PathBuf, #[getset(get = "pub", set = "pub")] + pub(crate) flatten: bool, + #[getset(get = "pub", set = "pub")] pub(crate) on_existing: OnExisting, + #[getset(get = "pub", set = "pub")] + pub(crate) ignore_manifest: bool, } impl Default for OciOptions { @@ -35,8 +39,10 @@ impl Default for OciOptions { allow_insecure: vec![], username: None, password: None, + flatten: false, cache_dir: xdg.global().cache().clone(), on_existing: OnExisting::Ignore, + ignore_manifest: false, } } } diff --git a/crates/wick/wick-oci-utils/src/package.rs b/crates/wick/wick-oci-utils/src/package.rs index 4e5f7acf..11d608a9 100644 --- a/crates/wick/wick-oci-utils/src/package.rs +++ b/crates/wick/wick-oci-utils/src/package.rs @@ -13,16 +13,16 @@ pub mod media_types; /// Represents a single file in a Wick package. #[derive(Debug, Clone)] pub struct PackageFile { - path: PathBuf, + package_path: PathBuf, hash: String, media_type: String, contents: bytes::Bytes, } impl PackageFile { - pub fn new(path: PathBuf, hash: String, media_type: String, contents: bytes::Bytes) -> Self { + pub fn new(package_path: PathBuf, hash: String, media_type: String, contents: bytes::Bytes) -> Self { Self { - path, + package_path, hash, media_type, contents, @@ -31,8 +31,8 @@ impl PackageFile { /// Get path for the file. #[must_use] - pub const fn path(&self) -> &PathBuf { - &self.path + pub const fn package_path(&self) -> &PathBuf { + &self.package_path } /// Get hash for the file. diff --git a/crates/wick/wick-oci-utils/src/package/pull.rs b/crates/wick/wick-oci-utils/src/package/pull.rs index 5f95a677..67a6e1dd 100644 --- a/crates/wick/wick-oci-utils/src/package/pull.rs +++ b/crates/wick/wick-oci-utils/src/package/pull.rs @@ -27,11 +27,15 @@ pub struct PullResult { pub async fn pull(reference: &str, options: &OciOptions) -> Result { let (image_ref, protocol) = crate::utils::parse_reference_and_protocol(reference, &options.allow_insecure)?; - let cache_dir = get_cache_directory(reference, &options.cache_dir)?; + let cache_dir = if options.flatten { + options.cache_dir.clone() + } else { + get_cache_directory(reference, &options.cache_dir)? + }; let manifest_file = cache_dir.join(AssetManifest::FILENAME); - if manifest_file.exists() { + if !options.ignore_manifest && manifest_file.exists() { debug!(cache_hit = true, "remote asset"); let json = tokio::fs::read_to_string(&manifest_file).await?; @@ -194,9 +198,12 @@ pub async fn pull(reference: &str, options: &OciOptions) -> Result = HashMap::new(); - annotations_map.insert(annotations::TITLE.to_owned(), file.path().display().to_string()); + annotations_map.insert(annotations::TITLE.to_owned(), file.package_path().display().to_string()); let media_type = file.media_type().to_owned(); let digest = file.hash().to_owned(); let data = file.into_contents(); diff --git a/crates/wick/wick-package/src/package.rs b/crates/wick/wick-package/src/package.rs index 22b964a3..f7fdcc94 100644 --- a/crates/wick/wick-package/src/package.rs +++ b/crates/wick/wick-package/src/package.rs @@ -122,6 +122,7 @@ pub struct WickPackage { absolute_path: PathBuf, registry: Option, root: String, + basedir: Option, } impl WickPackage { @@ -130,14 +131,17 @@ impl WickPackage { /// The provided path can be a file or directory. If it is a directory, the WickPackage will be created /// based on the files within the directory. #[allow(clippy::too_many_lines)] - pub async fn from_path(path: &Path) -> Result { + pub async fn from_path(basedir: Option, path: &Path) -> Result { + let path = basedir + .as_ref() + .map_or_else(|| path.to_path_buf(), |basedir| basedir.join(path)); //add check to see if its a path or directory and call appropriate api to find files based on that. if path.is_dir() { - return Err(Error::Directory(path.to_path_buf())); + return Err(Error::Directory(path)); } let options = wick_config::FetchOptions::default(); - let config = WickConfiguration::fetch(path, options).await?.into_inner(); + let config = WickConfiguration::fetch(&path, options).await?.into_inner(); if !matches!( config, WickConfiguration::App(_) | WickConfiguration::Component(_) | WickConfiguration::Types(_) @@ -146,7 +150,7 @@ impl WickPackage { } let full_path = path .normalize() - .map_err(|e| Error::ReadFile(path.to_path_buf(), e))? + .map_err(|e| Error::ReadFile(path.clone(), e))? .into_path_buf(); let parent_dir = full_path @@ -197,9 +201,7 @@ impl WickPackage { let assets = config.assets(); let mut wick_files: Vec = Vec::new(); - let root_bytes = fs::read(path) - .await - .map_err(|e| Error::ReadFile(path.to_path_buf(), e))?; + let root_bytes = fs::read(&path).await.map_err(|e| Error::ReadFile(path.clone(), e))?; let root_hash = format!("sha256:{}", digest(root_bytes.as_slice())); let root_file = PackageFile::new( @@ -231,7 +233,7 @@ impl WickPackage { let (_, return_assets) = process_assets(Default::default(), assets, parent_dir.clone(), parent_dir.clone()).await?; //merge return assets vector to wick_files wick_files.extend(return_assets); - trace!(files = ?wick_files.iter().map(|f| f.path()).collect::>(), + trace!(files = ?wick_files.iter().map(|f| f.package_path()).collect::>(), "package files" ); @@ -244,6 +246,7 @@ impl WickPackage { absolute_path: full_path, registry, root: root_file_path, + basedir, }) } @@ -253,6 +256,12 @@ impl WickPackage { self.files.iter().collect() } + #[must_use] + /// Return the base directory of the WickPackage if it came from the filesystem. + pub const fn basedir(&self) -> Option<&PathBuf> { + self.basedir.as_ref() + } + #[must_use] /// Returns a list of the files contained within the WickPackage. pub const fn path(&self) -> &PathBuf { @@ -321,7 +330,7 @@ impl WickPackage { pub async fn pull(reference: &str, options: &OciOptions) -> Result { let result = wick_oci_utils::package::pull(reference, options).await?; - let package = Self::from_path(&result.base_dir.join(Path::new(&result.root_path))).await; + let package = Self::from_path(Some(result.base_dir), &result.root_path).await; match package { Ok(package) => Ok(package), diff --git a/crates/wick/wick-package/tests/wick_package_integration.rs b/crates/wick/wick-package/tests/wick_package_integration.rs index b28e774a..f71728cf 100644 --- a/crates/wick/wick-package/tests/wick_package_integration.rs +++ b/crates/wick/wick-package/tests/wick_package_integration.rs @@ -28,7 +28,7 @@ mod integration_test { // Run the push operation let package_path = Path::new("./tests/files/jinja.wick"); println!("Package path: {:?}", package_path); - let mut package = WickPackage::from_path(package_path).await.unwrap(); + let mut package = WickPackage::from_path(None, package_path).await.unwrap(); // Set the host to the whatever docker registry the tests are using. package.registry_mut().map(|r| r.set_host(host)); @@ -71,22 +71,26 @@ mod integration_test { // Sort both the pushed_files and pulled_files by path let mut pushed_files_sorted = pushed_files.clone(); let mut pulled_files_sorted = pulled_files.clone(); - pushed_files_sorted.sort_by_key(|file| file.path()); - pulled_files_sorted.sort_by_key(|file| file.path()); + pushed_files_sorted.sort_by_key(|file| file.package_path()); + pulled_files_sorted.sort_by_key(|file| file.package_path()); for (pushed_file, pulled_file) in pushed_files_sorted.iter().zip(pulled_files_sorted.iter()) { - let pushed_file_path = pushed_file.path().to_str().unwrap().trim_start_matches(&crate_dir); + let pushed_file_path = pushed_file + .package_path() + .to_str() + .unwrap() + .trim_start_matches(&crate_dir); let pulled_file_path = pulled_file - .path() + .package_path() .to_str() .unwrap() .trim_start_matches(tempdir.to_str().unwrap()); assert_eq!(pushed_file_path, pulled_file_path, "Mismatch in file paths"); //if pushed_file.path() ends with .tar.gz, don't compare hashes - if pushed_file.path().to_str().unwrap().ends_with(".tar.gz") { + if pushed_file.package_path().to_str().unwrap().ends_with(".tar.gz") { continue; } - println!("Comparing hashes for file: {:?}", pushed_file.path()); + println!("Comparing hashes for file: {:?}", pushed_file.package_path()); assert_eq!(pushed_file.hash(), pulled_file.hash(), "Mismatch in file hashes"); assert_eq!( pushed_file.media_type(), diff --git a/crates/wick/wick-package/tests/wick_package_types_integration.rs b/crates/wick/wick-package/tests/wick_package_types_integration.rs index 6fdf9869..29ed6ddf 100644 --- a/crates/wick/wick-package/tests/wick_package_types_integration.rs +++ b/crates/wick/wick-package/tests/wick_package_types_integration.rs @@ -28,7 +28,7 @@ mod integration_test { // Run the push operation let package_path = Path::new("./tests/files/mytypes.wick"); println!("Package path: {:?}", package_path); - let mut package = WickPackage::from_path(package_path).await.unwrap(); + let mut package = WickPackage::from_path(None, package_path).await.unwrap(); // Set the host to the whatever docker registry the tests are using. package.registry_mut().map(|r| r.set_host(host)); @@ -71,22 +71,26 @@ mod integration_test { // Sort both the pushed_files and pulled_files by path let mut pushed_files_sorted = pushed_files.clone(); let mut pulled_files_sorted = pulled_files.clone(); - pushed_files_sorted.sort_by_key(|file| file.path()); - pulled_files_sorted.sort_by_key(|file| file.path()); + pushed_files_sorted.sort_by_key(|file| file.package_path()); + pulled_files_sorted.sort_by_key(|file| file.package_path()); for (pushed_file, pulled_file) in pushed_files_sorted.iter().zip(pulled_files_sorted.iter()) { - let pushed_file_path = pushed_file.path().to_str().unwrap().trim_start_matches(&crate_dir); + let pushed_file_path = pushed_file + .package_path() + .to_str() + .unwrap() + .trim_start_matches(&crate_dir); let pulled_file_path = pulled_file - .path() + .package_path() .to_str() .unwrap() .trim_start_matches(tempdir.to_str().unwrap()); assert_eq!(pushed_file_path, pulled_file_path, "Mismatch in file paths"); //if pushed_file.path() ends with .tar.gz, don't compare hashes - if pushed_file.path().to_str().unwrap().ends_with(".tar.gz") { + if pushed_file.package_path().to_str().unwrap().ends_with(".tar.gz") { continue; } - println!("Comparing hashes for file: {:?}", pushed_file.path()); + println!("Comparing hashes for file: {:?}", pushed_file.package_path()); assert_eq!(pushed_file.hash(), pulled_file.hash(), "Mismatch in file hashes"); assert_eq!( pushed_file.media_type(), diff --git a/src/commands/install.rs b/src/commands/install.rs index 02ed7ca7..ffb1eb39 100644 --- a/src/commands/install.rs +++ b/src/commands/install.rs @@ -39,7 +39,7 @@ pub(crate) async fn handle( let oci_opts = reconcile_fetch_options(&opts.application, &settings, opts.oci, None); let app_as_path = PathBuf::from(&opts.application); let package = if app_as_path.exists() { - WickPackage::from_path(&app_as_path).await? + WickPackage::from_path(None, &app_as_path).await? } else { crate::oci::pull(opts.application, oci_opts).await? }; diff --git a/src/commands/registry/pull.rs b/src/commands/registry/pull.rs index 18f467e5..2074fe98 100644 --- a/src/commands/registry/pull.rs +++ b/src/commands/registry/pull.rs @@ -21,6 +21,10 @@ pub(crate) struct Options { #[clap(flatten)] pub(crate) oci_opts: crate::options::oci::OciOptions, + + /// Write package files directly to output dir, don't use the wick heirachical structure. + #[clap(long = "flattened", short = 'F', action)] + pub(crate) flattened: bool, } #[allow(clippy::unused_async)] @@ -29,20 +33,38 @@ pub(crate) async fn handle( settings: wick_settings::Settings, span: tracing::Span, ) -> Result { - let oci_opts = reconcile_fetch_options(&opts.reference, &settings, opts.oci_opts, Some(opts.output)); + let flattened = opts.flattened; + let force = opts.oci_opts.force; + let mut oci_opts = reconcile_fetch_options(&opts.reference, &settings, opts.oci_opts, Some(opts.output)); + oci_opts.set_ignore_manifest(true); + if flattened { + if !force { + oci_opts.set_on_existing(wick_oci_utils::OnExisting::Error); + } + oci_opts.set_flatten(true); + } span.in_scope(|| debug!(options=?oci_opts, reference= opts.reference, "pulling reference")); - let pull_result = pull(opts.reference, oci_opts).instrument(span.clone()).await?; + let package = pull(opts.reference, oci_opts).instrument(span.clone()).await?; - let files = pull_result + let files = package .list_files() .iter() - .map(|f| f.path().display().to_string()) + .map(|f| f.package_path().display().to_string()) .collect::>(); + let basedir = package.basedir().unwrap(); + Ok(StructuredOutput::new( - format!("Pulled file: \n {}", files.join("\n")), + format!( + "Pulled file: \n{}", + files + .iter() + .map(|n| format!("- {}", basedir.join(n).display())) + .collect::>() + .join("\n") + ), serde_json::json!({"files": files}), )) } diff --git a/src/commands/registry/push.rs b/src/commands/registry/push.rs index 3fbd5d48..3ca4a750 100644 --- a/src/commands/registry/push.rs +++ b/src/commands/registry/push.rs @@ -35,7 +35,7 @@ pub(crate) async fn handle( ) -> Result { span.in_scope(|| debug!("push artifact")); - let mut package = wick_package::WickPackage::from_path(&opts.source) + let mut package = wick_package::WickPackage::from_path(None, &opts.source) .instrument(span.clone()) .await?;