diff --git a/contrib/nydusify/pkg/checker/rule/bootstrap.go b/contrib/nydusify/pkg/checker/rule/bootstrap.go index c36f4acc5d5..b5bdd031a17 100644 --- a/contrib/nydusify/pkg/checker/rule/bootstrap.go +++ b/contrib/nydusify/pkg/checker/rule/bootstrap.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "reflect" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -54,6 +53,16 @@ func (rule *BootstrapRule) Validate() error { return nil } + // Parse blob list from blob layers in Nydus manifest + blobListInLayer := map[string]bool{} + layers := rule.Parsed.NydusImage.Manifest.Layers + for i, layer := range layers { + if i != len(layers)-1 { + blobListInLayer[layer.Digest.Hex()] = true + } + } + + // Parse blob list from blob table of bootstrap var bootstrap bootstrapDebug bootstrapBytes, err := ioutil.ReadFile(rule.DebugOutputPath) if err != nil { @@ -62,27 +71,26 @@ func (rule *BootstrapRule) Validate() error { if err := json.Unmarshal(bootstrapBytes, &bootstrap); err != nil { return errors.Wrap(err, "unmarshal bootstrap output JSON") } - - // Parse blob list from blob layers in Nydus manifest - var blobListInLayer []string - layers := rule.Parsed.NydusImage.Manifest.Layers - for i, layer := range layers { - if i != len(layers)-1 { - blobListInLayer = append(blobListInLayer, layer.Digest.Hex()) + blobListInBootstrap := map[string]bool{} + lostInLayer := false + for _, blobID := range bootstrap.Blobs { + blobListInBootstrap[blobID] = true + if !blobListInLayer[blobID] { + lostInLayer = true } } - // Blob list recorded in manifest annotation should be equal with - // the blob list recorded in blob table of bootstrap - if !reflect.DeepEqual(bootstrap.Blobs, blobListInLayer) { - return fmt.Errorf( - "nydus blob list in bootstrap(%d) does not match with manifest(%d)'s, %v != %v", - len(bootstrap.Blobs), - len(blobListInLayer), - bootstrap.Blobs, - blobListInLayer, - ) + if !lostInLayer { + return nil } - return nil + // The blobs recorded in blob table of bootstrap should all appear + // in the layers. + return fmt.Errorf( + "nydus blobs in the blob table of bootstrap(%d) should all appear in the layers of manifest(%d), %v != %v", + len(blobListInBootstrap), + len(blobListInLayer), + blobListInBootstrap, + blobListInLayer, + ) } diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index 2360cb5f4e6..8e0b197a3dc 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -468,7 +468,7 @@ impl RafsSuper { } /// Load Rafs super block from a metadata file. - pub fn load_from_metadata(path: &str, mode: RafsMode, validate_digest: bool) -> Result { + pub fn load_from_metadata(path: &Path, mode: RafsMode, validate_digest: bool) -> Result { // open bootstrap file let file = OpenOptions::new().read(true).write(false).open(path)?; let mut rs = RafsSuper { @@ -483,7 +483,7 @@ impl RafsSuper { Ok(rs) } - pub fn load_chunk_dict_from_metadata(path: &str) -> Result { + pub fn load_chunk_dict_from_metadata(path: &Path) -> Result { // open bootstrap file let file = OpenOptions::new().read(true).write(false).open(path)?; let mut rs = RafsSuper { @@ -749,24 +749,20 @@ impl RafsSuper { cb: &mut dyn FnMut(&dyn RafsInode, &Path) -> anyhow::Result<()>, ) -> anyhow::Result<()> { let inode = self.get_inode(ino, false)?; - if !inode.is_dir() { - return Ok(()); - } - let parent_path = if let Some(parent) = parent { + let path = if let Some(parent) = parent { parent.join(inode.name()) } else { PathBuf::from("/") }; + cb(inode.as_ref(), &path)?; + if !inode.is_dir() { + return Ok(()); + } let child_count = inode.get_child_count(); for idx in 0..child_count { let child = inode.get_child_by_index(idx)?; let child_ino = child.ino(); - if child.is_dir() { - self.walk_inodes(child_ino, Some(&parent_path), cb)?; - } else { - let child_path = parent_path.join(child.name()); - cb(child.as_ref(), &child_path)?; - } + self.walk_inodes(child_ino, Some(&path), cb)?; } Ok(()) } diff --git a/src/bin/nydus-image/builder/diff.rs b/src/bin/nydus-image/builder/diff.rs index afb9693c31b..67465a84173 100644 --- a/src/bin/nydus-image/builder/diff.rs +++ b/src/bin/nydus-image/builder/diff.rs @@ -136,7 +136,7 @@ use crate::core::context::{ use crate::core::node::{ChunkSource, ChunkWrapper, Node, NodeChunk, Overlay}; use crate::core::tree::Tree; use nydus_utils::digest::RafsDigest; -use rafs::metadata::layout::{RafsBlobTable, RAFS_ROOT_INODE}; +use rafs::metadata::layout::RAFS_ROOT_INODE; use rafs::metadata::{RafsInode, RafsMode, RafsSuper}; use storage::device::BlobChunkInfo; @@ -305,7 +305,7 @@ fn dump_blob( blob_nodes: &mut Vec, chunk_dict: Arc, ) -> Result<(Option, ChunkMap)> { - let mut blob_ctx = BlobContext::new(blob_id, blob_storage)?; + let mut blob_ctx = BlobContext::new(blob_id, blob_storage, ctx.blob_offset)?; blob_ctx.set_chunk_dict(chunk_dict); blob_ctx.set_chunk_size(ctx.chunk_size); blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6); @@ -523,14 +523,16 @@ impl DiffBuilder { path: &Path| -> Result<()> { let mut chunks = Vec::new(); - inode.walk_chunks(&mut |cki: &dyn BlobChunkInfo| -> Result<()> { - let chunk = ChunkWrapper::from_chunk_info(cki); - chunks.push(NodeChunk { - source: ChunkSource::Parent, - inner: chunk, - }); - Ok(()) - })?; + if inode.is_reg() { + inode.walk_chunks(&mut |cki: &dyn BlobChunkInfo| -> Result<()> { + let chunk = ChunkWrapper::from_chunk_info(cki); + chunks.push(NodeChunk { + source: ChunkSource::Parent, + inner: chunk, + }); + Ok(()) + })?; + } self.chunk_map .insert(path.to_path_buf(), (chunks, inode.get_digest())); Ok(()) @@ -653,14 +655,7 @@ impl DiffBuilder { // Dump bootstrap file let blob_table = blob_mgr.to_blob_table(ctx)?; - match blob_table { - RafsBlobTable::V5(table) => { - bootstrap.dump_rafsv5(ctx, bootstrap_ctx, &table)?; - } - RafsBlobTable::V6(table) => { - bootstrap.dump_rafsv6(ctx, bootstrap_ctx, &table)?; - } - }; + bootstrap.dump(ctx, bootstrap_ctx, &blob_table)?; bootstrap_ctx.blobs = blob_mgr .get_blobs() .iter() @@ -861,7 +856,6 @@ pub mod tests { use super::*; use nydus_utils::exec; - use storage::RAFS_DEFAULT_CHUNK_SIZE; fn create_dir(path: &Path) { fs::create_dir_all(path).unwrap(); @@ -1002,10 +996,7 @@ pub mod tests { (Overlay::UpperAddition, PathBuf::from("/test-1-symlink")), (Overlay::UpperAddition, PathBuf::from("/test-2")), ]; - let ctx = BuildContext { - chunk_size: RAFS_DEFAULT_CHUNK_SIZE as u32, - ..Default::default() - }; + let ctx = BuildContext::default(); let nodes = walk_diff(&ctx, None, lower_dir.clone(), lower_dir.clone()).unwrap(); for (i, node) in nodes.into_iter().enumerate() { println!("lower node: {:?} {:?}", node.overlay, node.target()); diff --git a/src/bin/nydus-image/builder/directory.rs b/src/bin/nydus-image/builder/directory.rs index b7ed7e7c46d..ae86fd7d576 100644 --- a/src/bin/nydus-image/builder/directory.rs +++ b/src/bin/nydus-image/builder/directory.rs @@ -16,7 +16,6 @@ use crate::core::context::{ }; use crate::core::node::{Node, Overlay}; use crate::core::tree::Tree; -use rafs::metadata::layout::RafsBlobTable; struct FilesystemTreeBuilder {} @@ -31,6 +30,7 @@ impl FilesystemTreeBuilder { ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, parent: &mut Node, + layer_idx: u16, ) -> Result> { let mut result = Vec::new(); if !parent.is_dir() { @@ -44,7 +44,7 @@ impl FilesystemTreeBuilder { event_tracer!("load_from_directory", +children.len()); for child in children { let path = child.path(); - let child = Node::new( + let mut child = Node::new( ctx.fs_version, ctx.source_path.clone(), path.clone(), @@ -54,6 +54,7 @@ impl FilesystemTreeBuilder { true, ) .with_context(|| format!("failed to create node {:?}", path))?; + child.layer_idx = layer_idx; // as per OCI spec, whiteout file should not be present within final image // or filesystem, only existed in layers. @@ -65,7 +66,7 @@ impl FilesystemTreeBuilder { } let mut child = Tree::new(child); - child.children = self.load_children(ctx, bootstrap_ctx, &mut child.node)?; + child.children = self.load_children(ctx, bootstrap_ctx, &mut child.node, layer_idx)?; result.push(child); } @@ -85,6 +86,7 @@ impl DirectoryBuilder { &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, + layer_idx: u16, ) -> Result { let node = Node::new( ctx.fs_version, @@ -99,7 +101,7 @@ impl DirectoryBuilder { let tree_builder = FilesystemTreeBuilder::new(); tree.children = timing_tracer!( - { tree_builder.load_children(ctx, bootstrap_ctx, &mut tree.node) }, + { tree_builder.load_children(ctx, bootstrap_ctx, &mut tree.node, layer_idx) }, "load_from_directory" )?; @@ -116,7 +118,8 @@ impl Builder for DirectoryBuilder { ) -> Result { let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; // Scan source directory to build upper layer tree. - let mut tree = self.build_tree_from_fs(ctx, &mut bootstrap_ctx)?; + let layer_idx = if bootstrap_ctx.layered { 1u16 } else { 0u16 }; + let mut tree = self.build_tree_from_fs(ctx, &mut bootstrap_ctx, layer_idx)?; let mut bootstrap = Bootstrap::new()?; if bootstrap_ctx.layered { // Merge with lower layer if there's one, do not prepare `prefetch` list during merging. @@ -131,7 +134,11 @@ impl Builder for DirectoryBuilder { )?; // Dump blob file - let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone())?; + let mut blob_ctx = BlobContext::new( + ctx.blob_id.clone(), + ctx.blob_storage.clone(), + ctx.blob_offset, + )?; blob_ctx.set_chunk_dict(blob_mgr.get_chunk_dict()); blob_ctx.set_chunk_size(ctx.chunk_size); blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6); @@ -159,10 +166,7 @@ impl Builder for DirectoryBuilder { // Dump bootstrap file let blob_table = blob_mgr.to_blob_table(ctx)?; - match blob_table { - RafsBlobTable::V5(table) => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, &table)?, - RafsBlobTable::V6(table) => bootstrap.dump_rafsv6(ctx, &mut bootstrap_ctx, &table)?, - } + bootstrap.dump(ctx, &mut bootstrap_ctx, &blob_table)?; bootstrap_mgr.add(bootstrap_ctx); BuildOutput::new(&blob_mgr, &bootstrap_mgr) diff --git a/src/bin/nydus-image/builder/stargz.rs b/src/bin/nydus-image/builder/stargz.rs index c669c7486df..f7583ac7425 100644 --- a/src/bin/nydus-image/builder/stargz.rs +++ b/src/bin/nydus-image/builder/stargz.rs @@ -333,7 +333,7 @@ impl StargzIndexTreeBuilder { Ok(()) } - fn build(&mut self, ctx: &mut BuildContext) -> Result { + fn build(&mut self, ctx: &mut BuildContext, layer_idx: u16) -> Result { // Parse stargz TOC index from a file let toc_index = TocIndex::load(&ctx.source_path)?; if toc_index.entries.is_empty() { @@ -411,7 +411,7 @@ impl StargzIndexTreeBuilder { let mut lost_dirs = Vec::new(); self.make_lost_dirs(&entry, &mut lost_dirs)?; for dir in &lost_dirs { - let node = self.parse_node(dir, ctx.explicit_uidgid, ctx.fs_version)?; + let node = self.parse_node(dir, ctx.explicit_uidgid, ctx.fs_version, layer_idx)?; nodes.push(node); } @@ -419,7 +419,7 @@ impl StargzIndexTreeBuilder { hardlink_map.insert(entry.path()?, entry.hardlink_link_path()); } - let node = self.parse_node(entry, ctx.explicit_uidgid, ctx.fs_version)?; + let node = self.parse_node(entry, ctx.explicit_uidgid, ctx.fs_version, layer_idx)?; if entry.path()? == PathBuf::from("/") { tree = Some(Tree::new(node.clone())); } @@ -448,6 +448,7 @@ impl StargzIndexTreeBuilder { entry: &TocEntry, explicit_uidgid: bool, version: RafsVersion, + layer_idx: u16, ) -> Result { let chunks = Vec::new(); let entry_path = entry.path()?; @@ -548,6 +549,7 @@ impl StargzIndexTreeBuilder { chunks, symlink, xattrs, + layer_idx, ctime: 0, offset: 0, dirents: Vec::<(u64, OsString, u32)>::new(), @@ -574,7 +576,7 @@ impl StargzBuilder { let mut decompressed_blob_size = 0u64; let mut compressed_blob_size = 0u64; let blob_index = blob_mgr.alloc_index()?; - let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone())?; + let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone(), 0)?; blob_ctx.set_chunk_dict(blob_mgr.get_chunk_dict()); blob_ctx.set_chunk_size(ctx.chunk_size); @@ -615,10 +617,10 @@ impl StargzBuilder { Ok(()) } - fn build_tree_from_index(&mut self, ctx: &mut BuildContext) -> Result { + fn build_tree_from_index(&mut self, ctx: &mut BuildContext, layer_idx: u16) -> Result { let mut tree_builder = StargzIndexTreeBuilder::new(); tree_builder - .build(ctx) + .build(ctx, layer_idx) .context("failed to build tree from stargz index") } } @@ -632,7 +634,8 @@ impl Builder for StargzBuilder { ) -> Result { let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; // Build tree from source - let mut tree = self.build_tree_from_index(ctx)?; + let layer_idx = if bootstrap_ctx.layered { 1u16 } else { 0u16 }; + let mut tree = self.build_tree_from_index(ctx, layer_idx)?; let mut bootstrap = Bootstrap::new()?; if bootstrap_mgr.f_parent_bootstrap.is_some() { // Merge with lower layer if there's one. @@ -649,10 +652,10 @@ impl Builder for StargzBuilder { // Dump bootstrap file let blob_table = blob_mgr.to_blob_table(ctx)?; - match blob_table { - RafsBlobTable::V5(table) => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, &table)?, - RafsBlobTable::V6(_) => todo!(), + if let RafsBlobTable::V6(_) = blob_table { + todo!(); } + bootstrap.dump(ctx, &mut bootstrap_ctx, &blob_table)?; bootstrap_mgr.add(bootstrap_ctx); BuildOutput::new(&blob_mgr, &bootstrap_mgr) diff --git a/src/bin/nydus-image/core/blob_compact.rs b/src/bin/nydus-image/core/blob_compact.rs index fec7a6b8bf5..f6d93ecd49d 100644 --- a/src/bin/nydus-image/core/blob_compact.rs +++ b/src/bin/nydus-image/core/blob_compact.rs @@ -14,7 +14,6 @@ use crate::core::tree::Tree; use anyhow::Result; use nydus_utils::digest::RafsDigest; use nydus_utils::try_round_up_4k; -use rafs::metadata::layout::RafsBlobTable; use rafs::metadata::{RafsMode, RafsSuper}; use serde::{Deserialize, Serialize}; use sha2::Digest; @@ -517,7 +516,7 @@ impl BlobCompactor { } State::Rebuild(cs) => { let blob_storage = ArtifactStorage::FileDir(PathBuf::from(dir)); - let mut blob_ctx = BlobContext::new(String::from(""), Some(blob_storage))?; + let mut blob_ctx = BlobContext::new(String::from(""), Some(blob_storage), 0)?; blob_ctx.set_meta_info_enabled(self.is_v6()); let blob_idx = self.new_blob_mgr.alloc_index()?; let new_chunks = cs.dump( @@ -552,12 +551,12 @@ impl BlobCompactor { backend: Arc, cfg: &Config, ) -> Result> { - let rs = - RafsSuper::load_from_metadata(s_boostrap.to_str().unwrap(), RafsMode::Direct, true)?; + let rs = RafsSuper::load_from_metadata(&s_boostrap, RafsMode::Direct, true)?; info!("load bootstrap {:?} successfully", s_boostrap); let mut build_ctx = BuildContext::new( "".to_string(), false, + 0, rs.meta.get_compressor(), rs.meta.get_digester(), rs.meta.explicit_uidgid(), @@ -599,14 +598,7 @@ impl BlobCompactor { std::mem::swap(&mut bootstrap_ctx.nodes, &mut compactor.nodes); // blobs have already been dumped, dump bootstrap only let blob_table = compactor.new_blob_mgr.to_blob_table(&build_ctx)?; - match blob_table { - RafsBlobTable::V5(table) => { - bootstrap.dump_rafsv5(&mut build_ctx, &mut bootstrap_ctx, &table)?; - } - RafsBlobTable::V6(table) => { - bootstrap.dump_rafsv6(&mut build_ctx, &mut bootstrap_ctx, &table)?; - } - }; + bootstrap.dump(&mut build_ctx, &mut bootstrap_ctx, &blob_table)?; bootstrap_mgr.add(bootstrap_ctx); Ok(Some(BuildOutput::new( &compactor.new_blob_mgr, diff --git a/src/bin/nydus-image/core/bootstrap.rs b/src/bin/nydus-image/core/bootstrap.rs index 0f168aebcf8..5056cde3058 100644 --- a/src/bin/nydus-image/core/bootstrap.rs +++ b/src/bin/nydus-image/core/bootstrap.rs @@ -17,6 +17,7 @@ use rafs::metadata::layout::v6::{ align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6SuperBlock, RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE, }; +use rafs::metadata::layout::RafsBlobTable; use rafs::RafsIoWrite; use rafs::metadata::layout::RAFS_ROOT_INODE; @@ -66,13 +67,8 @@ impl Bootstrap { // user to pass the source root as prefetch hint. Check it here. ctx.prefetch.insert_if_need(&tree.node); - let inode_map = if tree.node.overlay.is_lower_layer() { - &mut bootstrap_ctx.lower_inode_map - } else { - &mut bootstrap_ctx.upper_inode_map - }; - inode_map.insert( - (tree.node.src_ino, tree.node.src_dev), + bootstrap_ctx.inode_map.insert( + (tree.node.layer_idx, tree.node.src_ino, tree.node.src_dev), vec![tree.node.index], ); @@ -120,8 +116,7 @@ impl Bootstrap { )?; // Clear all cached states for next upper layer build. - bootstrap_ctx.lower_inode_map.clear(); - bootstrap_ctx.upper_inode_map.clear(); + bootstrap_ctx.inode_map.clear(); ctx.prefetch.clear(); Ok(tree) @@ -169,12 +164,11 @@ impl Bootstrap { // Hardlink handle, all hardlink nodes' ino, nlink should be the same, // because the real_ino may be conflicted between different layers, // so we need to find hardlink node index list in the layer where the node is located. - let inode_map = if child.node.overlay.is_lower_layer() { - &mut bootstrap_ctx.lower_inode_map - } else { - &mut bootstrap_ctx.upper_inode_map - }; - if let Some(indexes) = inode_map.get_mut(&(child.node.src_ino, child.node.src_dev)) { + if let Some(indexes) = bootstrap_ctx.inode_map.get_mut(&( + child.node.layer_idx, + child.node.src_ino, + child.node.src_dev, + )) { let nlink = indexes.len() as u32 + 1; let first_index = indexes[0]; child.node.inode.set_ino(first_index); @@ -188,8 +182,8 @@ impl Bootstrap { child.node.inode.set_ino(index); child.node.inode.set_nlink(1); // Store inode real ino - inode_map.insert( - (child.node.src_ino, child.node.src_dev), + bootstrap_ctx.inode_map.insert( + (child.node.layer_idx, child.node.src_ino, child.node.src_dev), vec![child.node.index], ); } @@ -364,8 +358,20 @@ impl Bootstrap { } } + pub fn dump( + &mut self, + ctx: &mut BuildContext, + bootstrap_ctx: &mut BootstrapContext, + blob_table: &RafsBlobTable, + ) -> Result<()> { + match blob_table { + RafsBlobTable::V5(table) => self.dump_rafsv5(ctx, bootstrap_ctx, &table), + RafsBlobTable::V6(table) => self.dump_rafsv6(ctx, bootstrap_ctx, &table), + } + } + /// Dump bootstrap and blob file, return (Vec, blob_size) - pub fn dump_rafsv5( + fn dump_rafsv5( &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, @@ -400,8 +406,7 @@ impl Bootstrap { // Set super block let mut super_block = RafsV5SuperBlock::new(); - let inodes_count = - (bootstrap_ctx.lower_inode_map.len() + bootstrap_ctx.upper_inode_map.len()) as u64; + let inodes_count = bootstrap_ctx.inode_map.len() as u64; super_block.set_inodes_count(inodes_count); super_block.set_inode_table_offset(super_block_size as u64); super_block.set_inode_table_entries(inode_table_entries); @@ -498,7 +503,7 @@ impl Bootstrap { } /// Dump bootstrap and blob file, return (Vec, blob_size) - pub fn dump_rafsv6( + fn dump_rafsv6( &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, diff --git a/src/bin/nydus-image/core/chunk_dict.rs b/src/bin/nydus-image/core/chunk_dict.rs index a7cf97abac0..85b0dda51c2 100644 --- a/src/bin/nydus-image/core/chunk_dict.rs +++ b/src/bin/nydus-image/core/chunk_dict.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::mem::size_of; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; @@ -93,7 +94,7 @@ impl ChunkDict for HashChunkDict { } impl HashChunkDict { - fn from_bootstrap_file(path: &str) -> Result { + fn from_bootstrap_file(path: &Path) -> Result { let rs = RafsSuper::load_chunk_dict_from_metadata(path) .with_context(|| format!("failed to open bootstrap file {:?}", path))?; let mut d = HashChunkDict { @@ -138,7 +139,7 @@ impl HashChunkDict { } } -/// Load a chunk dictionary from external source. +/// Parse a chunk dictionary argument string. /// /// # Argument /// `arg` may be in inform of: @@ -150,22 +151,28 @@ impl HashChunkDict { /// image.boot /// ~/image/image.boot /// boltdb=/var/db/dict.db (not supported yet) -pub(crate) fn import_chunk_dict(arg: &str) -> Result> { +pub fn parse_chunk_dict_arg(arg: &str) -> Result { let (file_type, file_path) = match arg.find('=') { None => ("bootstrap", arg), Some(idx) => (&arg[0..idx], &arg[idx + 1..]), }; - info!("import chunk dict file {}={}", file_type, file_path); + info!("parse chunk dict argument {}={}", file_type, file_path); + match file_type { - "bootstrap" => { - HashChunkDict::from_bootstrap_file(file_path).map(|d| Arc::new(d) as Arc) + "bootstrap" => Ok(PathBuf::from(file_path)), + _ => { + bail!("invalid chunk dict type {}", file_type); } - _ => Err(std::io::Error::from_raw_os_error(libc::EINVAL)) - .with_context(|| format!("invalid chunk dict type {}", file_type)), } } +/// Load a chunk dictionary from external source. +pub(crate) fn import_chunk_dict(arg: &str) -> Result> { + let file_path = parse_chunk_dict_arg(arg)?; + HashChunkDict::from_bootstrap_file(&file_path).map(|d| Arc::new(d) as Arc) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs index 7ad59f41b1b..233b3155807 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/src/bin/nydus-image/core/context.rs @@ -284,18 +284,22 @@ impl Clone for BlobContext { } impl BlobContext { - pub fn new(blob_id: String, blob_stor: Option) -> Result { + pub fn new( + blob_id: String, + blob_stor: Option, + blob_offset: u64, + ) -> Result { let writer = if let Some(blob_stor) = blob_stor { Some(ArtifactBufferWriter::new(blob_stor)?) } else { None }; - Ok(Self::new_with_writer(blob_id, writer)) + Ok(Self::new_with_writer(blob_id, writer, blob_offset)) } pub fn from(blob: &BlobInfo, chunk_source: ChunkSource) -> Self { - let mut ctx = Self::new_with_writer(blob.blob_id().to_owned(), None); + let mut ctx = Self::new_with_writer(blob.blob_id().to_owned(), None, 0); ctx.blob_readahead_size = blob.readahead_size(); ctx.chunk_count = blob.chunk_count(); @@ -320,7 +324,11 @@ impl BlobContext { ctx } - pub fn new_with_writer(blob_id: String, writer: Option) -> Self { + pub fn new_with_writer( + blob_id: String, + writer: Option, + blob_offset: u64, + ) -> Self { let size = if writer.is_some() { RAFS_MAX_CHUNK_SIZE as usize } else { @@ -339,7 +347,7 @@ impl BlobContext { compressed_blob_size: 0, decompressed_blob_size: 0, - compress_offset: 0, + compress_offset: blob_offset, decompress_offset: 0, chunk_count: 0, @@ -591,9 +599,8 @@ impl BlobManager { pub struct BootstrapContext { /// This build has a parent bootstrap. pub layered: bool, - /// Cache node index for hardlinks, HashMap<(real_inode, dev), Vec>. - pub lower_inode_map: HashMap<(Inode, u64), Vec>, - pub upper_inode_map: HashMap<(Inode, u64), Vec>, + /// Cache node index for hardlinks, HashMap<(layer_index, real_inode, dev), Vec>. + pub inode_map: HashMap<(u16, Inode, u64), Vec>, /// Store all nodes in ascendant ordor, indexed by (node.index - 1). pub nodes: Vec, /// Current position to write in f_bootstrap @@ -609,8 +616,7 @@ impl BootstrapContext { pub fn new(storage: ArtifactStorage, layered: bool) -> Result { Ok(Self { layered, - lower_inode_map: HashMap::new(), - upper_inode_map: HashMap::new(), + inode_map: HashMap::new(), nodes: Vec::new(), offset: EROFS_BLOCK_SIZE, name: String::new(), @@ -678,7 +684,7 @@ impl BootstrapManager { } } -#[derive(Default, Clone)] +#[derive(Clone)] pub struct BuildContext { /// Blob id (user specified or sha256(blob)). pub blob_id: String, @@ -687,6 +693,8 @@ pub struct BuildContext { /// `decompress_offset` within chunk info. Therefore, provide a new flag /// to image tool thus to align chunks in blob with 4k size. pub aligned_chunk: bool, + /// Add a offset for compressed blob. + pub blob_offset: u64, /// Blob chunk compress flag. pub compressor: compress::Algorithm, /// Inode and chunk digest algorithm flag. @@ -721,6 +729,7 @@ impl BuildContext { pub fn new( blob_id: String, aligned_chunk: bool, + blob_offset: u64, compressor: compress::Algorithm, digester: digest::Algorithm, explicit_uidgid: bool, @@ -733,6 +742,7 @@ impl BuildContext { BuildContext { blob_id, aligned_chunk, + blob_offset, compressor, digester, explicit_uidgid, @@ -759,6 +769,30 @@ impl BuildContext { } } +impl Default for BuildContext { + fn default() -> Self { + Self { + blob_id: String::new(), + aligned_chunk: false, + blob_offset: 0, + compressor: compress::Algorithm::default(), + digester: digest::Algorithm::default(), + explicit_uidgid: true, + whiteout_spec: WhiteoutSpec::default(), + + chunk_size: RAFS_DEFAULT_CHUNK_SIZE as u32, + fs_version: RafsVersion::default(), + + source_type: SourceType::default(), + source_path: PathBuf::new(), + + prefetch: Prefetch::default(), + blob_storage: None, + has_xattr: true, + } + } +} + #[derive(Serialize, Deserialize, Default, Debug, Clone)] pub struct BuildOutputBlob { pub blob_id: String, diff --git a/src/bin/nydus-image/core/node.rs b/src/bin/nydus-image/core/node.rs index 202d3421fc1..b1c879fafac 100644 --- a/src/bin/nydus-image/core/node.rs +++ b/src/bin/nydus-image/core/node.rs @@ -15,7 +15,6 @@ use std::os::linux::fs::MetadataExt; use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::str::FromStr; -use std::sync::Arc; use anyhow::{Context, Error, Result}; use nix::sys::stat; @@ -47,7 +46,7 @@ use super::chunk_dict::ChunkDict; use super::context::{BlobContext, BootstrapContext, BuildContext, RafsVersion}; use super::tree::Tree; -const ROOT_PATH_NAME: &[u8] = &[b'/']; +pub const ROOT_PATH_NAME: &[u8] = &[b'/']; /// Prefix for OCI whiteout file. pub const OCISPEC_WHITEOUT_PREFIX: &str = ".wh."; @@ -103,6 +102,8 @@ pub enum WhiteoutSpec { Oci, /// "whiteouts and opaque directories" in https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt Overlayfs, + /// No whiteout spec, which will build all `.wh.*` and `.wh..wh..opq` files into bootstrap. + None, } impl Default for WhiteoutSpec { @@ -118,6 +119,7 @@ impl FromStr for WhiteoutSpec { match s { "oci" => Ok(Self::Oci), "overlayfs" => Ok(Self::Overlayfs), + "none" => Ok(Self::None), _ => Err(anyhow!("invalid whiteout spec")), } } @@ -222,6 +224,8 @@ pub struct Node { pub(crate) target: PathBuf, /// Parsed version of `target`. pub(crate) target_vec: Vec, + /// Layer index where node located. + pub layer_idx: u16, /// Last status change time of the file, in nanoseconds. pub ctime: i64, /// Used by rafsv6 inode datalayout @@ -283,6 +287,7 @@ impl Node { symlink: None, xattrs: RafsXAttrs::default(), explicit_uidgid, + layer_idx: 0, ctime: 0, offset: 0, dirents: Vec::new(), @@ -1022,6 +1027,9 @@ impl Node { return Some(WhiteoutType::OverlayFsOpaque); } } + WhiteoutSpec::None => { + return None; + } } None @@ -1193,7 +1201,7 @@ impl InodeWrapper { } } - pub fn from_inode_info(inode: &Arc) -> Self { + pub fn from_inode_info(inode: &dyn RafsInode) -> Self { if let Some(inode) = inode.as_any().downcast_ref::() { InodeWrapper::V5(to_rafsv5_inode(inode)) } else if let Some(inode) = inode.as_any().downcast_ref::() { diff --git a/src/bin/nydus-image/core/tree.rs b/src/bin/nydus-image/core/tree.rs index 7a65f8e0a4c..0d1f4390b26 100644 --- a/src/bin/nydus-image/core/tree.rs +++ b/src/bin/nydus-image/core/tree.rs @@ -18,7 +18,6 @@ use std::ffi::OsStr; use std::ffi::OsString; use std::path::PathBuf; -use std::sync::Arc; use anyhow::Result; use rafs::metadata::layout::{bytes_to_os_str, RafsXAttrs, RAFS_ROOT_INODE}; @@ -51,7 +50,8 @@ impl Tree { pub fn from_bootstrap(rs: &RafsSuper, chunk_dict: &mut T) -> Result { let tree_builder = MetadataTreeBuilder::new(&rs); let root_inode = rs.get_inode(RAFS_ROOT_INODE, true)?; - let root_node = tree_builder.parse_node(root_inode, PathBuf::from("/"))?; + let root_node = + MetadataTreeBuilder::parse_node(&rs, root_inode.as_ref(), PathBuf::from("/"))?; let mut tree = Tree::new(root_node); tree.children = timing_tracer!( @@ -240,7 +240,7 @@ impl Tree { } } -struct MetadataTreeBuilder<'a> { +pub struct MetadataTreeBuilder<'a> { rs: &'a RafsSuper, } @@ -277,7 +277,7 @@ impl<'a> MetadataTreeBuilder<'a> { let child = inode.get_child_by_index(idx)?; let child_ino = child.ino(); let child_path = parent_path.join(child.name()); - let child = self.parse_node(child, child_path)?; + let child = Self::parse_node(&self.rs, child.as_ref(), child_path)?; if child.is_reg() { for chunk in &child.chunks { @@ -297,7 +297,7 @@ impl<'a> MetadataTreeBuilder<'a> { } /// Convert a `RafsInode` object to an in-memory `Node` object. - fn parse_node(&self, inode: Arc, path: PathBuf) -> Result { + pub fn parse_node(rs: &RafsSuper, inode: &dyn RafsInode, path: PathBuf) -> Result { let chunks = if inode.is_reg() { let chunk_count = inode.get_chunk_count(); let mut chunks = Vec::with_capacity(chunk_count as usize); @@ -330,7 +330,7 @@ impl<'a> MetadataTreeBuilder<'a> { // to avoid breaking hardlink detecting logic. let src_dev = u64::MAX; - let inode_wrapper = InodeWrapper::from_inode_info(&inode); + let inode_wrapper = InodeWrapper::from_inode_info(inode); let source = PathBuf::from("/"); let target = Node::generate_target(&path, &source); let target_vec = Node::generate_target_vec(&target); @@ -341,7 +341,7 @@ impl<'a> MetadataTreeBuilder<'a> { src_dev, rdev: inode.rdev() as u64, overlay: Overlay::Lower, - explicit_uidgid: self.rs.meta.explicit_uidgid(), + explicit_uidgid: rs.meta.explicit_uidgid(), source, target, path, @@ -350,6 +350,7 @@ impl<'a> MetadataTreeBuilder<'a> { chunks, symlink, xattrs, + layer_idx: 0, ctime: 0, offset: 0, dirents: Vec::<(u64, OsString, u32)>::new(), diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index bed1a69b713..449bed0608e 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -30,7 +30,7 @@ use storage::RAFS_DEFAULT_CHUNK_SIZE; use crate::builder::{Builder, DiffBuilder, DirectoryBuilder, StargzBuilder}; use crate::core::blob_compact::BlobCompactor; -use crate::core::chunk_dict::import_chunk_dict; +use crate::core::chunk_dict::{import_chunk_dict, parse_chunk_dict_arg}; use crate::core::context::{ ArtifactStorage, BlobManager, BootstrapManager, BuildContext, BuildOutput, BuildOutputArtifact, RafsVersion, SourceType, @@ -38,6 +38,7 @@ use crate::core::context::{ use crate::core::node::{self, WhiteoutSpec}; use crate::core::prefetch::Prefetch; use crate::core::tree; +use crate::merge::Merger; use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass}; use crate::validator::Validator; use storage::factory::{BackendConfig, BlobFactory}; @@ -47,6 +48,7 @@ mod trace; mod builder; mod core; mod inspect; +mod merge; mod stat; mod validator; @@ -133,6 +135,20 @@ impl OutputSerializer { } fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { + let arg_chunk_dict = Arg::with_name("chunk-dict") + .long("chunk-dict") + .short("M") + .help("Specify a chunk dictionary for chunk deduplication") + .takes_value(true); + let arg_prefetch_policy = Arg::with_name("prefetch-policy") + .long("prefetch-policy") + .short("P") + .help("blob data prefetch policy") + .takes_value(true) + .required(false) + .default_value("none") + .possible_values(&["fs", "blob", "none"]); + // TODO: Try to use yaml to define below options App::new("") .version(bti_string.as_str()) @@ -245,14 +261,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .required(false), ) .arg( - Arg::with_name("prefetch-policy") - .long("prefetch-policy") - .short("P") - .help("prefetch policy:") - .takes_value(true) - .required(false) - .default_value("none") - .possible_values(&["fs", "blob", "none"]), + arg_prefetch_policy.clone(), ) .arg( Arg::with_name("repeatable") @@ -277,7 +286,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .takes_value(true) .required(true) .default_value("oci") - .possible_values(&["oci", "overlayfs"]) + .possible_values(&["oci", "overlayfs", "none"]) ) .arg( Arg::with_name("output-json") @@ -293,6 +302,13 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .help("Align data chunks to 4K") .takes_value(false) ) + .arg( + Arg::with_name("blob-offset") + .long("blob-offset") + .help("add an offset for compressed blob (is only used to put the blob in the tarball)") + .default_value("0") + .takes_value(true) + ) .arg( Arg::with_name("blob-dir") .long("blob-dir") @@ -301,11 +317,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .takes_value(true) ) .arg( - Arg::with_name("chunk-dict") - .long("chunk-dict") - .short("M") - .help("Specify a chunk dictionary for chunk deduplication") - .takes_value(true) + arg_chunk_dict.clone(), ) .arg( Arg::with_name("backend-type") @@ -322,6 +334,30 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .takes_value(true) ) ) + .subcommand( + SubCommand::with_name("merge") + .about("Merge multiple bootstraps into a overlaid bootstrap") + .arg( + Arg::with_name("bootstrap") + .long("bootstrap") + .short("B") + .help("output path of nydus overlaid bootstrap") + .required(true) + .takes_value(true), + ) + .arg( + arg_chunk_dict, + ) + .arg( + arg_prefetch_policy, + ) + .arg( + Arg::with_name("SOURCE") + .help("bootstrap paths (allow one or more)") + .required(true) + .multiple(true), + ) + ) .subcommand( SubCommand::with_name("check") .about("Validates nydus image's filesystem metadata") @@ -505,6 +541,8 @@ fn main() -> Result<()> { if let Some(matches) = cmd.subcommand_matches("create") { Command::create(matches, &build_info) + } else if let Some(matches) = cmd.subcommand_matches("merge") { + Command::merge(matches) } else if let Some(matches) = cmd.subcommand_matches("check") { Command::check(matches, &build_info) } else if let Some(matches) = cmd.subcommand_matches("inspect") { @@ -525,6 +563,7 @@ impl Command { fn create(matches: &clap::ArgMatches, build_info: &BuildTimeInfo) -> Result<()> { let blob_id = Self::get_blob_id(&matches)?; let chunk_size = Self::get_chunk_size(&matches)?; + let blob_offset = Self::get_blob_offset(&matches)?; let parent_bootstrap = Self::get_parent_bootstrap(&matches)?; let source_path = PathBuf::from(matches.value_of("SOURCE").unwrap()); let extra_paths: Vec = matches @@ -569,15 +608,12 @@ impl Command { } } - let prefetch_policy = matches - .value_of("prefetch-policy") - .unwrap_or_default() - .parse()?; - let prefetch = Prefetch::new(prefetch_policy)?; + let prefetch = Self::get_prefetch(matches)?; let mut build_ctx = BuildContext::new( blob_id, aligned_chunk, + blob_offset, compressor, digester, !repeatable, @@ -650,6 +686,29 @@ impl Command { Ok(()) } + fn merge(matches: &clap::ArgMatches) -> Result<()> { + let source_bootstrap_paths: Vec = matches + .values_of("SOURCE") + .map(|paths| paths.map(PathBuf::from).collect()) + .unwrap(); + let target_bootstrap_path = Self::get_bootstrap(&matches)?; + let chunk_dict_path = if let Some(arg) = matches.value_of("chunk-dict") { + Some(parse_chunk_dict_arg(arg)?) + } else { + None + }; + let mut ctx = BuildContext { + prefetch: Self::get_prefetch(matches)?, + ..Default::default() + }; + Merger::merge( + &mut ctx, + source_bootstrap_paths, + target_bootstrap_path.to_path_buf(), + chunk_dict_path, + ) + } + fn compact(matches: &clap::ArgMatches, build_info: &BuildTimeInfo) -> Result<()> { let bootstrap_path = PathBuf::from(Self::get_bootstrap(matches)?); let dst_bootstrap = match matches.value_of("output-bootstrap") { @@ -887,6 +946,23 @@ impl Command { } } + fn get_prefetch(matches: &clap::ArgMatches) -> Result { + let prefetch_policy = matches + .value_of("prefetch-policy") + .unwrap_or_default() + .parse()?; + Prefetch::new(prefetch_policy) + } + + fn get_blob_offset(matches: &clap::ArgMatches) -> Result { + match matches.value_of("blob-offset") { + None => Ok(0), + Some(v) => v + .parse::() + .context(format!("invalid blob offset {}", v)), + } + } + fn get_fs_version(matches: &clap::ArgMatches) -> Result { match matches.value_of("fs-version") { None => Ok(RafsVersion::V6), diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs new file mode 100644 index 00000000000..bc029ed007e --- /dev/null +++ b/src/bin/nydus-image/merge.rs @@ -0,0 +1,184 @@ +// Copyright (C) 2022 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashSet; +use std::convert::TryFrom; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +use nydus_utils::compress; +use nydus_utils::digest::{self}; +use rafs::metadata::layout::RAFS_ROOT_INODE; +use rafs::metadata::{RafsInode, RafsMode, RafsSuper, RafsSuperMeta}; + +use crate::core::bootstrap::Bootstrap; +use crate::core::chunk_dict::HashChunkDict; +use crate::core::context::ArtifactStorage; +use crate::core::context::{BlobContext, BlobManager, BootstrapContext, BuildContext}; +use crate::core::node::{ChunkSource, Overlay, WhiteoutSpec}; +use crate::core::tree::{MetadataTreeBuilder, Tree}; + +#[derive(Clone, Debug, Eq, PartialEq)] +struct Flags { + explicit_uidgid: bool, + has_xattr: bool, + compressor: compress::Algorithm, + digester: digest::Algorithm, +} + +impl Flags { + fn from_meta(meta: &RafsSuperMeta) -> Self { + Self { + explicit_uidgid: meta.explicit_uidgid(), + has_xattr: meta.has_xattr(), + compressor: meta.get_compressor(), + digester: meta.get_digester(), + } + } + + fn set_to_ctx(&self, ctx: &mut BuildContext) { + ctx.explicit_uidgid = self.explicit_uidgid; + ctx.has_xattr = self.has_xattr; + ctx.compressor = self.compressor; + ctx.digester = self.digester; + } +} + +/// Merger merge multiple bootstraps (generally come from nydus tar blob of +/// intermediate image layer) into one bootstrap (uses as final image layer). +pub struct Merger {} + +impl Merger { + /// Merge assumes the bootstrap name as the hash of whole tar blob. + fn get_blob_hash(bootstrap_path: &Path) -> Result { + let blob_hash = bootstrap_path + .file_name() + .ok_or_else(|| anyhow!("get file name"))? + .to_str() + .ok_or_else(|| anyhow!("convert to string"))?; + Ok(blob_hash.to_string()) + } + + pub fn merge( + ctx: &mut BuildContext, + sources: Vec, + target: PathBuf, + chunk_dict: Option, + ) -> Result<()> { + if sources.is_empty() { + bail!("please provide at least one source bootstrap path"); + } + + let mut tree: Option = None; + let mut blob_mgr = BlobManager::new(); + let mut flags: Option = None; + + // Get the blobs come from chunk dict bootstrap. + let mut chunk_dict_blobs = HashSet::new(); + if let Some(chunk_dict_path) = &chunk_dict { + let rs = RafsSuper::load_from_metadata(&chunk_dict_path, RafsMode::Direct, true) + .context(format!("load chunk dict bootstrap {:?}", chunk_dict_path))?; + for blob in rs.superblock.get_blob_infos() { + chunk_dict_blobs.insert(blob.blob_id().to_string()); + } + } + + for (layer_idx, bootstrap_path) in sources.iter().enumerate() { + let rs = RafsSuper::load_from_metadata(&bootstrap_path, RafsMode::Direct, true) + .context(format!("load bootstrap {:?}", bootstrap_path))?; + + let current_flags = Flags::from_meta(&rs.meta); + if let Some(flags) = &flags { + if flags != ¤t_flags { + bail!( + "inconsistent flags between bootstraps, current bootstrap {:?}, flags {:?}", + bootstrap_path, + current_flags, + ); + } + } else { + // Keep final bootstrap following superblock flags of source bootstraps. + current_flags.set_to_ctx(ctx); + flags = Some(current_flags); + } + + let parent_blobs = rs.superblock.get_blob_infos(); + let blob_hash = Self::get_blob_hash(bootstrap_path)?; + let mut blob_idx_map = Vec::new(); + let mut parent_blob_added = false; + + for blob in &parent_blobs { + let mut blob_ctx = BlobContext::from(blob, ChunkSource::Parent); + if chunk_dict_blobs.get(blob.blob_id()).is_none() { + // It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands + // use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but + // only at most one new blob, the other blobs should be from the chunk dict image. + if parent_blob_added { + bail!("invalid bootstrap, seems have multiple non-chunk-dict blobs in this bootstrap"); + } + // The blob id (blob sha256 hash) in parent bootstrap is invalid for nydusd + // runtime, should change it to the hash of whole tar blob. + blob_ctx.blob_id = blob_hash.to_owned(); + parent_blob_added = true; + } + blob_idx_map.push(blob_mgr.len() as u32); + blob_mgr.add(blob_ctx); + } + + if let Some(tree) = &mut tree { + let mut nodes = Vec::new(); + rs.walk_inodes(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode, + path: &Path| + -> Result<()> { + let mut node = MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf()) + .context(format!("parse node from bootstrap {:?}", bootstrap_path))?; + for chunk in &mut node.chunks { + let origin_blob_index = chunk.inner.blob_index() as usize; + // Set the blob index of chunk to real index in blob table of final bootstrap. + chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]); + } + // Set node's layer index to distinguish same inode number (from bootstrap) + // between different layers. + node.layer_idx = u16::try_from(layer_idx).context(format!( + "too many layers {}, limited to {}", + layer_idx, + u16::MAX + ))?; + node.overlay = Overlay::UpperAddition; + match node.whiteout_type(WhiteoutSpec::Oci) { + Some(_) => { + // Insert whiteouts at the head, so they will be handled first when + // applying to lower layer. + nodes.insert(0, node); + } + _ => { + nodes.push(node); + } + } + Ok(()) + })?; + for node in &nodes { + tree.apply(node, true, WhiteoutSpec::Oci)?; + } + } else { + let mut dict = HashChunkDict::default(); + tree = Some(Tree::from_bootstrap(&rs, &mut dict)?); + } + } + + // Safe to unwrap because there is at least one source bootstrap. + let mut tree = tree.unwrap(); + let mut bootstrap = Bootstrap::new()?; + let storage = ArtifactStorage::SingleFile(target.clone()); + let mut bootstrap_ctx = BootstrapContext::new(storage, false)?; + bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree)?; + let blob_table = blob_mgr.to_blob_table(&ctx)?; + bootstrap + .dump(ctx, &mut bootstrap_ctx, &blob_table) + .context(format!("dump bootstrap to {:?}", target))?; + + Ok(()) + } +} diff --git a/src/bin/nydus-image/stat.rs b/src/bin/nydus-image/stat.rs index 8ff25d29973..a067308a109 100644 --- a/src/bin/nydus-image/stat.rs +++ b/src/bin/nydus-image/stat.rs @@ -7,7 +7,7 @@ use std::fs::OpenOptions; use std::path::Path; use std::sync::atomic::Ordering; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use rafs::metadata::{RafsMode, RafsSuper}; use serde::Serialize; @@ -158,12 +158,7 @@ impl ImageStat { } pub fn stat(&mut self, path: &Path, is_base: bool) -> Result<()> { - let p = match path.to_str() { - None => bail!("invalid path to nydus image metadata blob"), - Some(v) => v, - }; - - let rs = RafsSuper::load_from_metadata(p, RafsMode::Direct, false)?; + let rs = RafsSuper::load_from_metadata(path, RafsMode::Direct, false)?; let mut dict = HashChunkDict::default(); let mut hardlinks = HashSet::new(); let tree = diff --git a/src/bin/nydus-image/validator.rs b/src/bin/nydus-image/validator.rs index dfa31ff871d..eaf2082d5ff 100644 --- a/src/bin/nydus-image/validator.rs +++ b/src/bin/nydus-image/validator.rs @@ -6,7 +6,7 @@ use std::path::Path; -use anyhow::{Context, Error, Result}; +use anyhow::{Context, Result}; use rafs::metadata::{RafsMode, RafsSuper}; use crate::tree::Tree; @@ -17,10 +17,7 @@ pub struct Validator { impl Validator { pub fn new(bootstrap_path: &Path) -> Result { - let path = bootstrap_path - .to_str() - .ok_or_else(|| Error::msg("bootstrap path is invalid"))?; - let sb = RafsSuper::load_from_metadata(path, RafsMode::Direct, true)?; + let sb = RafsSuper::load_from_metadata(bootstrap_path, RafsMode::Direct, true)?; Ok(Self { sb }) } diff --git a/tests/diff.rs b/tests/diff.rs index 8bada3caef7..5f2280ad25c 100644 --- a/tests/diff.rs +++ b/tests/diff.rs @@ -228,6 +228,7 @@ fn integration_test_diff_build_with_chunk_dict() { &snapshot_dir_4, )); let expected_chunk_dict_bootstrap = IntoIter::new([ + (PathBuf::from("/"), vec![]), ( PathBuf::from("/file-1"), vec![ @@ -324,21 +325,23 @@ fn integration_test_diff_build_with_chunk_dict() { let mut reader = Box::new(file) as RafsIoReader; rs.load(&mut reader).unwrap(); let mut actual = HashMap::new(); + let blobs = rs.superblock.get_blob_infos(); rs.walk_inodes(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode, path: &Path| -> Result<()> { let mut chunks = Vec::new(); - let blobs = rs.superblock.get_blob_infos(); - inode - .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { - chunks.push(( - chunk.blob_index(), - blobs[chunk.blob_index() as usize].blob_id().to_string(), - format!("{}", chunk.chunk_id()), - )); - Ok(()) - }) - .unwrap(); + if inode.is_reg() { + inode + .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { + chunks.push(( + chunk.blob_index(), + blobs[chunk.blob_index() as usize].blob_id().to_string(), + format!("{}", chunk.chunk_id()), + )); + Ok(()) + }) + .unwrap(); + } actual.insert(path.to_path_buf(), chunks); Ok(()) }) @@ -386,6 +389,7 @@ fn integration_test_diff_build_with_chunk_dict() { &snapshot_dir_6, )); let expected_bootstrap = IntoIter::new([ + (PathBuf::from("/"), vec![]), ( PathBuf::from("/file-8"), vec![ @@ -460,21 +464,23 @@ fn integration_test_diff_build_with_chunk_dict() { let mut reader = Box::new(file) as RafsIoReader; rs.load(&mut reader).unwrap(); let mut actual = HashMap::new(); + let blobs = rs.superblock.get_blob_infos(); rs.walk_inodes(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode, path: &Path| -> Result<()> { let mut chunks = Vec::new(); - let blobs = rs.superblock.get_blob_infos(); - inode - .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { - chunks.push(( - chunk.blob_index(), - blobs[chunk.blob_index() as usize].blob_id().to_string(), - format!("{}", chunk.chunk_id()), - )); - Ok(()) - }) - .unwrap(); + if inode.is_reg() { + inode + .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { + chunks.push(( + chunk.blob_index(), + blobs[chunk.blob_index() as usize].blob_id().to_string(), + format!("{}", chunk.chunk_id()), + )); + Ok(()) + }) + .unwrap(); + } actual.insert(path.to_path_buf(), chunks); Ok(()) }) @@ -539,6 +545,7 @@ fn integration_test_diff_build_with_chunk_dict() { )); let expected_bootstrap = IntoIter::new([ + (PathBuf::from("/"), vec![]), ( PathBuf::from("/file-1"), vec![ @@ -678,21 +685,23 @@ fn integration_test_diff_build_with_chunk_dict() { let mut reader = Box::new(file) as RafsIoReader; rs.load(&mut reader).unwrap(); let mut actual = HashMap::new(); + let blobs = rs.superblock.get_blob_infos(); rs.walk_inodes(RAFS_ROOT_INODE, None, &mut |inode: &dyn RafsInode, path: &Path| -> Result<()> { let mut chunks = Vec::new(); - let blobs = rs.superblock.get_blob_infos(); - inode - .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { - chunks.push(( - chunk.blob_index(), - blobs[chunk.blob_index() as usize].blob_id().to_string(), - format!("{}", chunk.chunk_id()), - )); - Ok(()) - }) - .unwrap(); + if inode.is_reg() { + inode + .walk_chunks(&mut |chunk: &dyn BlobChunkInfo| -> Result<()> { + chunks.push(( + chunk.blob_index(), + blobs[chunk.blob_index() as usize].blob_id().to_string(), + format!("{}", chunk.chunk_id()), + )); + Ok(()) + }) + .unwrap(); + } actual.insert(path.to_path_buf(), chunks); Ok(()) })