Merge pull request #351 from imeoer/nydus-tar-build

Builder: support tar build

contrib/nydusify/pkg/checker/rule/bootstrap.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"reflect"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -54,6 +53,16 @@ func (rule *BootstrapRule) Validate() error {
 		return nil
 	}
 
+	// Parse blob list from blob layers in Nydus manifest
+	blobListInLayer := map[string]bool{}
+	layers := rule.Parsed.NydusImage.Manifest.Layers
+	for i, layer := range layers {
+		if i != len(layers)-1 {
+			blobListInLayer[layer.Digest.Hex()] = true
+		}
+	}
+
+	// Parse blob list from blob table of bootstrap
 	var bootstrap bootstrapDebug
 	bootstrapBytes, err := ioutil.ReadFile(rule.DebugOutputPath)
 	if err != nil {
@@ -62,27 +71,26 @@ func (rule *BootstrapRule) Validate() error {
 	if err := json.Unmarshal(bootstrapBytes, &bootstrap); err != nil {
 		return errors.Wrap(err, "unmarshal bootstrap output JSON")
 	}
-
-	// Parse blob list from blob layers in Nydus manifest
-	var blobListInLayer []string
-	layers := rule.Parsed.NydusImage.Manifest.Layers
-	for i, layer := range layers {
-		if i != len(layers)-1 {
-			blobListInLayer = append(blobListInLayer, layer.Digest.Hex())
+	blobListInBootstrap := map[string]bool{}
+	lostInLayer := false
+	for _, blobID := range bootstrap.Blobs {
+		blobListInBootstrap[blobID] = true
+		if !blobListInLayer[blobID] {
+			lostInLayer = true
 		}
 	}
 
-	// Blob list recorded in manifest annotation should be equal with
-	// the blob list recorded in blob table of bootstrap
-	if !reflect.DeepEqual(bootstrap.Blobs, blobListInLayer) {
-		return fmt.Errorf(
-			"nydus blob list in bootstrap(%d) does not match with manifest(%d)'s, %v != %v",
-			len(bootstrap.Blobs),
-			len(blobListInLayer),
-			bootstrap.Blobs,
-			blobListInLayer,
-		)
+	if !lostInLayer {
+		return nil
 	}
 
-	return nil
+	// The blobs recorded in blob table of bootstrap should all appear
+	// in the layers.
+	return fmt.Errorf(
+		"nydus blobs in the blob table of bootstrap(%d) should all appear in the layers of manifest(%d), %v != %v",
+		len(blobListInBootstrap),
+		len(blobListInLayer),
+		blobListInBootstrap,
+		blobListInLayer,
+	)
 }

rafs/src/metadata/mod.rs

@@ -468,7 +468,7 @@ impl RafsSuper {
     }
 
     /// Load Rafs super block from a metadata file.
-    pub fn load_from_metadata(path: &str, mode: RafsMode, validate_digest: bool) -> Result<Self> {
+    pub fn load_from_metadata(path: &Path, mode: RafsMode, validate_digest: bool) -> Result<Self> {
         // open bootstrap file
         let file = OpenOptions::new().read(true).write(false).open(path)?;
         let mut rs = RafsSuper {
@@ -483,7 +483,7 @@ impl RafsSuper {
         Ok(rs)
     }
 
-    pub fn load_chunk_dict_from_metadata(path: &str) -> Result<Self> {
+    pub fn load_chunk_dict_from_metadata(path: &Path) -> Result<Self> {
         // open bootstrap file
         let file = OpenOptions::new().read(true).write(false).open(path)?;
         let mut rs = RafsSuper {
@@ -749,24 +749,20 @@ impl RafsSuper {
         cb: &mut dyn FnMut(&dyn RafsInode, &Path) -> anyhow::Result<()>,
     ) -> anyhow::Result<()> {
         let inode = self.get_inode(ino, false)?;
-        if !inode.is_dir() {
-            return Ok(());
-        }
-        let parent_path = if let Some(parent) = parent {
+        let path = if let Some(parent) = parent {
             parent.join(inode.name())
         } else {
             PathBuf::from("/")
         };
+        cb(inode.as_ref(), &path)?;
+        if !inode.is_dir() {
+            return Ok(());
+        }
         let child_count = inode.get_child_count();
         for idx in 0..child_count {
             let child = inode.get_child_by_index(idx)?;
             let child_ino = child.ino();
-            if child.is_dir() {
-                self.walk_inodes(child_ino, Some(&parent_path), cb)?;
-            } else {
-                let child_path = parent_path.join(child.name());
-                cb(child.as_ref(), &child_path)?;
-            }
+            self.walk_inodes(child_ino, Some(&path), cb)?;
         }
         Ok(())
     }

src/bin/nydus-image/builder/diff.rs

@@ -136,7 +136,7 @@ use crate::core::context::{
 use crate::core::node::{ChunkSource, ChunkWrapper, Node, NodeChunk, Overlay};
 use crate::core::tree::Tree;
 use nydus_utils::digest::RafsDigest;
-use rafs::metadata::layout::{RafsBlobTable, RAFS_ROOT_INODE};
+use rafs::metadata::layout::RAFS_ROOT_INODE;
 use rafs::metadata::{RafsInode, RafsMode, RafsSuper};
 use storage::device::BlobChunkInfo;
 
@@ -305,7 +305,7 @@ fn dump_blob(
     blob_nodes: &mut Vec<Node>,
     chunk_dict: Arc<dyn ChunkDict>,
 ) -> Result<(Option<BlobContext>, ChunkMap)> {
-    let mut blob_ctx = BlobContext::new(blob_id, blob_storage)?;
+    let mut blob_ctx = BlobContext::new(blob_id, blob_storage, ctx.blob_offset)?;
     blob_ctx.set_chunk_dict(chunk_dict);
     blob_ctx.set_chunk_size(ctx.chunk_size);
     blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6);
@@ -523,14 +523,16 @@ impl DiffBuilder {
                                                         path: &Path|
              -> Result<()> {
                 let mut chunks = Vec::new();
-                inode.walk_chunks(&mut |cki: &dyn BlobChunkInfo| -> Result<()> {
-                    let chunk = ChunkWrapper::from_chunk_info(cki);
-                    chunks.push(NodeChunk {
-                        source: ChunkSource::Parent,
-                        inner: chunk,
-                    });
-                    Ok(())
-                })?;
+                if inode.is_reg() {
+                    inode.walk_chunks(&mut |cki: &dyn BlobChunkInfo| -> Result<()> {
+                        let chunk = ChunkWrapper::from_chunk_info(cki);
+                        chunks.push(NodeChunk {
+                            source: ChunkSource::Parent,
+                            inner: chunk,
+                        });
+                        Ok(())
+                    })?;
+                }
                 self.chunk_map
                     .insert(path.to_path_buf(), (chunks, inode.get_digest()));
                 Ok(())
@@ -653,14 +655,7 @@ impl DiffBuilder {
 
         // Dump bootstrap file
         let blob_table = blob_mgr.to_blob_table(ctx)?;
-        match blob_table {
-            RafsBlobTable::V5(table) => {
-                bootstrap.dump_rafsv5(ctx, bootstrap_ctx, &table)?;
-            }
-            RafsBlobTable::V6(table) => {
-                bootstrap.dump_rafsv6(ctx, bootstrap_ctx, &table)?;
-            }
-        };
+        bootstrap.dump(ctx, bootstrap_ctx, &blob_table)?;
         bootstrap_ctx.blobs = blob_mgr
             .get_blobs()
             .iter()
@@ -861,7 +856,6 @@ pub mod tests {
 
     use super::*;
     use nydus_utils::exec;
-    use storage::RAFS_DEFAULT_CHUNK_SIZE;
 
     fn create_dir(path: &Path) {
         fs::create_dir_all(path).unwrap();
@@ -1002,10 +996,7 @@ pub mod tests {
             (Overlay::UpperAddition, PathBuf::from("/test-1-symlink")),
             (Overlay::UpperAddition, PathBuf::from("/test-2")),
         ];
-        let ctx = BuildContext {
-            chunk_size: RAFS_DEFAULT_CHUNK_SIZE as u32,
-            ..Default::default()
-        };
+        let ctx = BuildContext::default();
         let nodes = walk_diff(&ctx, None, lower_dir.clone(), lower_dir.clone()).unwrap();
         for (i, node) in nodes.into_iter().enumerate() {
             println!("lower node: {:?} {:?}", node.overlay, node.target());

src/bin/nydus-image/builder/directory.rs

@@ -16,7 +16,6 @@ use crate::core::context::{
 };
 use crate::core::node::{Node, Overlay};
 use crate::core::tree::Tree;
-use rafs::metadata::layout::RafsBlobTable;
 
 struct FilesystemTreeBuilder {}
 
@@ -31,6 +30,7 @@ impl FilesystemTreeBuilder {
         ctx: &mut BuildContext,
         bootstrap_ctx: &mut BootstrapContext,
         parent: &mut Node,
+        layer_idx: u16,
     ) -> Result<Vec<Tree>> {
         let mut result = Vec::new();
         if !parent.is_dir() {
@@ -44,7 +44,7 @@ impl FilesystemTreeBuilder {
         event_tracer!("load_from_directory", +children.len());
         for child in children {
             let path = child.path();
-            let child = Node::new(
+            let mut child = Node::new(
                 ctx.fs_version,
                 ctx.source_path.clone(),
                 path.clone(),
@@ -54,6 +54,7 @@ impl FilesystemTreeBuilder {
                 true,
             )
             .with_context(|| format!("failed to create node {:?}", path))?;
+            child.layer_idx = layer_idx;
 
             // as per OCI spec, whiteout file should not be present within final image
             // or filesystem, only existed in layers.
@@ -65,7 +66,7 @@ impl FilesystemTreeBuilder {
             }
 
             let mut child = Tree::new(child);
-            child.children = self.load_children(ctx, bootstrap_ctx, &mut child.node)?;
+            child.children = self.load_children(ctx, bootstrap_ctx, &mut child.node, layer_idx)?;
             result.push(child);
         }
 
@@ -85,6 +86,7 @@ impl DirectoryBuilder {
         &mut self,
         ctx: &mut BuildContext,
         bootstrap_ctx: &mut BootstrapContext,
+        layer_idx: u16,
     ) -> Result<Tree> {
         let node = Node::new(
             ctx.fs_version,
@@ -99,7 +101,7 @@ impl DirectoryBuilder {
         let tree_builder = FilesystemTreeBuilder::new();
 
         tree.children = timing_tracer!(
-            { tree_builder.load_children(ctx, bootstrap_ctx, &mut tree.node) },
+            { tree_builder.load_children(ctx, bootstrap_ctx, &mut tree.node, layer_idx) },
             "load_from_directory"
         )?;
 
@@ -116,7 +118,8 @@ impl Builder for DirectoryBuilder {
     ) -> Result<BuildOutput> {
         let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?;
         // Scan source directory to build upper layer tree.
-        let mut tree = self.build_tree_from_fs(ctx, &mut bootstrap_ctx)?;
+        let layer_idx = if bootstrap_ctx.layered { 1u16 } else { 0u16 };
+        let mut tree = self.build_tree_from_fs(ctx, &mut bootstrap_ctx, layer_idx)?;
         let mut bootstrap = Bootstrap::new()?;
         if bootstrap_ctx.layered {
             // Merge with lower layer if there's one, do not prepare `prefetch` list during merging.
@@ -131,7 +134,11 @@ impl Builder for DirectoryBuilder {
         )?;
 
         // Dump blob file
-        let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone())?;
+        let mut blob_ctx = BlobContext::new(
+            ctx.blob_id.clone(),
+            ctx.blob_storage.clone(),
+            ctx.blob_offset,
+        )?;
         blob_ctx.set_chunk_dict(blob_mgr.get_chunk_dict());
         blob_ctx.set_chunk_size(ctx.chunk_size);
         blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6);
@@ -159,10 +166,7 @@ impl Builder for DirectoryBuilder {
 
         // Dump bootstrap file
         let blob_table = blob_mgr.to_blob_table(ctx)?;
-        match blob_table {
-            RafsBlobTable::V5(table) => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, &table)?,
-            RafsBlobTable::V6(table) => bootstrap.dump_rafsv6(ctx, &mut bootstrap_ctx, &table)?,
-        }
+        bootstrap.dump(ctx, &mut bootstrap_ctx, &blob_table)?;
 
         bootstrap_mgr.add(bootstrap_ctx);
         BuildOutput::new(&blob_mgr, &bootstrap_mgr)

src/bin/nydus-image/builder/stargz.rs

@@ -333,7 +333,7 @@ impl StargzIndexTreeBuilder {
         Ok(())
     }
 
-    fn build(&mut self, ctx: &mut BuildContext) -> Result<Tree> {
+    fn build(&mut self, ctx: &mut BuildContext, layer_idx: u16) -> Result<Tree> {
         // Parse stargz TOC index from a file
         let toc_index = TocIndex::load(&ctx.source_path)?;
         if toc_index.entries.is_empty() {
@@ -411,15 +411,15 @@ impl StargzIndexTreeBuilder {
             let mut lost_dirs = Vec::new();
             self.make_lost_dirs(&entry, &mut lost_dirs)?;
             for dir in &lost_dirs {
-                let node = self.parse_node(dir, ctx.explicit_uidgid, ctx.fs_version)?;
+                let node = self.parse_node(dir, ctx.explicit_uidgid, ctx.fs_version, layer_idx)?;
                 nodes.push(node);
             }
 
             if entry.is_hardlink() {
                 hardlink_map.insert(entry.path()?, entry.hardlink_link_path());
             }
 
-            let node = self.parse_node(entry, ctx.explicit_uidgid, ctx.fs_version)?;
+            let node = self.parse_node(entry, ctx.explicit_uidgid, ctx.fs_version, layer_idx)?;
             if entry.path()? == PathBuf::from("/") {
                 tree = Some(Tree::new(node.clone()));
             }
@@ -448,6 +448,7 @@ impl StargzIndexTreeBuilder {
         entry: &TocEntry,
         explicit_uidgid: bool,
         version: RafsVersion,
+        layer_idx: u16,
     ) -> Result<Node> {
         let chunks = Vec::new();
         let entry_path = entry.path()?;
@@ -548,6 +549,7 @@ impl StargzIndexTreeBuilder {
             chunks,
             symlink,
             xattrs,
+            layer_idx,
             ctime: 0,
             offset: 0,
             dirents: Vec::<(u64, OsString, u32)>::new(),
@@ -574,7 +576,7 @@ impl StargzBuilder {
         let mut decompressed_blob_size = 0u64;
         let mut compressed_blob_size = 0u64;
         let blob_index = blob_mgr.alloc_index()?;
-        let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone())?;
+        let mut blob_ctx = BlobContext::new(ctx.blob_id.clone(), ctx.blob_storage.clone(), 0)?;
         blob_ctx.set_chunk_dict(blob_mgr.get_chunk_dict());
         blob_ctx.set_chunk_size(ctx.chunk_size);
 
@@ -615,10 +617,10 @@ impl StargzBuilder {
         Ok(())
     }
 
-    fn build_tree_from_index(&mut self, ctx: &mut BuildContext) -> Result<Tree> {
+    fn build_tree_from_index(&mut self, ctx: &mut BuildContext, layer_idx: u16) -> Result<Tree> {
         let mut tree_builder = StargzIndexTreeBuilder::new();
         tree_builder
-            .build(ctx)
+            .build(ctx, layer_idx)
             .context("failed to build tree from stargz index")
     }
 }
@@ -632,7 +634,8 @@ impl Builder for StargzBuilder {
     ) -> Result<BuildOutput> {
         let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?;
         // Build tree from source
-        let mut tree = self.build_tree_from_index(ctx)?;
+        let layer_idx = if bootstrap_ctx.layered { 1u16 } else { 0u16 };
+        let mut tree = self.build_tree_from_index(ctx, layer_idx)?;
         let mut bootstrap = Bootstrap::new()?;
         if bootstrap_mgr.f_parent_bootstrap.is_some() {
             // Merge with lower layer if there's one.
@@ -649,10 +652,10 @@ impl Builder for StargzBuilder {
 
         // Dump bootstrap file
         let blob_table = blob_mgr.to_blob_table(ctx)?;
-        match blob_table {
-            RafsBlobTable::V5(table) => bootstrap.dump_rafsv5(ctx, &mut bootstrap_ctx, &table)?,
-            RafsBlobTable::V6(_) => todo!(),
+        if let RafsBlobTable::V6(_) = blob_table {
+            todo!();
         }
+        bootstrap.dump(ctx, &mut bootstrap_ctx, &blob_table)?;
 
         bootstrap_mgr.add(bootstrap_ctx);
         BuildOutput::new(&blob_mgr, &bootstrap_mgr)