diff --git a/Cargo.lock b/Cargo.lock index 60b1d1b7039..715d363c269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -447,6 +447,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "equivalent" version = "1.0.1" @@ -569,8 +575,7 @@ dependencies = [ [[package]] name = "fuse-backend-rs" version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a63a89f40ec26a0a1434e89de3f4ee939a920eae15d641053ee09ee6ed44b" +source = "git+https://github.com/weizhang555/fuse-backend-rs.git?branch=overlay-impl#b9a541f392576a39b54f3179e42519e2db393456" dependencies = [ "arc-swap", "bitflags 1.3.2", @@ -582,6 +587,7 @@ dependencies = [ "log", "mio", "nix", + "radix_trie", "versionize", "versionize_derive", "vhost", @@ -1153,6 +1159,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.24.2" @@ -1658,6 +1673,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" diff --git a/Cargo.toml b/Cargo.toml index fe729262beb..4f9f1543c00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,9 @@ time = { version = "0.3.14", features = ["formatting"] } xattr = "1.0.1" vmm-sys-util = "0.11.0" +[patch.crates-io] +fuse-backend-rs = { git = 'https://github.com/weizhang555/fuse-backend-rs.git', branch = 'overlay-impl' } + [features] default = [ "fuse-backend-rs/fusedev", diff --git a/api/src/config.rs b/api/src/config.rs index 0dd77fd0d8f..7a40e0e4211 100644 --- a/api/src/config.rs +++ b/api/src/config.rs @@ -29,6 +29,8 @@ pub struct ConfigV2 { pub cache: Option, /// Configuration information for RAFS filesystem. pub rafs: Option, + /// Overlay configuration information for the instance. + pub overlay: Option, /// Internal runtime configuration. #[serde(skip)] pub internal: ConfigV2Internal, @@ -42,6 +44,7 @@ impl Default for ConfigV2 { backend: None, cache: None, rafs: None, + overlay: None, internal: ConfigV2Internal::default(), } } @@ -56,6 +59,7 @@ impl ConfigV2 { backend: None, cache: None, rafs: None, + overlay: None, internal: ConfigV2Internal::default(), } } @@ -1024,6 +1028,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 { backend: Some(c.backend.clone()), cache: Some(c.cache.clone()), rafs: None, + overlay: None, internal: ConfigV2Internal::default(), } } @@ -1395,6 +1400,7 @@ impl TryFrom for ConfigV2 { backend: Some(backend), cache: Some(cache), rafs: Some(rafs), + overlay: None, internal: ConfigV2Internal::default(), }) } @@ -1523,6 +1529,15 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 { } } +/// Configuration information for Overlay filesystem. +/// OverlayConfig is used to configure the writable layer(upper layer), +/// The filesystem will be writable when OverlayConfig is set. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct OverlayConfig { + pub upper_dir: String, + pub work_dir: String, +} + #[cfg(test)] mod tests { use super::*; diff --git a/builder/src/core/context.rs b/builder/src/core/context.rs index dceb066dc87..14f33db855c 100644 --- a/builder/src/core/context.rs +++ b/builder/src/core/context.rs @@ -1516,6 +1516,7 @@ mod tests { id: "id".to_owned(), cache: None, rafs: None, + overlay: None, internal: ConfigV2Internal { blob_accessible: Arc::new(AtomicBool::new(true)), }, diff --git a/rafs/src/fs.rs b/rafs/src/fs.rs index 15a1848c4ce..a3119477c3d 100644 --- a/rafs/src/fs.rs +++ b/rafs/src/fs.rs @@ -291,7 +291,8 @@ impl Rafs { // since nydusify gives root directory permission of 0o750 and fuse mount // options `rootmode=` does not affect root directory's permission bits, ending // up with preventing other users from accessing the container rootfs. - if attr.ino == self.root_ino() { + let root_ino = self.root_ino(); + if attr.ino == root_ino { attr.mode = attr.mode & !0o777 | 0o755; } @@ -684,9 +685,9 @@ impl FileSystem for Rafs { _inode: Self::Inode, _flags: u32, _fuse_flags: u32, - ) -> Result<(Option, OpenOptions)> { + ) -> Result<(Option, OpenOptions, Option)> { // Keep cache since we are readonly - Ok((None, OpenOptions::KEEP_CACHE)) + Ok((None, OpenOptions::KEEP_CACHE, None)) } fn release( @@ -886,6 +887,14 @@ impl FileSystem for Rafs { } } +#[cfg(target_os = "linux")] +// Let Rafs works as an OverlayFs layer. +impl Layer for Rafs { + fn root_inode(&self) -> Self::Inode { + self.root_ino() + } +} + #[cfg(all(test, feature = "backend-oss"))] pub(crate) mod tests { use super::*; diff --git a/service/src/fs_service.rs b/service/src/fs_service.rs index a1ee0a6b683..b37128e6f14 100644 --- a/service/src/fs_service.rs +++ b/service/src/fs_service.rs @@ -13,10 +13,14 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Arc, MutexGuard}; +#[cfg(target_os = "linux")] +use fuse_backend_rs::api::filesystem::{FileSystem, FsOptions, Layer}; use fuse_backend_rs::api::vfs::VfsError; use fuse_backend_rs::api::{BackFileSystem, Vfs}; #[cfg(target_os = "linux")] -use fuse_backend_rs::passthrough::{Config, PassthroughFs}; +use fuse_backend_rs::overlayfs::{config::Config as overlay_config, OverlayFs}; +#[cfg(target_os = "linux")] +use fuse_backend_rs::passthrough::{CachePolicy, Config as passthrough_config, PassthroughFs}; use nydus_api::ConfigV2; use nydus_rafs::fs::Rafs; use nydus_rafs::{RafsError, RafsIoRead}; @@ -244,8 +248,77 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result { let config = Arc::new(config); let (mut rafs, reader) = Rafs::new(&config, &cmd.mountpoint, Path::new(&cmd.source))?; rafs.import(reader, prefetch_files)?; - info!("RAFS filesystem imported"); - Ok(Box::new(rafs)) + + // Put a writable upper layer above the rafs to create an OverlayFS with two layers. + match &config.overlay { + Some(ovl_conf) => { + // check workdir and upperdir params. + if ovl_conf.work_dir.is_empty() || ovl_conf.upper_dir.is_empty() { + return Err(Error::InvalidArguments(String::from( + "workdir and upperdir must be specified for overlayfs", + ))); + } + + // Create an overlay upper layer with passthroughfs. + #[cfg(target_os = "macos")] + return Err(Error::InvalidArguments(String::from( + "not support OverlayFs since passthroughfs isn't supported on MacOS", + ))); + #[cfg(target_os = "linux")] + { + let fs_cfg = passthrough_config { + // Use upper_dir as root_dir as rw layer. + root_dir: ovl_conf.upper_dir.clone(), + do_import: true, + writeback: true, + no_open: true, + no_opendir: true, + xattr: true, + cache_policy: CachePolicy::Always, + ..Default::default() + }; + let fsopts = FsOptions::WRITEBACK_CACHE + | FsOptions::ZERO_MESSAGE_OPEN + | FsOptions::ZERO_MESSAGE_OPENDIR; + + let passthrough_fs = PassthroughFs::<()>::new(fs_cfg) + .map_err(|e| Error::InvalidConfig(format!("{}", e)))?; + passthrough_fs.init(fsopts).map_err(Error::PassthroughFs)?; + + type BoxedLayer = Box + Send + Sync>; + let upper_layer = Arc::new(Box::new(passthrough_fs) as BoxedLayer); + + // Create overlay lower layer with rafs, use lower_dir as root_dir of rafs. + let lower_layers = vec![Arc::new(Box::new(rafs) as BoxedLayer)]; + + let overlay_config = overlay_config { + work: ovl_conf.work_dir.clone(), + mountpoint: cmd.mountpoint.clone(), + do_import: false, + no_open: true, + no_opendir: true, + ..Default::default() + }; + let overlayfs = + OverlayFs::new(Some(upper_layer), lower_layers, overlay_config) + .map_err(|e| Error::InvalidConfig(format!("{}", e)))?; + info!( + "init overlay fs inode, upper {}, work {}\n", + ovl_conf.upper_dir.clone(), + ovl_conf.work_dir.clone() + ); + overlayfs + .import() + .map_err(|e| Error::InvalidConfig(format!("{}", e)))?; + info!("Overlay filesystem imported"); + Ok(Box::new(overlayfs)) + } + } + None => { + info!("RAFS filesystem imported"); + Ok(Box::new(rafs)) + } + } } FsBackendType::PassthroughFs => { #[cfg(target_os = "macos")] @@ -257,7 +330,7 @@ fn fs_backend_factory(cmd: &FsBackendMountCmd) -> Result { // Vfs by default enables no_open and writeback, passthroughfs // needs to specify them explicitly. // TODO(liubo): enable no_open_dir. - let fs_cfg = Config { + let fs_cfg = passthrough_config { root_dir: cmd.source.to_string(), do_import: false, writeback: true, diff --git a/smoke/tests/overlay_fs_test.go b/smoke/tests/overlay_fs_test.go new file mode 100644 index 00000000000..574e9b1839a --- /dev/null +++ b/smoke/tests/overlay_fs_test.go @@ -0,0 +1,110 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/containerd/log" + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/dragonflyoss/nydus/smoke/tests/texture" + "github.com/dragonflyoss/nydus/smoke/tests/tool" + "github.com/dragonflyoss/nydus/smoke/tests/tool/test" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +type OverlayFsTestSuite struct { + t *testing.T +} + +func (ovl_ts *OverlayFsTestSuite) prepareTestEnv(t *testing.T) *tool.Context { + ctx := tool.DefaultContext(t) + ctx.PrepareWorkDir(t) + + packOption := converter.PackOption{ + BuilderPath: ctx.Binary.Builder, + Compressor: ctx.Build.Compressor, + FsVersion: ctx.Build.FSVersion, + ChunkSize: ctx.Build.ChunkSize, + } + + lowerLayer := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "lower")) + lowerBlobDigest := lowerLayer.Pack(t, packOption, ctx.Env.BlobDir) + mergeOption := converter.MergeOption{ + BuilderPath: ctx.Binary.Builder, + ChunkDictPath: "", + OCIRef: true, + } + actualDigests, lowerBootstrap := tool.MergeLayers(t, *ctx, mergeOption, []converter.Layer{ + { + Digest: lowerBlobDigest, + }, + }) + require.Equal(t, []digest.Digest{lowerBlobDigest}, actualDigests) + + // Verify lower layer mounted by nydusd + ctx.Env.BootstrapPath = lowerBootstrap + tool.Verify(t, *ctx, lowerLayer.FileTree) + return ctx +} + +func (ovl_ts *OverlayFsTestSuite) TestSimpleOverlayFs(t *testing.T) { + ctx := ovl_ts.prepareTestEnv(t) + fmt.Printf("Workdir is %v\n", ctx.Env.WorkDir) + //defer ctx.Destroy(t) + + nydusd, err := tool.NewNydusdWithOverlay(tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + BootstrapPath: ctx.Env.BootstrapPath, + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + BackendType: "localfs", + BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), + EnablePrefetch: ctx.Runtime.EnablePrefetch, + BlobCacheDir: ctx.Env.CacheDir, + CacheType: ctx.Runtime.CacheType, + CacheCompressed: ctx.Runtime.CacheCompressed, + RafsMode: ctx.Runtime.RafsMode, + OvlUpperDir: ctx.Env.OvlUpperDir, + OvlWorkDir: ctx.Env.OvlWorkDir, + DigestValidate: false, + Writable: true, + }) + require.NoError(t, err) + + err = nydusd.Mount() + require.NoError(t, err) + defer func() { + if err := nydusd.Umount(); err != nil { + log.L.WithError(err).Errorf("umount") + } + }() + + // Write some file under mounted dir. + mountedDir := ctx.Env.MountDir + file := filepath.Join(mountedDir, "test.txt") + err = os.WriteFile(file, []byte("hello world"), 0644) + require.NoError(t, err) + + // Read it back + data, err := os.ReadFile(file) + require.NoError(t, err) + require.Equal(t, "hello world", string(data)) + + // Try to read from upper dir. + upperFile := filepath.Join(ctx.Env.OvlUpperDir, "test.txt") + data, err = os.ReadFile(upperFile) + require.NoError(t, err) + require.Equal(t, "hello world", string(data)) +} + +func TestOverlayFs(t *testing.T) { + test.Run(t, &OverlayFsTestSuite{t: t}) +} diff --git a/smoke/tests/tool/context.go b/smoke/tests/tool/context.go index 2d74ac02e97..04cf6d851cf 100644 --- a/smoke/tests/tool/context.go +++ b/smoke/tests/tool/context.go @@ -47,6 +47,8 @@ type EnvContext struct { CacheDir string MountDir string BootstrapPath string + OvlUpperDir string + OvlWorkDir string } type Context struct { @@ -98,11 +100,21 @@ func (ctx *Context) PrepareWorkDir(t *testing.T) { err = os.MkdirAll(mountDir, 0755) require.NoError(t, err) + // For overlay fs + ovlUpperDir := filepath.Join(workDir, "upper") + err = os.MkdirAll(ovlUpperDir, 0755) + require.NoError(t, err) + ovlWorkDir := filepath.Join(workDir, "work") + err = os.MkdirAll(ovlWorkDir, 0755) + require.NoError(t, err) + ctx.Env = EnvContext{ - WorkDir: workDir, - BlobDir: blobDir, - CacheDir: cacheDir, - MountDir: mountDir, + WorkDir: workDir, + BlobDir: blobDir, + CacheDir: cacheDir, + MountDir: mountDir, + OvlUpperDir: ovlUpperDir, + OvlWorkDir: ovlWorkDir, } } diff --git a/smoke/tests/tool/layer.go b/smoke/tests/tool/layer.go index a6f1290ac08..523ea301198 100644 --- a/smoke/tests/tool/layer.go +++ b/smoke/tests/tool/layer.go @@ -230,7 +230,7 @@ func (l *Layer) Overlay(_ *testing.T, upper *Layer) *Layer { func (l *Layer) recordFileTree(t *testing.T) { l.FileTree = map[string]*File{} - filepath.Walk(l.workDir, func(path string, fi os.FileInfo, err error) error { + filepath.Walk(l.workDir, func(path string, _ os.FileInfo, _ error) error { targetPath := l.TargetPath(t, path) l.FileTree[targetPath] = NewFile(t, path, targetPath) return nil diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index a56b85aeeb0..8cbcf98275d 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -69,6 +69,10 @@ type NydusdConfig struct { AccessPattern bool PrefetchFiles []string AmplifyIO uint64 + // Overlay config. + OvlUpperDir string + OvlWorkDir string + Writable bool } type Nydusd struct { @@ -109,6 +113,30 @@ var configTpl = ` } ` +var configOvlTpl = ` + { + "version": 2, + "backend": { + "type": "localfs", + "localfs": {{.BackendConfig}} + }, + "cache": { + "type": "blobcache", + "filecache": { + "work_dir": "{{.BlobCacheDir}}" + } + }, + "rafs": { + "mode": "{{.RafsMode}}", + "enable_xattr": true + }, + "overlay": { + "upper_dir": "{{.OvlUpperDir}}", + "work_dir": "{{.OvlWorkDir}}" + } +} + ` + func makeConfig(conf NydusdConfig) error { tpl := template.Must(template.New("").Parse(configTpl)) @@ -124,6 +152,21 @@ func makeConfig(conf NydusdConfig) error { return nil } +func makeOvlConfig(conf NydusdConfig) error { + tpl := template.Must(template.New("").Parse(configOvlTpl)) + + var ret bytes.Buffer + if err := tpl.Execute(&ret, conf); err != nil { + return errors.New("prepare config template for Nydusd") + } + + if err := os.WriteFile(conf.ConfigPath, ret.Bytes(), 0600); err != nil { + return errors.Wrapf(err, "write config file for Nydusd") + } + + return nil +} + func CheckReady(ctx context.Context, sock string) <-chan bool { ready := make(chan bool) @@ -188,6 +231,15 @@ func NewNydusd(conf NydusdConfig) (*Nydusd, error) { }, nil } +func NewNydusdWithOverlay(conf NydusdConfig) (*Nydusd, error) { + if err := makeOvlConfig(conf); err != nil { + return nil, errors.Wrap(err, "create config file for Nydusd") + } + return &Nydusd{ + NydusdConfig: conf, + }, nil +} + func (nydusd *Nydusd) Mount() error { _ = nydusd.Umount() @@ -205,6 +257,9 @@ func (nydusd *Nydusd) Mount() error { if len(nydusd.BootstrapPath) > 0 { args = append(args, "--bootstrap", nydusd.BootstrapPath) } + if nydusd.Writable { + args = append(args, "--writable") + } cmd := exec.Command(nydusd.NydusdPath, args...) cmd.Stdout = os.Stdout diff --git a/smoke/tests/tool/verify.go b/smoke/tests/tool/verify.go index e78c49ebe35..b3dd177d373 100644 --- a/smoke/tests/tool/verify.go +++ b/smoke/tests/tool/verify.go @@ -43,7 +43,7 @@ func Verify(t *testing.T, ctx Context, expectedFiles map[string]*File) { }() actualFiles := map[string]*File{} - err = filepath.WalkDir(ctx.Env.MountDir, func(path string, entry fs.DirEntry, err error) error { + err = filepath.WalkDir(ctx.Env.MountDir, func(path string, _ fs.DirEntry, err error) error { require.Nil(t, err) targetPath, err := filepath.Rel(ctx.Env.MountDir, path) diff --git a/storage/src/device.rs b/storage/src/device.rs index 3eb45d0e215..0800f1c59c3 100644 --- a/storage/src/device.rs +++ b/storage/src/device.rs @@ -1338,12 +1338,9 @@ impl FileReadWriteVolatile for BlobDeviceIoVec<'_> { unimplemented!(); } - fn read_at_volatile( - &mut self, - _slice: FileVolatileSlice, - _offset: u64, - ) -> Result { - unimplemented!(); + fn read_at_volatile(&mut self, slice: FileVolatileSlice, offset: u64) -> Result { + let buffers = [slice]; + self.read_vectored_at_volatile(&buffers, offset) } // The default read_vectored_at_volatile only read to the first slice, so we have to overload it.