From 7ef6d7bebe7c970b1f09fc01c0ba6221b58c88f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 13 Mar 2023 15:18:44 +0100 Subject: [PATCH] Extend the metadata backend to read the attributes from a locked source --- .../utils/decomposedfs/lookup/lookup.go | 10 ++-- .../metadata/messagepack_backend.go | 48 +++++++++++-------- .../utils/decomposedfs/metadata/metadata.go | 13 ++++- .../decomposedfs/metadata/xattrs_backend.go | 7 +++ 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/lookup/lookup.go b/pkg/storage/utils/decomposedfs/lookup/lookup.go index 018bb0f087d..fe411c80575 100644 --- a/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -295,15 +295,15 @@ func (lu *Lookup) CopyMetadata(src, target string, filter func(attributeName str // The optional filter function can be used to filter by attribute name, e.g. by checking a prefix // For the source file, a matching lockedfile is required. // NOTE: target resource will be write locked! -func (lu *Lookup) CopyMetadataWithSourceLock(source, target string, filter func(attributeName string) bool, readLock *lockedfile.File) (err error) { +func (lu *Lookup) CopyMetadataWithSourceLock(sourcePath, targetPath string, filter func(attributeName string) bool, lockedSource *lockedfile.File) (err error) { switch { - case readLock == nil: + case lockedSource == nil: return errors.New("no lock provided") - case readLock.File.Name() != lu.MetadataBackend().MetadataPath(source): + case lockedSource.File.Name() != lu.MetadataBackend().MetadataPath(sourcePath): return errors.New("lockpath does not match filepath") } - attrs, err := lu.metadataBackend.All(source) + attrs, err := lu.metadataBackend.AllWithLockedSource(sourcePath, lockedSource) if err != nil { return err } @@ -315,5 +315,5 @@ func (lu *Lookup) CopyMetadataWithSourceLock(source, target string, filter func( } } - return lu.MetadataBackend().SetMultiple(target, newAttrs, true) + return lu.MetadataBackend().SetMultiple(targetPath, newAttrs, true) } diff --git a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index 9d13cdab337..2d203f2a995 100644 --- a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -57,14 +57,14 @@ func NewMessagePackBackend(rootPath string, o options.CacheOptions) MessagePackB func (b MessagePackBackend) All(path string) (map[string][]byte, error) { path = b.MetadataPath(path) - return b.loadAttributes(path) + return b.loadAttributes(path, nil) } // Get an extended attribute value for the given key func (b MessagePackBackend) Get(path, key string) ([]byte, error) { path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path) + attribs, err := b.loadAttributes(path, nil) if err != nil { return []byte{}, err } @@ -79,7 +79,7 @@ func (b MessagePackBackend) Get(path, key string) ([]byte, error) { func (b MessagePackBackend) GetInt64(path, key string) (int64, error) { path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path) + attribs, err := b.loadAttributes(path, nil) if err != nil { return 0, err } @@ -99,7 +99,7 @@ func (b MessagePackBackend) GetInt64(path, key string) (int64, error) { func (b MessagePackBackend) List(path string) ([]string, error) { path = b.MetadataPath(path) - attribs, err := b.loadAttributes(path) + attribs, err := b.loadAttributes(path, nil) if err != nil { return nil, err } @@ -125,6 +125,13 @@ func (b MessagePackBackend) Remove(path, key string) error { return b.saveAttributes(path, nil, []string{key}, true) } +// AllWithLockedSource reads all extended attributes from the given reader (if possible). +// The path argument is used for storing the data in the cache +func (b MessagePackBackend) AllWithLockedSource(path string, source io.Reader) (map[string][]byte, error) { + path = b.MetadataPath(path) + return b.loadAttributes(path, source) +} + func (b MessagePackBackend) saveAttributes(path string, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error { var ( f readWriteCloseSeekTruncater @@ -188,31 +195,34 @@ func (b MessagePackBackend) saveAttributes(path string, setAttribs map[string][] return b.metaCache.PushToCache(b.cacheKey(path), attribs) } -func (b MessagePackBackend) loadAttributes(path string) (map[string][]byte, error) { +func (b MessagePackBackend) loadAttributes(path string, source io.Reader) (map[string][]byte, error) { attribs := map[string][]byte{} err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) if err == nil { return attribs, err } - lockedFile, err := lockedfile.Open(path) - - // // No cached entry found. Read from storage and store in cache - if err != nil { - if os.IsNotExist(err) { - // some of the caller rely on ENOTEXISTS to be returned when the - // actual file (not the metafile) does not exist in order to - // determine whether a node exists or not -> stat the actual node - _, err := os.Stat(strings.TrimSuffix(path, ".mpk")) - if err != nil { - return nil, err + if source == nil { + fmt.Println("lockedfile ro open", path, time.Now()) + source, err = lockedfile.Open(path) + fmt.Println("lockedfile ro done", path, time.Now()) + // // No cached entry found. Read from storage and store in cache + if err != nil { + if os.IsNotExist(err) { + // some of the caller rely on ENOTEXISTS to be returned when the + // actual file (not the metafile) does not exist in order to + // determine whether a node exists or not -> stat the actual node + _, err := os.Stat(strings.TrimSuffix(path, ".mpk")) + if err != nil { + return nil, err + } + return attribs, nil // no attributes set yet } - return attribs, nil // no attributes set yet } + defer source.(*lockedfile.File).Close() } - defer lockedFile.Close() - msgBytes, err := io.ReadAll(lockedFile) + msgBytes, err := io.ReadAll(source) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata.go b/pkg/storage/utils/decomposedfs/metadata/metadata.go index d7685e50f33..23e25984417 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata.go @@ -18,7 +18,10 @@ package metadata -import "errors" +import ( + "errors" + "io" +) var errUnconfiguredError = errors.New("no metadata backend configured. Bailing out") @@ -36,6 +39,8 @@ type Backend interface { Rename(oldPath, newPath string) error IsMetaFile(path string) bool MetadataPath(path string) string + + AllWithLockedSource(path string, source io.Reader) (map[string][]byte, error) } // NullBackend is the default stub backend, used to enforce the configuration of a proper backend @@ -76,3 +81,9 @@ func (NullBackend) Rename(oldPath, newPath string) error { return errUnconfigure // MetadataPath returns the path of the file holding the metadata for the given path func (NullBackend) MetadataPath(path string) string { return "" } + +// AllWithLockedSource reads all extended attributes from the given reader +// The path argument is used for storing the data in the cache +func (NullBackend) AllWithLockedSource(path string, source io.Reader) (map[string][]byte, error) { + return nil, errUnconfiguredError +} diff --git a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index b05dc099c8e..5442f3809d1 100644 --- a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -19,6 +19,7 @@ package metadata import ( + "io" "os" "path/filepath" "strconv" @@ -166,3 +167,9 @@ func cleanupLockfile(f *lockedfile.File) { _ = f.Close() _ = os.Remove(f.Name()) } + +// AllFromSource reads all extended attributes from the given reader. +// The path argument is used for storing the data in the cache +func (b XattrsBackend) AllWithLockedSource(path string, _ io.Reader) (map[string][]byte, error) { + return b.All(path) +}