From 371c1d7335e919c4286a0bb4ec06fd2be5601775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 10 Mar 2023 11:44:06 +0100 Subject: [PATCH] Create dedicated type for node attributes, based on map[string][]byte --- .../utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/decomposedfs/grants.go | 4 +- pkg/storage/utils/decomposedfs/grants_test.go | 2 +- .../utils/decomposedfs/lookup/lookup.go | 21 +-- pkg/storage/utils/decomposedfs/metadata.go | 2 +- .../decomposedfs/metadata/ini_backend.go | 39 ++--- .../utils/decomposedfs/metadata/metadata.go | 16 +- .../decomposedfs/metadata/metadata_test.go | 40 ++--- .../decomposedfs/metadata/xattrs_backend.go | 25 ++- pkg/storage/utils/decomposedfs/node/node.go | 146 +++++++++--------- pkg/storage/utils/decomposedfs/node/xattrs.go | 73 ++++++++- pkg/storage/utils/decomposedfs/recycle.go | 8 +- pkg/storage/utils/decomposedfs/spaces.go | 79 +++++----- .../utils/decomposedfs/testhelpers/helpers.go | 2 +- .../utils/decomposedfs/tree/migrations.go | 2 +- pkg/storage/utils/decomposedfs/tree/tree.go | 52 +++---- .../utils/decomposedfs/tree/tree_test.go | 2 +- .../utils/decomposedfs/upload/processing.go | 14 +- .../utils/decomposedfs/upload/upload.go | 8 +- pkg/storage/utils/decomposedfs/upload_test.go | 2 +- 20 files changed, 291 insertions(+), 248 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 0f6a988f4a..bdb5fd657e 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -575,7 +575,7 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { // mark the home node as the end of propagation - if err = n.SetXattr(prefixes.PropagationAttr, "1"); err != nil { + if err = n.SetXattrString(prefixes.PropagationAttr, "1"); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") // FIXME: This does not return an error at all, but results in a severe situation that the diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index 6a10bafb24..f22ba94e8d 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -134,7 +134,7 @@ func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference) return nil, errtypes.NotFound(f) } log := appctx.GetLogger(ctx) - var attrs map[string]string + var attrs node.Attributes if attrs, err = grantNode.Xattrs(); err != nil { log.Error().Err(err).Msg("error listing attributes") return nil, err @@ -312,7 +312,7 @@ func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provide // set the grant e := ace.FromGrant(g) principal, value := e.Marshal() - if err := n.SetXattr(prefixes.GrantPrefix+principal, string(value)); err != nil { + if err := n.SetXattr(prefixes.GrantPrefix+principal, value); err != nil { appctx.GetLogger(ctx).Error().Err(err). Str("principal", principal).Msg("Could not set grant for principal") return err diff --git a/pkg/storage/utils/decomposedfs/grants_test.go b/pkg/storage/utils/decomposedfs/grants_test.go index b5ad54290b..142ff632a8 100644 --- a/pkg/storage/utils/decomposedfs/grants_test.go +++ b/pkg/storage/utils/decomposedfs/grants_test.go @@ -138,7 +138,7 @@ var _ = Describe("Grants", func() { Path: "/dir1", }) Expect(err).ToNot(HaveOccurred()) - attr, err := n.Xattr(prefixes.GrantUserAcePrefix + grant.Grantee.GetUserId().OpaqueId) + attr, err := n.XattrString(prefixes.GrantUserAcePrefix + grant.Grantee.GetUserId().OpaqueId) Expect(err).ToNot(HaveOccurred()) Expect(attr).To(Equal(fmt.Sprintf("\x00t=A:f=:p=rw:c=%s:e=0\n", o.GetOpaqueId()))) // NOTE: this tests ace package }) diff --git a/pkg/storage/utils/decomposedfs/lookup/lookup.go b/pkg/storage/utils/decomposedfs/lookup/lookup.go index 4769dcc0b4..c075781520 100644 --- a/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -23,7 +23,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -59,14 +58,10 @@ func (lu *Lookup) MetadataBackend() metadata.Backend { // ReadBlobSizeAttr reads the blobsize from the xattrs func (lu *Lookup) ReadBlobSizeAttr(path string) (int64, error) { - attr, err := lu.metadataBackend.Get(path, prefixes.BlobsizeAttr) + blobSize, err := lu.metadataBackend.GetInt64(path, prefixes.BlobsizeAttr) if err != nil { return 0, errors.Wrapf(err, "error reading blobsize xattr") } - blobSize, err := strconv.ParseInt(attr, 10, 64) - if err != nil { - return 0, errors.Wrapf(err, "invalid blobsize xattr format") - } return blobSize, nil } @@ -76,22 +71,18 @@ func (lu *Lookup) ReadBlobIDAttr(path string) (string, error) { if err != nil { return "", errors.Wrapf(err, "error reading blobid xattr") } - return attr, nil + return string(attr), nil } // TypeFromPath returns the type of the node at the given path func (lu *Lookup) TypeFromPath(path string) provider.ResourceType { // Try to read from xattrs - typeAttr, err := lu.metadataBackend.Get(path, prefixes.TypeAttr) - t := provider.ResourceType_RESOURCE_TYPE_INVALID + typeAttr, err := lu.metadataBackend.GetInt64(path, prefixes.TypeAttr) if err == nil { - typeInt, err := strconv.ParseInt(typeAttr, 10, 32) - if err != nil { - return t - } - return provider.ResourceType(typeInt) + return provider.ResourceType(int32(typeAttr)) } + t := provider.ResourceType_RESOURCE_TYPE_INVALID // Fall back to checking on disk fi, err := os.Lstat(path) if err != nil { @@ -317,7 +308,7 @@ func (lu *Lookup) CopyMetadataWithSourceLock(source, target string, filter func( return err } - newAttrs := make(map[string]string, 0) + newAttrs := make(map[string][]byte, 0) for attrName, val := range attrs { if filter == nil || filter(attrName) { newAttrs[attrName] = val diff --git a/pkg/storage/utils/decomposedfs/metadata.go b/pkg/storage/utils/decomposedfs/metadata.go index 686a4bdf30..31c96fef0a 100644 --- a/pkg/storage/utils/decomposedfs/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata.go @@ -111,7 +111,7 @@ func (fs *Decomposedfs) SetArbitraryMetadata(ctx context.Context, ref *provider. } for k, v := range md.Metadata { attrName := prefixes.MetadataPrefix + k - if err = n.SetXattr(attrName, v); err != nil { + if err = n.SetXattrString(attrName, v); err != nil { errs = append(errs, errors.Wrap(err, "Decomposedfs: could not set metadata attribute "+attrName+" to "+k)) } } diff --git a/pkg/storage/utils/decomposedfs/metadata/ini_backend.go b/pkg/storage/utils/decomposedfs/metadata/ini_backend.go index bc68c0470f..3e3dbe022d 100644 --- a/pkg/storage/utils/decomposedfs/metadata/ini_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/ini_backend.go @@ -61,23 +61,23 @@ func NewIniBackend(rootPath string, o options.CacheOptions) IniBackend { } // All reads all extended attributes for a node -func (b IniBackend) All(path string) (map[string]string, error) { +func (b IniBackend) All(path string) (map[string][]byte, error) { path = b.MetadataPath(path) return b.loadMeta(path) } // Get an extended attribute value for the given key -func (b IniBackend) Get(path, key string) (string, error) { +func (b IniBackend) Get(path, key string) ([]byte, error) { path = b.MetadataPath(path) attribs, err := b.loadMeta(path) if err != nil { - return "", err + return []byte{}, err } val, ok := attribs[key] if !ok { - return "", &xattr.Error{Op: "ini.get", Path: path, Name: key, Err: xattr.ENOATTR} + return []byte{}, &xattr.Error{Op: "ini.get", Path: path, Name: key, Err: xattr.ENOATTR} } return val, nil } @@ -94,7 +94,7 @@ func (b IniBackend) GetInt64(path, key string) (int64, error) { if !ok { return 0, &xattr.Error{Op: "ini.get", Path: path, Name: key, Err: xattr.ENOATTR} } - i, err := strconv.ParseInt(val, 10, 64) + i, err := strconv.ParseInt(string(val), 10, 64) if err != nil { return 0, err } @@ -118,12 +118,12 @@ func (b IniBackend) List(path string) ([]string, error) { } // Set sets one attribute for the given path -func (b IniBackend) Set(path, key, val string) error { - return b.SetMultiple(path, map[string]string{key: val}, true) +func (b IniBackend) Set(path, key string, val []byte) error { + return b.SetMultiple(path, map[string][]byte{key: val}, true) } // SetMultiple sets a set of attribute for the given path -func (b IniBackend) SetMultiple(path string, attribs map[string]string, acquireLock bool) error { +func (b IniBackend) SetMultiple(path string, attribs map[string][]byte, acquireLock bool) error { return b.saveIni(path, attribs, nil, acquireLock) } @@ -132,7 +132,7 @@ func (b IniBackend) Remove(path, key string) error { return b.saveIni(path, nil, []string{key}, true) } -func (b IniBackend) saveIni(path string, setAttribs map[string]string, deleteAttribs []string, acquireLock bool) error { +func (b IniBackend) saveIni(path string, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error { var ( f readWriteCloseSeekTruncater err error @@ -199,8 +199,8 @@ func (b IniBackend) saveIni(path string, setAttribs map[string]string, deleteAtt return b.metaCache.PushToCache(b.cacheKey(path), iniAttribs) } -func (b IniBackend) loadMeta(path string) (map[string]string, error) { - var attribs map[string]string +func (b IniBackend) loadMeta(path string) (map[string][]byte, error) { + var attribs map[string][]byte err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) if err == nil { return attribs, err @@ -295,20 +295,20 @@ func needsEncoding(s []byte) bool { return false } -func encodeAttribs(attribs map[string]string) map[string]string { +func encodeAttribs(attribs map[string][]byte) map[string]string { encAttribs := map[string]string{} for key, val := range attribs { - if needsEncoding([]byte(val)) { - encAttribs["base64:"+key] = base64.StdEncoding.EncodeToString([]byte(val)) + if needsEncoding(val) { + encAttribs["base64:"+key] = base64.StdEncoding.EncodeToString(val) } else { - encAttribs[key] = val + encAttribs[key] = string(val) } } return encAttribs } -func decodeAttribs(iniFile *ini.File) (map[string]string, error) { - decodedAttributes := map[string]string{} +func decodeAttribs(iniFile *ini.File) (map[string][]byte, error) { + decodedAttributes := map[string][]byte{} for key, val := range iniFile.Section("").KeysHash() { if strings.HasPrefix(key, "base64:") { key = strings.TrimPrefix(key, "base64:") @@ -316,9 +316,10 @@ func decodeAttribs(iniFile *ini.File) (map[string]string, error) { if err != nil { return nil, err } - val = string(valBytes) + decodedAttributes[key] = valBytes + } else { + decodedAttributes[key] = []byte(val) } - decodedAttributes[key] = val } return decodedAttributes, nil } diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata.go b/pkg/storage/utils/decomposedfs/metadata/metadata.go index 6507ee01e3..d7685e50f3 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata.go @@ -24,12 +24,12 @@ var errUnconfiguredError = errors.New("no metadata backend configured. Bailing o // Backend defines the interface for file attribute backends type Backend interface { - All(path string) (map[string]string, error) - Get(path, key string) (string, error) + All(path string) (map[string][]byte, error) + Get(path, key string) ([]byte, error) GetInt64(path, key string) (int64, error) List(path string) (attribs []string, err error) - Set(path, key, val string) error - SetMultiple(path string, attribs map[string]string, acquireLock bool) error + Set(path, key string, val []byte) error + SetMultiple(path string, attribs map[string][]byte, acquireLock bool) error Remove(path, key string) error Purge(path string) error @@ -42,10 +42,10 @@ type Backend interface { type NullBackend struct{} // All reads all extended attributes for a node -func (NullBackend) All(path string) (map[string]string, error) { return nil, errUnconfiguredError } +func (NullBackend) All(path string) (map[string][]byte, error) { return nil, errUnconfiguredError } // Get an extended attribute value for the given key -func (NullBackend) Get(path, key string) (string, error) { return "", errUnconfiguredError } +func (NullBackend) Get(path, key string) ([]byte, error) { return []byte{}, errUnconfiguredError } // GetInt64 reads a string as int64 from the xattrs func (NullBackend) GetInt64(path, key string) (int64, error) { return 0, errUnconfiguredError } @@ -55,10 +55,10 @@ func (NullBackend) GetInt64(path, key string) (int64, error) { return 0, errUnco func (NullBackend) List(path string) ([]string, error) { return nil, errUnconfiguredError } // Set sets one attribute for the given path -func (NullBackend) Set(path string, key string, val string) error { return errUnconfiguredError } +func (NullBackend) Set(path string, key string, val []byte) error { return errUnconfiguredError } // SetMultiple sets a set of attribute for the given path -func (NullBackend) SetMultiple(path string, attribs map[string]string, acquireLock bool) error { +func (NullBackend) SetMultiple(path string, attribs map[string][]byte, acquireLock bool) error { return errUnconfiguredError } diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata_test.go b/pkg/storage/utils/decomposedfs/metadata/metadata_test.go index d1fc4a04fd..34a049afcd 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata_test.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata_test.go @@ -64,7 +64,7 @@ var _ = Describe("Backend", func() { Describe("Set", func() { It("sets an attribute", func() { - err := backend.Set(file, "foo", "bar") + err := backend.Set(file, "foo", []byte("bar")) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) @@ -73,9 +73,9 @@ var _ = Describe("Backend", func() { }) It("updates an attribute", func() { - err := backend.Set(file, "foo", "bar") + err := backend.Set(file, "foo", []byte("bar")) Expect(err).ToNot(HaveOccurred()) - err = backend.Set(file, "foo", "baz") + err = backend.Set(file, "foo", []byte("baz")) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) @@ -84,14 +84,14 @@ var _ = Describe("Backend", func() { }) It("encodes where needed", func() { - err := backend.Set(file, "user.ocis.cs.foo", "bar") + err := backend.Set(file, "user.ocis.cs.foo", []byte("bar")) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) Expect(err).ToNot(HaveOccurred()) Expect(string(content)).To(Equal("user.ocis.cs.foo = bar\n")) - err = backend.Set(file, "user.ocis.cs.foo", string([]byte{200, 201, 202})) + err = backend.Set(file, "user.ocis.cs.foo", []byte{200, 201, 202}) Expect(err).ToNot(HaveOccurred()) content, err = os.ReadFile(metafile) @@ -100,14 +100,14 @@ var _ = Describe("Backend", func() { }) It("doesn't encode already encoded attributes", func() { - err := backend.Set(file, "user.ocis.cs.foo", string([]byte{200, 201, 202})) + err := backend.Set(file, "user.ocis.cs.foo", []byte{200, 201, 202}) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) Expect(err).ToNot(HaveOccurred()) Expect(string(content)).To(Equal("`base64:user.ocis.cs.foo` = yMnK\n")) - err = backend.Set(file, "user.something", "doesn'tmatter") + err = backend.Set(file, "user.something", []byte("doesn'tmatter")) Expect(err).ToNot(HaveOccurred()) content, err = os.ReadFile(metafile) @@ -119,7 +119,7 @@ var _ = Describe("Backend", func() { _, err := backend.Get(file, "foo") Expect(err).To(HaveOccurred()) - err = backend.Set(file, "foo", "") + err = backend.Set(file, "foo", []byte{}) Expect(err).ToNot(HaveOccurred()) v, err := backend.Get(file, "foo") @@ -130,7 +130,7 @@ var _ = Describe("Backend", func() { Describe("SetMultiple", func() { It("sets attributes", func() { - err := backend.SetMultiple(file, map[string]string{"foo": "bar", "baz": "qux"}, true) + err := backend.SetMultiple(file, map[string][]byte{"foo": []byte("bar"), "baz": []byte("qux")}, true) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) @@ -140,9 +140,9 @@ var _ = Describe("Backend", func() { }) It("updates an attribute", func() { - err := backend.Set(file, "foo", "bar") + err := backend.Set(file, "foo", []byte("bar")) Expect(err).ToNot(HaveOccurred()) - err = backend.SetMultiple(file, map[string]string{"foo": "bar", "baz": "qux"}, true) + err = backend.SetMultiple(file, map[string][]byte{"foo": []byte("bar"), "baz": []byte("qux")}, true) Expect(err).ToNot(HaveOccurred()) content, err := os.ReadFile(metafile) @@ -152,11 +152,11 @@ var _ = Describe("Backend", func() { }) It("encodes where needed", func() { - err := backend.SetMultiple(file, map[string]string{ - "user.ocis.something.foo": "bar", - "user.ocis.cs.foo": string([]byte{200, 201, 202}), - "user.ocis.md.foo": string([]byte{200, 201, 202}), - "user.ocis.grant.foo": "bar", + err := backend.SetMultiple(file, map[string][]byte{ + "user.ocis.something.foo": []byte("bar"), + "user.ocis.cs.foo": []byte{200, 201, 202}, + "user.ocis.md.foo": []byte{200, 201, 202}, + "user.ocis.grant.foo": []byte("bar"), }, true) Expect(err).ToNot(HaveOccurred()) @@ -180,7 +180,7 @@ var _ = Describe("Backend", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(v)).To(Equal(2)) Expect(v["foo"]).To(Equal("123")) - Expect(v["bar"]).To(Equal("baz")) + Expect(v["bar"]).To(Equal([]byte("baz"))) }) It("returns an empty map", func() { @@ -197,7 +197,7 @@ var _ = Describe("Backend", func() { v, err := backend.List(file) Expect(err).ToNot(HaveOccurred()) - Expect(v).To(ConsistOf("foo", "bar")) + Expect(v).To(ConsistOf("foo", []byte("bar"))) }) It("returns an empty list", func() { @@ -214,7 +214,7 @@ var _ = Describe("Backend", func() { v, err := backend.Get(file, "foo") Expect(err).ToNot(HaveOccurred()) - Expect(v).To(Equal("bar")) + Expect(v).To(Equal([]byte("bar"))) }) It("returns an error on unknown attributes", func() { @@ -246,7 +246,7 @@ var _ = Describe("Backend", func() { v, err := backend.Get(file, "foo") Expect(err).ToNot(HaveOccurred()) - Expect(v).To(Equal("bar")) + Expect(v).To(Equal([]byte("bar"))) err = backend.Remove(file, "foo") Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index 88c99083e7..b05dc099c8 100644 --- a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -35,13 +35,8 @@ type XattrsBackend struct{} // Get an extended attribute value for the given key // No file locking is involved here as reading a single xattr is // considered to be atomic. -func (b XattrsBackend) Get(filePath, key string) (string, error) { - v, err := xattr.Get(filePath, key) - if err != nil { - return "", err - } - val := string(v) - return val, nil +func (b XattrsBackend) Get(filePath, key string) ([]byte, error) { + return xattr.Get(filePath, key) } // GetInt64 reads a string as int64 from the xattrs @@ -50,7 +45,7 @@ func (b XattrsBackend) GetInt64(filePath, key string) (int64, error) { if err != nil { return 0, err } - v, err := strconv.ParseInt(attr, 10, 64) + v, err := strconv.ParseInt(string(attr), 10, 64) if err != nil { return 0, err } @@ -76,7 +71,7 @@ func (XattrsBackend) List(filePath string) (attribs []string, err error) { // All reads all extended attributes for a node, protected by a // shared file lock -func (b XattrsBackend) All(filePath string) (attribs map[string]string, err error) { +func (b XattrsBackend) All(filePath string) (attribs map[string][]byte, err error) { attrNames, err := b.List(filePath) if err != nil { @@ -89,13 +84,13 @@ func (b XattrsBackend) All(filePath string) (attribs map[string]string, err erro ) // error handling: Count if there are errors while reading all attribs. // if there were any, return an error. - attribs = make(map[string]string, len(attrNames)) + attribs = make(map[string][]byte, len(attrNames)) for _, name := range attrNames { var val []byte if val, xerr = xattr.Get(filePath, name); xerr != nil { xerrs++ } else { - attribs[name] = string(val) + attribs[name] = val } } @@ -107,12 +102,12 @@ func (b XattrsBackend) All(filePath string) (attribs map[string]string, err erro } // Set sets one attribute for the given path -func (b XattrsBackend) Set(path string, key string, val string) (err error) { - return b.SetMultiple(path, map[string]string{key: val}, true) +func (b XattrsBackend) Set(path string, key string, val []byte) (err error) { + return b.SetMultiple(path, map[string][]byte{key: val}, true) } // SetMultiple sets a set of attribute for the given path -func (XattrsBackend) SetMultiple(path string, attribs map[string]string, acquireLock bool) (err error) { +func (XattrsBackend) SetMultiple(path string, attribs map[string][]byte, acquireLock bool) (err error) { if acquireLock { err := os.MkdirAll(filepath.Dir(path), 0600) if err != nil { @@ -132,7 +127,7 @@ func (XattrsBackend) SetMultiple(path string, attribs map[string]string, acquire xerr error ) for key, val := range attribs { - if xerr = xattr.Set(path, key, []byte(val)); xerr != nil { + if xerr = xattr.Set(path, key, val); xerr != nil { // log xerrs++ } diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 51712a7f0a..9725ae2193 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -87,7 +87,7 @@ type Node struct { SpaceRoot *Node lu PathLookup - xattrsCache map[string]string + xattrsCache map[string][]byte nodeType *provider.ResourceType } @@ -128,13 +128,9 @@ func (n *Node) Type() provider.ResourceType { t := provider.ResourceType_RESOURCE_TYPE_INVALID // Try to read from xattrs - typeAttr, err := n.Xattr(prefixes.TypeAttr) + typeAttr, err := n.XattrInt32(prefixes.TypeAttr) if err == nil { - typeInt, err := strconv.ParseInt(typeAttr, 10, 32) - if err != nil { - return t - } - t = provider.ResourceType(typeInt) + t = provider.ResourceType(typeAttr) n.nodeType = &t return t } @@ -172,9 +168,10 @@ func (n *Node) SetType(t provider.ResourceType) { func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { n.SpaceRoot.owner = new - var attribs = map[string]string{prefixes.OwnerIDAttr: new.OpaqueId, - prefixes.OwnerIDPAttr: new.Idp, - prefixes.OwnerTypeAttr: utils.UserTypeToString(new.Type)} + attribs := Attributes{} + attribs.SetString(prefixes.OwnerIDAttr, new.OpaqueId) + attribs.SetString(prefixes.OwnerIDPAttr, new.Idp) + attribs.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(new.Type)) if err := n.SpaceRoot.SetXattrs(attribs, true); err != nil { return err @@ -185,13 +182,12 @@ func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { // WriteAllNodeMetadata writes the Node metadata to disk func (n *Node) WriteAllNodeMetadata(ctx context.Context) (err error) { - attribs := make(map[string]string) - - attribs[prefixes.TypeAttr] = strconv.FormatInt(int64(n.Type()), 10) - attribs[prefixes.ParentidAttr] = n.ParentID - attribs[prefixes.NameAttr] = n.Name - attribs[prefixes.BlobIDAttr] = n.BlobID - attribs[prefixes.BlobsizeAttr] = strconv.FormatInt(n.Blobsize, 10) + attribs := Attributes{} + attribs.SetInt64(prefixes.TypeAttr, int64(n.Type())) + attribs.SetString(prefixes.ParentidAttr, n.ParentID) + attribs.SetString(prefixes.NameAttr, n.Name) + attribs.SetString(prefixes.BlobIDAttr, n.BlobID) + attribs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) return n.SetXattrs(attribs, true) } @@ -199,11 +195,12 @@ func (n *Node) WriteAllNodeMetadata(ctx context.Context) (err error) { // WriteOwner writes the space owner func (n *Node) WriteOwner(owner *userpb.UserId) error { n.SpaceRoot.owner = owner - attribs := map[string]string{ - prefixes.OwnerIDAttr: owner.OpaqueId, - prefixes.OwnerIDPAttr: owner.Idp, - prefixes.OwnerTypeAttr: utils.UserTypeToString(owner.Type), - } + + attribs := Attributes{} + attribs.SetString(prefixes.OwnerIDAttr, owner.OpaqueId) + attribs.SetString(prefixes.OwnerIDPAttr, owner.Idp) + attribs.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(owner.Type)) + if err := n.SpaceRoot.SetXattrs(attribs, true); err != nil { return err } @@ -254,7 +251,7 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis r.Exists = true // lookup name in extended attributes - r.Name, err = r.Xattr(prefixes.NameAttr) + r.Name, err = r.XattrString(prefixes.NameAttr) if err != nil { return nil, err } @@ -312,8 +309,8 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis } n.Exists = true - n.Name = attrs[prefixes.NameAttr] - n.ParentID = attrs[prefixes.ParentidAttr] + n.Name = attrs.String(prefixes.NameAttr) + n.ParentID = attrs.String(prefixes.ParentidAttr) if n.ParentID == "" { d, _ := os.ReadFile(n.InternalPath() + ".ini") appctx.GetLogger(ctx).Error().Str("nodeid", n.ID).Str("parentid", n.ParentID).Interface("attrs", attrs).Str("ini", string(d)).Msg("missing parent id") @@ -350,8 +347,8 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis } if revisionSuffix == "" { - n.BlobID = attrs[prefixes.BlobIDAttr] - blobSize, err := strconv.ParseInt(attrs[prefixes.BlobsizeAttr], 10, 64) + n.BlobID = attrs.String(prefixes.BlobIDAttr) + blobSize, err := attrs.Int64(prefixes.BlobsizeAttr) if err != nil { return nil, err } @@ -440,8 +437,8 @@ func (n *Node) Parent() (p *Node, err error) { } // lookup name and parent id in extended attributes - p.ParentID, _ = p.Xattr(prefixes.ParentidAttr) - p.Name, _ = p.Xattr(prefixes.NameAttr) + p.ParentID, _ = p.XattrString(prefixes.ParentidAttr) + p.Name, _ = p.XattrString(prefixes.NameAttr) // check node exists if _, err := os.Stat(p.InternalPath()); err == nil { @@ -464,10 +461,10 @@ func (n *Node) readOwner() (*userpb.UserId, error) { var attr string var err error // lookup ID in extended attributes - attr, err = n.SpaceRoot.Xattr(prefixes.OwnerIDAttr) + attr, err = n.SpaceRoot.XattrString(prefixes.OwnerIDAttr) switch { case err == nil: - owner.OpaqueId = attr + owner.OpaqueId = string(attr) case metadata.IsAttrUnset(err): // ignore default: @@ -475,10 +472,10 @@ func (n *Node) readOwner() (*userpb.UserId, error) { } // lookup IDP in extended attributes - attr, err = n.SpaceRoot.Xattr(prefixes.OwnerIDPAttr) + attr, err = n.SpaceRoot.XattrString(prefixes.OwnerIDPAttr) switch { case err == nil: - owner.Idp = attr + owner.Idp = string(attr) case metadata.IsAttrUnset(err): // ignore default: @@ -486,10 +483,10 @@ func (n *Node) readOwner() (*userpb.UserId, error) { } // lookup type in extended attributes - attr, err = n.SpaceRoot.Xattr(prefixes.OwnerTypeAttr) + attr, err = n.SpaceRoot.XattrString(prefixes.OwnerTypeAttr) switch { case err == nil: - owner.Type = utils.UserTypeMap(attr) + owner.Type = utils.UserTypeMap(string(attr)) case metadata.IsAttrUnset(err): // ignore default: @@ -598,7 +595,7 @@ func (n *Node) SetEtag(ctx context.Context, val string) (err error) { return nil } // etag is only valid until the calculated etag changes, is part of propagation - return n.SetXattr(prefixes.TmpEtagAttr, val) + return n.SetXattrString(prefixes.TmpEtagAttr, val) } // SetFavorite sets the favorite for the current user @@ -621,13 +618,13 @@ func (n *Node) SetEtag(ctx context.Context, val string) (err error) { func (n *Node) SetFavorite(uid *userpb.UserId, val string) error { // the favorite flag is specific to the user, so we need to incorporate the userid fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - return n.SetXattr(fa, val) + return n.SetXattrString(fa, val) } // IsDir returns true if the node is a directory func (n *Node) IsDir() bool { - attr, _ := n.Xattr(prefixes.TypeAttr) - return attr == strconv.FormatInt(int64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) + attr, _ := n.XattrInt32(prefixes.TypeAttr) + return attr == int32(provider.ResourceType_RESOURCE_TYPE_CONTAINER) } // AsResourceInfo return the node as CS3 ResourceInfo @@ -639,7 +636,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi var target string if nodeType == provider.ResourceType_RESOURCE_TYPE_REFERENCE { - target, _ = n.Xattr(prefixes.ReferenceAttr) + target, _ = n.XattrString(prefixes.ReferenceAttr) } id := &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID} @@ -699,8 +696,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } // use temporary etag if it is set - if b, err := n.Xattr(prefixes.TmpEtagAttr); err == nil && b != "" { - ri.Etag = fmt.Sprintf(`"%x"`, b) // TODO why do we convert string(b)? is the temporary etag stored as string? -> should we use bytes? use hex.EncodeToString? + if b, err := n.XattrString(prefixes.TmpEtagAttr); err == nil && b != "" { + ri.Etag = fmt.Sprintf(`"%x"`, b) } else if ri.Etag, err = calculateEtag(n.ID, tmTime); err != nil { sublog.Debug().Err(err).Msg("could not calculate etag") } @@ -742,7 +739,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // the favorite flag is specific to the user, so we need to incorporate the userid if uid := u.GetId(); uid != nil { fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if val, err := n.Xattr(fa); err == nil { + if val, err := n.XattrString(fa); err == nil { sublog.Debug(). Str("favorite", fa). Msg("found favorite flag") @@ -816,7 +813,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // only read when key was requested k := key[len(prefixes.MetadataPrefix):] if _, ok := mdKeysMap[k]; returnAllMetadata || ok { - metadata[k] = value + metadata[k] = string(value) } } @@ -874,7 +871,7 @@ func (n *Node) readChecksumIntoOpaque(ctx context.Context, algo string, ri *prov // quota is always stored on the root node func (n *Node) readQuotaIntoOpaque(ctx context.Context, ri *provider.ResourceInfo) { - v, err := n.Xattr(prefixes.QuotaAttr) + v, err := n.XattrString(prefixes.QuotaAttr) switch { case err == nil: // make sure we have a proper signed int @@ -904,7 +901,7 @@ func (n *Node) readQuotaIntoOpaque(ctx context.Context, ri *provider.ResourceInf // HasPropagation checks if the propagation attribute exists and is set to "1" func (n *Node) HasPropagation() (propagation bool) { - if b, err := n.Xattr(prefixes.PropagationAttr); err == nil { + if b, err := n.XattrString(prefixes.PropagationAttr); err == nil { return b == "1" } return false @@ -912,7 +909,7 @@ func (n *Node) HasPropagation() (propagation bool) { // GetTMTime reads the tmtime from the extended attributes func (n *Node) GetTMTime() (time.Time, error) { - b, err := n.Xattr(prefixes.TreeMTimeAttr) + b, err := n.XattrString(prefixes.TreeMTimeAttr) if err == nil { return time.Parse(time.RFC3339Nano, b) } @@ -930,12 +927,12 @@ func (n *Node) SetTMTime(t *time.Time) (err error) { if t == nil { return n.RemoveXattr(prefixes.TreeMTimeAttr) } - return n.SetXattr(prefixes.TreeMTimeAttr, t.UTC().Format(time.RFC3339Nano)) + return n.SetXattrString(prefixes.TreeMTimeAttr, t.UTC().Format(time.RFC3339Nano)) } // GetDTime reads the dtime from the extended attributes func (n *Node) GetDTime() (tmTime time.Time, err error) { - b, err := n.Xattr(prefixes.DTimeAttr) + b, err := n.XattrString(prefixes.DTimeAttr) if err != nil { return time.Time{}, err } @@ -947,7 +944,7 @@ func (n *Node) SetDTime(t *time.Time) (err error) { if t == nil { return n.RemoveXattr(prefixes.DTimeAttr) } - return n.SetXattr(prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) + return n.SetXattrString(prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) } // IsDisabled returns true when the node has a dmtime attribute set @@ -962,30 +959,30 @@ func (n *Node) IsDisabled() bool { // GetTreeSize reads the treesize from the extended attributes func (n *Node) GetTreeSize() (treesize uint64, err error) { - var b string - if b, err = n.Xattr(prefixes.TreesizeAttr); err != nil { - return + s, err := n.XattrInt64(prefixes.TreesizeAttr) + if err != nil { + return 0, err } - return strconv.ParseUint(b, 10, 64) + return uint64(s), nil } // SetTreeSize writes the treesize to the extended attributes func (n *Node) SetTreeSize(ts uint64) (err error) { - return n.SetXattr(prefixes.TreesizeAttr, strconv.FormatUint(ts, 10)) + return n.SetXattrString(prefixes.TreesizeAttr, strconv.FormatUint(ts, 10)) } // GetBlobSize reads the blobsize from the extended attributes func (n *Node) GetBlobSize() (treesize uint64, err error) { - var b string - if b, err = n.Xattr(prefixes.BlobsizeAttr); err != nil { - return + s, err := n.XattrInt64(prefixes.BlobsizeAttr) + if err != nil { + return 0, err } - return strconv.ParseUint(b, 10, 64) + return uint64(s), nil } // SetChecksum writes the checksum with the given checksum type to the extended attributes func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { - return n.SetXattr(prefixes.ChecksumPrefix+csType, string(h.Sum(nil))) + return n.SetXattr(prefixes.ChecksumPrefix+csType, h.Sum(nil)) } // UnsetTempEtag removes the temporary etag attribute @@ -1195,7 +1192,7 @@ func (n *Node) FindStorageSpaceRoot() error { // UnmarkProcessing removes the processing flag from the node func (n *Node) UnmarkProcessing(uploadID string) error { - v, _ := n.Xattr(prefixes.StatusPrefix) + v, _ := n.XattrString(prefixes.StatusPrefix) if v != ProcessingStatus+uploadID { // file started another postprocessing later - do not remove return nil @@ -1205,7 +1202,7 @@ func (n *Node) UnmarkProcessing(uploadID string) error { // IsProcessing returns true if the node is currently being processed func (n *Node) IsProcessing() bool { - v, err := n.Xattr(prefixes.StatusPrefix) + v, err := n.XattrString(prefixes.StatusPrefix) return err == nil && strings.HasPrefix(v, ProcessingStatus) } @@ -1217,15 +1214,15 @@ func (n *Node) IsSpaceRoot() bool { // SetScanData sets the virus scan info to the node func (n *Node) SetScanData(info string, date time.Time) error { - return n.SetXattrs(map[string]string{ - prefixes.ScanStatusPrefix: info, - prefixes.ScanDatePrefix: date.Format(time.RFC3339Nano), - }, true) + attribs := Attributes{} + attribs.SetString(prefixes.ScanStatusPrefix, info) + attribs.SetString(prefixes.ScanDatePrefix, date.Format(time.RFC3339Nano)) + return n.SetXattrs(attribs, true) } // ScanData returns scanning information of the node func (n *Node) ScanData() (scanned bool, virus string, scantime time.Time) { - ti, _ := n.Xattr(prefixes.ScanDatePrefix) + ti, _ := n.XattrString(prefixes.ScanDatePrefix) if ti == "" { return // not scanned yet } @@ -1235,7 +1232,7 @@ func (n *Node) ScanData() (scanned bool, virus string, scantime time.Time) { return } - i, err := n.Xattr(prefixes.ScanStatusPrefix) + i, err := n.XattrString(prefixes.ScanStatusPrefix) if err != nil { return } @@ -1253,10 +1250,19 @@ var CheckQuota = func(spaceRoot *Node, overwrite bool, oldSize, newSize uint64) if !enoughDiskSpace(spaceRoot.InternalPath(), newSize) { return false, errtypes.InsufficientStorage("disk full") } - quotaByteStr, _ := spaceRoot.Xattr(prefixes.QuotaAttr) - if quotaByteStr == "" || quotaByteStr == QuotaUnlimited { + quotaByteStr, _ := spaceRoot.XattrString(prefixes.QuotaAttr) + switch quotaByteStr { + case "": // if quota is not set, it means unlimited return true, nil + case QuotaUnlimited: + return true, nil + case QuotaUncalculated: + // treat it as unlimited + return true, nil + case QuotaUnknown: + // treat it as unlimited + return true, nil } quotaByte, _ := strconv.ParseUint(quotaByteStr, 10, 64) if overwrite { diff --git a/pkg/storage/utils/decomposedfs/node/xattrs.go b/pkg/storage/utils/decomposedfs/node/xattrs.go index f7cb51a964..917bfaf743 100644 --- a/pkg/storage/utils/decomposedfs/node/xattrs.go +++ b/pkg/storage/utils/decomposedfs/node/xattrs.go @@ -19,11 +19,31 @@ package node import ( + "strconv" + "github.com/pkg/xattr" ) +type Attributes map[string][]byte + +func (md Attributes) String(key string) string { + return string(md[key]) +} + +func (md Attributes) SetString(key, val string) { + md[key] = []byte(val) +} + +func (md Attributes) Int64(key string) (int64, error) { + return strconv.ParseInt(string(md[key]), 10, 64) +} + +func (md Attributes) SetInt64(key string, val int64) { + md[key] = []byte(strconv.FormatInt(val, 10)) +} + // SetXattrs sets multiple extended attributes on the write-through cache/node -func (n *Node) SetXattrs(attribs map[string]string, acquireLock bool) (err error) { +func (n *Node) SetXattrs(attribs map[string][]byte, acquireLock bool) (err error) { if n.xattrsCache != nil { for k, v := range attribs { n.xattrsCache[k] = v @@ -34,7 +54,7 @@ func (n *Node) SetXattrs(attribs map[string]string, acquireLock bool) (err error } // SetXattr sets an extended attribute on the write-through cache/node -func (n *Node) SetXattr(key, val string) (err error) { +func (n *Node) SetXattr(key string, val []byte) (err error) { if n.xattrsCache != nil { n.xattrsCache[key] = val } @@ -42,6 +62,15 @@ func (n *Node) SetXattr(key, val string) (err error) { return n.lu.MetadataBackend().Set(n.InternalPath(), key, val) } +// SetXattrString sets a string extended attribute on the write-through cache/node +func (n *Node) SetXattrString(key, val string) (err error) { + if n.xattrsCache != nil { + n.xattrsCache[key] = []byte(val) + } + + return n.lu.MetadataBackend().Set(n.InternalPath(), key, []byte(val)) +} + // RemoveXattr removes an extended attribute from the write-through cache/node func (n *Node) RemoveXattr(key string) error { if n.xattrsCache != nil { @@ -52,7 +81,7 @@ func (n *Node) RemoveXattr(key string) error { // Xattrs returns the extended attributes of the node. If the attributes have already // been cached they are not read from disk again. -func (n *Node) Xattrs() (map[string]string, error) { +func (n *Node) Xattrs() (Attributes, error) { if n.xattrsCache != nil { return n.xattrsCache, nil } @@ -67,11 +96,11 @@ func (n *Node) Xattrs() (map[string]string, error) { // Xattr returns an extended attribute of the node. If the attributes have already // been cached it is not read from disk again. -func (n *Node) Xattr(key string) (string, error) { +func (n *Node) Xattr(key string) ([]byte, error) { if n.xattrsCache == nil { attrs, err := n.lu.MetadataBackend().All(n.InternalPath()) if err != nil { - return "", err + return []byte{}, err } n.xattrsCache = attrs } @@ -80,5 +109,37 @@ func (n *Node) Xattr(key string) (string, error) { return val, nil } // wrap the error as xattr does - return "", &xattr.Error{Op: "xattr.get", Path: n.InternalPath(), Name: key, Err: xattr.ENOATTR} + return []byte{}, &xattr.Error{Op: "xattr.get", Path: n.InternalPath(), Name: key, Err: xattr.ENOATTR} +} + +// XattrString returns the string representation of an attribute +func (n *Node) XattrString(key string) (string, error) { + b, err := n.Xattr(key) + if err != nil { + return "", err + } + return string(b), nil +} + +// XattrInt32 returns the int32 representation of an attribute +func (n *Node) XattrInt32(key string) (int32, error) { + b, err := n.XattrString(key) + if err != nil { + return 0, err + } + + typeInt, err := strconv.ParseInt(b, 10, 32) + if err != nil { + return 0, err + } + return int32(typeInt), nil +} + +// XattrInt64 returns the int64 representation of an attribute +func (n *Node) XattrInt64(key string) (int64, error) { + b, err := n.XattrString(key) + if err != nil { + return 0, err + } + return strconv.ParseInt(b, 10, 64) } diff --git a/pkg/storage/utils/decomposedfs/recycle.go b/pkg/storage/utils/decomposedfs/recycle.go index 398a78bc4a..18598ef980 100644 --- a/pkg/storage/utils/decomposedfs/recycle.go +++ b/pkg/storage/utils/decomposedfs/recycle.go @@ -94,7 +94,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference } // lookup origin path in extended attributes if attrBytes, ok := attrs[prefixes.TrashOriginAttr]; ok { - origin = attrBytes + origin = string(attrBytes) } else { sublog.Error().Err(err).Str("space", spaceID).Msg("could not read origin path, skipping") return nil, err @@ -114,7 +114,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference nodeType := fs.lu.TypeFromPath(originalPath) if nodeType != provider.ResourceType_RESOURCE_TYPE_CONTAINER { // this is the case when we want to directly list a file in the trashbin - blobsize, err := strconv.ParseInt(attrs[prefixes.BlobsizeAttr], 10, 64) + blobsize, err := strconv.ParseInt(string(attrs[prefixes.BlobsizeAttr]), 10, 64) if err != nil { return items, err } @@ -168,7 +168,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") continue } - size, err = strconv.ParseInt(attr, 10, 64) + size, err = strconv.ParseInt(string(attr), 10, 64) if err != nil { sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") continue @@ -263,7 +263,7 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p // lookup origin path in extended attributes if attr, ok := attrs[prefixes.TrashOriginAttr]; ok { - item.Ref = &provider.Reference{Path: attr} + item.Ref = &provider.Reference{Path: string(attr)} } else { log.Error().Str("trashRoot", trashRoot).Str("item", itemPath).Str("node", nodeID).Str("dtime", timeSuffix).Msg("could not read origin path, skipping") continue diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 5135aa71dc..710800b9a5 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -25,7 +25,6 @@ import ( "math" "os" "path/filepath" - "strconv" "strings" "time" @@ -120,16 +119,16 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return nil, err } - metadata := make(map[string]string, 6) + metadata := make(node.Attributes, 6) // always enable propagation on the storage space root // mark the space root node as the end of propagation - metadata[prefixes.PropagationAttr] = "1" - metadata[prefixes.NameAttr] = req.Name - metadata[prefixes.SpaceNameAttr] = req.Name + metadata.SetString(prefixes.PropagationAttr, "1") + metadata.SetString(prefixes.NameAttr, req.Name) + metadata.SetString(prefixes.SpaceNameAttr, req.Name) if req.Type != "" { - metadata[prefixes.SpaceTypeAttr] = req.Type + metadata.SetString(prefixes.SpaceTypeAttr, req.Type) } if q := req.GetQuota(); q != nil { @@ -137,19 +136,19 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr if fs.o.MaxQuota != quotaUnrestricted && q.GetQuotaMaxBytes() > fs.o.MaxQuota { return nil, errtypes.BadRequest("decompsedFS: requested quota is higher than allowed") } - metadata[prefixes.QuotaAttr] = strconv.FormatUint(q.QuotaMaxBytes, 10) + metadata.SetInt64(prefixes.QuotaAttr, int64(q.QuotaMaxBytes)) } else if fs.o.MaxQuota != quotaUnrestricted { // If no quota was requested but a max quota was set then the the storage space has a quota // of max quota. - metadata[prefixes.QuotaAttr] = strconv.FormatUint(fs.o.MaxQuota, 10) + metadata.SetInt64(prefixes.QuotaAttr, int64(fs.o.MaxQuota)) } if description != "" { - metadata[prefixes.SpaceDescriptionAttr] = description + metadata.SetString(prefixes.SpaceDescriptionAttr, description) } if alias != "" { - metadata[prefixes.SpaceAliasAttr] = alias + metadata.SetString(prefixes.SpaceAliasAttr, alias) } if err := root.SetXattrs(metadata, true); err != nil { @@ -438,10 +437,10 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up space := req.StorageSpace _, spaceID, _, _ := storagespace.SplitID(space.Id.OpaqueId) - metadata := make(map[string]string, 5) + metadata := make(node.Attributes, 5) if space.Name != "" { - metadata[prefixes.NameAttr] = space.Name - metadata[prefixes.SpaceNameAttr] = space.Name + metadata.SetString(prefixes.NameAttr, space.Name) + metadata.SetString(prefixes.SpaceNameAttr, space.Name) } if space.Quota != nil { @@ -450,16 +449,16 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up Status: &v1beta11.Status{Code: v1beta11.Code_CODE_INVALID_ARGUMENT, Message: "decompsedFS: requested quota is higher than allowed"}, }, nil } - metadata[prefixes.QuotaAttr] = strconv.FormatUint(space.Quota.QuotaMaxBytes, 10) + metadata.SetInt64(prefixes.QuotaAttr, int64(space.Quota.QuotaMaxBytes)) } // TODO also return values which are not in the request if space.Opaque != nil { if description, ok := space.Opaque.Map["description"]; ok { - metadata[prefixes.SpaceDescriptionAttr] = string(description.Value) + metadata[prefixes.SpaceDescriptionAttr] = description.Value } if alias := utils.ReadPlainFromOpaque(space.Opaque, "spaceAlias"); alias != "" { - metadata[prefixes.SpaceAliasAttr] = alias + metadata.SetString(prefixes.SpaceAliasAttr, alias) } if image := utils.ReadPlainFromOpaque(space.Opaque, "image"); image != "" { imageID, err := storagespace.ParseID(image) @@ -468,7 +467,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND, Message: "decomposedFS: space image resource not found"}, }, nil } - metadata[prefixes.SpaceImageAttr] = imageID.OpaqueId + metadata.SetString(prefixes.SpaceImageAttr, imageID.OpaqueId) } if readme := utils.ReadPlainFromOpaque(space.Opaque, "readme"); readme != "" { readmeID, err := storagespace.ParseID(readme) @@ -477,7 +476,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND, Message: "decomposedFS: space readme resource not found"}, }, nil } - metadata[prefixes.SpaceReadmeAttr] = readmeID.OpaqueId + metadata.SetString(prefixes.SpaceReadmeAttr, readmeID.OpaqueId) } } @@ -576,7 +575,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return err } - st, err := n.SpaceRoot.Xattr(prefixes.SpaceTypeAttr) + st, err := n.SpaceRoot.XattrString(prefixes.SpaceTypeAttr) if err != nil { return errtypes.InternalError(fmt.Sprintf("space %s does not have a spacetype, possible corrupt decompsedfs", n.ID)) } @@ -606,7 +605,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return errtypes.NewErrtypeFromStatus(status.NewInvalid(ctx, "can't purge enabled space")) } - spaceType, err := n.Xattr(prefixes.SpaceTypeAttr) + spaceType, err := n.XattrString(prefixes.SpaceTypeAttr) if err != nil { return err } @@ -712,7 +711,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, var err error // TODO apply more filters var sname string - if sname, err = n.SpaceRoot.Xattr(prefixes.SpaceNameAttr); err != nil { + if sname, err = n.SpaceRoot.XattrString(prefixes.SpaceNameAttr); err != nil { // FIXME: Is that a severe problem? appctx.GetLogger(ctx).Debug().Err(err).Msg("space does not have a name attribute") } @@ -817,7 +816,8 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, // Mtime is set either as node.tmtime or as fi.mtime below } - if space.SpaceType, err = n.SpaceRoot.Xattr(prefixes.SpaceTypeAttr); err != nil { + space.SpaceType, err = n.SpaceRoot.XattrString(prefixes.SpaceTypeAttr) + if err != nil { appctx.GetLogger(ctx).Debug().Err(err).Msg("space does not have a type attribute") } @@ -866,41 +866,32 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, } // quota - quotaAttr, ok := spaceAttributes[prefixes.QuotaAttr] - if ok { + if q, err := spaceAttributes.Int64(prefixes.QuotaAttr); err == nil && q >= 0 { // make sure we have a proper signed int // we use the same magic numbers to indicate: // -1 = uncalculated // -2 = unknown // -3 = unlimited - if quota, err := strconv.ParseUint(quotaAttr, 10, 64); err == nil { - space.Quota = &provider.Quota{ - QuotaMaxBytes: quota, - QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited? - } - } else { - return nil, err + space.Quota = &provider.Quota{ + QuotaMaxBytes: uint64(q), + QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited? } } - spaceImage, ok := spaceAttributes[prefixes.SpaceImageAttr] - if ok { + if si := spaceAttributes.String(prefixes.SpaceImageAttr); si != "" { space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "image", storagespace.FormatResourceID( - provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: spaceImage}, + provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: si}, )) } - spaceDescription, ok := spaceAttributes[prefixes.SpaceDescriptionAttr] - if ok { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "description", spaceDescription) + if sd := spaceAttributes.String(prefixes.SpaceDescriptionAttr); sd != "" { + space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "description", sd) } - spaceReadme, ok := spaceAttributes[prefixes.SpaceReadmeAttr] - if ok { + if sr := spaceAttributes.String(prefixes.SpaceReadmeAttr); sr != "" { space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "readme", storagespace.FormatResourceID( - provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: spaceReadme}, + provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: sr}, )) } - spaceAlias, ok := spaceAttributes[prefixes.SpaceAliasAttr] - if ok { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "spaceAlias", spaceAlias) + if sa := spaceAttributes.String(prefixes.SpaceAliasAttr); sa != "" { + space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "spaceAlias", sa) } // add rootinfo @@ -909,7 +900,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, return space, nil } -func mapHasKey(checkMap map[string]string, keys ...string) bool { +func mapHasKey(checkMap map[string][]byte, keys ...string) bool { for _, key := range keys { if _, hasKey := checkMap[key]; hasKey { return true diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index 3e71933031..26a6d61f48 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -277,7 +277,7 @@ func (t *TestEnv) CreateTestStorageSpace(typ string, quota *providerv1beta1.Quot if err != nil { return nil, err } - if err = h.SetXattr(prefixes.SpaceNameAttr, "username"); err != nil { + if err = h.SetXattr(prefixes.SpaceNameAttr, []byte("username")); err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/tree/migrations.go b/pkg/storage/utils/decomposedfs/tree/migrations.go index eb5032d07c..61235ba113 100644 --- a/pkg/storage/utils/decomposedfs/tree/migrations.go +++ b/pkg/storage/utils/decomposedfs/tree/migrations.go @@ -59,7 +59,7 @@ func (t *Tree) migration0001Nodes() error { nodePath := filepath.Join(nodesPath, n.Name()) attr, err := t.lookup.MetadataBackend().Get(nodePath, prefixes.ParentidAttr) - if err == nil && attr == node.RootID { + if err == nil && string(attr) == node.RootID { if err := t.moveNode(n.Name(), n.Name()); err != nil { logger.New().Error().Err(err). Str("space", n.Name()). diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index aeb0f53f68..9ab30331f5 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -367,7 +367,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } // update name attribute - if err := oldNode.SetXattr(prefixes.NameAttr, newNode.Name); err != nil { + if err := oldNode.SetXattrString(prefixes.NameAttr, newNode.Name); err != nil { return errors.Wrap(err, "Decomposedfs: could not set name attribute") } @@ -387,10 +387,10 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } // update target parentid and name - if err := oldNode.SetXattrs(map[string]string{ - prefixes.ParentidAttr: newNode.ParentID, - prefixes.NameAttr: newNode.Name, - }, true); err != nil { + attribs := node.Attributes{} + attribs.SetString(prefixes.ParentidAttr, newNode.ParentID) + attribs.SetString(prefixes.NameAttr, newNode.Name) + if err := oldNode.SetXattrs(attribs, true); err != nil { return errors.Wrap(err, "Decomposedfs: could not update old node attributes") } @@ -488,7 +488,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // set origin location in metadata nodePath := n.InternalPath() - if err := n.SetXattr(prefixes.TrashOriginAttr, origin); err != nil { + if err := n.SetXattrString(prefixes.TrashOriginAttr, origin); err != nil { return err } @@ -618,13 +618,11 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa targetNode.Exists = true - attrs := map[string]string{ - // update name attribute - prefixes.NameAttr: targetNode.Name, - } + attrs := node.Attributes{} + attrs.SetString(prefixes.NameAttr, targetNode.Name) if trashPath != "" { // set ParentidAttr to restorePath's node parent id - attrs[prefixes.ParentidAttr] = targetNode.ParentID + attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID) } if err = recycleNode.SetXattrs(attrs, true); err != nil { @@ -778,7 +776,7 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err } if t.treeTimeAccounting || (t.treeSizeAccounting && sizeDiff != 0) { - attrs := map[string]string{} + attrs := node.Attributes{} var f *lockedfile.File // lock node before reading treesize or tree time @@ -830,10 +828,10 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err if updateSyncTime { // update the tree time of the parent node - attrs[prefixes.TreeMTimeAttr] = sTime.UTC().Format(time.RFC3339Nano) + attrs.SetString(prefixes.TreeMTimeAttr, sTime.UTC().Format(time.RFC3339Nano)) } - attrs[prefixes.TmpEtagAttr] = "" + attrs.SetString(prefixes.TmpEtagAttr, "") } // size accounting @@ -860,7 +858,7 @@ func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err } // update the tree size of the node - attrs[prefixes.TreesizeAttr] = strconv.FormatUint(newSize, 10) + attrs.SetString(prefixes.TreesizeAttr, strconv.FormatUint(newSize, 10)) sublog.Debug().Uint64("newSize", newSize).Msg("updated treesize of parent node") } @@ -912,10 +910,10 @@ func (t *Tree) calculateTreeSize(ctx context.Context, childrenPath string) (uint continue // continue after an error } sizeAttr := "" - if attribs[prefixes.TypeAttr] == strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) { - sizeAttr = attribs[prefixes.BlobsizeAttr] + if string(attribs[prefixes.TypeAttr]) == strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) { + sizeAttr = string(attribs[prefixes.BlobsizeAttr]) } else { - sizeAttr = attribs[prefixes.TreesizeAttr] + sizeAttr = string(attribs[prefixes.TreesizeAttr]) } csize, err := strconv.ParseInt(sizeAttr, 10, 64) if err != nil { @@ -993,10 +991,10 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( } recycleNode.SetType(t.lookup.TypeFromPath(recycleNode.InternalPath())) - var attrStr string + var attrBytes []byte // lookup blobID in extended attributes - if attrStr, err = backend.Get(deletedNodePath, prefixes.BlobIDAttr); err == nil { - recycleNode.BlobID = attrStr + if attrBytes, err = backend.Get(deletedNodePath, prefixes.BlobIDAttr); err == nil { + recycleNode.BlobID = string(attrBytes) } else { return } @@ -1007,15 +1005,15 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( } // lookup parent id in extended attributes - if attrStr, err = backend.Get(deletedNodePath, prefixes.ParentidAttr); err == nil { - recycleNode.ParentID = attrStr + if attrBytes, err = backend.Get(deletedNodePath, prefixes.ParentidAttr); err == nil { + recycleNode.ParentID = string(attrBytes) } else { return } // lookup name in extended attributes - if attrStr, err = backend.Get(deletedNodePath, prefixes.NameAttr); err == nil { - recycleNode.Name = attrStr + if attrBytes, err = backend.Get(deletedNodePath, prefixes.NameAttr); err == nil { + recycleNode.Name = string(attrBytes) } else { return } @@ -1024,8 +1022,8 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( origin = "/" // lookup origin path in extended attributes - if attrStr, err = backend.Get(resolvedTrashItem, prefixes.TrashOriginAttr); err == nil { - origin = filepath.Join(attrStr, path) + if attrBytes, err = backend.Get(resolvedTrashItem, prefixes.TrashOriginAttr); err == nil { + origin = filepath.Join(string(attrBytes), path) } else { log.Error().Err(err).Str("trashItem", trashItem).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") } diff --git a/pkg/storage/utils/decomposedfs/tree/tree_test.go b/pkg/storage/utils/decomposedfs/tree/tree_test.go index c1ad5b1931..c026811c4e 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree_test.go +++ b/pkg/storage/utils/decomposedfs/tree/tree_test.go @@ -401,7 +401,7 @@ var _ = Describe("Tree", func() { stopdir, err := env.CreateTestDir("testdir/stophere", &provider.Reference{ResourceId: env.SpaceRootRes}) Expect(err).ToNot(HaveOccurred()) - err = stopdir.SetXattr(prefixes.PropagationAttr, "0") + err = stopdir.SetXattrString(prefixes.PropagationAttr, "0") Expect(err).ToNot(HaveOccurred()) otherdir, err := env.CreateTestDir("testdir/stophere/lotsofbytes", &provider.Reference{ResourceId: env.SpaceRootRes}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/storage/utils/decomposedfs/upload/processing.go b/pkg/storage/utils/decomposedfs/upload/processing.go index 187757bb4e..33fe4501fb 100644 --- a/pkg/storage/utils/decomposedfs/upload/processing.go +++ b/pkg/storage/utils/decomposedfs/upload/processing.go @@ -237,7 +237,7 @@ func Get(ctx context.Context, id string, lu *lookup.Lookup, tp Tree, fsRoot stri } // CreateNodeForUpload will create the target node for the Upload -func CreateNodeForUpload(upload *Upload, initAttrs map[string]string) (*node.Node, error) { +func CreateNodeForUpload(upload *Upload, initAttrs node.Attributes) (*node.Node, error) { fi, err := os.Stat(upload.binPath) if err != nil { return nil, err @@ -283,12 +283,12 @@ func CreateNodeForUpload(upload *Upload, initAttrs map[string]string) (*node.Nod } // overwrite technical information - initAttrs[prefixes.TypeAttr] = strconv.FormatInt(int64(n.Type()), 10) - initAttrs[prefixes.ParentidAttr] = n.ParentID - initAttrs[prefixes.NameAttr] = n.Name - initAttrs[prefixes.BlobIDAttr] = n.BlobID - initAttrs[prefixes.BlobsizeAttr] = strconv.FormatInt(n.Blobsize, 10) - initAttrs[prefixes.StatusPrefix] = node.ProcessingStatus + upload.Info.ID + initAttrs.SetInt64(prefixes.TypeAttr, int64(n.Type())) + initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) + initAttrs.SetString(prefixes.NameAttr, n.Name) + initAttrs.SetString(prefixes.BlobIDAttr, n.BlobID) + initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) + initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+upload.Info.ID) // update node metadata with new blobid etc err = n.SetXattrs(initAttrs, false) diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index d8ab3b411e..d576cb2314 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -229,10 +229,10 @@ func (upload *Upload) FinishUpload(_ context.Context) error { } // update checksums - attrs := map[string]string{ - prefixes.ChecksumPrefix + "sha1": string(sha1h.Sum(nil)), - prefixes.ChecksumPrefix + "md5": string(md5h.Sum(nil)), - prefixes.ChecksumPrefix + "adler32": string(adler32h.Sum(nil)), + attrs := node.Attributes{ + prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), + prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), + prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), } n, err := CreateNodeForUpload(upload, attrs) diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go index b1b247cf48..5c5f8e9a1d 100644 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ b/pkg/storage/utils/decomposedfs/upload_test.go @@ -173,7 +173,7 @@ var _ = Describe("File uploads", func() { // the space name attribute is the stop condition in the lookup h, err := lu.NodeFromResource(ctx, rootRef) Expect(err).ToNot(HaveOccurred()) - err = h.SetXattr(prefixes.SpaceNameAttr, "username") + err = h.SetXattrString(prefixes.SpaceNameAttr, "username") Expect(err).ToNot(HaveOccurred()) permissions.On("AssemblePermissions", mock.Anything, mock.Anything, mock.Anything).Return(provider.ResourcePermissions{ Stat: true,