From a5d6c5bbea09d972cae3ed0ceaf14f29a6471f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 20 Jul 2022 11:58:15 +0000 Subject: [PATCH 01/94] initial jsoncs3 share manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 734 ++++++++++++++++++ .../manager/jsoncs3/jsoncs3_suite_test.go | 31 + pkg/share/manager/jsoncs3/jsoncs3_test.go | 178 +++++ pkg/share/manager/loader/loader.go | 1 + 4 files changed, 944 insertions(+) create mode 100644 pkg/share/manager/jsoncs3/jsoncs3.go create mode 100644 pkg/share/manager/jsoncs3/jsoncs3_suite_test.go create mode 100644 pkg/share/manager/jsoncs3/jsoncs3_test.go diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go new file mode 100644 index 0000000000..de63735df5 --- /dev/null +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -0,0 +1,734 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package jsoncs3 + +import ( + "context" + "encoding/json" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/bluele/gcache" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/share" + "github.com/golang/protobuf/proto" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages + "github.com/google/uuid" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "google.golang.org/genproto/protobuf/field_mask" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/encoding/prototext" + + "github.com/cs3org/reva/v2/pkg/share/manager/registry" + "github.com/cs3org/reva/v2/pkg/utils" +) + +/* + The sharded json driver splits the json file per storage space. Similar to fileids shareids are prefixed with the spaceid. + + FAQ + Q: Why not split shares by user and have a list per user? + A: While shares are created by users, they are persisted as grants on a file. + If we persist shares by their creator/owner they would vanish if a user is deprovisioned: shares + in project spaces could not be managed collaboratively. + By splitting by space, we are in fact not only splitting by user, but more granular, per space. + +*/ + +func init() { + registry.Register("jsoncs3", New) +} + +// New returns a new mgr. +func New(m map[string]interface{}) (share.Manager, error) { + c, err := parseConfig(m) + if err != nil { + err = errors.Wrap(err, "error creating a new manager") + return nil, err + } + + if c.GatewayAddr == "" { + return nil, errors.New("share manager config is missing gateway address") + } + + c.init() + + // load or create file + model, err := loadOrCreate(c.File) + if err != nil { + err = errors.Wrap(err, "error loading the file containing the shares") + return nil, err + } + + return &mgr{ + c: c, + model: model, + storagesCache: gcache.New(1000000).LFU().Build(), + }, nil + +} + +func loadOrCreate(file string) (*shareModel, error) { + if info, err := os.Stat(file); errors.Is(err, fs.ErrNotExist) || info.Size() == 0 { + if err := ioutil.WriteFile(file, []byte("{}"), 0700); err != nil { + err = errors.Wrap(err, "error opening/creating the file: "+file) + return nil, err + } + } + + fd, err := os.OpenFile(file, os.O_CREATE, 0644) + if err != nil { + err = errors.Wrap(err, "error opening/creating the file: "+file) + return nil, err + } + defer fd.Close() + + data, err := ioutil.ReadAll(fd) + if err != nil { + err = errors.Wrap(err, "error reading the data") + return nil, err + } + + j := &jsonEncoding{} + if err := json.Unmarshal(data, j); err != nil { + err = errors.Wrap(err, "error decoding data from json") + return nil, err + } + + m := &shareModel{State: j.State, MountPoint: j.MountPoint} + for _, s := range j.Shares { + var decShare collaboration.Share + if err = utils.UnmarshalJSONToProtoV1([]byte(s), &decShare); err != nil { + return nil, errors.Wrap(err, "error decoding share from json") + } + m.Shares = append(m.Shares, &decShare) + } + + if m.State == nil { + m.State = map[string]map[string]collaboration.ShareState{} + } + if m.MountPoint == nil { + m.MountPoint = map[string]map[string]*provider.Reference{} + } + + m.file = file + return m, nil +} + +type shareModel struct { + file string + State map[string]map[string]collaboration.ShareState `json:"state"` // map[username]map[share_id]ShareState + MountPoint map[string]map[string]*provider.Reference `json:"mount_point"` // map[username]map[share_id]MountPoint + Shares []*collaboration.Share `json:"shares"` +} + +type jsonEncoding struct { + State map[string]map[string]collaboration.ShareState `json:"state"` // map[username]map[share_id]ShareState + MountPoint map[string]map[string]*provider.Reference `json:"mount_point"` // map[username]map[share_id]MountPoint + Shares []string `json:"shares"` +} + +func (m *shareModel) Save() error { + j := &jsonEncoding{State: m.State, MountPoint: m.MountPoint} + for _, s := range m.Shares { + encShare, err := utils.MarshalProtoV1ToJSON(s) + if err != nil { + return errors.Wrap(err, "error encoding to json") + } + j.Shares = append(j.Shares, string(encShare)) + } + + data, err := json.Marshal(j) + if err != nil { + err = errors.Wrap(err, "error encoding to json") + return err + } + + if err := ioutil.WriteFile(m.file, data, 0644); err != nil { + err = errors.Wrap(err, "error writing to file: "+m.file) + return err + } + + return nil +} + +type mgr struct { + c *config + sync.Mutex // concurrent access to the file + model *shareModel + serviceUser *userv1beta1.User + SpaceRoot *provider.ResourceId + storagesCache gcache.Cache + + initialized bool +} + +// We store the shares in the metadata storage under /{storageid}/{spaceid}.json +// To determine the spaces a user has access to we maintain an empty /{userid}/{storageid}/{spaceid} file +// that we persist when initially traversing all shares in the metadata /{storageid}/{spaceid}.json files +// when a user creates a new share the jsoncs3 manager touches a new /{userid}/{storageid}/{spaceid} file +// - the changed mtime can be used to determine when a space needs to be reread for redundant setups + +// when initializing we only initialize per user: +// - we list /{userid}/*, for every space we fetch /{storageid}/{spaceid}.json if we +// have not cached it yet, or if the /{userid}/{storageid}/{spaceid} etag changed +// - if it does not exist we query the registry for every storage provider id, then +// we traverse /{storageid}/ in the metadata storage to +// 1. create /{userid}/ +// 3. touch /{userid}/{storageid}/{spaceid} +// 2. touch /{userid}/{storageid} (not needed when mtime propagation is enabled) + +// we need to split the two lists: +// /{userid}/received/{storageid}/{spaceid} +// /{userid}/created/{storageid}/{spaceid} + +func (m *mgr) initialize(ctx context.Context) error { + // if local copy is invalid fetch a new one + // invalid = net set || etag changed + if m.initialized { + return nil + } + + m.Lock() + defer m.Unlock() + + if m.initialized { // check if initialization happened while grabbing the lock + return nil + } + + user := ctxpkg.ContextMustGetUser(ctx) + + client, err := m.metadataClient() + if err != nil { + return err + } + + ctx, err = m.getAuthContext(context.Background()) + if err != nil { + return err + } + spaceid := "shares-space" + + // FIXME only create space if ListContainer fails + cssr, err := client.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ + Opaque: &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "spaceid": { + Decoder: "plain", + Value: []byte(spaceid), + }, + }, + }, + Owner: m.serviceUser, + Name: "Shares", + Type: "metadata", + }) + switch { + case err != nil: + return err + case cssr.Status.Code == rpcv1beta1.Code_CODE_OK: + m.SpaceRoot = cssr.StorageSpace.Root + case cssr.Status.Code == rpcv1beta1.Code_CODE_ALREADY_EXISTS: + // TODO make CreateStorageSpace return existing space? + m.SpaceRoot = &provider.ResourceId{SpaceId: spaceid, OpaqueId: spaceid} + default: + return errtypes.NewErrtypeFromStatus(cssr.Status) + } + + // we list /{userid}/* || /{userid}/received ? created? + res, err := client.ListContainer(ctx, &provider.ListContainerRequest{ + Ref: &provider.Reference{ResourceId: m.SpaceRoot, Path: filepath.Join(user.Id.OpaqueId, "created")}, + }) + + switch { + case err != nil: + return err + case cssr.Status.Code == rpcv1beta1.Code_CODE_OK: + // for every space we fetch /{storageid}/{spaceid}.json if we + // have not cached it yet, or if the /{userid}/created/{storageid}/{spaceid} etag changed + for _, storageInfo := range res.Infos { + // do we have spaces for this storage cached? + if spaces := m.storagesCache.Get(storageInfo.Name); spaces != nil { + // TODO return space if etag is same + } + // TODO update cache + } + case cssr.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND: + // if it does not exist we query the registry for every storage provider id, then + // we traverse /{storageid}/ in the metadata storage to + // 1. create /{userid}/ + // 3. touch /{userid}/{storageid}/{spaceid} + // 2. touch /{userid}/{storageid} (not needed when mtime propagation is enabled) + default: + return errtypes.NewErrtypeFromStatus(cssr.Status) + } + + return nil +} + +func (m *mgr) metadataClient() (provider.ProviderAPIClient, error) { + return pool.GetStorageProviderServiceClient(m.c.ProviderAddr) +} + +func (m *mgr) getAuthContext(ctx context.Context) (context.Context, error) { + client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) + if err != nil { + return nil, err + } + + authCtx := ctxpkg.ContextSetUser(context.Background(), m.serviceUser) + authRes, err := client.Authenticate(authCtx, &gateway.AuthenticateRequest{ + Type: "machine", + ClientId: "userid:" + m.serviceUser.Id.OpaqueId, + ClientSecret: m.c.MachineAuthAPIKey, + }) + if err != nil { + return nil, err + } + if authRes.GetStatus().GetCode() != rpcv1beta1.Code_CODE_OK { + return nil, errtypes.NewErrtypeFromStatus(authRes.GetStatus()) + } + authCtx = metadata.AppendToOutgoingContext(authCtx, ctxpkg.TokenHeader, authRes.Token) + return authCtx, nil +} + +func (m *mgr) getClient() (gateway.GatewayAPIClient, error) { + return pool.GetGatewayServiceClient(m.c.GatewayAddr) +} + +type config struct { + File string `mapstructure:"file"` + GatewayAddr string `mapstructure:"gateway_addr"` + // ProviderAddr is the address of the metadata storage provider + ProviderAddr string `mapstructure:"provider_addr"` + ServiceUser string `mapstructure:"service_user"` + MachineAuthAPIKey string `mapstructure:"machine_auth_api_key"` +} + +func (c *config) init() { + if c.File == "" { + c.File = "/var/tmp/reva/shares.json" + } +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + return c, nil +} + +func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { + id := uuid.NewString() + user := ctxpkg.ContextMustGetUser(ctx) + now := time.Now().UnixNano() + ts := &typespb.Timestamp{ + Seconds: uint64(now / int64(time.Second)), + Nanos: uint32(now % int64(time.Second)), + } + + // do not allow share to myself or the owner if share is for a user + // TODO(labkode): should not this be caught already at the gw level? + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER && + (utils.UserEqual(g.Grantee.GetUserId(), user.Id) || utils.UserEqual(g.Grantee.GetUserId(), md.Owner)) { + return nil, errors.New("json: owner/creator and grantee are the same") + } + + // check if share already exists. + key := &collaboration.ShareKey{ + Owner: md.Owner, + ResourceId: md.Id, + Grantee: g.Grantee, + } + + m.Lock() + defer m.Unlock() + _, _, err := m.getByKey(key) + if err == nil { + // share already exists + return nil, errtypes.AlreadyExists(key.String()) + } + + s := &collaboration.Share{ + Id: &collaboration.ShareId{ + OpaqueId: id, + }, + ResourceId: md.Id, + Permissions: g.Permissions, + Grantee: g.Grantee, + Owner: md.Owner, + Creator: user.Id, + Ctime: ts, + Mtime: ts, + } + + m.model.Shares = append(m.model.Shares, s) + if err := m.model.Save(); err != nil { + err = errors.Wrap(err, "error saving model") + return nil, err + } + + return s, nil +} + +// getByID must be called in a lock-controlled block. +func (m *mgr) getByID(id *collaboration.ShareId) (int, *collaboration.Share, error) { + for i, s := range m.model.Shares { + if s.GetId().OpaqueId == id.OpaqueId { + return i, s, nil + } + } + return -1, nil, errtypes.NotFound(id.String()) +} + +// getByKey must be called in a lock-controlled block. +func (m *mgr) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, error) { + for i, s := range m.model.Shares { + if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) && + utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) { + return i, s, nil + } + } + return -1, nil, errtypes.NotFound(key.String()) +} + +// get must be called in a lock-controlled block. +func (m *mgr) get(ref *collaboration.ShareReference) (idx int, s *collaboration.Share, err error) { + switch { + case ref.GetId() != nil: + idx, s, err = m.getByID(ref.GetId()) + case ref.GetKey() != nil: + idx, s, err = m.getByKey(ref.GetKey()) + default: + err = errtypes.NotFound(ref.String()) + } + return +} + +func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { + m.Lock() + defer m.Unlock() + _, s, err := m.get(ref) + if err != nil { + return nil, err + } + // check if we are the owner or the grantee + user := ctxpkg.ContextMustGetUser(ctx) + if share.IsCreatedByUser(s, user) || share.IsGrantedToUser(s, user) { + return s, nil + } + // we return not found to not disclose information + return nil, errtypes.NotFound(ref.String()) +} + +func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { + m.Lock() + defer m.Unlock() + user := ctxpkg.ContextMustGetUser(ctx) + + idx, s, err := m.get(ref) + if err != nil { + return err + } + if !share.IsCreatedByUser(s, user) { + return errtypes.NotFound(ref.String()) + } + + last := len(m.model.Shares) - 1 + m.model.Shares[idx] = m.model.Shares[last] + // explicitly nil the reference to prevent memory leaks + // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order + m.model.Shares[last] = nil + m.model.Shares = m.model.Shares[:last] + if err := m.model.Save(); err != nil { + err = errors.Wrap(err, "error saving model") + return err + } + return nil +} + +func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { + m.Lock() + defer m.Unlock() + idx, s, err := m.get(ref) + if err != nil { + return nil, err + } + + user := ctxpkg.ContextMustGetUser(ctx) + if !share.IsCreatedByUser(s, user) { + return nil, errtypes.NotFound(ref.String()) + } + + now := time.Now().UnixNano() + m.model.Shares[idx].Permissions = p + m.model.Shares[idx].Mtime = &typespb.Timestamp{ + Seconds: uint64(now / int64(time.Second)), + Nanos: uint32(now % int64(time.Second)), + } + + if err := m.model.Save(); err != nil { + err = errors.Wrap(err, "error saving model") + return nil, err + } + return m.model.Shares[idx], nil +} + +func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { + if err := m.initialize(); err != nil { + return nil, err + } + + m.Lock() + defer m.Unlock() + log := appctx.GetLogger(ctx) + user := ctxpkg.ContextMustGetUser(ctx) + + client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) + if err != nil { + return nil, errors.Wrap(err, "failed to list shares") + } + cache := make(map[string]struct{}) + var ss []*collaboration.Share + for _, s := range m.model.Shares { + if share.MatchesFilters(s, filters) { + // Only add the share if the share was created by the user or if + // the user has ListGrants permissions on the shared resource. + // The ListGrants check is necessary when a space member wants + // to list shares in a space. + // We are using a cache here so that we don't have to stat a + // resource multiple times. + key := strings.Join([]string{s.ResourceId.StorageId, s.ResourceId.OpaqueId}, "!") + if _, hit := cache[key]; !hit && !share.IsCreatedByUser(s, user) { + sRes, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: s.ResourceId}}) + if err != nil || sRes.Status.Code != rpcv1beta1.Code_CODE_OK { + log.Error(). + Err(err). + Interface("status", sRes.Status). + Interface("resource_id", s.ResourceId). + Msg("ListShares: could not stat resource") + continue + } + if !sRes.Info.PermissionSet.ListGrants { + continue + } + cache[key] = struct{}{} + } + ss = append(ss, s) + } + } + return ss, nil +} + +// we list the shares that are targeted to the user in context or to the user groups. +func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { + m.Lock() + defer m.Unlock() + + user := ctxpkg.ContextMustGetUser(ctx) + mem := make(map[string]int) + var rss []*collaboration.ReceivedShare + for _, s := range m.model.Shares { + if !share.IsCreatedByUser(s, user) && + share.IsGrantedToUser(s, user) && + share.MatchesFilters(s, filters) { + + rs := m.convert(user.Id, s) + idx, seen := mem[s.ResourceId.OpaqueId] + if !seen { + rss = append(rss, rs) + mem[s.ResourceId.OpaqueId] = len(rss) - 1 + continue + } + + // When we arrive here there was already a share for this resource. + // if there is a mix-up of shares of type group and shares of type user we need to deduplicate them, since it points + // to the same resource. Leave the more explicit and hide the less explicit. In this case we hide the group shares + // and return the user share to the user. + other := rss[idx] + if other.Share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP && s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { + if other.State == rs.State { + rss[idx] = rs + } else { + rss = append(rss, rs) + } + } + } + } + + return rss, nil +} + +// convert must be called in a lock-controlled block. +func (m *mgr) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { + rs := &collaboration.ReceivedShare{ + Share: s, + State: collaboration.ShareState_SHARE_STATE_PENDING, + } + if v, ok := m.model.State[currentUser.String()]; ok { + if state, ok := v[s.Id.String()]; ok { + rs.State = state + } + } + if v, ok := m.model.MountPoint[currentUser.String()]; ok { + if mp, ok := v[s.Id.String()]; ok { + rs.MountPoint = mp + } + } + return rs +} + +func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { + return m.getReceived(ctx, ref) +} + +func (m *mgr) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { + m.Lock() + defer m.Unlock() + _, s, err := m.get(ref) + if err != nil { + return nil, err + } + user := ctxpkg.ContextMustGetUser(ctx) + if !share.IsGrantedToUser(s, user) { + return nil, errtypes.NotFound(ref.String()) + } + return m.convert(user.Id, s), nil +} + +func (m *mgr) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { + rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}}) + if err != nil { + return nil, err + } + + m.Lock() + defer m.Unlock() + + for i := range fieldMask.Paths { + switch fieldMask.Paths[i] { + case "state": + rs.State = receivedShare.State + case "mount_point": + rs.MountPoint = receivedShare.MountPoint + default: + return nil, errtypes.NotSupported("updating " + fieldMask.Paths[i] + " is not supported") + } + } + + user := ctxpkg.ContextMustGetUser(ctx) + // Persist state + if v, ok := m.model.State[user.Id.String()]; ok { + v[rs.Share.Id.String()] = rs.State + m.model.State[user.Id.String()] = v + } else { + a := map[string]collaboration.ShareState{ + rs.Share.Id.String(): rs.State, + } + m.model.State[user.Id.String()] = a + } + + // Persist mount point + if v, ok := m.model.MountPoint[user.Id.String()]; ok { + v[rs.Share.Id.String()] = rs.MountPoint + m.model.MountPoint[user.Id.String()] = v + } else { + a := map[string]*provider.Reference{ + rs.Share.Id.String(): rs.MountPoint, + } + m.model.MountPoint[user.Id.String()] = a + } + + if err := m.model.Save(); err != nil { + err = errors.Wrap(err, "error saving model") + return nil, err + } + + return rs, nil +} + +// Dump exports shares and received shares to channels (e.g. during migration) +func (m *mgr) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { + log := appctx.GetLogger(ctx) + for _, s := range m.model.Shares { + shareChan <- s + } + + for userIDString, states := range m.model.State { + userMountPoints := m.model.MountPoint[userIDString] + id := &userv1beta1.UserId{} + mV2 := proto.MessageV2(id) + if err := prototext.Unmarshal([]byte(userIDString), mV2); err != nil { + log.Error().Err(err).Msg("error unmarshalling the user id") + continue + } + + for shareIDString, state := range states { + sid := &collaboration.ShareId{} + mV2 := proto.MessageV2(sid) + if err := prototext.Unmarshal([]byte(shareIDString), mV2); err != nil { + log.Error().Err(err).Msg("error unmarshalling the user id") + continue + } + + var s *collaboration.Share + for _, is := range m.model.Shares { + if is.Id.OpaqueId == sid.OpaqueId { + s = is + break + } + } + if s == nil { + log.Warn().Str("share id", sid.OpaqueId).Msg("Share not found") + continue + } + + var mp *provider.Reference + if userMountPoints != nil { + mp = userMountPoints[shareIDString] + } + + receivedShareChan <- share.ReceivedShareWithUser{ + UserID: id, + ReceivedShare: &collaboration.ReceivedShare{ + Share: s, + State: state, + MountPoint: mp, + }, + } + } + } + + return nil +} diff --git a/pkg/share/manager/jsoncs3/jsoncs3_suite_test.go b/pkg/share/manager/jsoncs3/jsoncs3_suite_test.go new file mode 100644 index 0000000000..d51fd31569 --- /dev/null +++ b/pkg/share/manager/jsoncs3/jsoncs3_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package jsoncs3_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJson(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Json Suite") +} diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go new file mode 100644 index 0000000000..8901496c34 --- /dev/null +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -0,0 +1,178 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package jsoncs3_test + +import ( + "context" + "io/ioutil" + "os" + "sync" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" + "github.com/cs3org/reva/v2/pkg/share" + "github.com/cs3org/reva/v2/pkg/share/manager/json" + "google.golang.org/protobuf/types/known/fieldmaskpb" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Json", func() { + var ( + user1 = &userpb.User{ + Id: &userpb.UserId{ + Idp: "https://localhost:9200", + OpaqueId: "admin", + }, + } + user2 = &userpb.User{ + Id: &userpb.UserId{ + Idp: "https://localhost:9200", + OpaqueId: "einstein", + }, + } + + sharedResource = &providerv1beta1.ResourceInfo{ + Id: &providerv1beta1.ResourceId{ + StorageId: "storageid", + OpaqueId: "opaqueid", + }, + } + + m share.Manager + tmpFile *os.File + ctx context.Context + granteeCtx context.Context + ) + + BeforeEach(func() { + var err error + tmpFile, err = ioutil.TempFile("", "reva-unit-test-*.json") + Expect(err).ToNot(HaveOccurred()) + + config := map[string]interface{}{ + "file": tmpFile.Name(), + "gateway_addr": "https://localhost:9200", + } + m, err = json.New(config) + Expect(err).ToNot(HaveOccurred()) + + ctx = ctxpkg.ContextSetUser(context.Background(), user1) + granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) + }) + + AfterEach(func() { + os.Remove(tmpFile.Name()) + }) + + Describe("Dump", func() { + JustBeforeEach(func() { + share, err := m.Share(ctx, sharedResource, &collaboration.ShareGrant{ + Grantee: &providerv1beta1.Grantee{ + Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, + Id: &providerv1beta1.Grantee_UserId{UserId: user2.Id}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Id}}) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED + rs.MountPoint = &providerv1beta1.Reference{Path: "newPath/"} + + _, err = m.UpdateReceivedShare(granteeCtx, + rs, &fieldmaskpb.FieldMask{Paths: []string{"state", "mount_point"}}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("dumps all shares", func() { + sharesChan := make(chan *collaboration.Share) + receivedChan := make(chan share.ReceivedShareWithUser) + + shares := []*collaboration.Share{} + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + for s := range sharesChan { + if s != nil { + shares = append(shares, s) + } + } + wg.Done() + }() + go func() { + for range receivedChan { + } + wg.Done() + }() + err := m.(share.DumpableManager).Dump(ctx, sharesChan, receivedChan) + Expect(err).ToNot(HaveOccurred()) + close(sharesChan) + close(receivedChan) + wg.Wait() + + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].Creator).To(Equal(user1.Id)) + Expect(shares[0].Grantee.GetUserId()).To(Equal(user2.Id)) + Expect(shares[0].ResourceId).To(Equal(sharedResource.Id)) + }) + + It("dumps all received shares", func() { + sharesChan := make(chan *collaboration.Share) + receivedChan := make(chan share.ReceivedShareWithUser) + + shares := []share.ReceivedShareWithUser{} + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + for range sharesChan { + } + wg.Done() + }() + go func() { + for rs := range receivedChan { + if rs.UserID != nil && rs.ReceivedShare != nil { + shares = append(shares, rs) + } + } + + wg.Done() + }() + err := m.(share.DumpableManager).Dump(ctx, sharesChan, receivedChan) + Expect(err).ToNot(HaveOccurred()) + close(sharesChan) + close(receivedChan) + wg.Wait() + + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].UserID).To(Equal(user2.Id)) + Expect(shares[0].ReceivedShare.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + Expect(shares[0].ReceivedShare.MountPoint.Path).To(Equal("newPath/")) + Expect(shares[0].ReceivedShare.Share.Creator).To(Equal(user1.Id)) + Expect(shares[0].ReceivedShare.Share.Grantee.GetUserId()).To(Equal(user2.Id)) + Expect(shares[0].ReceivedShare.Share.ResourceId).To(Equal(sharedResource.Id)) + }) + }) +}) diff --git a/pkg/share/manager/loader/loader.go b/pkg/share/manager/loader/loader.go index b29c6ad31a..e67719976d 100644 --- a/pkg/share/manager/loader/loader.go +++ b/pkg/share/manager/loader/loader.go @@ -22,6 +22,7 @@ import ( // Load core share manager drivers. _ "github.com/cs3org/reva/v2/pkg/share/manager/cs3" _ "github.com/cs3org/reva/v2/pkg/share/manager/json" + _ "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" _ "github.com/cs3org/reva/v2/pkg/share/manager/memory" _ "github.com/cs3org/reva/v2/pkg/share/manager/owncloudsql" // Add your own here From ab2bfd38fce36f0726ee0aa3a62c68dcff852f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 20 Jul 2022 21:32:25 +0000 Subject: [PATCH 02/94] explore and note thoughts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 67 +++++++++++++++++++--------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index de63735df5..844d5c44bb 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -91,9 +91,9 @@ func New(m map[string]interface{}) (share.Manager, error) { } return &mgr{ - c: c, - model: model, - storagesCache: gcache.New(1000000).LFU().Build(), + c: c, + model: model, + spaceETags: gcache.New(1000000).LFU().Build(), }, nil } @@ -183,12 +183,12 @@ func (m *shareModel) Save() error { } type mgr struct { - c *config - sync.Mutex // concurrent access to the file - model *shareModel - serviceUser *userv1beta1.User - SpaceRoot *provider.ResourceId - storagesCache gcache.Cache + c *config + sync.Mutex // concurrent access to the file + model *shareModel + serviceUser *userv1beta1.User + SpaceRoot *provider.ResourceId + spaceETags gcache.Cache initialized bool } @@ -200,13 +200,13 @@ type mgr struct { // - the changed mtime can be used to determine when a space needs to be reread for redundant setups // when initializing we only initialize per user: -// - we list /{userid}/*, for every space we fetch /{storageid}/{spaceid}.json if we -// have not cached it yet, or if the /{userid}/{storageid}/{spaceid} etag changed +// - we list /{userid}/created/*, for every space we fetch /{storageid}/{spaceid}.json if we +// have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed // - if it does not exist we query the registry for every storage provider id, then // we traverse /{storageid}/ in the metadata storage to -// 1. create /{userid}/ -// 3. touch /{userid}/{storageid}/{spaceid} -// 2. touch /{userid}/{storageid} (not needed when mtime propagation is enabled) +// 1. create /{userid}/created +// 2. touch /{userid}/created/{storageid}${spaceid} +// TODO 3. split storageid from spaceid touch /{userid}/created/{storageid} && /{userid}/created/{storageid}/{spaceid} (not needed when mtime propagation is enabled) // we need to split the two lists: // /{userid}/received/{storageid}/{spaceid} @@ -214,7 +214,7 @@ type mgr struct { func (m *mgr) initialize(ctx context.Context) error { // if local copy is invalid fetch a new one - // invalid = net set || etag changed + // invalid = not set || etag changed if m.initialized { return nil } @@ -275,13 +275,31 @@ func (m *mgr) initialize(ctx context.Context) error { return err case cssr.Status.Code == rpcv1beta1.Code_CODE_OK: // for every space we fetch /{storageid}/{spaceid}.json if we - // have not cached it yet, or if the /{userid}/created/{storageid}/{spaceid} etag changed + // have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed for _, storageInfo := range res.Infos { // do we have spaces for this storage cached? - if spaces := m.storagesCache.Get(storageInfo.Name); spaces != nil { - // TODO return space if etag is same + etag := m.getCachedSpaceETag(storageInfo.Name) + if etag == "" || etag != storageInfo.Etag { + + // TODO update cache + // reread /{storageid}/{spaceid}.json ? + // hmm the dir listing for a /einstein-id/created/{storageid}${spaceid} might have a different + // etag than the one for /marie-id/created/{storageid}${spaceid} + // do we also need the mtime in addition to the etag? so we can determine which one is younger? + // currently if einstein creates a share in space a we do a stat for every + // other user with access to the space because we update the cached space etag AND we touch the + // /einstein-id/created/{storageid}${spaceid} ... which updates the mtime ... so we don't need + // the etag, but only the mtime of /einstein-id/created/{storageid}${spaceid} ? which we set to + // the /{storageid}/{spaceid}.json mtime. since we always do the mtime setting ... this should work + // well .. if cs3 touch allows setting the mtime + client.TouchFile(ctx, &provider.TouchFileRequest{ + Ref: &provider.Reference{}, + Opaque: &typespb.Opaque{ /*TODO allow setting the mtime with touch*/ }, + }) + // maybe we need SetArbitraryMetadata to set the mtime } - // TODO update cache + // + // TODO use space if etag is same } case cssr.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND: // if it does not exist we query the registry for every storage provider id, then @@ -296,6 +314,15 @@ func (m *mgr) initialize(ctx context.Context) error { return nil } +func (m *mgr) getCachedSpaceETag(spaceid string) string { + if e, err := m.spaceETags.Get(spaceid); err != gcache.KeyNotFoundError { + if etag, ok := e.(string); ok { + return etag + } + } + return "" +} + func (m *mgr) metadataClient() (provider.ProviderAPIClient, error) { return pool.GetStorageProviderServiceClient(m.c.ProviderAddr) } @@ -506,7 +533,7 @@ func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference } func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { - if err := m.initialize(); err != nil { + if err := m.initialize(ctx); err != nil { return nil, err } From f82c775b6ee5b1dd3ffb1559a4f6fe451ac117cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 25 Jul 2022 09:42:58 +0000 Subject: [PATCH 03/94] more thoughts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 844d5c44bb..3c3573417e 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -194,9 +194,25 @@ type mgr struct { } // We store the shares in the metadata storage under /{storageid}/{spaceid}.json -// To determine the spaces a user has access to we maintain an empty /{userid}/{storageid}/{spaceid} file + +// To persist the mountpoints of group shares the /{userid}/received/{storageid}/{spaceid}.json file is used. +// - it allows every user to update his own mountpoint without having to update&reread the /{storageid}/{spaceid}.json file + +// To persist the accepted / pending state of shares the /{userid}/received/{storageid}/{spaceid}.json file is used. +// - it allows every user to update his own mountpoint without having to update&reread the /{storageid}/{spaceid}.json file + +// To determine access to group shares a /{groupid}/received/{storageid}/{spaceid} file is used. + +// Whenever a share is created, the share manager has to +// 1. update the /{storageid}/{spaceid}.json file, +// 2. touch /{userid}/created/{storageid}/{spaceid} and +// 3. touch /{userid}/received/{storageid}/{spaceid}.json or /{groupid}/received/{storageid}/{spaceid} +// - The /{userid}/received/{storageid}/{spaceid}.json file persists mountpoints and accepted / rejected state +// - (optional) to wrap these three steps in a transaction the share manager can create a transaction file befor the first step and clean it up when all steps succeded + +// To determine the spaces a user has access to we maintain an empty /{userid}/(received|created)/{storageid}/{spaceid} folder // that we persist when initially traversing all shares in the metadata /{storageid}/{spaceid}.json files -// when a user creates a new share the jsoncs3 manager touches a new /{userid}/{storageid}/{spaceid} file +// when a user creates a new share the jsoncs3 manager touches a new /{userid}/(received|created)/{storageid}/{spaceid} folder // - the changed mtime can be used to determine when a space needs to be reread for redundant setups // when initializing we only initialize per user: From 9fe892f1064c3d089ead52524fc7718c39f6b71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 26 Jul 2022 09:43:57 +0200 Subject: [PATCH 04/94] Lay some groundwork --- pkg/share/manager/jsoncs3/jsoncs3.go | 230 +++++----------------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 119 +---------- 2 files changed, 59 insertions(+), 290 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 3c3573417e..f1ae8b8765 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -21,16 +21,14 @@ package jsoncs3 import ( "context" "encoding/json" - "io/fs" + "fmt" "io/ioutil" - "os" "path/filepath" "strings" "sync" "time" "github.com/bluele/gcache" - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" @@ -41,12 +39,12 @@ import ( "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/share" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/golang/protobuf/proto" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" - "google.golang.org/grpc/metadata" "google.golang.org/protobuf/encoding/prototext" "github.com/cs3org/reva/v2/pkg/share/manager/registry" @@ -66,83 +64,50 @@ import ( */ func init() { - registry.Register("jsoncs3", New) + registry.Register("jsoncs3", NewDefault) } -// New returns a new mgr. -func New(m map[string]interface{}) (share.Manager, error) { - c, err := parseConfig(m) - if err != nil { - err = errors.Wrap(err, "error creating a new manager") - return nil, err - } - - if c.GatewayAddr == "" { - return nil, errors.New("share manager config is missing gateway address") - } - - c.init() - - // load or create file - model, err := loadOrCreate(c.File) - if err != nil { - err = errors.Wrap(err, "error loading the file containing the shares") - return nil, err - } +type config struct { + GatewayAddr string `mapstructure:"gateway_addr"` + ProviderAddr string `mapstructure:"provider_addr"` + ServiceUserID string `mapstructure:"service_user_id"` + ServiceUserIdp string `mapstructure:"service_user_idp"` + MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` +} - return &mgr{ - c: c, - model: model, - spaceETags: gcache.New(1000000).LFU().Build(), - }, nil +type manager struct { + sync.RWMutex + storage metadata.Storage + spaceETags gcache.Cache + serviceUser *userv1beta1.User + SpaceRoot *provider.ResourceId + initialized bool } -func loadOrCreate(file string) (*shareModel, error) { - if info, err := os.Stat(file); errors.Is(err, fs.ErrNotExist) || info.Size() == 0 { - if err := ioutil.WriteFile(file, []byte("{}"), 0700); err != nil { - err = errors.Wrap(err, "error opening/creating the file: "+file) - return nil, err - } - } - - fd, err := os.OpenFile(file, os.O_CREATE, 0644) - if err != nil { - err = errors.Wrap(err, "error opening/creating the file: "+file) +// NewDefault returns a new manager instance with default dependencies +func NewDefault(m map[string]interface{}) (share.Manager, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error creating a new manager") return nil, err } - defer fd.Close() - data, err := ioutil.ReadAll(fd) + s, err := metadata.NewCS3Storage(c.GatewayAddr, c.ProviderAddr, c.ServiceUserID, c.ServiceUserIdp, c.MachineAuthAPIKey) if err != nil { - err = errors.Wrap(err, "error reading the data") return nil, err } - j := &jsonEncoding{} - if err := json.Unmarshal(data, j); err != nil { - err = errors.Wrap(err, "error decoding data from json") - return nil, err - } - - m := &shareModel{State: j.State, MountPoint: j.MountPoint} - for _, s := range j.Shares { - var decShare collaboration.Share - if err = utils.UnmarshalJSONToProtoV1([]byte(s), &decShare); err != nil { - return nil, errors.Wrap(err, "error decoding share from json") - } - m.Shares = append(m.Shares, &decShare) - } + return New(s) +} - if m.State == nil { - m.State = map[string]map[string]collaboration.ShareState{} - } - if m.MountPoint == nil { - m.MountPoint = map[string]map[string]*provider.Reference{} - } +// New returns a new manager instance. +func New(s metadata.Storage) (share.Manager, error) { + return &manager{ + storage: s, + spaceETags: gcache.New(1_000_000).LFU().Build(), + }, nil - m.file = file - return m, nil } type shareModel struct { @@ -182,17 +147,6 @@ func (m *shareModel) Save() error { return nil } -type mgr struct { - c *config - sync.Mutex // concurrent access to the file - model *shareModel - serviceUser *userv1beta1.User - SpaceRoot *provider.ResourceId - spaceETags gcache.Cache - - initialized bool -} - // We store the shares in the metadata storage under /{storageid}/{spaceid}.json // To persist the mountpoints of group shares the /{userid}/received/{storageid}/{spaceid}.json file is used. @@ -228,7 +182,7 @@ type mgr struct { // /{userid}/received/{storageid}/{spaceid} // /{userid}/created/{storageid}/{spaceid} -func (m *mgr) initialize(ctx context.Context) error { +func (m *manager) initialize(ctx context.Context) error { // if local copy is invalid fetch a new one // invalid = not set || etag changed if m.initialized { @@ -242,44 +196,15 @@ func (m *mgr) initialize(ctx context.Context) error { return nil } - user := ctxpkg.ContextMustGetUser(ctx) - - client, err := m.metadataClient() - if err != nil { - return err + user, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + return fmt.Errorf("missing user in context") } - ctx, err = m.getAuthContext(context.Background()) + err := m.storage.Init(context.Background(), "jsoncs3-share-manager-metadata") if err != nil { return err } - spaceid := "shares-space" - - // FIXME only create space if ListContainer fails - cssr, err := client.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ - Opaque: &typespb.Opaque{ - Map: map[string]*typespb.OpaqueEntry{ - "spaceid": { - Decoder: "plain", - Value: []byte(spaceid), - }, - }, - }, - Owner: m.serviceUser, - Name: "Shares", - Type: "metadata", - }) - switch { - case err != nil: - return err - case cssr.Status.Code == rpcv1beta1.Code_CODE_OK: - m.SpaceRoot = cssr.StorageSpace.Root - case cssr.Status.Code == rpcv1beta1.Code_CODE_ALREADY_EXISTS: - // TODO make CreateStorageSpace return existing space? - m.SpaceRoot = &provider.ResourceId{SpaceId: spaceid, OpaqueId: spaceid} - default: - return errtypes.NewErrtypeFromStatus(cssr.Status) - } // we list /{userid}/* || /{userid}/received ? created? res, err := client.ListContainer(ctx, &provider.ListContainerRequest{ @@ -330,7 +255,7 @@ func (m *mgr) initialize(ctx context.Context) error { return nil } -func (m *mgr) getCachedSpaceETag(spaceid string) string { +func (m *manager) getCachedSpaceETag(spaceid string) string { if e, err := m.spaceETags.Get(spaceid); err != gcache.KeyNotFoundError { if etag, ok := e.(string); ok { return etag @@ -339,60 +264,7 @@ func (m *mgr) getCachedSpaceETag(spaceid string) string { return "" } -func (m *mgr) metadataClient() (provider.ProviderAPIClient, error) { - return pool.GetStorageProviderServiceClient(m.c.ProviderAddr) -} - -func (m *mgr) getAuthContext(ctx context.Context) (context.Context, error) { - client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) - if err != nil { - return nil, err - } - - authCtx := ctxpkg.ContextSetUser(context.Background(), m.serviceUser) - authRes, err := client.Authenticate(authCtx, &gateway.AuthenticateRequest{ - Type: "machine", - ClientId: "userid:" + m.serviceUser.Id.OpaqueId, - ClientSecret: m.c.MachineAuthAPIKey, - }) - if err != nil { - return nil, err - } - if authRes.GetStatus().GetCode() != rpcv1beta1.Code_CODE_OK { - return nil, errtypes.NewErrtypeFromStatus(authRes.GetStatus()) - } - authCtx = metadata.AppendToOutgoingContext(authCtx, ctxpkg.TokenHeader, authRes.Token) - return authCtx, nil -} - -func (m *mgr) getClient() (gateway.GatewayAPIClient, error) { - return pool.GetGatewayServiceClient(m.c.GatewayAddr) -} - -type config struct { - File string `mapstructure:"file"` - GatewayAddr string `mapstructure:"gateway_addr"` - // ProviderAddr is the address of the metadata storage provider - ProviderAddr string `mapstructure:"provider_addr"` - ServiceUser string `mapstructure:"service_user"` - MachineAuthAPIKey string `mapstructure:"machine_auth_api_key"` -} - -func (c *config) init() { - if c.File == "" { - c.File = "/var/tmp/reva/shares.json" - } -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - -func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { +func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { id := uuid.NewString() user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() @@ -446,7 +318,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collabora } // getByID must be called in a lock-controlled block. -func (m *mgr) getByID(id *collaboration.ShareId) (int, *collaboration.Share, error) { +func (m *manager) getByID(id *collaboration.ShareId) (int, *collaboration.Share, error) { for i, s := range m.model.Shares { if s.GetId().OpaqueId == id.OpaqueId { return i, s, nil @@ -456,7 +328,7 @@ func (m *mgr) getByID(id *collaboration.ShareId) (int, *collaboration.Share, err } // getByKey must be called in a lock-controlled block. -func (m *mgr) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, error) { +func (m *manager) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, error) { for i, s := range m.model.Shares { if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) && utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) { @@ -467,7 +339,7 @@ func (m *mgr) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, } // get must be called in a lock-controlled block. -func (m *mgr) get(ref *collaboration.ShareReference) (idx int, s *collaboration.Share, err error) { +func (m *manager) get(ref *collaboration.ShareReference) (idx int, s *collaboration.Share, err error) { switch { case ref.GetId() != nil: idx, s, err = m.getByID(ref.GetId()) @@ -479,7 +351,7 @@ func (m *mgr) get(ref *collaboration.ShareReference) (idx int, s *collaboration. return } -func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { +func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { m.Lock() defer m.Unlock() _, s, err := m.get(ref) @@ -495,7 +367,7 @@ func (m *mgr) GetShare(ctx context.Context, ref *collaboration.ShareReference) ( return nil, errtypes.NotFound(ref.String()) } -func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { +func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() user := ctxpkg.ContextMustGetUser(ctx) @@ -521,7 +393,7 @@ func (m *mgr) Unshare(ctx context.Context, ref *collaboration.ShareReference) er return nil } -func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { +func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() idx, s, err := m.get(ref) @@ -548,7 +420,7 @@ func (m *mgr) UpdateShare(ctx context.Context, ref *collaboration.ShareReference return m.model.Shares[idx], nil } -func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { +func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { if err := m.initialize(ctx); err != nil { return nil, err } @@ -595,7 +467,7 @@ func (m *mgr) ListShares(ctx context.Context, filters []*collaboration.Filter) ( } // we list the shares that are targeted to the user in context or to the user groups. -func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { +func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() @@ -634,7 +506,7 @@ func (m *mgr) ListReceivedShares(ctx context.Context, filters []*collaboration.F } // convert must be called in a lock-controlled block. -func (m *mgr) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { +func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { rs := &collaboration.ReceivedShare{ Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, @@ -652,11 +524,11 @@ func (m *mgr) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) * return rs } -func (m *mgr) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { +func (m *manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { return m.getReceived(ctx, ref) } -func (m *mgr) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { +func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() _, s, err := m.get(ref) @@ -670,7 +542,7 @@ func (m *mgr) getReceived(ctx context.Context, ref *collaboration.ShareReference return m.convert(user.Id, s), nil } -func (m *mgr) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { +func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}}) if err != nil { return nil, err @@ -722,7 +594,7 @@ func (m *mgr) UpdateReceivedShare(ctx context.Context, receivedShare *collaborat } // Dump exports shares and received shares to channels (e.g. during migration) -func (m *mgr) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { +func (m *manager) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { log := appctx.GetLogger(ctx) for _, s := range m.model.Shares { shareChan <- s diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 8901496c34..72b4143d34 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -20,17 +20,13 @@ package jsoncs3_test import ( "context" - "io/ioutil" - "os" - "sync" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share" - "github.com/cs3org/reva/v2/pkg/share/manager/json" - "google.golang.org/protobuf/types/known/fieldmaskpb" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" + "github.com/stretchr/testify/mock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -58,121 +54,22 @@ var _ = Describe("Json", func() { }, } + storage *storagemocks.Storage m share.Manager - tmpFile *os.File ctx context.Context granteeCtx context.Context ) BeforeEach(func() { - var err error - tmpFile, err = ioutil.TempFile("", "reva-unit-test-*.json") - Expect(err).ToNot(HaveOccurred()) + storage = &storagemocks.Storage{} + storage.On("Init", mock.Anything, mock.Anything).Return(nil) + storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) + storage.On("SimpleUpload", mock.Anything, mock.Anything, mock.Anything).Return(nil) - config := map[string]interface{}{ - "file": tmpFile.Name(), - "gateway_addr": "https://localhost:9200", - } - m, err = json.New(config) + m, err = jsoncs3.New(storage) Expect(err).ToNot(HaveOccurred()) ctx = ctxpkg.ContextSetUser(context.Background(), user1) granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) }) - - AfterEach(func() { - os.Remove(tmpFile.Name()) - }) - - Describe("Dump", func() { - JustBeforeEach(func() { - share, err := m.Share(ctx, sharedResource, &collaboration.ShareGrant{ - Grantee: &providerv1beta1.Grantee{ - Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, - Id: &providerv1beta1.Grantee_UserId{UserId: user2.Id}, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: share.Id}}) - Expect(err).ToNot(HaveOccurred()) - Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) - rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED - rs.MountPoint = &providerv1beta1.Reference{Path: "newPath/"} - - _, err = m.UpdateReceivedShare(granteeCtx, - rs, &fieldmaskpb.FieldMask{Paths: []string{"state", "mount_point"}}) - Expect(err).ToNot(HaveOccurred()) - }) - - It("dumps all shares", func() { - sharesChan := make(chan *collaboration.Share) - receivedChan := make(chan share.ReceivedShareWithUser) - - shares := []*collaboration.Share{} - - wg := sync.WaitGroup{} - wg.Add(2) - go func() { - for s := range sharesChan { - if s != nil { - shares = append(shares, s) - } - } - wg.Done() - }() - go func() { - for range receivedChan { - } - wg.Done() - }() - err := m.(share.DumpableManager).Dump(ctx, sharesChan, receivedChan) - Expect(err).ToNot(HaveOccurred()) - close(sharesChan) - close(receivedChan) - wg.Wait() - - Expect(len(shares)).To(Equal(1)) - Expect(shares[0].Creator).To(Equal(user1.Id)) - Expect(shares[0].Grantee.GetUserId()).To(Equal(user2.Id)) - Expect(shares[0].ResourceId).To(Equal(sharedResource.Id)) - }) - - It("dumps all received shares", func() { - sharesChan := make(chan *collaboration.Share) - receivedChan := make(chan share.ReceivedShareWithUser) - - shares := []share.ReceivedShareWithUser{} - - wg := sync.WaitGroup{} - wg.Add(2) - go func() { - for range sharesChan { - } - wg.Done() - }() - go func() { - for rs := range receivedChan { - if rs.UserID != nil && rs.ReceivedShare != nil { - shares = append(shares, rs) - } - } - - wg.Done() - }() - err := m.(share.DumpableManager).Dump(ctx, sharesChan, receivedChan) - Expect(err).ToNot(HaveOccurred()) - close(sharesChan) - close(receivedChan) - wg.Wait() - - Expect(len(shares)).To(Equal(1)) - Expect(shares[0].UserID).To(Equal(user2.Id)) - Expect(shares[0].ReceivedShare.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) - Expect(shares[0].ReceivedShare.MountPoint.Path).To(Equal("newPath/")) - Expect(shares[0].ReceivedShare.Share.Creator).To(Equal(user1.Id)) - Expect(shares[0].ReceivedShare.Share.Grantee.GetUserId()).To(Equal(user2.Id)) - Expect(shares[0].ReceivedShare.Share.ResourceId).To(Equal(sharedResource.Id)) - }) - }) }) From ac6751c099c640d0bba3fd908844e4391c76f7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 27 Jul 2022 11:44:19 +0200 Subject: [PATCH 05/94] WIP --- pkg/share/manager/jsoncs3/jsoncs3.go | 555 +++++++++----------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 66 ++- pkg/storage/utils/metadata/cs3.go | 20 +- pkg/storage/utils/metadata/disk.go | 30 ++ pkg/storage/utils/metadata/mocks/Storage.go | 25 + pkg/storage/utils/metadata/storage.go | 3 + 6 files changed, 384 insertions(+), 315 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index f1ae8b8765..21080a73ed 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -20,32 +20,24 @@ package jsoncs3 import ( "context" - "encoding/json" "fmt" - "io/ioutil" "path/filepath" - "strings" "sync" "time" "github.com/bluele/gcache" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/v2/pkg/appctx" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/share" - "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" - "github.com/golang/protobuf/proto" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" - "google.golang.org/protobuf/encoding/prototext" "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/utils" @@ -75,9 +67,14 @@ type config struct { MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` } +type shareCache map[string]providerSpaces +type providerSpaces map[string]spaceShares +type spaceShares []*collaboration.Share + type manager struct { sync.RWMutex + cache shareCache storage metadata.Storage spaceETags gcache.Cache serviceUser *userv1beta1.User @@ -104,48 +101,19 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ + cache: shareCache{}, storage: s, spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil - -} - -type shareModel struct { - file string - State map[string]map[string]collaboration.ShareState `json:"state"` // map[username]map[share_id]ShareState - MountPoint map[string]map[string]*provider.Reference `json:"mount_point"` // map[username]map[share_id]MountPoint - Shares []*collaboration.Share `json:"shares"` -} - -type jsonEncoding struct { - State map[string]map[string]collaboration.ShareState `json:"state"` // map[username]map[share_id]ShareState - MountPoint map[string]map[string]*provider.Reference `json:"mount_point"` // map[username]map[share_id]MountPoint - Shares []string `json:"shares"` } -func (m *shareModel) Save() error { - j := &jsonEncoding{State: m.State, MountPoint: m.MountPoint} - for _, s := range m.Shares { - encShare, err := utils.MarshalProtoV1ToJSON(s) - if err != nil { - return errors.Wrap(err, "error encoding to json") - } - j.Shares = append(j.Shares, string(encShare)) - } - - data, err := json.Marshal(j) - if err != nil { - err = errors.Wrap(err, "error encoding to json") - return err - } - - if err := ioutil.WriteFile(m.file, data, 0644); err != nil { - err = errors.Wrap(err, "error writing to file: "+m.file) - return err - } - - return nil -} +// File structure in the jsoncs3 space: +// +// /shares/{shareid.json} // points to {storageid}/{spaceid} for looking up the share information +// /storages/{storageid}/{spaceid.json} // contains all share information of all shares in that space +// /users/{userid}/created/{storageid}/{spaceid} // points to a space the user created shares in +// /users/{userid}/received/{storageid}/{spaceid}.json // holds the states of received shares of the users in the according space +// /groups/{groupid}/received/{storageid}/{spaceid} // points to a space the group has received shares in // We store the shares in the metadata storage under /{storageid}/{spaceid}.json @@ -206,50 +174,36 @@ func (m *manager) initialize(ctx context.Context) error { return err } - // we list /{userid}/* || /{userid}/received ? created? - res, err := client.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ResourceId: m.SpaceRoot, Path: filepath.Join(user.Id.OpaqueId, "created")}, - }) - - switch { - case err != nil: + infos, err := m.storage.ListDir(ctx, filepath.Join("users", user.Id.OpaqueId, "created")) + if err != nil { return err - case cssr.Status.Code == rpcv1beta1.Code_CODE_OK: - // for every space we fetch /{storageid}/{spaceid}.json if we - // have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed - for _, storageInfo := range res.Infos { - // do we have spaces for this storage cached? - etag := m.getCachedSpaceETag(storageInfo.Name) - if etag == "" || etag != storageInfo.Etag { - - // TODO update cache - // reread /{storageid}/{spaceid}.json ? - // hmm the dir listing for a /einstein-id/created/{storageid}${spaceid} might have a different - // etag than the one for /marie-id/created/{storageid}${spaceid} - // do we also need the mtime in addition to the etag? so we can determine which one is younger? - // currently if einstein creates a share in space a we do a stat for every - // other user with access to the space because we update the cached space etag AND we touch the - // /einstein-id/created/{storageid}${spaceid} ... which updates the mtime ... so we don't need - // the etag, but only the mtime of /einstein-id/created/{storageid}${spaceid} ? which we set to - // the /{storageid}/{spaceid}.json mtime. since we always do the mtime setting ... this should work - // well .. if cs3 touch allows setting the mtime - client.TouchFile(ctx, &provider.TouchFileRequest{ - Ref: &provider.Reference{}, - Opaque: &typespb.Opaque{ /*TODO allow setting the mtime with touch*/ }, - }) - // maybe we need SetArbitraryMetadata to set the mtime - } - // - // TODO use space if etag is same + } + // for every space we fetch /{storageid}/{spaceid}.json if we + // have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed + for _, storageInfo := range infos { + // do we have spaces for this storage cached? + etag := m.getCachedSpaceETag(storageInfo.Name) + if etag == "" || etag != storageInfo.Etag { + + // TODO update cache + // reread /{storageid}/{spaceid}.json ? + // hmm the dir listing for a /einstein-id/created/{storageid}${spaceid} might have a different + // etag than the one for /marie-id/created/{storageid}${spaceid} + // do we also need the mtime in addition to the etag? so we can determine which one is younger? + // currently if einstein creates a share in space a we do a stat for every + // other user with access to the space because we update the cached space etag AND we touch the + // /einstein-id/created/{storageid}${spaceid} ... which updates the mtime ... so we don't need + // the etag, but only the mtime of /einstein-id/created/{storageid}${spaceid} ? which we set to + // the /{storageid}/{spaceid}.json mtime. since we always do the mtime setting ... this should work + // well .. if cs3 touch allows setting the mtime + // client.TouchFile(ctx, &provider.TouchFileRequest{ + // Ref: &provider.Reference{}, + // Opaque: &typespb.Opaque{ /*TODO allow setting the mtime with touch*/ }, + // }) + // maybe we need SetArbitraryMetadata to set the mtime } - case cssr.Status.Code == rpcv1beta1.Code_CODE_NOT_FOUND: - // if it does not exist we query the registry for every storage provider id, then - // we traverse /{storageid}/ in the metadata storage to - // 1. create /{userid}/ - // 3. touch /{userid}/{storageid}/{spaceid} - // 2. touch /{userid}/{storageid} (not needed when mtime propagation is enabled) - default: - return errtypes.NewErrtypeFromStatus(cssr.Status) + // + // TODO use space if etag is same } return nil @@ -308,33 +262,35 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Mtime: ts, } - m.model.Shares = append(m.model.Shares, s) - if err := m.model.Save(); err != nil { - err = errors.Wrap(err, "error saving model") - return nil, err + if m.cache[md.Id.StorageId] == nil { + m.cache[md.Id.StorageId] = providerSpaces{} } + if m.cache[md.Id.StorageId][md.Id.SpaceId] == nil { + m.cache[md.Id.StorageId][md.Id.SpaceId] = spaceShares{} + } + m.cache[md.Id.StorageId][md.Id.SpaceId] = append(m.cache[md.Id.StorageId][md.Id.SpaceId], s) return s, nil } // getByID must be called in a lock-controlled block. func (m *manager) getByID(id *collaboration.ShareId) (int, *collaboration.Share, error) { - for i, s := range m.model.Shares { - if s.GetId().OpaqueId == id.OpaqueId { - return i, s, nil - } - } + // for i, s := range m.model.Shares { + // if s.GetId().OpaqueId == id.OpaqueId { + // return i, s, nil + // } + // } return -1, nil, errtypes.NotFound(id.String()) } // getByKey must be called in a lock-controlled block. func (m *manager) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, error) { - for i, s := range m.model.Shares { - if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) && - utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) { - return i, s, nil - } - } + // for i, s := range m.model.Shares { + // if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) && + // utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) { + // return i, s, nil + // } + // } return -1, nil, errtypes.NotFound(key.String()) } @@ -370,54 +326,55 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) - - idx, s, err := m.get(ref) - if err != nil { - return err - } - if !share.IsCreatedByUser(s, user) { - return errtypes.NotFound(ref.String()) - } - - last := len(m.model.Shares) - 1 - m.model.Shares[idx] = m.model.Shares[last] - // explicitly nil the reference to prevent memory leaks - // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - m.model.Shares[last] = nil - m.model.Shares = m.model.Shares[:last] - if err := m.model.Save(); err != nil { - err = errors.Wrap(err, "error saving model") - return err - } + // user := ctxpkg.ContextMustGetUser(ctx) + + // idx, s, err := m.get(ref) + // if err != nil { + // return err + // } + // if !share.IsCreatedByUser(s, user) { + // return errtypes.NotFound(ref.String()) + // } + + // last := len(m.model.Shares) - 1 + // m.model.Shares[idx] = m.model.Shares[last] + // // explicitly nil the reference to prevent memory leaks + // // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order + // m.model.Shares[last] = nil + // m.model.Shares = m.model.Shares[:last] + // if err := m.model.Save(); err != nil { + // err = errors.Wrap(err, "error saving model") + // return err + // } return nil } func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - idx, s, err := m.get(ref) - if err != nil { - return nil, err - } - - user := ctxpkg.ContextMustGetUser(ctx) - if !share.IsCreatedByUser(s, user) { - return nil, errtypes.NotFound(ref.String()) - } - - now := time.Now().UnixNano() - m.model.Shares[idx].Permissions = p - m.model.Shares[idx].Mtime = &typespb.Timestamp{ - Seconds: uint64(now / int64(time.Second)), - Nanos: uint32(now % int64(time.Second)), - } - - if err := m.model.Save(); err != nil { - err = errors.Wrap(err, "error saving model") - return nil, err - } - return m.model.Shares[idx], nil + // idx, s, err := m.get(ref) + // if err != nil { + // return nil, err + // } + + // user := ctxpkg.ContextMustGetUser(ctx) + // if !share.IsCreatedByUser(s, user) { + // return nil, errtypes.NotFound(ref.String()) + // } + + // now := time.Now().UnixNano() + // m.model.Shares[idx].Permissions = p + // m.model.Shares[idx].Mtime = &typespb.Timestamp{ + // Seconds: uint64(now / int64(time.Second)), + // Nanos: uint32(now % int64(time.Second)), + // } + + // if err := m.model.Save(); err != nil { + // err = errors.Wrap(err, "error saving model") + // return nil, err + // } + // return m.model.Shares[idx], nil + return nil, nil } func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { @@ -427,42 +384,42 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte m.Lock() defer m.Unlock() - log := appctx.GetLogger(ctx) - user := ctxpkg.ContextMustGetUser(ctx) - - client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) - if err != nil { - return nil, errors.Wrap(err, "failed to list shares") - } - cache := make(map[string]struct{}) + // log := appctx.GetLogger(ctx) + // user := ctxpkg.ContextMustGetUser(ctx) + + // client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) + // if err != nil { + // return nil, errors.Wrap(err, "failed to list shares") + // } + // cache := make(map[string]struct{}) var ss []*collaboration.Share - for _, s := range m.model.Shares { - if share.MatchesFilters(s, filters) { - // Only add the share if the share was created by the user or if - // the user has ListGrants permissions on the shared resource. - // The ListGrants check is necessary when a space member wants - // to list shares in a space. - // We are using a cache here so that we don't have to stat a - // resource multiple times. - key := strings.Join([]string{s.ResourceId.StorageId, s.ResourceId.OpaqueId}, "!") - if _, hit := cache[key]; !hit && !share.IsCreatedByUser(s, user) { - sRes, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: s.ResourceId}}) - if err != nil || sRes.Status.Code != rpcv1beta1.Code_CODE_OK { - log.Error(). - Err(err). - Interface("status", sRes.Status). - Interface("resource_id", s.ResourceId). - Msg("ListShares: could not stat resource") - continue - } - if !sRes.Info.PermissionSet.ListGrants { - continue - } - cache[key] = struct{}{} - } - ss = append(ss, s) - } - } + // for _, s := range m.model.Shares { + // if share.MatchesFilters(s, filters) { + // // Only add the share if the share was created by the user or if + // // the user has ListGrants permissions on the shared resource. + // // The ListGrants check is necessary when a space member wants + // // to list shares in a space. + // // We are using a cache here so that we don't have to stat a + // // resource multiple times. + // key := strings.Join([]string{s.ResourceId.StorageId, s.ResourceId.OpaqueId}, "!") + // if _, hit := cache[key]; !hit && !share.IsCreatedByUser(s, user) { + // sRes, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: s.ResourceId}}) + // if err != nil || sRes.Status.Code != rpcv1beta1.Code_CODE_OK { + // log.Error(). + // Err(err). + // Interface("status", sRes.Status). + // Interface("resource_id", s.ResourceId). + // Msg("ListShares: could not stat resource") + // continue + // } + // if !sRes.Info.PermissionSet.ListGrants { + // continue + // } + // cache[key] = struct{}{} + // } + // ss = append(ss, s) + // } + // } return ss, nil } @@ -471,36 +428,36 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati m.Lock() defer m.Unlock() - user := ctxpkg.ContextMustGetUser(ctx) - mem := make(map[string]int) + // user := ctxpkg.ContextMustGetUser(ctx) + // mem := make(map[string]int) var rss []*collaboration.ReceivedShare - for _, s := range m.model.Shares { - if !share.IsCreatedByUser(s, user) && - share.IsGrantedToUser(s, user) && - share.MatchesFilters(s, filters) { - - rs := m.convert(user.Id, s) - idx, seen := mem[s.ResourceId.OpaqueId] - if !seen { - rss = append(rss, rs) - mem[s.ResourceId.OpaqueId] = len(rss) - 1 - continue - } - - // When we arrive here there was already a share for this resource. - // if there is a mix-up of shares of type group and shares of type user we need to deduplicate them, since it points - // to the same resource. Leave the more explicit and hide the less explicit. In this case we hide the group shares - // and return the user share to the user. - other := rss[idx] - if other.Share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP && s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { - if other.State == rs.State { - rss[idx] = rs - } else { - rss = append(rss, rs) - } - } - } - } + // for _, s := range m.model.Shares { + // if !share.IsCreatedByUser(s, user) && + // share.IsGrantedToUser(s, user) && + // share.MatchesFilters(s, filters) { + + // rs := m.convert(user.Id, s) + // idx, seen := mem[s.ResourceId.OpaqueId] + // if !seen { + // rss = append(rss, rs) + // mem[s.ResourceId.OpaqueId] = len(rss) - 1 + // continue + // } + + // // When we arrive here there was already a share for this resource. + // // if there is a mix-up of shares of type group and shares of type user we need to deduplicate them, since it points + // // to the same resource. Leave the more explicit and hide the less explicit. In this case we hide the group shares + // // and return the user share to the user. + // other := rss[idx] + // if other.Share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP && s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { + // if other.State == rs.State { + // rss[idx] = rs + // } else { + // rss = append(rss, rs) + // } + // } + // } + // } return rss, nil } @@ -511,16 +468,16 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - if v, ok := m.model.State[currentUser.String()]; ok { - if state, ok := v[s.Id.String()]; ok { - rs.State = state - } - } - if v, ok := m.model.MountPoint[currentUser.String()]; ok { - if mp, ok := v[s.Id.String()]; ok { - rs.MountPoint = mp - } - } + // if v, ok := m.model.State[currentUser.String()]; ok { + // if state, ok := v[s.Id.String()]; ok { + // rs.State = state + // } + // } + // if v, ok := m.model.MountPoint[currentUser.String()]; ok { + // if mp, ok := v[s.Id.String()]; ok { + // rs.MountPoint = mp + // } + // } return rs } @@ -562,88 +519,88 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab } } - user := ctxpkg.ContextMustGetUser(ctx) + // user := ctxpkg.ContextMustGetUser(ctx) // Persist state - if v, ok := m.model.State[user.Id.String()]; ok { - v[rs.Share.Id.String()] = rs.State - m.model.State[user.Id.String()] = v - } else { - a := map[string]collaboration.ShareState{ - rs.Share.Id.String(): rs.State, - } - m.model.State[user.Id.String()] = a - } - - // Persist mount point - if v, ok := m.model.MountPoint[user.Id.String()]; ok { - v[rs.Share.Id.String()] = rs.MountPoint - m.model.MountPoint[user.Id.String()] = v - } else { - a := map[string]*provider.Reference{ - rs.Share.Id.String(): rs.MountPoint, - } - m.model.MountPoint[user.Id.String()] = a - } - - if err := m.model.Save(); err != nil { - err = errors.Wrap(err, "error saving model") - return nil, err - } + // if v, ok := m.model.State[user.Id.String()]; ok { + // v[rs.Share.Id.String()] = rs.State + // m.model.State[user.Id.String()] = v + // } else { + // a := map[string]collaboration.ShareState{ + // rs.Share.Id.String(): rs.State, + // } + // m.model.State[user.Id.String()] = a + // } + + // // Persist mount point + // if v, ok := m.model.MountPoint[user.Id.String()]; ok { + // v[rs.Share.Id.String()] = rs.MountPoint + // m.model.MountPoint[user.Id.String()] = v + // } else { + // a := map[string]*provider.Reference{ + // rs.Share.Id.String(): rs.MountPoint, + // } + // m.model.MountPoint[user.Id.String()] = a + // } + + // if err := m.model.Save(); err != nil { + // err = errors.Wrap(err, "error saving model") + // return nil, err + // } return rs, nil } -// Dump exports shares and received shares to channels (e.g. during migration) -func (m *manager) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { - log := appctx.GetLogger(ctx) - for _, s := range m.model.Shares { - shareChan <- s - } - - for userIDString, states := range m.model.State { - userMountPoints := m.model.MountPoint[userIDString] - id := &userv1beta1.UserId{} - mV2 := proto.MessageV2(id) - if err := prototext.Unmarshal([]byte(userIDString), mV2); err != nil { - log.Error().Err(err).Msg("error unmarshalling the user id") - continue - } - - for shareIDString, state := range states { - sid := &collaboration.ShareId{} - mV2 := proto.MessageV2(sid) - if err := prototext.Unmarshal([]byte(shareIDString), mV2); err != nil { - log.Error().Err(err).Msg("error unmarshalling the user id") - continue - } - - var s *collaboration.Share - for _, is := range m.model.Shares { - if is.Id.OpaqueId == sid.OpaqueId { - s = is - break - } - } - if s == nil { - log.Warn().Str("share id", sid.OpaqueId).Msg("Share not found") - continue - } - - var mp *provider.Reference - if userMountPoints != nil { - mp = userMountPoints[shareIDString] - } - - receivedShareChan <- share.ReceivedShareWithUser{ - UserID: id, - ReceivedShare: &collaboration.ReceivedShare{ - Share: s, - State: state, - MountPoint: mp, - }, - } - } - } - - return nil -} +// // Dump exports shares and received shares to channels (e.g. during migration) +// func (m *manager) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { +// log := appctx.GetLogger(ctx) +// for _, s := range m.model.Shares { +// shareChan <- s +// } + +// for userIDString, states := range m.model.State { +// userMountPoints := m.model.MountPoint[userIDString] +// id := &userv1beta1.UserId{} +// mV2 := proto.MessageV2(id) +// if err := prototext.Unmarshal([]byte(userIDString), mV2); err != nil { +// log.Error().Err(err).Msg("error unmarshalling the user id") +// continue +// } + +// for shareIDString, state := range states { +// sid := &collaboration.ShareId{} +// mV2 := proto.MessageV2(sid) +// if err := prototext.Unmarshal([]byte(shareIDString), mV2); err != nil { +// log.Error().Err(err).Msg("error unmarshalling the user id") +// continue +// } + +// var s *collaboration.Share +// for _, is := range m.model.Shares { +// if is.Id.OpaqueId == sid.OpaqueId { +// s = is +// break +// } +// } +// if s == nil { +// log.Warn().Str("share id", sid.OpaqueId).Msg("Share not found") +// continue +// } + +// var mp *provider.Reference +// if userMountPoints != nil { +// mp = userMountPoints[shareIDString] +// } + +// receivedShareChan <- share.ReceivedShareWithUser{ +// UserID: id, +// ReceivedShare: &collaboration.ReceivedShare{ +// Share: s, +// State: state, +// MountPoint: mp, +// }, +// } +// } +// } + +// return nil +// } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 72b4143d34..9ab52263f8 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -22,10 +22,13 @@ import ( "context" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" + storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" "github.com/stretchr/testify/mock" . "github.com/onsi/ginkgo/v2" @@ -40,12 +43,12 @@ var _ = Describe("Json", func() { OpaqueId: "admin", }, } - user2 = &userpb.User{ - Id: &userpb.UserId{ - Idp: "https://localhost:9200", - OpaqueId: "einstein", - }, - } + // user2 = &userpb.User{ + // Id: &userpb.UserId{ + // Idp: "https://localhost:9200", + // OpaqueId: "einstein", + // }, + // } sharedResource = &providerv1beta1.ResourceInfo{ Id: &providerv1beta1.ResourceId{ @@ -54,10 +57,33 @@ var _ = Describe("Json", func() { }, } - storage *storagemocks.Storage - m share.Manager - ctx context.Context - granteeCtx context.Context + grantee = &userpb.User{ + Id: &userpb.UserId{ + Idp: "localhost:1111", + OpaqueId: "2", + }, + Groups: []string{"users"}, + } + grant = &collaboration.ShareGrant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{UserId: grantee.GetId()}, + }, + Permissions: &collaboration.SharePermissions{ + Permissions: &provider.ResourcePermissions{ + GetPath: true, + InitiateFileDownload: true, + ListFileVersions: true, + ListContainer: true, + Stat: true, + }, + }, + } + + storage *storagemocks.Storage + m share.Manager + ctx context.Context + // granteeCtx context.Context ) BeforeEach(func() { @@ -66,10 +92,28 @@ var _ = Describe("Json", func() { storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) storage.On("SimpleUpload", mock.Anything, mock.Anything, mock.Anything).Return(nil) + var err error m, err = jsoncs3.New(storage) Expect(err).ToNot(HaveOccurred()) ctx = ctxpkg.ContextSetUser(context.Background(), user1) - granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) + // granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) + }) + + Describe("Share", func() { + It("fails if the share already exists", func() { + _, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + _, err = m.Share(ctx, sharedResource, grant) + Expect(err).To(HaveOccurred()) + }) + + It("creates a share", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + Expect(share).ToNot(BeNil()) + Expect(share.ResourceId).To(Equal(sharedResource.Id)) + }) }) }) diff --git a/pkg/storage/utils/metadata/cs3.go b/pkg/storage/utils/metadata/cs3.go index f8ae1157ef..ade6a72d66 100644 --- a/pkg/storage/utils/metadata/cs3.go +++ b/pkg/storage/utils/metadata/cs3.go @@ -255,6 +255,20 @@ func (cs3 *CS3) Delete(ctx context.Context, path string) error { // ReadDir returns the entries in a given directory func (cs3 *CS3) ReadDir(ctx context.Context, path string) ([]string, error) { + infos, err := cs3.ListDir(ctx, path) + if err != nil { + return nil, err + } + + entries := []string{} + for _, ri := range infos { + entries = append(entries, ri.Path) + } + return entries, nil +} + +// ListDir returns a list of ResourceInfos for the entries in a given directory +func (cs3 *CS3) ListDir(ctx context.Context, path string) ([]*provider.ResourceInfo, error) { client, err := cs3.providerClient() if err != nil { return nil, err @@ -279,11 +293,7 @@ func (cs3 *CS3) ReadDir(ctx context.Context, path string) ([]string, error) { return nil, errtypes.NewErrtypeFromStatus(res.Status) } - entries := []string{} - for _, ri := range res.Infos { - entries = append(entries, ri.Path) - } - return entries, nil + return res.Infos, nil } // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context. diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index 81062562ce..53b138c290 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -24,6 +24,9 @@ import ( "io/ioutil" "os" "path" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ) // Disk represents a disk metadata storage @@ -80,6 +83,33 @@ func (disk *Disk) ReadDir(_ context.Context, p string) ([]string, error) { return entries, nil } +// ListDir returns a list of ResourceInfos for the entries in a given directory +func (disk *Disk) ListDir(ctx context.Context, path string) ([]*provider.ResourceInfo, error) { + infos, err := ioutil.ReadDir(disk.targetPath(path)) + if err != nil { + if _, ok := err.(*fs.PathError); ok { + return []*provider.ResourceInfo{}, nil + } + return nil, err + } + + entries := make([]*provider.ResourceInfo, 0, len(infos)) + for _, info := range infos { + entry := &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Path: "./" + info.Name(), + Name: info.Name(), + Mtime: &typesv1beta1.Timestamp{Seconds: uint64(info.ModTime().Unix()), Nanos: uint32(info.ModTime().Nanosecond())}, + } + if info.IsDir() { + entry.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + entry.Type = provider.ResourceType_RESOURCE_TYPE_FILE + entries = append(entries, entry) + } + return entries, nil +} + // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context. func (disk *Disk) MakeDirIfNotExist(_ context.Context, path string) error { return os.MkdirAll(disk.targetPath(path), 0777) diff --git a/pkg/storage/utils/metadata/mocks/Storage.go b/pkg/storage/utils/metadata/mocks/Storage.go index f783ed274a..aaa37c9e76 100644 --- a/pkg/storage/utils/metadata/mocks/Storage.go +++ b/pkg/storage/utils/metadata/mocks/Storage.go @@ -25,6 +25,8 @@ import ( mock "github.com/stretchr/testify/mock" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + testing "testing" ) @@ -89,6 +91,29 @@ func (_m *Storage) Init(ctx context.Context, name string) error { return r0 } +// ListDir provides a mock function with given fields: ctx, path +func (_m *Storage) ListDir(ctx context.Context, path string) ([]*providerv1beta1.ResourceInfo, error) { + ret := _m.Called(ctx, path) + + var r0 []*providerv1beta1.ResourceInfo + if rf, ok := ret.Get(0).(func(context.Context, string) []*providerv1beta1.ResourceInfo); ok { + r0 = rf(ctx, path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*providerv1beta1.ResourceInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, path) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // MakeDirIfNotExist provides a mock function with given fields: ctx, name func (_m *Storage) MakeDirIfNotExist(ctx context.Context, name string) error { ret := _m.Called(ctx, name) diff --git a/pkg/storage/utils/metadata/storage.go b/pkg/storage/utils/metadata/storage.go index dda9bc6726..53e4513e9f 100644 --- a/pkg/storage/utils/metadata/storage.go +++ b/pkg/storage/utils/metadata/storage.go @@ -20,6 +20,8 @@ package metadata import ( "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) //go:generate make --no-print-directory -C ../../../.. mockery NAME=Storage @@ -34,6 +36,7 @@ type Storage interface { Delete(ctx context.Context, path string) error ReadDir(ctx context.Context, path string) ([]string, error) + ListDir(ctx context.Context, path string) ([]*provider.ResourceInfo, error) CreateSymlink(ctx context.Context, oldname, newname string) error ResolveSymlink(ctx context.Context, name string) (string, error) From b080c7077817bbc68a7495fe52a7604a8a086bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Jul 2022 10:23:47 +0000 Subject: [PATCH 06/94] implement getByKey and getByID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 57 +++++++++++++++++----------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 21080a73ed..fdbce53f61 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages + "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -236,14 +237,14 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // check if share already exists. key := &collaboration.ShareKey{ - Owner: md.Owner, + //Owner: md.Owner, owner not longer matters as it belongs to the space ResourceId: md.Id, Grantee: g.Grantee, } m.Lock() defer m.Unlock() - _, _, err := m.getByKey(key) + _, err := m.getByKey(key) if err == nil { // share already exists return nil, errtypes.AlreadyExists(key.String()) @@ -274,33 +275,45 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } // getByID must be called in a lock-controlled block. -func (m *manager) getByID(id *collaboration.ShareId) (int, *collaboration.Share, error) { - // for i, s := range m.model.Shares { - // if s.GetId().OpaqueId == id.OpaqueId { - // return i, s, nil - // } - // } - return -1, nil, errtypes.NotFound(id.String()) +func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, error) { + shareid, err := storagespace.ParseID(id.OpaqueId) + if err != nil { + // invalid share id, does not exist + return nil, errtypes.NotFound(id.String()) + } + if providerSpaces, ok := m.cache[shareid.StorageId]; ok { + if spaceShares, ok := providerSpaces[shareid.SpaceId]; ok { + for _, share := range spaceShares { + if share.GetId().OpaqueId == shareid.OpaqueId { + return share, nil + } + } + } + } + return nil, errtypes.NotFound(id.String()) } // getByKey must be called in a lock-controlled block. -func (m *manager) getByKey(key *collaboration.ShareKey) (int, *collaboration.Share, error) { - // for i, s := range m.model.Shares { - // if (utils.UserEqual(key.Owner, s.Owner) || utils.UserEqual(key.Owner, s.Creator)) && - // utils.ResourceIDEqual(key.ResourceId, s.ResourceId) && utils.GranteeEqual(key.Grantee, s.Grantee) { - // return i, s, nil - // } - // } - return -1, nil, errtypes.NotFound(key.String()) +func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { + if providerSpaces, ok := m.cache[key.ResourceId.StorageId]; ok { + if spaceShares, ok := providerSpaces[key.ResourceId.SpaceId]; ok { + for _, share := range spaceShares { + if utils.GranteeEqual(key.Grantee, share.Grantee) { + return share, nil + } + } + } + } + return nil, errtypes.NotFound(key.String()) } // get must be called in a lock-controlled block. -func (m *manager) get(ref *collaboration.ShareReference) (idx int, s *collaboration.Share, err error) { +func (m *manager) get(ref *collaboration.ShareReference) (s *collaboration.Share, err error) { switch { case ref.GetId() != nil: - idx, s, err = m.getByID(ref.GetId()) + s, err = m.getByID(ref.GetId()) case ref.GetKey() != nil: - idx, s, err = m.getByKey(ref.GetKey()) + s, err = m.getByKey(ref.GetKey()) default: err = errtypes.NotFound(ref.String()) } @@ -310,7 +323,7 @@ func (m *manager) get(ref *collaboration.ShareReference) (idx int, s *collaborat func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - _, s, err := m.get(ref) + s, err := m.get(ref) if err != nil { return nil, err } @@ -488,7 +501,7 @@ func (m *manager) GetReceivedShare(ctx context.Context, ref *collaboration.Share func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() - _, s, err := m.get(ref) + s, err := m.get(ref) if err != nil { return nil, err } From 1a706435c68bd20f2b99b23f99fd4d96399b363e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Jul 2022 10:47:20 +0000 Subject: [PATCH 07/94] GetShare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 18 +++++++++--- pkg/share/manager/jsoncs3/jsoncs3_test.go | 36 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index fdbce53f61..71338e6dbb 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -220,7 +220,7 @@ func (m *manager) getCachedSpaceETag(spaceid string) string { } func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { - id := uuid.NewString() + user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() ts := &typespb.Timestamp{ @@ -249,7 +249,17 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // share already exists return nil, errtypes.AlreadyExists(key.String()) } - + shareReference := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: md.GetId().StorageId, + SpaceId: md.GetId().SpaceId, + OpaqueId: uuid.NewString(), + }, + } + id, err := storagespace.FormatReference(shareReference) + if err != nil { + return nil, err + } s := &collaboration.Share{ Id: &collaboration.ShareId{ OpaqueId: id, @@ -284,7 +294,7 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro if providerSpaces, ok := m.cache[shareid.StorageId]; ok { if spaceShares, ok := providerSpaces[shareid.SpaceId]; ok { for _, share := range spaceShares { - if share.GetId().OpaqueId == shareid.OpaqueId { + if share.GetId().OpaqueId == id.OpaqueId { return share, nil } } @@ -327,7 +337,7 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc if err != nil { return nil, err } - // check if we are the owner or the grantee + // check if we are the creator or the grantee user := ctxpkg.ContextMustGetUser(ctx) if share.IsCreatedByUser(s, user) || share.IsGrantedToUser(s, user) { return s, nil diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 9ab52263f8..eb7d4b1c2d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -53,6 +53,7 @@ var _ = Describe("Json", func() { sharedResource = &providerv1beta1.ResourceInfo{ Id: &providerv1beta1.ResourceId{ StorageId: "storageid", + SpaceId: "spaceid", OpaqueId: "opaqueid", }, } @@ -116,4 +117,39 @@ var _ = Describe("Json", func() { Expect(share.ResourceId).To(Equal(sharedResource.Id)) }) }) + + Describe("GetShare", func() { + It("retrieves an existing share by id", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(share.ResourceId).To(Equal(sharedResource.Id)) + }) + + It("retrieves an existing share by key", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.ResourceId).To(Equal(sharedResource.Id)) + Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) + }) + }) }) From 7fa2872783a96903e68bc177ba2e856fa5a55ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Jul 2022 11:20:42 +0000 Subject: [PATCH 08/94] Unshare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 52 +++++++++++----------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 54 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 27 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 71338e6dbb..9517d3cae9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -70,7 +70,7 @@ type config struct { type shareCache map[string]providerSpaces type providerSpaces map[string]spaceShares -type spaceShares []*collaboration.Share +type spaceShares gcache.Cache type manager struct { sync.RWMutex @@ -277,9 +277,9 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.cache[md.Id.StorageId] = providerSpaces{} } if m.cache[md.Id.StorageId][md.Id.SpaceId] == nil { - m.cache[md.Id.StorageId][md.Id.SpaceId] = spaceShares{} + m.cache[md.Id.StorageId][md.Id.SpaceId] = gcache.New(-1).Simple().Build() } - m.cache[md.Id.StorageId][md.Id.SpaceId] = append(m.cache[md.Id.StorageId][md.Id.SpaceId], s) + m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) return s, nil } @@ -293,9 +293,11 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } if providerSpaces, ok := m.cache[shareid.StorageId]; ok { if spaceShares, ok := providerSpaces[shareid.SpaceId]; ok { - for _, share := range spaceShares { - if share.GetId().OpaqueId == id.OpaqueId { - return share, nil + for _, value := range spaceShares.GetALL(false) { + if share, ok := value.(*collaboration.Share); ok { + if share.GetId().OpaqueId == id.OpaqueId { + return share, nil + } } } } @@ -307,9 +309,11 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { if providerSpaces, ok := m.cache[key.ResourceId.StorageId]; ok { if spaceShares, ok := providerSpaces[key.ResourceId.SpaceId]; ok { - for _, share := range spaceShares { - if utils.GranteeEqual(key.Grantee, share.Grantee) { - return share, nil + for _, value := range spaceShares.GetALL(false) { + if share, ok := value.(*collaboration.Share); ok { + if utils.GranteeEqual(key.Grantee, share.Grantee) { + return share, nil + } } } } @@ -349,26 +353,20 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() - // user := ctxpkg.ContextMustGetUser(ctx) + user := ctxpkg.ContextMustGetUser(ctx) - // idx, s, err := m.get(ref) - // if err != nil { - // return err - // } - // if !share.IsCreatedByUser(s, user) { - // return errtypes.NotFound(ref.String()) - // } + s, err := m.get(ref) + if err != nil { + return err + } + if !share.IsCreatedByUser(s, user) { + // TODO why not permission denied? + return errtypes.NotFound(ref.String()) + } + + shareid, err := storagespace.ParseID(s.Id.OpaqueId) + m.cache[shareid.StorageId][shareid.SpaceId].Remove(s.Id.OpaqueId) - // last := len(m.model.Shares) - 1 - // m.model.Shares[idx] = m.model.Shares[last] - // // explicitly nil the reference to prevent memory leaks - // // https://github.com/golang/go/wiki/SliceTricks#delete-without-preserving-order - // m.model.Shares[last] = nil - // m.model.Shares = m.model.Shares[:last] - // if err := m.model.Save(); err != nil { - // err = errors.Wrap(err, "error saving model") - // return err - // } return nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index eb7d4b1c2d..7d8eacb01b 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -122,6 +122,7 @@ var _ = Describe("Json", func() { It("retrieves an existing share by id", func() { share, err := m.Share(ctx, sharedResource, grant) Expect(err).ToNot(HaveOccurred()) + s, err := m.GetShare(ctx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ Id: &collaboration.ShareId{ @@ -152,4 +153,57 @@ var _ = Describe("Json", func() { Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) }) }) + + Describe("UnShare", func() { + It("removes an existing share by id", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + err = m.Unshare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) + }) + + It("removes an existing share by key", func() { + _, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + err = m.Unshare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) + }) + }) }) From afa977e16ae6021733bd1990a2758239cc7edf98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Jul 2022 11:28:37 +0000 Subject: [PATCH 09/94] UpdateShare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 32 +++++----- pkg/share/manager/jsoncs3/jsoncs3_test.go | 71 +++++++++++++++++++++++ 2 files changed, 87 insertions(+), 16 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 9517d3cae9..95d1faa7e2 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -373,29 +373,29 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - // idx, s, err := m.get(ref) - // if err != nil { - // return nil, err - // } + s, err := m.get(ref) + if err != nil { + return nil, err + } - // user := ctxpkg.ContextMustGetUser(ctx) - // if !share.IsCreatedByUser(s, user) { - // return nil, errtypes.NotFound(ref.String()) - // } + user := ctxpkg.ContextMustGetUser(ctx) + if !share.IsCreatedByUser(s, user) { + return nil, errtypes.NotFound(ref.String()) + } - // now := time.Now().UnixNano() - // m.model.Shares[idx].Permissions = p - // m.model.Shares[idx].Mtime = &typespb.Timestamp{ - // Seconds: uint64(now / int64(time.Second)), - // Nanos: uint32(now % int64(time.Second)), - // } + now := time.Now().UnixNano() + s.Permissions = p + s.Mtime = &typespb.Timestamp{ + Seconds: uint64(now / int64(time.Second)), + Nanos: uint32(now % int64(time.Second)), + } + // FIXME actually persist // if err := m.model.Save(); err != nil { // err = errors.Wrap(err, "error saving model") // return nil, err // } - // return m.model.Shares[idx], nil - return nil, nil + return s, nil } func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 7d8eacb01b..e7d22b0f8e 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -206,4 +206,75 @@ var _ = Describe("Json", func() { Expect(s).To(BeNil()) }) }) + + Describe("UpdateShare", func() { + It("updates an existing share by id", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }, &collaboration.SharePermissions{ + Permissions: &providerv1beta1.ResourcePermissions{ + Stat: true, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(us).ToNot(BeNil()) + Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + }) + + It("updates an existing share by key", func() { + _, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }, &collaboration.SharePermissions{ + Permissions: &providerv1beta1.ResourcePermissions{ + Stat: true, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(us).ToNot(BeNil()) + Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + }) + }) }) From f5e83fc9cdf82bea57c05ea2dcd641728237d819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Jul 2022 13:18:20 +0000 Subject: [PATCH 10/94] ListShares without filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 68 ++++++++++++++++++++--- pkg/share/manager/jsoncs3/jsoncs3_test.go | 13 +++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 95d1faa7e2..049cffd543 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -72,10 +72,15 @@ type shareCache map[string]providerSpaces type providerSpaces map[string]spaceShares type spaceShares gcache.Cache +type createdCache map[string]createdSpaces +type createdSpaces gcache.Cache + type manager struct { sync.RWMutex - cache shareCache + cache shareCache + createdCache createdCache + storage metadata.Storage spaceETags gcache.Cache serviceUser *userv1beta1.User @@ -102,9 +107,10 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ - cache: shareCache{}, - storage: s, - spaceETags: gcache.New(1_000_000).LFU().Build(), + cache: shareCache{}, + createdCache: createdCache{}, + storage: s, + spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil } @@ -281,6 +287,15 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) + if m.createdCache[user.Id.OpaqueId] == nil { + m.createdCache[user.Id.OpaqueId] = gcache.New(-1).Simple().Build() + } + // set flag for creator to have access to space + m.createdCache[user.Id.OpaqueId].Set(storagespace.FormatResourceID(provider.ResourceId{ + StorageId: md.Id.StorageId, + SpaceId: md.Id.SpaceId, + }), time.Now()) + return s, nil } @@ -398,22 +413,57 @@ func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer return s, nil } +// ListShares func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { - if err := m.initialize(ctx); err != nil { + /*if err := m.initialize(ctx); err != nil { return nil, err - } + }*/ m.Lock() defer m.Unlock() - // log := appctx.GetLogger(ctx) - // user := ctxpkg.ContextMustGetUser(ctx) + //log := appctx.GetLogger(ctx) + user := ctxpkg.ContextMustGetUser(ctx) + var ss []*collaboration.Share + + if m.createdCache[user.Id.OpaqueId] == nil { + return ss, nil + } + + for key, value := range m.createdCache[user.Id.OpaqueId].GetALL(false) { + var ssid string + var mtime time.Time + var ok bool + if ssid, ok = key.(string); !ok { + continue + } + if mtime, ok = value.(time.Time); !ok { + continue + } + if mtime.Sub(time.Now()) > time.Second*30 { + // TODO reread from disk + } + providerid, spaceid, _, err := storagespace.SplitID(ssid) + if err != nil { + continue + } + if providerSpaces, ok := m.cache[providerid]; ok { + if spaceShares, ok := providerSpaces[spaceid]; ok { + for _, value := range spaceShares.GetALL(false) { + if share, ok := value.(*collaboration.Share); ok { + if utils.UserEqual(user.GetId(), share.GetCreator()) { + ss = append(ss, share) + } + } + } + } + } + } // client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) // if err != nil { // return nil, errors.Wrap(err, "failed to list shares") // } // cache := make(map[string]struct{}) - var ss []*collaboration.Share // for _, s := range m.model.Shares { // if share.MatchesFilters(s, filters) { // // Only add the share if the share was created by the user or if diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index e7d22b0f8e..9aba0e75e9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -277,4 +277,17 @@ var _ = Describe("Json", func() { Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) }) }) + + Describe("ListShares", func() { + It("lists an existing share", func() { + share, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + shares, err := m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(1)) + + Expect(shares[0].Id).To(Equal(share.Id)) + }) + }) }) From 449ee9b55ae72dea6556c35e536b28cd8254b072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Jul 2022 10:02:26 +0200 Subject: [PATCH 11/94] DRY up tests --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 270 ++++++++++------------ 1 file changed, 128 insertions(+), 142 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 9aba0e75e9..277e4e0bbe 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -85,6 +85,18 @@ var _ = Describe("Json", func() { m share.Manager ctx context.Context // granteeCtx context.Context + + // helper functions + shareBykey = func(key *collaboration.ShareKey) *collaboration.Share { + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: key, + }, + }) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, s).ToNot(BeNil()) + return s + } ) BeforeEach(func() { @@ -118,176 +130,150 @@ var _ = Describe("Json", func() { }) }) - Describe("GetShare", func() { - It("retrieves an existing share by id", func() { - share, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: share.Id.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(s).ToNot(BeNil()) - Expect(share.ResourceId).To(Equal(sharedResource.Id)) - }) + Context("with an existing share", func() { + var ( + share *collaboration.Share + ) - It("retrieves an existing share by key", func() { - share, err := m.Share(ctx, sharedResource, grant) + BeforeEach(func() { + var err error + share, err = m.Share(ctx, sharedResource, grant) Expect(err).ToNot(HaveOccurred()) - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(s).ToNot(BeNil()) - Expect(s.ResourceId).To(Equal(sharedResource.Id)) - Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) }) - }) - - Describe("UnShare", func() { - It("removes an existing share by id", func() { - share, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - err = m.Unshare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: share.Id.OpaqueId, + Describe("GetShare", func() { + It("retrieves an existing share by id", func() { + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, }, - }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(share.ResourceId).To(Equal(sharedResource.Id)) }) - Expect(err).ToNot(HaveOccurred()) - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, - }, - }, + It("retrieves an existing share by key", func() { + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.ResourceId).To(Equal(sharedResource.Id)) + Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) }) - Expect(err).To(HaveOccurred()) - Expect(s).To(BeNil()) }) - It("removes an existing share by key", func() { - _, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - - err = m.Unshare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + Describe("UnShare", func() { + It("removes an existing share by id", func() { + err := m.Unshare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + }) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, }, - }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) }) - Expect(err).To(HaveOccurred()) - Expect(s).To(BeNil()) - }) - }) - - Describe("UpdateShare", func() { - It("updates an existing share by id", func() { - share, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: share.Id.OpaqueId, + It("removes an existing share by key", func() { + err := m.Unshare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, }, - }, - }, &collaboration.SharePermissions{ - Permissions: &providerv1beta1.ResourcePermissions{ - Stat: true, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) - - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + }) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, }, - }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) }) - Expect(err).ToNot(HaveOccurred()) - Expect(s).ToNot(BeNil()) - Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) }) - It("updates an existing share by key", func() { - _, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - - us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + Describe("UpdateShare", func() { + It("updates an existing share by id", func() { + us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, }, - }, - }, &collaboration.SharePermissions{ - Permissions: &providerv1beta1.ResourcePermissions{ - Stat: true, - }, + }, &collaboration.SharePermissions{ + Permissions: &providerv1beta1.ResourcePermissions{ + Stat: true, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(us).ToNot(BeNil()) + Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) }) - Expect(err).ToNot(HaveOccurred()) - Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + It("updates an existing share by key", func() { + us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, }, - }, + }, &collaboration.SharePermissions{ + Permissions: &providerv1beta1.ResourcePermissions{ + Stat: true, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(us).ToNot(BeNil()) + Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) + Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) }) - Expect(err).ToNot(HaveOccurred()) - Expect(s).ToNot(BeNil()) - Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) }) - }) - Describe("ListShares", func() { - It("lists an existing share", func() { - share, err := m.Share(ctx, sharedResource, grant) - Expect(err).ToNot(HaveOccurred()) - - shares, err := m.ListShares(ctx, nil) - Expect(err).ToNot(HaveOccurred()) - Expect(shares).To(HaveLen(1)) + Describe("ListShares", func() { + It("lists an existing share", func() { + shares, err := m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(1)) - Expect(shares[0].Id).To(Equal(share.Id)) + Expect(shares[0].Id).To(Equal(share.Id)) + }) }) }) }) From defcb219660d839852cd670bb807da89136b4302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Jul 2022 10:21:40 +0200 Subject: [PATCH 12/94] Extend and cleanup test suite --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 94 +++++++++-------------- 1 file changed, 38 insertions(+), 56 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 277e4e0bbe..7889f12a08 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -65,19 +65,28 @@ var _ = Describe("Json", func() { }, Groups: []string{"users"}, } + readPermissions = &provider.ResourcePermissions{ + GetPath: true, + InitiateFileDownload: true, + ListFileVersions: true, + ListContainer: true, + Stat: true, + } + writePermissions = &provider.ResourcePermissions{ + GetPath: true, + InitiateFileDownload: true, + InitiateFileUpload: true, + ListFileVersions: true, + ListContainer: true, + Stat: true, + } grant = &collaboration.ShareGrant{ Grantee: &provider.Grantee{ Type: provider.GranteeType_GRANTEE_TYPE_USER, Id: &provider.Grantee_UserId{UserId: grantee.GetId()}, }, Permissions: &collaboration.SharePermissions{ - Permissions: &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - }, + Permissions: readPermissions, }, } @@ -139,7 +148,6 @@ var _ = Describe("Json", func() { var err error share, err = m.Share(ctx, sharedResource, grant) Expect(err).ToNot(HaveOccurred()) - }) Describe("GetShare", func() { @@ -167,7 +175,7 @@ var _ = Describe("Json", func() { }) Describe("UnShare", func() { - It("removes an existing share by id", func() { + It("removes an existing share", func() { err := m.Unshare(ctx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ Id: &collaboration.ShareId{ @@ -188,33 +196,17 @@ var _ = Describe("Json", func() { Expect(err).To(HaveOccurred()) Expect(s).To(BeNil()) }) - - It("removes an existing share by key", func() { - err := m.Unshare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, - }, - }, - }) - Expect(err).To(HaveOccurred()) - Expect(s).To(BeNil()) - }) }) Describe("UpdateShare", func() { - It("updates an existing share by id", func() { + It("updates an existing share", func() { + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + + // enhance privileges us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ Id: &collaboration.ShareId{ @@ -222,47 +214,37 @@ var _ = Describe("Json", func() { }, }, }, &collaboration.SharePermissions{ - Permissions: &providerv1beta1.ResourcePermissions{ - Stat: true, - }, + Permissions: writePermissions, }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + Expect(us.GetPermissions().GetPermissions()).To(Equal(writePermissions)) - s := shareBykey(&collaboration.ShareKey{ + s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) - }) + Expect(s.GetPermissions().GetPermissions()).To(Equal(writePermissions)) - It("updates an existing share by key", func() { - us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Key{ - Key: &collaboration.ShareKey{ - ResourceId: sharedResource.Id, - Grantee: grant.Grantee, + // reduce privileges + us, err = m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, }, }, }, &collaboration.SharePermissions{ - Permissions: &providerv1beta1.ResourcePermissions{ - Stat: true, - }, + Permissions: readPermissions, }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(us.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + Expect(us.GetPermissions().GetPermissions()).To(Equal(readPermissions)) - s := shareBykey(&collaboration.ShareKey{ + s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions().Stat).To(BeTrue()) - Expect(s.GetPermissions().GetPermissions().ListContainer).To(BeFalse()) + Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) }) }) From 4b800a2fa673683a930881e7867dd9fdbfa92864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Jul 2022 10:42:33 +0200 Subject: [PATCH 13/94] Mininmal implementation for ListReceivedShares --- pkg/share/manager/jsoncs3/jsoncs3.go | 127 ++++++++++------------ pkg/share/manager/jsoncs3/jsoncs3_test.go | 40 ++++--- 2 files changed, 82 insertions(+), 85 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 049cffd543..e85e1fe57f 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -72,14 +72,15 @@ type shareCache map[string]providerSpaces type providerSpaces map[string]spaceShares type spaceShares gcache.Cache -type createdCache map[string]createdSpaces -type createdSpaces gcache.Cache +type accessCache map[string]spaceTimes +type spaceTimes gcache.Cache type manager struct { sync.RWMutex - cache shareCache - createdCache createdCache + cache shareCache + createdCache accessCache + receivedCache accessCache storage metadata.Storage spaceETags gcache.Cache @@ -107,10 +108,11 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ - cache: shareCache{}, - createdCache: createdCache{}, - storage: s, - spaceETags: gcache.New(1_000_000).LFU().Build(), + cache: shareCache{}, + createdCache: accessCache{}, + receivedCache: accessCache{}, + storage: s, + spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil } @@ -287,15 +289,24 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) + // set flag for creator to have access to space if m.createdCache[user.Id.OpaqueId] == nil { m.createdCache[user.Id.OpaqueId] = gcache.New(-1).Simple().Build() } - // set flag for creator to have access to space m.createdCache[user.Id.OpaqueId].Set(storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, }), time.Now()) + // set flag for grantee to have access to space + if m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] == nil { + m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] = gcache.New(-1).Simple().Build() + } + m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(storagespace.FormatResourceID(provider.ResourceId{ + StorageId: md.Id.StorageId, + SpaceId: md.Id.SpaceId, + }), time.Now()) + return s, nil } @@ -459,38 +470,6 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte } } - // client, err := pool.GetGatewayServiceClient(m.c.GatewayAddr) - // if err != nil { - // return nil, errors.Wrap(err, "failed to list shares") - // } - // cache := make(map[string]struct{}) - // for _, s := range m.model.Shares { - // if share.MatchesFilters(s, filters) { - // // Only add the share if the share was created by the user or if - // // the user has ListGrants permissions on the shared resource. - // // The ListGrants check is necessary when a space member wants - // // to list shares in a space. - // // We are using a cache here so that we don't have to stat a - // // resource multiple times. - // key := strings.Join([]string{s.ResourceId.StorageId, s.ResourceId.OpaqueId}, "!") - // if _, hit := cache[key]; !hit && !share.IsCreatedByUser(s, user) { - // sRes, err := client.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{ResourceId: s.ResourceId}}) - // if err != nil || sRes.Status.Code != rpcv1beta1.Code_CODE_OK { - // log.Error(). - // Err(err). - // Interface("status", sRes.Status). - // Interface("resource_id", s.ResourceId). - // Msg("ListShares: could not stat resource") - // continue - // } - // if !sRes.Info.PermissionSet.ListGrants { - // continue - // } - // cache[key] = struct{}{} - // } - // ss = append(ss, s) - // } - // } return ss, nil } @@ -499,37 +478,45 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati m.Lock() defer m.Unlock() - // user := ctxpkg.ContextMustGetUser(ctx) - // mem := make(map[string]int) var rss []*collaboration.ReceivedShare - // for _, s := range m.model.Shares { - // if !share.IsCreatedByUser(s, user) && - // share.IsGrantedToUser(s, user) && - // share.MatchesFilters(s, filters) { - - // rs := m.convert(user.Id, s) - // idx, seen := mem[s.ResourceId.OpaqueId] - // if !seen { - // rss = append(rss, rs) - // mem[s.ResourceId.OpaqueId] = len(rss) - 1 - // continue - // } - - // // When we arrive here there was already a share for this resource. - // // if there is a mix-up of shares of type group and shares of type user we need to deduplicate them, since it points - // // to the same resource. Leave the more explicit and hide the less explicit. In this case we hide the group shares - // // and return the user share to the user. - // other := rss[idx] - // if other.Share.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP && s.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { - // if other.State == rs.State { - // rss[idx] = rs - // } else { - // rss = append(rss, rs) - // } - // } - // } - // } + user := ctxpkg.ContextMustGetUser(ctx) + + if m.receivedCache[user.Id.OpaqueId] == nil { + return rss, nil + } + for key, value := range m.receivedCache[user.Id.OpaqueId].GetALL(false) { + var ssid string + var mtime time.Time + var ok bool + if ssid, ok = key.(string); !ok { + continue + } + if mtime, ok = value.(time.Time); !ok { + continue + } + if mtime.Sub(time.Now()) > time.Second*30 { + // TODO reread from disk + } + providerid, spaceid, _, err := storagespace.SplitID(ssid) + if err != nil { + continue + } + if providerSpaces, ok := m.cache[providerid]; ok { + if spaceShares, ok := providerSpaces[spaceid]; ok { + for _, value := range spaceShares.GetALL(false) { + if share, ok := value.(*collaboration.Share); ok { + if utils.UserEqual(user.GetId(), share.GetGrantee().GetUserId()) { + rs := &collaboration.ReceivedShare{ + Share: share, + } + rss = append(rss, rs) + } + } + } + } + } + } return rss, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 7889f12a08..4e59503e10 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -43,12 +43,12 @@ var _ = Describe("Json", func() { OpaqueId: "admin", }, } - // user2 = &userpb.User{ - // Id: &userpb.UserId{ - // Idp: "https://localhost:9200", - // OpaqueId: "einstein", - // }, - // } + user2 = &userpb.User{ + Id: &userpb.UserId{ + Idp: "https://localhost:9200", + OpaqueId: "einstein", + }, + } sharedResource = &providerv1beta1.ResourceInfo{ Id: &providerv1beta1.ResourceId{ @@ -59,10 +59,7 @@ var _ = Describe("Json", func() { } grantee = &userpb.User{ - Id: &userpb.UserId{ - Idp: "localhost:1111", - OpaqueId: "2", - }, + Id: user2.Id, Groups: []string{"users"}, } readPermissions = &provider.ResourcePermissions{ @@ -90,10 +87,10 @@ var _ = Describe("Json", func() { }, } - storage *storagemocks.Storage - m share.Manager - ctx context.Context - // granteeCtx context.Context + storage *storagemocks.Storage + m share.Manager + ctx context.Context + granteeCtx context.Context // helper functions shareBykey = func(key *collaboration.ShareKey) *collaboration.Share { @@ -119,7 +116,7 @@ var _ = Describe("Json", func() { Expect(err).ToNot(HaveOccurred()) ctx = ctxpkg.ContextSetUser(context.Background(), user1) - // granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) + granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) }) Describe("Share", func() { @@ -257,5 +254,18 @@ var _ = Describe("Json", func() { Expect(shares[0].Id).To(Equal(share.Id)) }) }) + + Describe("ListReceivedShares", func() { + PIt("filters by resource id") + PIt("filters by owner") + PIt("filters by creator") + PIt("filters by grantee type") + + It("lists the received shares", func() { + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + }) + }) }) }) From b7a937c7b60267ff0c6c0933bd5590d3eb2d280b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Jul 2022 11:51:10 +0200 Subject: [PATCH 14/94] Add a basic cache for the received shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 102 ++++++++++++---------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 40 +++++++++ 2 files changed, 98 insertions(+), 44 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index e85e1fe57f..657beddc08 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -68,19 +68,30 @@ type config struct { MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` } -type shareCache map[string]providerSpaces +type providerCache map[string]providerSpaces type providerSpaces map[string]spaceShares type spaceShares gcache.Cache -type accessCache map[string]spaceTimes +type createdCache map[string]spaceTimes type spaceTimes gcache.Cache +type receivedCache map[string]receivedSpaces +type receivedSpaces gcache.Cache +type receivedSpace struct { + mtime int64 + receivedShareStates map[string]receivedShareState +} +type receivedShareState struct { + state collaboration.ShareState + mountPoint *provider.Reference +} + type manager struct { sync.RWMutex - cache shareCache - createdCache accessCache - receivedCache accessCache + cache providerCache + createdCache createdCache + receivedCache receivedCache storage metadata.Storage spaceETags gcache.Cache @@ -108,9 +119,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ - cache: shareCache{}, - createdCache: accessCache{}, - receivedCache: accessCache{}, + cache: providerCache{}, + createdCache: createdCache{}, + receivedCache: receivedCache{}, storage: s, spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil @@ -300,12 +311,35 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // set flag for grantee to have access to space if m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] == nil { - m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] = gcache.New(-1).Simple().Build() + m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] = gcache.New(-1).Simple().Build() // receivedSpaces } - m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(storagespace.FormatResourceID(provider.ResourceId{ + spaceId := storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, - }), time.Now()) + }) + if !m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Has(spaceId) { + err := m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(spaceId, receivedSpace{}) + if err != nil { + return nil, err + } + } + val, err := m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].GetIFPresent(spaceId) + if err != nil && err != gcache.KeyNotFoundError { + return nil, err + } + receivedSpace, ok := val.(receivedSpace) + if !ok { + return nil, errtypes.InternalError("invalid type in cache") + } + receivedSpace.mtime = now + if receivedSpace.receivedShareStates == nil { + receivedSpace.receivedShareStates = map[string]receivedShareState{} + } + receivedSpace.receivedShareStates[id] = receivedShareState{ + state: collaboration.ShareState_SHARE_STATE_PENDING, + // mountpoint stays empty until user accepts the share + } + m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(spaceId, receivedSpace) return s, nil } @@ -487,28 +521,36 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati for key, value := range m.receivedCache[user.Id.OpaqueId].GetALL(false) { var ssid string - var mtime time.Time + var rspace receivedSpace var ok bool if ssid, ok = key.(string); !ok { continue } - if mtime, ok = value.(time.Time); !ok { + if rspace, ok = value.(receivedSpace); !ok { continue } - if mtime.Sub(time.Now()) > time.Second*30 { + + if rspace.mtime < time.Now().Add(-30*time.Second).UnixNano() { // TODO reread from disk } + providerid, spaceid, _, err := storagespace.SplitID(ssid) if err != nil { continue } if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { - for _, value := range spaceShares.GetALL(false) { + for shareId, state := range rspace.receivedShareStates { + value, err := spaceShares.Get(shareId) + if err != nil { + continue + } if share, ok := value.(*collaboration.Share); ok { if utils.UserEqual(user.GetId(), share.GetGrantee().GetUserId()) { rs := &collaboration.ReceivedShare{ - Share: share, + Share: share, + State: state.state, + MountPoint: state.mountPoint, } rss = append(rss, rs) } @@ -577,34 +619,6 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab } } - // user := ctxpkg.ContextMustGetUser(ctx) - // Persist state - // if v, ok := m.model.State[user.Id.String()]; ok { - // v[rs.Share.Id.String()] = rs.State - // m.model.State[user.Id.String()] = v - // } else { - // a := map[string]collaboration.ShareState{ - // rs.Share.Id.String(): rs.State, - // } - // m.model.State[user.Id.String()] = a - // } - - // // Persist mount point - // if v, ok := m.model.MountPoint[user.Id.String()]; ok { - // v[rs.Share.Id.String()] = rs.MountPoint - // m.model.MountPoint[user.Id.String()] = v - // } else { - // a := map[string]*provider.Reference{ - // rs.Share.Id.String(): rs.MountPoint, - // } - // m.model.MountPoint[user.Id.String()] = a - // } - - // if err := m.model.Save(); err != nil { - // err = errors.Wrap(err, "error saving model") - // return nil, err - // } - return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 4e59503e10..f6980819f4 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -30,6 +30,7 @@ import ( "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" "github.com/stretchr/testify/mock" + "google.golang.org/protobuf/types/known/fieldmaskpb" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -265,6 +266,45 @@ var _ = Describe("Json", func() { received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) Expect(err).ToNot(HaveOccurred()) Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.ResourceId).To(Equal(sharedResource.Id)) + Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + }) + }) + + Describe("GetReceivedShare", func() { + It("gets the state", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + }) + }) + + Describe("UpdateReceivedShare", func() { + It("updates the state", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + + rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED + rs, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"state"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + + rs, err = m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) }) }) }) From da4a2398067b01abd32a0c7c80d7fb8004e1f999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 28 Jul 2022 10:27:10 +0000 Subject: [PATCH 15/94] UpdateReceivedShare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 73 ++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 657beddc08..e801b5cdc9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -304,19 +304,16 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla if m.createdCache[user.Id.OpaqueId] == nil { m.createdCache[user.Id.OpaqueId] = gcache.New(-1).Simple().Build() } - m.createdCache[user.Id.OpaqueId].Set(storagespace.FormatResourceID(provider.ResourceId{ + spaceId := storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, - }), time.Now()) + }) + m.createdCache[user.Id.OpaqueId].Set(spaceId, time.Now()) // set flag for grantee to have access to space if m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] == nil { m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] = gcache.New(-1).Simple().Build() // receivedSpaces } - spaceId := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: md.Id.StorageId, - SpaceId: md.Id.SpaceId, - }) if !m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Has(spaceId) { err := m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(spaceId, receivedSpace{}) if err != nil { @@ -568,16 +565,25 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - // if v, ok := m.model.State[currentUser.String()]; ok { - // if state, ok := v[s.Id.String()]; ok { - // rs.State = state - // } - // } - // if v, ok := m.model.MountPoint[currentUser.String()]; ok { - // if mp, ok := v[s.Id.String()]; ok { - // rs.MountPoint = mp - // } - // } + + providerid, spaceid, _, err := storagespace.SplitID(s.Id.OpaqueId) + if err != nil { + return rs + } + spaceId := storagespace.FormatResourceID(provider.ResourceId{ + StorageId: providerid, + SpaceId: spaceid, + }) + v, err := m.receivedCache[currentUser.OpaqueId].Get(spaceId) + if err != nil { + return rs + } + if rspace, ok := v.(receivedSpace); ok { + if state, ok := rspace.receivedShareStates[s.Id.OpaqueId]; ok { + rs.State = state.state + rs.MountPoint = state.mountPoint + } + } return rs } @@ -619,6 +625,41 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab } } + newState := receivedShareState{ + state: rs.State, + mountPoint: rs.MountPoint, + } + + // write back + currentUser := ctxpkg.ContextMustGetUser(ctx) + spaceId := storagespace.FormatResourceID(provider.ResourceId{ + StorageId: rs.Share.ResourceId.StorageId, + SpaceId: rs.Share.ResourceId.SpaceId, + }) + v, err := m.receivedCache[currentUser.Id.OpaqueId].Get(spaceId) + switch { + case err == gcache.KeyNotFoundError: + // add new entry + m.receivedCache[currentUser.Id.OpaqueId].Set(spaceId, + receivedSpace{ + mtime: time.Now().UnixNano(), + receivedShareStates: map[string]receivedShareState{ + rs.Share.Id.OpaqueId: newState, + }, + }) + case err != nil: + // something went horribly wrong + return nil, err + + default: + // update entry + if rspace, ok := v.(receivedSpace); ok { + rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState + rspace.mtime = time.Now().UnixNano() + m.receivedCache[currentUser.Id.OpaqueId].Set(spaceId, rspace) + } + } + return rs, nil } From 0db511fb4ea2cefdc8a1e383b3cbe2ea7815010b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 28 Jul 2022 10:58:38 +0000 Subject: [PATCH 16/94] filter by id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 24 ++++++----- pkg/share/manager/jsoncs3/jsoncs3_test.go | 51 ++++++++++++++++++++++- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index e801b5cdc9..5ee399afc3 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -491,9 +491,11 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { for _, value := range spaceShares.GetALL(false) { - if share, ok := value.(*collaboration.Share); ok { - if utils.UserEqual(user.GetId(), share.GetCreator()) { - ss = append(ss, share) + if s, ok := value.(*collaboration.Share); ok { + if utils.UserEqual(user.GetId(), s.GetCreator()) { + if share.MatchesFilters(s, filters) { + ss = append(ss, s) + } } } } @@ -542,14 +544,16 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } - if share, ok := value.(*collaboration.Share); ok { - if utils.UserEqual(user.GetId(), share.GetGrantee().GetUserId()) { - rs := &collaboration.ReceivedShare{ - Share: share, - State: state.state, - MountPoint: state.mountPoint, + if s, ok := value.(*collaboration.Share); ok { + if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { + if share.MatchesFilters(s, filters) { + rs := &collaboration.ReceivedShare{ + Share: s, + State: state.state, + MountPoint: state.mountPoint, + } + rss = append(rss, rs) } - rss = append(rss, rs) } } } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index f6980819f4..88b89b5979 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -59,6 +59,14 @@ var _ = Describe("Json", func() { }, } + sharedResource2 = &providerv1beta1.ResourceInfo{ + Id: &providerv1beta1.ResourceId{ + StorageId: "storageid2", + SpaceId: "spaceid2", + OpaqueId: "opaqueid2", + }, + } + grantee = &userpb.User{ Id: user2.Id, Groups: []string{"users"}, @@ -128,13 +136,15 @@ var _ = Describe("Json", func() { Expect(err).To(HaveOccurred()) }) - It("creates a share", func() { + It("creates a user share", func() { share, err := m.Share(ctx, sharedResource, grant) Expect(err).ToNot(HaveOccurred()) Expect(share).ToNot(BeNil()) Expect(share.ResourceId).To(Equal(sharedResource.Id)) }) + + PIt("creates a group share") }) Context("with an existing share", func() { @@ -257,11 +267,48 @@ var _ = Describe("Json", func() { }) Describe("ListReceivedShares", func() { - PIt("filters by resource id") PIt("filters by owner") PIt("filters by creator") PIt("filters by grantee type") + It("filters by resource id", func() { + share2, err := m.Share(ctx, sharedResource2, grant) + Expect(err).ToNot(HaveOccurred()) + + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(2)) + + received, err = m.ListReceivedShares(granteeCtx, []*collaboration.Filter{ + { + Type: collaboration.Filter_TYPE_RESOURCE_ID, + Term: &collaboration.Filter_ResourceId{ + ResourceId: sharedResource.Id, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.ResourceId).To(Equal(sharedResource.Id)) + Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + Expect(received[0].Share.Id).To(Equal(share.Id)) + + received, err = m.ListReceivedShares(granteeCtx, []*collaboration.Filter{ + { + Type: collaboration.Filter_TYPE_RESOURCE_ID, + Term: &collaboration.Filter_ResourceId{ + ResourceId: sharedResource2.Id, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.ResourceId).To(Equal(sharedResource2.Id)) + Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + Expect(received[0].Share.Id).To(Equal(share2.Id)) + + }) + It("lists the received shares", func() { received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) Expect(err).ToNot(HaveOccurred()) From 33ac3a21e7167ec5a0c4093e63cc3405bd36d58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 28 Jul 2022 11:34:13 +0000 Subject: [PATCH 17/94] initial group accessCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 105 ++++++++++++++++++--------- 1 file changed, 71 insertions(+), 34 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 5ee399afc3..de36126db4 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -72,7 +72,7 @@ type providerCache map[string]providerSpaces type providerSpaces map[string]spaceShares type spaceShares gcache.Cache -type createdCache map[string]spaceTimes +type accessedCache map[string]spaceTimes type spaceTimes gcache.Cache type receivedCache map[string]receivedSpaces @@ -89,9 +89,10 @@ type receivedShareState struct { type manager struct { sync.RWMutex - cache providerCache - createdCache createdCache - receivedCache receivedCache + cache providerCache + createdCache accessedCache + receivedCache receivedCache + groupReceivedCache accessedCache storage metadata.Storage spaceETags gcache.Cache @@ -119,11 +120,12 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ - cache: providerCache{}, - createdCache: createdCache{}, - receivedCache: receivedCache{}, - storage: s, - spaceETags: gcache.New(1_000_000).LFU().Build(), + cache: providerCache{}, + createdCache: accessedCache{}, + receivedCache: receivedCache{}, + groupReceivedCache: accessedCache{}, + storage: s, + spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil } @@ -311,32 +313,42 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.createdCache[user.Id.OpaqueId].Set(spaceId, time.Now()) // set flag for grantee to have access to space - if m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] == nil { - m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()] = gcache.New(-1).Simple().Build() // receivedSpaces - } - if !m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Has(spaceId) { - err := m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(spaceId, receivedSpace{}) - if err != nil { + switch g.Grantee.Type { + case provider.GranteeType_GRANTEE_TYPE_USER: + userid := g.Grantee.GetUserId().GetOpaqueId() + if m.receivedCache[userid] == nil { + m.receivedCache[userid] = gcache.New(-1).Simple().Build() // receivedSpaces + } + if !m.receivedCache[userid].Has(spaceId) { + err := m.receivedCache[userid].Set(spaceId, receivedSpace{}) + if err != nil { + return nil, err + } + } + val, err := m.receivedCache[userid].GetIFPresent(spaceId) + if err != nil && err != gcache.KeyNotFoundError { return nil, err } + receivedSpace, ok := val.(receivedSpace) + if !ok { + return nil, errtypes.InternalError("invalid type in cache") + } + receivedSpace.mtime = now + if receivedSpace.receivedShareStates == nil { + receivedSpace.receivedShareStates = map[string]receivedShareState{} + } + receivedSpace.receivedShareStates[id] = receivedShareState{ + state: collaboration.ShareState_SHARE_STATE_PENDING, + // mountpoint stays empty until user accepts the share + } + m.receivedCache[userid].Set(spaceId, receivedSpace) + case provider.GranteeType_GRANTEE_TYPE_GROUP: + groupid := g.Grantee.GetGroupId().GetOpaqueId() + if m.groupReceivedCache[groupid] == nil { + m.groupReceivedCache[groupid] = gcache.New(-1).Simple().Build() + } + m.groupReceivedCache[groupid].Set(spaceId, time.Now()) } - val, err := m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].GetIFPresent(spaceId) - if err != nil && err != gcache.KeyNotFoundError { - return nil, err - } - receivedSpace, ok := val.(receivedSpace) - if !ok { - return nil, errtypes.InternalError("invalid type in cache") - } - receivedSpace.mtime = now - if receivedSpace.receivedShareStates == nil { - receivedSpace.receivedShareStates = map[string]receivedShareState{} - } - receivedSpace.receivedShareStates[id] = receivedShareState{ - state: collaboration.ShareState_SHARE_STATE_PENDING, - // mountpoint stays empty until user accepts the share - } - m.receivedCache[g.Grantee.GetUserId().GetOpaqueId()].Set(spaceId, receivedSpace) return s, nil } @@ -514,10 +526,30 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati var rss []*collaboration.ReceivedShare user := ctxpkg.ContextMustGetUser(ctx) - if m.receivedCache[user.Id.OpaqueId] == nil { - return rss, nil + ssids := map[string]receivedSpace{} + + // first collect all spaceids the user has access to as a group member + for _, group := range user.Groups { + for key, value := range m.groupReceivedCache[group].GetALL(false) { + var ssid string + var mtime time.Time + var ok bool + if ssid, ok = key.(string); !ok { + continue + } + if mtime, ok = value.(time.Time); !ok { + continue + } + if mtime.Sub(time.Now()) > time.Second*30 { + // TODO reread from disk + } + // add an empty entry, it will be overwritten with actual receivedSpace entries + // from the receivedCache per user below + ssids[ssid] = receivedSpace{} + } } + // add all spaces the user has receved shares for, this includes mount points and share stat for groups for key, value := range m.receivedCache[user.Id.OpaqueId].GetALL(false) { var ssid string var rspace receivedSpace @@ -532,6 +564,10 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if rspace.mtime < time.Now().Add(-30*time.Second).UnixNano() { // TODO reread from disk } + ssids[ssid] = rspace + } + + for ssid, rspace := range ssids { providerid, spaceid, _, err := storagespace.SplitID(ssid) if err != nil { @@ -560,6 +596,7 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati } } } + return rss, nil } From 247070e69238ae6eff378da29748276af3b09415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 28 Jul 2022 11:45:27 +0000 Subject: [PATCH 18/94] thoughts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index de36126db4..cb02791d1c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -568,13 +568,14 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati } for ssid, rspace := range ssids { - providerid, spaceid, _, err := storagespace.SplitID(ssid) if err != nil { continue } if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { + // FIXME we need to iterate over all shares to pick up pending group shares here + // or we use a receivedCache for groups as well ... with a groupid$userid key? for shareId, state := range rspace.receivedShareStates { value, err := spaceShares.Get(shareId) if err != nil { From 8eee4725564f3712c153f8b915bf2bab28c3119b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 28 Jul 2022 14:34:48 +0000 Subject: [PATCH 19/94] refactor shareCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 101 +++++++++++---------------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index cb02791d1c..13fcfa5baa 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -69,14 +69,9 @@ type config struct { } type providerCache map[string]providerSpaces -type providerSpaces map[string]spaceShares -type spaceShares gcache.Cache +type providerSpaces map[string]gcache.Cache -type accessedCache map[string]spaceTimes -type spaceTimes gcache.Cache - -type receivedCache map[string]receivedSpaces -type receivedSpaces gcache.Cache +type receivedCache map[string]gcache.Cache type receivedSpace struct { mtime int64 receivedShareStates map[string]receivedShareState @@ -89,10 +84,14 @@ type receivedShareState struct { type manager struct { sync.RWMutex - cache providerCache - createdCache accessedCache - receivedCache receivedCache - groupReceivedCache accessedCache + // cache holds the all shares, sharded by provider id and space id + cache providerCache + // createdCache holds the list of shares a user has created, sharded by user id and space id + createdCache shareCache + // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id + groupReceivedCache shareCache + // userReceivedCache holds the state of shares a user has received, sharded by user id and space id + userReceivedCache receivedCache storage metadata.Storage spaceETags gcache.Cache @@ -121,9 +120,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { func New(s metadata.Storage) (share.Manager, error) { return &manager{ cache: providerCache{}, - createdCache: accessedCache{}, - receivedCache: receivedCache{}, - groupReceivedCache: accessedCache{}, + createdCache: NewShareCache(), + userReceivedCache: receivedCache{}, + groupReceivedCache: NewShareCache(), storage: s, spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil @@ -303,29 +302,28 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) // set flag for creator to have access to space - if m.createdCache[user.Id.OpaqueId] == nil { - m.createdCache[user.Id.OpaqueId] = gcache.New(-1).Simple().Build() + if err := m.createdCache.Add(user.Id.OpaqueId, id); err != nil { + return nil, err } + spaceId := storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, }) - m.createdCache[user.Id.OpaqueId].Set(spaceId, time.Now()) - // set flag for grantee to have access to space switch g.Grantee.Type { case provider.GranteeType_GRANTEE_TYPE_USER: userid := g.Grantee.GetUserId().GetOpaqueId() - if m.receivedCache[userid] == nil { - m.receivedCache[userid] = gcache.New(-1).Simple().Build() // receivedSpaces + if m.userReceivedCache[userid] == nil { + m.userReceivedCache[userid] = gcache.New(-1).Simple().Build() // receivedSpaces } - if !m.receivedCache[userid].Has(spaceId) { - err := m.receivedCache[userid].Set(spaceId, receivedSpace{}) + if !m.userReceivedCache[userid].Has(spaceId) { + err := m.userReceivedCache[userid].Set(spaceId, receivedSpace{}) if err != nil { return nil, err } } - val, err := m.receivedCache[userid].GetIFPresent(spaceId) + val, err := m.userReceivedCache[userid].GetIFPresent(spaceId) if err != nil && err != gcache.KeyNotFoundError { return nil, err } @@ -341,13 +339,12 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla state: collaboration.ShareState_SHARE_STATE_PENDING, // mountpoint stays empty until user accepts the share } - m.receivedCache[userid].Set(spaceId, receivedSpace) + m.userReceivedCache[userid].Set(spaceId, receivedSpace) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() - if m.groupReceivedCache[groupid] == nil { - m.groupReceivedCache[groupid] = gcache.New(-1).Simple().Build() + if err := m.groupReceivedCache.Add(groupid, id); err != nil { + return nil, err } - m.groupReceivedCache[groupid].Set(spaceId, time.Now()) } return s, nil @@ -479,21 +476,8 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte user := ctxpkg.ContextMustGetUser(ctx) var ss []*collaboration.Share - if m.createdCache[user.Id.OpaqueId] == nil { - return ss, nil - } - - for key, value := range m.createdCache[user.Id.OpaqueId].GetALL(false) { - var ssid string - var mtime time.Time - var ok bool - if ssid, ok = key.(string); !ok { - continue - } - if mtime, ok = value.(time.Time); !ok { - continue - } - if mtime.Sub(time.Now()) > time.Second*30 { + for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { + if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { // TODO reread from disk } providerid, spaceid, _, err := storagespace.SplitID(ssid) @@ -502,8 +486,12 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte } if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { - for _, value := range spaceShares.GetALL(false) { - if s, ok := value.(*collaboration.Share); ok { + for shareid, _ := range spaceShareIDs.IDs { + v, err := spaceShares.Get(shareid) + if err != nil { + continue + } + if s, ok := v.(*collaboration.Share); ok { if utils.UserEqual(user.GetId(), s.GetCreator()) { if share.MatchesFilters(s, filters) { ss = append(ss, s) @@ -530,17 +518,8 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { - for key, value := range m.groupReceivedCache[group].GetALL(false) { - var ssid string - var mtime time.Time - var ok bool - if ssid, ok = key.(string); !ok { - continue - } - if mtime, ok = value.(time.Time); !ok { - continue - } - if mtime.Sub(time.Now()) > time.Second*30 { + for ssid, spaceShareIDs := range m.groupReceivedCache.List(group) { + if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { // TODO reread from disk } // add an empty entry, it will be overwritten with actual receivedSpace entries @@ -549,8 +528,8 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati } } - // add all spaces the user has receved shares for, this includes mount points and share stat for groups - for key, value := range m.receivedCache[user.Id.OpaqueId].GetALL(false) { + // add all spaces the user has receved shares for, this includes mount points and share state for groups + for key, value := range m.userReceivedCache[user.Id.OpaqueId].GetALL(false) { var ssid string var rspace receivedSpace var ok bool @@ -616,7 +595,7 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar StorageId: providerid, SpaceId: spaceid, }) - v, err := m.receivedCache[currentUser.OpaqueId].Get(spaceId) + v, err := m.userReceivedCache[currentUser.OpaqueId].Get(spaceId) if err != nil { return rs } @@ -678,11 +657,11 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab StorageId: rs.Share.ResourceId.StorageId, SpaceId: rs.Share.ResourceId.SpaceId, }) - v, err := m.receivedCache[currentUser.Id.OpaqueId].Get(spaceId) + v, err := m.userReceivedCache[currentUser.Id.OpaqueId].Get(spaceId) switch { case err == gcache.KeyNotFoundError: // add new entry - m.receivedCache[currentUser.Id.OpaqueId].Set(spaceId, + m.userReceivedCache[currentUser.Id.OpaqueId].Set(spaceId, receivedSpace{ mtime: time.Now().UnixNano(), receivedShareStates: map[string]receivedShareState{ @@ -698,7 +677,7 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab if rspace, ok := v.(receivedSpace); ok { rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState rspace.mtime = time.Now().UnixNano() - m.receivedCache[currentUser.Id.OpaqueId].Set(spaceId, rspace) + m.userReceivedCache[currentUser.Id.OpaqueId].Set(spaceId, rspace) } } From 3757085c21614819f0124996f000875d6dc9dde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 08:57:49 +0000 Subject: [PATCH 20/94] add todos for permission changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 13fcfa5baa..b8a80d1574 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -408,6 +408,7 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc return nil, err } // check if we are the creator or the grantee + // TODO allow manager to get shares in a space created by other users user := ctxpkg.ContextMustGetUser(ctx) if share.IsCreatedByUser(s, user) || share.IsGrantedToUser(s, user) { return s, nil @@ -425,6 +426,7 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference if err != nil { return err } + // TODO allow manager to unshare shares in a space created by other users if !share.IsCreatedByUser(s, user) { // TODO why not permission denied? return errtypes.NotFound(ref.String()) From 06b3f0a536a547f9aa186758fd18900ed81e2f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 09:27:54 +0000 Subject: [PATCH 21/94] add sharecache implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/sharecache.go | 95 +++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 pkg/share/manager/jsoncs3/sharecache.go diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go new file mode 100644 index 0000000000..9646429cba --- /dev/null +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -0,0 +1,95 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package jsoncs3 + +import ( + "errors" + "time" + + "github.com/bluele/gcache" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storagespace" +) + +type shareCache struct { + userShares map[string]gcache.Cache +} + +type spaceShareIDs struct { + mtime time.Time + IDs map[string]struct{} +} + +func NewShareCache() shareCache { + return shareCache{ + userShares: map[string]gcache.Cache{}, + } +} + +func (c *shareCache) Add(userid, shareID string) error { + if c.userShares[userid] == nil { + c.userShares[userid] = gcache.New(-1).Simple().Build() + } + storageid, spaceid, _, err := storagespace.SplitID(shareID) + if err != nil { + return err + } + spaceId := storagespace.FormatResourceID(provider.ResourceId{ + StorageId: storageid, + SpaceId: spaceid, + }) + v, err := c.userShares[userid].Get(spaceId) + switch { + case err == gcache.KeyNotFoundError: + // create new entry + return c.userShares[userid].Set(spaceId, &spaceShareIDs{ + mtime: time.Now(), + IDs: map[string]struct{}{shareID: {}}, + }) + case err != nil: + return err + } + // update list + spaceShareIDs, ok := v.(*spaceShareIDs) + if !ok { + return errors.New("invalid type") + } + spaceShareIDs.IDs[shareID] = struct{}{} + return nil +} +func (c *shareCache) List(userid string) map[string]spaceShareIDs { + r := make(map[string]spaceShareIDs) + for k, v := range c.userShares[userid].GetALL(false) { + + var ssid string + var cached *spaceShareIDs + var ok bool + if ssid, ok = k.(string); !ok { + continue + } + if cached, ok = v.(*spaceShareIDs); !ok { + continue + } + r[ssid] = spaceShareIDs{ + mtime: cached.mtime, + IDs: cached.IDs, + } + } + return r +} From 005831bb3c6b40ca54b30cbdcc7aea901d641a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 09:53:16 +0000 Subject: [PATCH 22/94] use group cache for pending state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 56 ++++++++++++++++++---------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index b8a80d1574..faf509859a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -90,8 +90,8 @@ type manager struct { createdCache shareCache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id groupReceivedCache shareCache - // userReceivedCache holds the state of shares a user has received, sharded by user id and space id - userReceivedCache receivedCache + // userReceivedStates holds the state of shares a user has received, sharded by user id and space id + userReceivedStates receivedCache storage metadata.Storage spaceETags gcache.Cache @@ -121,7 +121,7 @@ func New(s metadata.Storage) (share.Manager, error) { return &manager{ cache: providerCache{}, createdCache: NewShareCache(), - userReceivedCache: receivedCache{}, + userReceivedStates: receivedCache{}, groupReceivedCache: NewShareCache(), storage: s, spaceETags: gcache.New(1_000_000).LFU().Build(), @@ -314,16 +314,16 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla switch g.Grantee.Type { case provider.GranteeType_GRANTEE_TYPE_USER: userid := g.Grantee.GetUserId().GetOpaqueId() - if m.userReceivedCache[userid] == nil { - m.userReceivedCache[userid] = gcache.New(-1).Simple().Build() // receivedSpaces + if m.userReceivedStates[userid] == nil { + m.userReceivedStates[userid] = gcache.New(-1).Simple().Build() // receivedSpaces } - if !m.userReceivedCache[userid].Has(spaceId) { - err := m.userReceivedCache[userid].Set(spaceId, receivedSpace{}) + if !m.userReceivedStates[userid].Has(spaceId) { + err := m.userReceivedStates[userid].Set(spaceId, receivedSpace{}) if err != nil { return nil, err } } - val, err := m.userReceivedCache[userid].GetIFPresent(spaceId) + val, err := m.userReceivedStates[userid].GetIFPresent(spaceId) if err != nil && err != gcache.KeyNotFoundError { return nil, err } @@ -339,7 +339,7 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla state: collaboration.ShareState_SHARE_STATE_PENDING, // mountpoint stays empty until user accepts the share } - m.userReceivedCache[userid].Set(spaceId, receivedSpace) + m.userReceivedStates[userid].Set(spaceId, receivedSpace) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() if err := m.groupReceivedCache.Add(groupid, id); err != nil { @@ -524,14 +524,24 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { // TODO reread from disk } - // add an empty entry, it will be overwritten with actual receivedSpace entries - // from the receivedCache per user below - ssids[ssid] = receivedSpace{} + // add a pending entry, the state will be updated + // when reading the received shares below if they have already been accepted or denied + rs := receivedSpace{ + mtime: spaceShareIDs.mtime.UnixNano(), + receivedShareStates: make(map[string]receivedShareState, len(spaceShareIDs.IDs)), + } + + for shareid, _ := range spaceShareIDs.IDs { + rs.receivedShareStates[shareid] = receivedShareState{ + state: collaboration.ShareState_SHARE_STATE_PENDING, + } + } + ssids[ssid] = rs } } // add all spaces the user has receved shares for, this includes mount points and share state for groups - for key, value := range m.userReceivedCache[user.Id.OpaqueId].GetALL(false) { + for key, value := range m.userReceivedStates[user.Id.OpaqueId].GetALL(false) { var ssid string var rspace receivedSpace var ok bool @@ -545,7 +555,15 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if rspace.mtime < time.Now().Add(-30*time.Second).UnixNano() { // TODO reread from disk } - ssids[ssid] = rspace + // TODO use younger mtime to determine if + if rs, ok := ssids[ssid]; ok { + for shareid, state := range rspace.receivedShareStates { + // overwrite state + rs.receivedShareStates[shareid] = state + } + } else { + ssids[ssid] = rspace + } } for ssid, rspace := range ssids { @@ -555,8 +573,6 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati } if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { - // FIXME we need to iterate over all shares to pick up pending group shares here - // or we use a receivedCache for groups as well ... with a groupid$userid key? for shareId, state := range rspace.receivedShareStates { value, err := spaceShares.Get(shareId) if err != nil { @@ -597,7 +613,7 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar StorageId: providerid, SpaceId: spaceid, }) - v, err := m.userReceivedCache[currentUser.OpaqueId].Get(spaceId) + v, err := m.userReceivedStates[currentUser.OpaqueId].Get(spaceId) if err != nil { return rs } @@ -659,11 +675,11 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab StorageId: rs.Share.ResourceId.StorageId, SpaceId: rs.Share.ResourceId.SpaceId, }) - v, err := m.userReceivedCache[currentUser.Id.OpaqueId].Get(spaceId) + v, err := m.userReceivedStates[currentUser.Id.OpaqueId].Get(spaceId) switch { case err == gcache.KeyNotFoundError: // add new entry - m.userReceivedCache[currentUser.Id.OpaqueId].Set(spaceId, + m.userReceivedStates[currentUser.Id.OpaqueId].Set(spaceId, receivedSpace{ mtime: time.Now().UnixNano(), receivedShareStates: map[string]receivedShareState{ @@ -679,7 +695,7 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab if rspace, ok := v.(receivedSpace); ok { rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState rspace.mtime = time.Now().UnixNano() - m.userReceivedCache[currentUser.Id.OpaqueId].Set(spaceId, rspace) + m.userReceivedStates[currentUser.Id.OpaqueId].Set(spaceId, rspace) } } From af8127ea8b1a44f6f9d9562f35da233ca672f0df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 10:14:27 +0000 Subject: [PATCH 23/94] add plan for storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index faf509859a..c97d4d4fca 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -472,6 +472,27 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte return nil, err }*/ + // Q: how do we detect that a created list changed? + // Option 1: we rely on etag propagation on the storage to bubble up changes in any space to a single created list + // - drawback should stop etag propagation at /{userid}/ to prevent further propagation to the root of the share provider space + // - we could use the user.ocis.propagation xattr in decomposedfs or the eos equivalent to optimize the storage + // - pro: more efficient, more elegant + // - con: more complex, does not work on plain posix + // Option 2: we touch /{userid}/created/{storageid}/{spaceid}, + // /{userid}/created/{storageid} and + // /{userid}/created ourself + // - pro: easier to implement, works on plain posix + // - con: more requests + // Can this be hidden behind the metadata storage interface? + // Decision: use touch for now as it works withe plain posix and is easier to test + + // TODO check mtime of /users/{userid}/created/ + // check if a created or owned filter is set + // - do we have a cached list of created shares for the user in memory? + // - y: set if-not-match etag or mtime header to only get a listing if it changed + // - PROPFIND /users/{userid}/created/ + // - update cached list of created shares for the user in memory if changed + m.Lock() defer m.Unlock() //log := appctx.GetLogger(ctx) From 781a5262b06ad2d3dbb9caf7d173dcd9132147ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 12:56:15 +0000 Subject: [PATCH 24/94] add storage.Stat() and sketch out refresh from disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 49 ++++++++++++++------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 8 ++++ pkg/share/manager/jsoncs3/sharecache.go | 33 ++++++++++---- pkg/storage/utils/metadata/cs3.go | 28 ++++++++++++ pkg/storage/utils/metadata/disk.go | 22 ++++++++- pkg/storage/utils/metadata/mocks/Storage.go | 23 ++++++++++ pkg/storage/utils/metadata/storage.go | 16 +++++++ 7 files changed, 155 insertions(+), 24 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index c97d4d4fca..50e011e663 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -472,32 +472,46 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte return nil, err }*/ + m.Lock() + defer m.Unlock() + //log := appctx.GetLogger(ctx) + user := ctxpkg.ContextMustGetUser(ctx) + var ss []*collaboration.Share + // Q: how do we detect that a created list changed? // Option 1: we rely on etag propagation on the storage to bubble up changes in any space to a single created list // - drawback should stop etag propagation at /{userid}/ to prevent further propagation to the root of the share provider space // - we could use the user.ocis.propagation xattr in decomposedfs or the eos equivalent to optimize the storage // - pro: more efficient, more elegant // - con: more complex, does not work on plain posix - // Option 2: we touch /{userid}/created/{storageid}/{spaceid}, - // /{userid}/created/{storageid} and - // /{userid}/created ourself - // - pro: easier to implement, works on plain posix - // - con: more requests + // Option 2: we stat users/{userid}/created.json + // - pro: easier to implement, works on plain posix, no folders // Can this be hidden behind the metadata storage interface? // Decision: use touch for now as it works withe plain posix and is easier to test - // TODO check mtime of /users/{userid}/created/ - // check if a created or owned filter is set + // TODO check if a created or owned filter is set + userid := user.Id.OpaqueId + + var mtime time.Time // - do we have a cached list of created shares for the user in memory? - // - y: set if-not-match etag or mtime header to only get a listing if it changed - // - PROPFIND /users/{userid}/created/ - // - update cached list of created shares for the user in memory if changed + if usc := m.createdCache.Get(userid); usc != nil { + mtime = usc.mtime + // - y: set If-Modified-Since header to only download if it changed + } + // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed + userCreatedPath := filepath.Join("/users", userid, "created.json") + // check mtime of /users/{userid}/created.json + info, err := m.storage.Stat(ctx, userCreatedPath) + if err != nil { + // TODO check other cases, we currently only assume it does not exist + return ss, nil + } + if utils.TSToTime(info.Mtime).After(mtime) { + // - update cached list of created shares for the user in memory if changed + m.storage.SimpleDownload(ctx, userCreatedPath) + // rip out gcache so we can marshal / unmarshal the shareCache struct - m.Lock() - defer m.Unlock() - //log := appctx.GetLogger(ctx) - user := ctxpkg.ContextMustGetUser(ctx) - var ss []*collaboration.Share + } for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { @@ -537,6 +551,11 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati var rss []*collaboration.ReceivedShare user := ctxpkg.ContextMustGetUser(ctx) + // Q: how do we detect that a received list changed? + // - similar to the created.json we stat and download a received.json + // con: when adding a received share we MUST have if-match for the initiate-file-upload request + // to ensure consistency / prevent lost updates + ssids := map[string]receivedSpace{} // first collect all spaceids the user has access to as a group member diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 88b89b5979..896b572506 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -25,6 +25,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" @@ -95,6 +96,12 @@ var _ = Describe("Json", func() { Permissions: readPermissions, }, } + cacheStatInfo = &provider.ResourceInfo{ + Etag: "someetag", + Name: "created.json", + Size: 10, + Mtime: &typesv1beta1.Timestamp{}, + } storage *storagemocks.Storage m share.Manager @@ -119,6 +126,7 @@ var _ = Describe("Json", func() { storage.On("Init", mock.Anything, mock.Anything).Return(nil) storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) storage.On("SimpleUpload", mock.Anything, mock.Anything, mock.Anything).Return(nil) + storage.On("Stat", mock.Anything, mock.Anything).Return(cacheStatInfo, nil) var err error m, err = jsoncs3.New(storage) diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 9646429cba..6a94c7c682 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -28,7 +28,12 @@ import ( ) type shareCache struct { - userShares map[string]gcache.Cache + userShares map[string]*userShareCache +} + +type userShareCache struct { + mtime time.Time + userShares gcache.Cache } type spaceShareIDs struct { @@ -38,14 +43,25 @@ type spaceShareIDs struct { func NewShareCache() shareCache { return shareCache{ - userShares: map[string]gcache.Cache{}, + userShares: map[string]*userShareCache{}, } } +func (c *shareCache) Has(userid string) bool { + return c.userShares[userid] != nil +} +func (c *shareCache) Get(userid string) *userShareCache { + return c.userShares[userid] +} + func (c *shareCache) Add(userid, shareID string) error { - if c.userShares[userid] == nil { - c.userShares[userid] = gcache.New(-1).Simple().Build() + now := time.Now() + if _, ok := c.userShares[userid]; !ok { + c.userShares[userid] = &userShareCache{ + userShares: gcache.New(-1).Simple().Build(), + } } + c.userShares[userid].mtime = now storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -54,12 +70,12 @@ func (c *shareCache) Add(userid, shareID string) error { StorageId: storageid, SpaceId: spaceid, }) - v, err := c.userShares[userid].Get(spaceId) + v, err := c.userShares[userid].userShares.Get(spaceId) switch { case err == gcache.KeyNotFoundError: // create new entry - return c.userShares[userid].Set(spaceId, &spaceShareIDs{ - mtime: time.Now(), + return c.userShares[userid].userShares.Set(spaceId, &spaceShareIDs{ + mtime: now, IDs: map[string]struct{}{shareID: {}}, }) case err != nil: @@ -73,9 +89,10 @@ func (c *shareCache) Add(userid, shareID string) error { spaceShareIDs.IDs[shareID] = struct{}{} return nil } + func (c *shareCache) List(userid string) map[string]spaceShareIDs { r := make(map[string]spaceShareIDs) - for k, v := range c.userShares[userid].GetALL(false) { + for k, v := range c.userShares[userid].userShares.GetALL(false) { var ssid string var cached *spaceShareIDs diff --git a/pkg/storage/utils/metadata/cs3.go b/pkg/storage/utils/metadata/cs3.go index ade6a72d66..63fb0276d7 100644 --- a/pkg/storage/utils/metadata/cs3.go +++ b/pkg/storage/utils/metadata/cs3.go @@ -163,6 +163,34 @@ func (cs3 *CS3) SimpleUpload(ctx context.Context, uploadpath string, content []b return resp.Body.Close() } +func (cs3 *CS3) Stat(ctx context.Context, path string) (*provider.ResourceInfo, error) { + client, err := cs3.providerClient() + if err != nil { + return nil, err + } + ctx, err = cs3.getAuthContext(ctx) + if err != nil { + return nil, err + } + + req := provider.StatRequest{ + Ref: &provider.Reference{ + ResourceId: cs3.SpaceRoot, + Path: utils.MakeRelativePath(path), + }, + } + + res, err := client.Stat(ctx, &req) + if err != nil { + return nil, err + } + if res.Status.Code != rpc.Code_CODE_OK { + return nil, errtypes.NewErrtypeFromStatus(res.Status) + } + + return res.Info, nil +} + // SimpleDownload reads a file from the metadata storage func (cs3 *CS3) SimpleDownload(ctx context.Context, downloadpath string) (content []byte, err error) { client, err := cs3.providerClient() diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index 53b138c290..261e29e5b8 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -51,6 +51,27 @@ func (disk *Disk) Backend() string { return "disk" } +func (disk *Disk) Stat(ctx context.Context, path string) (*provider.ResourceInfo, error) { + info, err := os.Stat(disk.targetPath(path)) + if err != nil { + return nil, err + } + entry := &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Path: "./" + info.Name(), + Name: info.Name(), + Mtime: &typesv1beta1.Timestamp{Seconds: uint64(info.ModTime().Unix()), Nanos: uint32(info.ModTime().Nanosecond())}, + } + if info.IsDir() { + entry.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + entry.Etag, err = calcEtag(info.ModTime(), info.Size()) + if err != nil { + return nil, err + } + return entry, nil +} + // SimpleUpload stores a file on disk func (disk *Disk) SimpleUpload(_ context.Context, uploadpath string, content []byte) error { return os.WriteFile(disk.targetPath(uploadpath), content, 0644) @@ -104,7 +125,6 @@ func (disk *Disk) ListDir(ctx context.Context, path string) ([]*provider.Resourc if info.IsDir() { entry.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER } - entry.Type = provider.ResourceType_RESOURCE_TYPE_FILE entries = append(entries, entry) } return entries, nil diff --git a/pkg/storage/utils/metadata/mocks/Storage.go b/pkg/storage/utils/metadata/mocks/Storage.go index aaa37c9e76..37aba4a3b7 100644 --- a/pkg/storage/utils/metadata/mocks/Storage.go +++ b/pkg/storage/utils/metadata/mocks/Storage.go @@ -209,6 +209,29 @@ func (_m *Storage) SimpleUpload(ctx context.Context, uploadpath string, content return r0 } +// Stat provides a mock function with given fields: ctx, path +func (_m *Storage) Stat(ctx context.Context, path string) (*providerv1beta1.ResourceInfo, error) { + ret := _m.Called(ctx, path) + + var r0 *providerv1beta1.ResourceInfo + if rf, ok := ret.Get(0).(func(context.Context, string) *providerv1beta1.ResourceInfo); ok { + r0 = rf(ctx, path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*providerv1beta1.ResourceInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, path) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // NewStorage creates a new instance of Storage. It also registers a cleanup function to assert the mocks expectations. func NewStorage(t testing.TB) *Storage { mock := &Storage{} diff --git a/pkg/storage/utils/metadata/storage.go b/pkg/storage/utils/metadata/storage.go index 53e4513e9f..45064e5116 100644 --- a/pkg/storage/utils/metadata/storage.go +++ b/pkg/storage/utils/metadata/storage.go @@ -20,6 +20,10 @@ package metadata import ( "context" + "crypto/md5" + "encoding/binary" + "fmt" + "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) @@ -34,6 +38,7 @@ type Storage interface { SimpleUpload(ctx context.Context, uploadpath string, content []byte) error SimpleDownload(ctx context.Context, path string) ([]byte, error) Delete(ctx context.Context, path string) error + Stat(ctx context.Context, path string) (*provider.ResourceInfo, error) ReadDir(ctx context.Context, path string) ([]string, error) ListDir(ctx context.Context, path string) ([]*provider.ResourceInfo, error) @@ -43,3 +48,14 @@ type Storage interface { MakeDirIfNotExist(ctx context.Context, name string) error } + +func calcEtag(mtime time.Time, size int64) (string, error) { + h := md5.New() + if err := binary.Write(h, binary.BigEndian, mtime.UnixNano()); err != nil { + return "", err + } + if err := binary.Write(h, binary.BigEndian, size); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} From b94855b9aa478a4716fd51ff83d4ca2e68dce3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 13:28:13 +0000 Subject: [PATCH 25/94] read user cache from storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 16 ++++++- pkg/share/manager/jsoncs3/sharecache.go | 58 +++++++++---------------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 50e011e663..3a80fd8982 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -20,6 +20,7 @@ package jsoncs3 import ( "context" + "encoding/json" "fmt" "path/filepath" "sync" @@ -508,8 +509,19 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte } if utils.TSToTime(info.Mtime).After(mtime) { // - update cached list of created shares for the user in memory if changed - m.storage.SimpleDownload(ctx, userCreatedPath) - // rip out gcache so we can marshal / unmarshal the shareCache struct + createdBlob, err := m.storage.SimpleDownload(ctx, userCreatedPath) + if err == nil { + newShareCache := userShareCache{} + err := json.Unmarshal(createdBlob, &newShareCache) + if err != nil { + // TODO log error but continue? + // data corrupted, admin needs to take action + // the service still has data. dump it before ding? + } + m.createdCache.SetShareCache(userid, &newShareCache) + } else { + // TODO log error but continue with current cached data + } } diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 6a94c7c682..5878c788aa 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -19,10 +19,8 @@ package jsoncs3 import ( - "errors" "time" - "github.com/bluele/gcache" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/storagespace" ) @@ -33,7 +31,7 @@ type shareCache struct { type userShareCache struct { mtime time.Time - userShares gcache.Cache + userShares map[string]*spaceShareIDs } type spaceShareIDs struct { @@ -54,55 +52,41 @@ func (c *shareCache) Get(userid string) *userShareCache { return c.userShares[userid] } +func (c *shareCache) SetShareCache(userid string, shareCache *userShareCache) { + c.userShares[userid] = shareCache +} + func (c *shareCache) Add(userid, shareID string) error { - now := time.Now() - if _, ok := c.userShares[userid]; !ok { - c.userShares[userid] = &userShareCache{ - userShares: gcache.New(-1).Simple().Build(), - } - } - c.userShares[userid].mtime = now storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err } - spaceId := storagespace.FormatResourceID(provider.ResourceId{ + ssid := storagespace.FormatResourceID(provider.ResourceId{ StorageId: storageid, SpaceId: spaceid, }) - v, err := c.userShares[userid].userShares.Get(spaceId) - switch { - case err == gcache.KeyNotFoundError: - // create new entry - return c.userShares[userid].userShares.Set(spaceId, &spaceShareIDs{ - mtime: now, - IDs: map[string]struct{}{shareID: {}}, - }) - case err != nil: - return err + + now := time.Now() + if c.userShares[userid] == nil { + c.userShares[userid] = &userShareCache{ + userShares: map[string]*spaceShareIDs{}, + } } - // update list - spaceShareIDs, ok := v.(*spaceShareIDs) - if !ok { - return errors.New("invalid type") + if c.userShares[userid].userShares[ssid] == nil { + c.userShares[userid].userShares[ssid] = &spaceShareIDs{ + IDs: map[string]struct{}{}, + } } - spaceShareIDs.IDs[shareID] = struct{}{} + // add share id + c.userShares[userid].mtime = now + c.userShares[userid].userShares[ssid].mtime = now + c.userShares[userid].userShares[ssid].IDs[shareID] = struct{}{} return nil } func (c *shareCache) List(userid string) map[string]spaceShareIDs { r := make(map[string]spaceShareIDs) - for k, v := range c.userShares[userid].userShares.GetALL(false) { - - var ssid string - var cached *spaceShareIDs - var ok bool - if ssid, ok = k.(string); !ok { - continue - } - if cached, ok = v.(*spaceShareIDs); !ok { - continue - } + for ssid, cached := range c.userShares[userid].userShares { r[ssid] = spaceShareIDs{ mtime: cached.mtime, IDs: cached.IDs, From f25e074f88c2e9275bfad4c717d6761b8ba7b46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 13:45:31 +0000 Subject: [PATCH 26/94] write user share cache on share MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 40 +++++++++++++++++-------- pkg/share/manager/jsoncs3/sharecache.go | 2 +- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 3a80fd8982..5ed53f929f 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -277,13 +277,13 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla OpaqueId: uuid.NewString(), }, } - id, err := storagespace.FormatReference(shareReference) + shareID, err := storagespace.FormatReference(shareReference) if err != nil { return nil, err } s := &collaboration.Share{ Id: &collaboration.ShareId{ - OpaqueId: id, + OpaqueId: shareID, }, ResourceId: md.Id, Permissions: g.Permissions, @@ -303,11 +303,19 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) // set flag for creator to have access to space - if err := m.createdCache.Add(user.Id.OpaqueId, id); err != nil { + if err := m.createdCache.Add(user.Id.OpaqueId, shareID); err != nil { + return nil, err + } + createdBytes, err := json.Marshal(m.createdCache.GetShareCache(user.Id.OpaqueId)) + if err != nil { + return nil, err + } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := m.storage.SimpleUpload(ctx, userCreatedPath(user.Id.OpaqueId), createdBytes); err != nil { return nil, err } - spaceId := storagespace.FormatResourceID(provider.ResourceId{ + spaceID := storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, }) @@ -318,13 +326,13 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla if m.userReceivedStates[userid] == nil { m.userReceivedStates[userid] = gcache.New(-1).Simple().Build() // receivedSpaces } - if !m.userReceivedStates[userid].Has(spaceId) { - err := m.userReceivedStates[userid].Set(spaceId, receivedSpace{}) + if !m.userReceivedStates[userid].Has(spaceID) { + err := m.userReceivedStates[userid].Set(spaceID, receivedSpace{}) if err != nil { return nil, err } } - val, err := m.userReceivedStates[userid].GetIFPresent(spaceId) + val, err := m.userReceivedStates[userid].GetIFPresent(spaceID) if err != nil && err != gcache.KeyNotFoundError { return nil, err } @@ -336,14 +344,14 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla if receivedSpace.receivedShareStates == nil { receivedSpace.receivedShareStates = map[string]receivedShareState{} } - receivedSpace.receivedShareStates[id] = receivedShareState{ + receivedSpace.receivedShareStates[shareID] = receivedShareState{ state: collaboration.ShareState_SHARE_STATE_PENDING, // mountpoint stays empty until user accepts the share } - m.userReceivedStates[userid].Set(spaceId, receivedSpace) + m.userReceivedStates[userid].Set(spaceID, receivedSpace) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() - if err := m.groupReceivedCache.Add(groupid, id); err != nil { + if err := m.groupReceivedCache.Add(groupid, shareID); err != nil { return nil, err } } @@ -495,12 +503,14 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte var mtime time.Time // - do we have a cached list of created shares for the user in memory? - if usc := m.createdCache.Get(userid); usc != nil { + if usc := m.createdCache.GetShareCache(userid); usc != nil { mtime = usc.mtime // - y: set If-Modified-Since header to only download if it changed + } else { + mtime = time.Now() } // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed - userCreatedPath := filepath.Join("/users", userid, "created.json") + userCreatedPath := userCreatedPath(userid) // check mtime of /users/{userid}/created.json info, err := m.storage.Stat(ctx, userCreatedPath) if err != nil { @@ -522,7 +532,6 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte } else { // TODO log error but continue with current cached data } - } for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { @@ -555,6 +564,11 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte return ss, nil } +func userCreatedPath(userid string) string { + userCreatedPath := filepath.Join("/users", userid, "created.json") + return userCreatedPath +} + // we list the shares that are targeted to the user in context or to the user groups. func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { m.Lock() diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 5878c788aa..80498398cd 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -48,7 +48,7 @@ func NewShareCache() shareCache { func (c *shareCache) Has(userid string) bool { return c.userShares[userid] != nil } -func (c *shareCache) Get(userid string) *userShareCache { +func (c *shareCache) GetShareCache(userid string) *userShareCache { return c.userShares[userid] } From 61d73114db5e1433b78318d4cfc2c59fe442c621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 13:58:13 +0000 Subject: [PATCH 27/94] persist on unshare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 15 +++++++++++++++ pkg/share/manager/jsoncs3/sharecache.go | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 5ed53f929f..2ce4ec2db9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -444,6 +444,21 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference shareid, err := storagespace.ParseID(s.Id.OpaqueId) m.cache[shareid.StorageId][shareid.SpaceId].Remove(s.Id.OpaqueId) + // remove from created cache + if err := m.createdCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId); err != nil { + return err + } + createdBytes, err := json.Marshal(m.createdCache.GetShareCache(user.Id.OpaqueId)) + if err != nil { + return err + } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := m.storage.SimpleUpload(ctx, userCreatedPath(user.Id.OpaqueId), createdBytes); err != nil { + return err + } + + // remove from grantee cache + return nil } diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 80498398cd..3ebce25d25 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -84,6 +84,28 @@ func (c *shareCache) Add(userid, shareID string) error { return nil } +func (c *shareCache) Remove(userid, shareID string) error { + storageid, spaceid, _, err := storagespace.SplitID(shareID) + if err != nil { + return err + } + ssid := storagespace.FormatResourceID(provider.ResourceId{ + StorageId: storageid, + SpaceId: spaceid, + }) + + if c.userShares[userid] != nil { + if c.userShares[userid].userShares[ssid] != nil { + // remove share id + now := time.Now() + c.userShares[userid].mtime = now + c.userShares[userid].userShares[ssid].mtime = now + delete(c.userShares[userid].userShares[ssid].IDs, shareID) + } + } + return nil +} + func (c *shareCache) List(userid string) map[string]spaceShareIDs { r := make(map[string]spaceShareIDs) for ssid, cached := range c.userShares[userid].userShares { From e24690e18e7ee6bbcaf7ff6b995b978f969f5d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 14:47:23 +0000 Subject: [PATCH 28/94] extract set/removeCreatedCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 57 ++++++++++++++--------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 2 +- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 2ce4ec2db9..3acc1f14fc 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -302,24 +302,16 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) - // set flag for creator to have access to space - if err := m.createdCache.Add(user.Id.OpaqueId, shareID); err != nil { - return nil, err - } - createdBytes, err := json.Marshal(m.createdCache.GetShareCache(user.Id.OpaqueId)) + err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { return nil, err } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := m.storage.SimpleUpload(ctx, userCreatedPath(user.Id.OpaqueId), createdBytes); err != nil { - return nil, err - } spaceID := storagespace.FormatResourceID(provider.ResourceId{ StorageId: md.Id.StorageId, SpaceId: md.Id.SpaceId, }) - // set flag for grantee to have access to space + // set flag for grantee to have access to share switch g.Grantee.Type { case provider.GranteeType_GRANTEE_TYPE_USER: userid := g.Grantee.GetUserId().GetOpaqueId() @@ -445,20 +437,28 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference m.cache[shareid.StorageId][shareid.SpaceId].Remove(s.Id.OpaqueId) // remove from created cache - if err := m.createdCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId); err != nil { + err = m.removeFromCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + if err != nil { return err } - createdBytes, err := json.Marshal(m.createdCache.GetShareCache(user.Id.OpaqueId)) + + // TODO remove from grantee cache + + return nil +} + +func (m *manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid string) error { + if err := m.createdCache.Remove(creatorid, shareid); err != nil { + return err + } + createdBytes, err := json.Marshal(m.createdCache.GetShareCache(creatorid)) if err != nil { return err } // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := m.storage.SimpleUpload(ctx, userCreatedPath(user.Id.OpaqueId), createdBytes); err != nil { + if err := m.storage.SimpleUpload(ctx, userCreatedPath(creatorid), createdBytes); err != nil { return err } - - // remove from grantee cache - return nil } @@ -482,14 +482,29 @@ func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer Nanos: uint32(now % int64(time.Second)), } - // FIXME actually persist - // if err := m.model.Save(); err != nil { - // err = errors.Wrap(err, "error saving model") - // return nil, err - // } + err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + if err != nil { + return nil, err + } + return s, nil } +func (m *manager) setCreatedCache(ctx context.Context, creatorid, shareid string) error { + if err := m.createdCache.Add(creatorid, shareid); err != nil { + return err + } + createdBytes, err := json.Marshal(m.createdCache.GetShareCache(creatorid)) + if err != nil { + return err + } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := m.storage.SimpleUpload(ctx, userCreatedPath(creatorid), createdBytes); err != nil { + return err + } + return nil +} + // ListShares func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { /*if err := m.initialize(ctx); err != nil { diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 896b572506..1f5ae190b1 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -37,7 +37,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Json", func() { +var _ = Describe("Jsoncs3", func() { var ( user1 = &userpb.User{ Id: &userpb.UserId{ From d1f36373dfd6d22745b10d593d24942b9fc69c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 14:56:37 +0000 Subject: [PATCH 29/94] replace provider space gcache with map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 57 ++++++++++++---------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 3acc1f14fc..1c4f266401 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -70,7 +70,8 @@ type config struct { } type providerCache map[string]providerSpaces -type providerSpaces map[string]gcache.Cache +type providerSpaces map[string]providerShares +type providerShares map[string]*collaboration.Share type receivedCache map[string]gcache.Cache type receivedSpace struct { @@ -298,9 +299,9 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.cache[md.Id.StorageId] = providerSpaces{} } if m.cache[md.Id.StorageId][md.Id.SpaceId] == nil { - m.cache[md.Id.StorageId][md.Id.SpaceId] = gcache.New(-1).Simple().Build() + m.cache[md.Id.StorageId][md.Id.SpaceId] = providerShares{} } - m.cache[md.Id.StorageId][md.Id.SpaceId].Set(s.Id.OpaqueId, s) + m.cache[md.Id.StorageId][md.Id.SpaceId][s.Id.OpaqueId] = s err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -360,11 +361,9 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } if providerSpaces, ok := m.cache[shareid.StorageId]; ok { if spaceShares, ok := providerSpaces[shareid.SpaceId]; ok { - for _, value := range spaceShares.GetALL(false) { - if share, ok := value.(*collaboration.Share); ok { - if share.GetId().OpaqueId == id.OpaqueId { - return share, nil - } + for _, share := range spaceShares { + if share.GetId().OpaqueId == id.OpaqueId { + return share, nil } } } @@ -376,11 +375,9 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { if providerSpaces, ok := m.cache[key.ResourceId.StorageId]; ok { if spaceShares, ok := providerSpaces[key.ResourceId.SpaceId]; ok { - for _, value := range spaceShares.GetALL(false) { - if share, ok := value.(*collaboration.Share); ok { - if utils.GranteeEqual(key.Grantee, share.Grantee) { - return share, nil - } + for _, share := range spaceShares { + if utils.GranteeEqual(key.Grantee, share.Grantee) { + return share, nil } } } @@ -434,7 +431,7 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference } shareid, err := storagespace.ParseID(s.Id.OpaqueId) - m.cache[shareid.StorageId][shareid.SpaceId].Remove(s.Id.OpaqueId) + delete(m.cache[shareid.StorageId][shareid.SpaceId], s.Id.OpaqueId) // remove from created cache err = m.removeFromCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) @@ -575,15 +572,13 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { for shareid, _ := range spaceShareIDs.IDs { - v, err := spaceShares.Get(shareid) - if err != nil { + s := spaceShares[shareid] + if s == nil { continue } - if s, ok := v.(*collaboration.Share); ok { - if utils.UserEqual(user.GetId(), s.GetCreator()) { - if share.MatchesFilters(s, filters) { - ss = append(ss, s) - } + if utils.UserEqual(user.GetId(), s.GetCreator()) { + if share.MatchesFilters(s, filters) { + ss = append(ss, s) } } } @@ -670,20 +665,18 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if providerSpaces, ok := m.cache[providerid]; ok { if spaceShares, ok := providerSpaces[spaceid]; ok { for shareId, state := range rspace.receivedShareStates { - value, err := spaceShares.Get(shareId) - if err != nil { + s := spaceShares[shareId] + if s == nil { continue } - if s, ok := value.(*collaboration.Share); ok { - if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { - if share.MatchesFilters(s, filters) { - rs := &collaboration.ReceivedShare{ - Share: s, - State: state.state, - MountPoint: state.mountPoint, - } - rss = append(rss, rs) + if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { + if share.MatchesFilters(s, filters) { + rs := &collaboration.ReceivedShare{ + Share: s, + State: state.state, + MountPoint: state.mountPoint, } + rss = append(rss, rs) } } } From 35cd86acee1cfd37e74513836ec1146048da7723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Jul 2022 15:08:16 +0000 Subject: [PATCH 30/94] drop gcache and unused code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 147 ++++----------------------- 1 file changed, 20 insertions(+), 127 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 1c4f266401..2fdfcf9d7d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -21,12 +21,10 @@ package jsoncs3 import ( "context" "encoding/json" - "fmt" "path/filepath" "sync" "time" - "github.com/bluele/gcache" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -73,7 +71,8 @@ type providerCache map[string]providerSpaces type providerSpaces map[string]providerShares type providerShares map[string]*collaboration.Share -type receivedCache map[string]gcache.Cache +type receivedCache map[string]receivedSpaces +type receivedSpaces map[string]*receivedSpace type receivedSpace struct { mtime int64 receivedShareStates map[string]receivedShareState @@ -96,7 +95,6 @@ type manager struct { userReceivedStates receivedCache storage metadata.Storage - spaceETags gcache.Cache serviceUser *userv1beta1.User SpaceRoot *provider.ResourceId initialized bool @@ -126,7 +124,6 @@ func New(s metadata.Storage) (share.Manager, error) { userReceivedStates: receivedCache{}, groupReceivedCache: NewShareCache(), storage: s, - spaceETags: gcache.New(1_000_000).LFU().Build(), }, nil } @@ -173,74 +170,6 @@ func New(s metadata.Storage) (share.Manager, error) { // /{userid}/received/{storageid}/{spaceid} // /{userid}/created/{storageid}/{spaceid} -func (m *manager) initialize(ctx context.Context) error { - // if local copy is invalid fetch a new one - // invalid = not set || etag changed - if m.initialized { - return nil - } - - m.Lock() - defer m.Unlock() - - if m.initialized { // check if initialization happened while grabbing the lock - return nil - } - - user, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return fmt.Errorf("missing user in context") - } - - err := m.storage.Init(context.Background(), "jsoncs3-share-manager-metadata") - if err != nil { - return err - } - - infos, err := m.storage.ListDir(ctx, filepath.Join("users", user.Id.OpaqueId, "created")) - if err != nil { - return err - } - // for every space we fetch /{storageid}/{spaceid}.json if we - // have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed - for _, storageInfo := range infos { - // do we have spaces for this storage cached? - etag := m.getCachedSpaceETag(storageInfo.Name) - if etag == "" || etag != storageInfo.Etag { - - // TODO update cache - // reread /{storageid}/{spaceid}.json ? - // hmm the dir listing for a /einstein-id/created/{storageid}${spaceid} might have a different - // etag than the one for /marie-id/created/{storageid}${spaceid} - // do we also need the mtime in addition to the etag? so we can determine which one is younger? - // currently if einstein creates a share in space a we do a stat for every - // other user with access to the space because we update the cached space etag AND we touch the - // /einstein-id/created/{storageid}${spaceid} ... which updates the mtime ... so we don't need - // the etag, but only the mtime of /einstein-id/created/{storageid}${spaceid} ? which we set to - // the /{storageid}/{spaceid}.json mtime. since we always do the mtime setting ... this should work - // well .. if cs3 touch allows setting the mtime - // client.TouchFile(ctx, &provider.TouchFileRequest{ - // Ref: &provider.Reference{}, - // Opaque: &typespb.Opaque{ /*TODO allow setting the mtime with touch*/ }, - // }) - // maybe we need SetArbitraryMetadata to set the mtime - } - // - // TODO use space if etag is same - } - - return nil -} - -func (m *manager) getCachedSpaceETag(spaceid string) string { - if e, err := m.spaceETags.Get(spaceid); err != gcache.KeyNotFoundError { - if etag, ok := e.(string); ok { - return etag - } - } - return "" -} - func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := ctxpkg.ContextMustGetUser(ctx) @@ -317,22 +246,13 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla case provider.GranteeType_GRANTEE_TYPE_USER: userid := g.Grantee.GetUserId().GetOpaqueId() if m.userReceivedStates[userid] == nil { - m.userReceivedStates[userid] = gcache.New(-1).Simple().Build() // receivedSpaces + m.userReceivedStates[userid] = receivedSpaces{} // receivedSpaces } - if !m.userReceivedStates[userid].Has(spaceID) { - err := m.userReceivedStates[userid].Set(spaceID, receivedSpace{}) - if err != nil { - return nil, err - } - } - val, err := m.userReceivedStates[userid].GetIFPresent(spaceID) - if err != nil && err != gcache.KeyNotFoundError { - return nil, err - } - receivedSpace, ok := val.(receivedSpace) - if !ok { - return nil, errtypes.InternalError("invalid type in cache") + if m.userReceivedStates[userid][spaceID] == nil { + m.userReceivedStates[userid][spaceID] = &receivedSpace{} } + receivedSpace := m.userReceivedStates[userid][spaceID] + receivedSpace.mtime = now if receivedSpace.receivedShareStates == nil { receivedSpace.receivedShareStates = map[string]receivedShareState{} @@ -341,7 +261,6 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla state: collaboration.ShareState_SHARE_STATE_PENDING, // mountpoint stays empty until user accepts the share } - m.userReceivedStates[userid].Set(spaceID, receivedSpace) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() if err := m.groupReceivedCache.Add(groupid, shareID); err != nil { @@ -607,7 +526,7 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati // con: when adding a received share we MUST have if-match for the initiate-file-upload request // to ensure consistency / prevent lost updates - ssids := map[string]receivedSpace{} + ssids := map[string]*receivedSpace{} // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { @@ -627,22 +546,12 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati state: collaboration.ShareState_SHARE_STATE_PENDING, } } - ssids[ssid] = rs + ssids[ssid] = &rs } } // add all spaces the user has receved shares for, this includes mount points and share state for groups - for key, value := range m.userReceivedStates[user.Id.OpaqueId].GetALL(false) { - var ssid string - var rspace receivedSpace - var ok bool - if ssid, ok = key.(string); !ok { - continue - } - if rspace, ok = value.(receivedSpace); !ok { - continue - } - + for ssid, rspace := range m.userReceivedStates[user.Id.OpaqueId] { if rspace.mtime < time.Now().Add(-30*time.Second).UnixNano() { // TODO reread from disk } @@ -702,11 +611,7 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar StorageId: providerid, SpaceId: spaceid, }) - v, err := m.userReceivedStates[currentUser.OpaqueId].Get(spaceId) - if err != nil { - return rs - } - if rspace, ok := v.(receivedSpace); ok { + if rspace := m.userReceivedStates[currentUser.OpaqueId][spaceId]; rspace != nil { if state, ok := rspace.receivedShareStates[s.Id.OpaqueId]; ok { rs.State = state.state rs.MountPoint = state.mountPoint @@ -764,29 +669,17 @@ func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab StorageId: rs.Share.ResourceId.StorageId, SpaceId: rs.Share.ResourceId.SpaceId, }) - v, err := m.userReceivedStates[currentUser.Id.OpaqueId].Get(spaceId) - switch { - case err == gcache.KeyNotFoundError: - // add new entry - m.userReceivedStates[currentUser.Id.OpaqueId].Set(spaceId, - receivedSpace{ - mtime: time.Now().UnixNano(), - receivedShareStates: map[string]receivedShareState{ - rs.Share.Id.OpaqueId: newState, - }, - }) - case err != nil: - // something went horribly wrong - return nil, err + rspace := m.userReceivedStates[currentUser.Id.OpaqueId][spaceId] + if rspace == nil { + m.userReceivedStates[currentUser.Id.OpaqueId][spaceId] = + &receivedSpace{ + receivedShareStates: map[string]receivedShareState{}, + } - default: - // update entry - if rspace, ok := v.(receivedSpace); ok { - rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState - rspace.mtime = time.Now().UnixNano() - m.userReceivedStates[currentUser.Id.OpaqueId].Set(spaceId, rspace) - } } + // update entry + rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState + rspace.mtime = time.Now().UnixNano() return rs, nil } From 924d145a0be3472d7ee5ab0ba11e00807b34b62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 1 Aug 2022 10:17:02 +0200 Subject: [PATCH 31/94] Increase test coverage --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 99 ++++++++++++++++++++--- 1 file changed, 90 insertions(+), 9 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 1f5ae190b1..4e32c01a7d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -21,6 +21,7 @@ package jsoncs3_test import ( "context" + groupv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -96,6 +97,18 @@ var _ = Describe("Jsoncs3", func() { Permissions: readPermissions, }, } + + groupGrant = &collaboration.ShareGrant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_GROUP, + Id: &provider.Grantee_GroupId{GroupId: &groupv1beta1.GroupId{ + OpaqueId: "group1", + }}, + }, + Permissions: &collaboration.SharePermissions{ + Permissions: readPermissions, + }, + } cacheStatInfo = &provider.ResourceInfo{ Etag: "someetag", Name: "created.json", @@ -103,10 +116,12 @@ var _ = Describe("Jsoncs3", func() { Mtime: &typesv1beta1.Timestamp{}, } - storage *storagemocks.Storage - m share.Manager - ctx context.Context - granteeCtx context.Context + storage *storagemocks.Storage + m share.Manager + + ctx = ctxpkg.ContextSetUser(context.Background(), user1) + granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) + otherCtx = ctxpkg.ContextSetUser(context.Background(), &userpb.User{Id: &userpb.UserId{OpaqueId: "otheruser"}}) // helper functions shareBykey = func(key *collaboration.ShareKey) *collaboration.Share { @@ -131,9 +146,6 @@ var _ = Describe("Jsoncs3", func() { var err error m, err = jsoncs3.New(storage) Expect(err).ToNot(HaveOccurred()) - - ctx = ctxpkg.ContextSetUser(context.Background(), user1) - granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) }) Describe("Share", func() { @@ -152,7 +164,13 @@ var _ = Describe("Jsoncs3", func() { Expect(share.ResourceId).To(Equal(sharedResource.Id)) }) - PIt("creates a group share") + It("creates a group share", func() { + share, err := m.Share(ctx, sharedResource, groupGrant) + Expect(err).ToNot(HaveOccurred()) + + Expect(share).ToNot(BeNil()) + Expect(share.ResourceId).To(Equal(sharedResource.Id)) + }) }) Context("with an existing share", func() { @@ -167,6 +185,33 @@ var _ = Describe("Jsoncs3", func() { }) Describe("GetShare", func() { + It("handles unknown ids", func() { + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: "unknown-id", + }, + }, + }) + Expect(s).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + + It("handles unknown keys", func() { + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: &providerv1beta1.ResourceId{ + OpaqueId: "unknown", + }, + Grantee: grant.Grantee, + }, + }, + }) + Expect(s).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + It("retrieves an existing share by id", func() { s, err := m.GetShare(ctx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ @@ -188,9 +233,33 @@ var _ = Describe("Jsoncs3", func() { Expect(s.ResourceId).To(Equal(sharedResource.Id)) Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) }) + + It("does not return other users' shares", func() { + s, err := m.GetShare(otherCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) + }) }) Describe("UnShare", func() { + It("does not remove shares of other users", func() { + err := m.Unshare(otherCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }) + + Expect(err).To(HaveOccurred()) + }) + It("removes an existing share", func() { err := m.Unshare(ctx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ @@ -215,6 +284,19 @@ var _ = Describe("Jsoncs3", func() { }) Describe("UpdateShare", func() { + It("does not update shares of other users", func() { + _, err := m.UpdateShare(otherCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }, &collaboration.SharePermissions{ + Permissions: writePermissions, + }) + Expect(err).To(HaveOccurred()) + }) + It("updates an existing share", func() { s := shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, @@ -263,7 +345,6 @@ var _ = Describe("Jsoncs3", func() { Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) }) }) - Describe("ListShares", func() { It("lists an existing share", func() { shares, err := m.ListShares(ctx, nil) From 61d21b77edd5a83a18a9ba50d4af82c615e9b309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 1 Aug 2022 14:41:19 +0200 Subject: [PATCH 32/94] Do not choke when the usercache is empty --- pkg/share/manager/jsoncs3/sharecache.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 3ebce25d25..0ce35b9e0a 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -107,7 +107,11 @@ func (c *shareCache) Remove(userid, shareID string) error { } func (c *shareCache) List(userid string) map[string]spaceShareIDs { - r := make(map[string]spaceShareIDs) + r := map[string]spaceShareIDs{} + if c.userShares[userid] == nil { + return r + } + for ssid, cached := range c.userShares[userid].userShares { r[ssid] = spaceShareIDs{ mtime: cached.mtime, From 8635e06604a71123081e7f09728132b3d57ea561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 1 Aug 2022 14:41:50 +0200 Subject: [PATCH 33/94] Load the created.json when there's no in-memory cache yet --- pkg/share/manager/jsoncs3/jsoncs3.go | 2 +- pkg/share/manager/jsoncs3/jsoncs3_test.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 2fdfcf9d7d..9a37c4378f 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -453,7 +453,7 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte mtime = usc.mtime // - y: set If-Modified-Since header to only download if it changed } else { - mtime = time.Now() + mtime = time.Time{} } // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed userCreatedPath := userCreatedPath(userid) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 4e32c01a7d..03563a5dcf 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -110,10 +110,9 @@ var _ = Describe("Jsoncs3", func() { }, } cacheStatInfo = &provider.ResourceInfo{ - Etag: "someetag", Name: "created.json", Size: 10, - Mtime: &typesv1beta1.Timestamp{}, + Mtime: &typesv1beta1.Timestamp{Seconds: 100}, } storage *storagemocks.Storage @@ -346,6 +345,15 @@ var _ = Describe("Jsoncs3", func() { }) }) Describe("ListShares", func() { + It("loads the list of created shares if it hasn't been cashed yet", func() { + storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte{}, nil) + + shares, err := m.ListShares(otherCtx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(0)) + storage.AssertCalled(GinkgoT(), "SimpleDownload", mock.Anything, "/users/otheruser/created.json") + }) + It("lists an existing share", func() { shares, err := m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) From 03f9b98daae64acb9f14da9bae26d60d0ced86e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 2 Aug 2022 15:03:49 +0200 Subject: [PATCH 34/94] Add a ProviderCache --- pkg/share/manager/jsoncs3/jsoncs3.go | 118 ++++++++------------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 77 +++++++++++--- pkg/share/manager/jsoncs3/providercache.go | 76 +++++++++++++ pkg/share/manager/jsoncs3/sharecache.go | 74 ++++++------- 4 files changed, 222 insertions(+), 123 deletions(-) create mode 100644 pkg/share/manager/jsoncs3/providercache.go diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 9a37c4378f..9617d1cb3c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -67,10 +67,6 @@ type config struct { MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` } -type providerCache map[string]providerSpaces -type providerSpaces map[string]providerShares -type providerShares map[string]*collaboration.Share - type receivedCache map[string]receivedSpaces type receivedSpaces map[string]*receivedSpace type receivedSpace struct { @@ -86,11 +82,11 @@ type manager struct { sync.RWMutex // cache holds the all shares, sharded by provider id and space id - cache providerCache + cache ProviderCache // createdCache holds the list of shares a user has created, sharded by user id and space id - createdCache shareCache + createdCache ShareCache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id - groupReceivedCache shareCache + groupReceivedCache ShareCache // userReceivedStates holds the state of shares a user has received, sharded by user id and space id userReceivedStates receivedCache @@ -119,7 +115,7 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (share.Manager, error) { return &manager{ - cache: providerCache{}, + cache: NewProviderCache(), createdCache: NewShareCache(), userReceivedStates: receivedCache{}, groupReceivedCache: NewShareCache(), @@ -224,13 +220,7 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Mtime: ts, } - if m.cache[md.Id.StorageId] == nil { - m.cache[md.Id.StorageId] = providerSpaces{} - } - if m.cache[md.Id.StorageId][md.Id.SpaceId] == nil { - m.cache[md.Id.StorageId][md.Id.SpaceId] = providerShares{} - } - m.cache[md.Id.StorageId][md.Id.SpaceId][s.Id.OpaqueId] = s + m.cache.Add(md.Id.StorageId, md.Id.SpaceId, shareID, s) err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -278,27 +268,19 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro // invalid share id, does not exist return nil, errtypes.NotFound(id.String()) } - if providerSpaces, ok := m.cache[shareid.StorageId]; ok { - if spaceShares, ok := providerSpaces[shareid.SpaceId]; ok { - for _, share := range spaceShares { - if share.GetId().OpaqueId == id.OpaqueId { - return share, nil - } - } - } + share := m.cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) + if share == nil { + return nil, errtypes.NotFound(id.String()) } - return nil, errtypes.NotFound(id.String()) + return share, nil } // getByKey must be called in a lock-controlled block. func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { - if providerSpaces, ok := m.cache[key.ResourceId.StorageId]; ok { - if spaceShares, ok := providerSpaces[key.ResourceId.SpaceId]; ok { - for _, share := range spaceShares { - if utils.GranteeEqual(key.Grantee, share.Grantee) { - return share, nil - } - } + spaceShares := m.cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) + for _, share := range spaceShares.Shares { + if utils.GranteeEqual(key.Grantee, share.Grantee) { + return share, nil } } return nil, errtypes.NotFound(key.String()) @@ -350,7 +332,7 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference } shareid, err := storagespace.ParseID(s.Id.OpaqueId) - delete(m.cache[shareid.StorageId][shareid.SpaceId], s.Id.OpaqueId) + m.cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) // remove from created cache err = m.removeFromCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) @@ -423,14 +405,12 @@ func (m *manager) setCreatedCache(ctx context.Context, creatorid, shareid string // ListShares func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { - /*if err := m.initialize(ctx); err != nil { - return nil, err - }*/ - m.Lock() defer m.Unlock() + //log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) + userid := user.Id.OpaqueId var ss []*collaboration.Share // Q: how do we detect that a created list changed? @@ -445,29 +425,29 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte // Decision: use touch for now as it works withe plain posix and is easier to test // TODO check if a created or owned filter is set - userid := user.Id.OpaqueId var mtime time.Time // - do we have a cached list of created shares for the user in memory? if usc := m.createdCache.GetShareCache(userid); usc != nil { - mtime = usc.mtime + mtime = usc.Mtime // - y: set If-Modified-Since header to only download if it changed } else { - mtime = time.Time{} + mtime = time.Time{} // Set zero time so that data from storage always takes precedence } + // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed userCreatedPath := userCreatedPath(userid) - // check mtime of /users/{userid}/created.json info, err := m.storage.Stat(ctx, userCreatedPath) if err != nil { // TODO check other cases, we currently only assume it does not exist return ss, nil } + // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { // - update cached list of created shares for the user in memory if changed createdBlob, err := m.storage.SimpleDownload(ctx, userCreatedPath) if err == nil { - newShareCache := userShareCache{} + newShareCache := UserShareCache{} err := json.Unmarshal(createdBlob, &newShareCache) if err != nil { // TODO log error but continue? @@ -481,25 +461,22 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte } for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { - if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { + if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } providerid, spaceid, _, err := storagespace.SplitID(ssid) if err != nil { continue } - if providerSpaces, ok := m.cache[providerid]; ok { - if spaceShares, ok := providerSpaces[spaceid]; ok { - for shareid, _ := range spaceShareIDs.IDs { - s := spaceShares[shareid] - if s == nil { - continue - } - if utils.UserEqual(user.GetId(), s.GetCreator()) { - if share.MatchesFilters(s, filters) { - ss = append(ss, s) - } - } + spaceShares := m.cache.ListSpace(providerid, spaceid) + for shareid, _ := range spaceShareIDs.IDs { + s := spaceShares.Shares[shareid] + if s == nil { + continue + } + if utils.UserEqual(user.GetId(), s.GetCreator()) { + if share.MatchesFilters(s, filters) { + ss = append(ss, s) } } } @@ -531,13 +508,13 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { for ssid, spaceShareIDs := range m.groupReceivedCache.List(group) { - if time.Now().Sub(spaceShareIDs.mtime) > time.Second*30 { + if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } // add a pending entry, the state will be updated // when reading the received shares below if they have already been accepted or denied rs := receivedSpace{ - mtime: spaceShareIDs.mtime.UnixNano(), + mtime: spaceShareIDs.Mtime.UnixNano(), receivedShareStates: make(map[string]receivedShareState, len(spaceShareIDs.IDs)), } @@ -571,23 +548,20 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } - if providerSpaces, ok := m.cache[providerid]; ok { - if spaceShares, ok := providerSpaces[spaceid]; ok { - for shareId, state := range rspace.receivedShareStates { - s := spaceShares[shareId] - if s == nil { - continue - } - if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { - if share.MatchesFilters(s, filters) { - rs := &collaboration.ReceivedShare{ - Share: s, - State: state.state, - MountPoint: state.mountPoint, - } - rss = append(rss, rs) - } + spaceShares := m.cache.ListSpace(providerid, spaceid) + for shareId, state := range rspace.receivedShareStates { + s := spaceShares.Shares[shareId] + if s == nil { + continue + } + if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { + if share.MatchesFilters(s, filters) { + rs := &collaboration.ReceivedShare{ + Share: s, + State: state.state, + MountPoint: state.mountPoint, } + rss = append(rss, rs) } } } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 03563a5dcf..fbdef97fa2 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -20,6 +20,8 @@ package jsoncs3_test import ( "context" + "encoding/json" + "time" groupv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -31,6 +33,7 @@ import ( "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" + "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/stretchr/testify/mock" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -109,14 +112,9 @@ var _ = Describe("Jsoncs3", func() { Permissions: readPermissions, }, } - cacheStatInfo = &provider.ResourceInfo{ - Name: "created.json", - Size: 10, - Mtime: &typesv1beta1.Timestamp{Seconds: 100}, - } - - storage *storagemocks.Storage - m share.Manager + cacheStatInfo *provider.ResourceInfo + storage *storagemocks.Storage + m share.Manager ctx = ctxpkg.ContextSetUser(context.Background(), user1) granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) @@ -136,6 +134,12 @@ var _ = Describe("Jsoncs3", func() { ) BeforeEach(func() { + cacheStatInfo = &provider.ResourceInfo{ + Name: "created.json", + Size: 10, + Mtime: &typesv1beta1.Timestamp{Seconds: 100}, + } + storage = &storagemocks.Storage{} storage.On("Init", mock.Anything, mock.Anything).Return(nil) storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) @@ -345,21 +349,66 @@ var _ = Describe("Jsoncs3", func() { }) }) Describe("ListShares", func() { + It("lists an existing share", func() { + shares, err := m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(1)) + + Expect(shares[0].Id).To(Equal(share.Id)) + }) + It("loads the list of created shares if it hasn't been cashed yet", func() { - storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte{}, nil) + storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte("{}"), nil) - shares, err := m.ListShares(otherCtx, nil) + emptyCtx := ctxpkg.ContextSetUser(context.Background(), &userpb.User{Id: &userpb.UserId{OpaqueId: "emptyuser"}}) + shares, err := m.ListShares(emptyCtx, nil) Expect(err).ToNot(HaveOccurred()) Expect(shares).To(HaveLen(0)) - storage.AssertCalled(GinkgoT(), "SimpleDownload", mock.Anything, "/users/otheruser/created.json") + storage.AssertCalled(GinkgoT(), "SimpleDownload", mock.Anything, "/users/emptyuser/created.json") }) - It("lists an existing share", func() { + It("reloads the shares only if the cache was invalidated", func() { + storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte("{}"), nil) + + _, err := m.ListShares(ctx, nil) // data in storage is older -> no download + Expect(err).ToNot(HaveOccurred()) + + cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) + + _, err = m.ListShares(ctx, nil) // data in storage is younger -> download + Expect(err).ToNot(HaveOccurred()) + + storage.AssertNumberOfCalls(GinkgoT(), "SimpleDownload", 1) + }) + + It("uses the data from the storage after reload", func() { shares, err := m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) - Expect(shares).To(HaveLen(1)) + Expect(len(shares)).To(Equal(1)) - Expect(shares[0].Id).To(Equal(share.Id)) + providerid, spaceid, _, err := storagespace.SplitID(shares[0].Id.OpaqueId) + Expect(err).ToNot(HaveOccurred()) + + cache := jsoncs3.UserShareCache{ + Mtime: time.Now(), + UserShares: map[string]*jsoncs3.SpaceShareIDs{ + providerid + "$" + spaceid: { + Mtime: time.Now(), + IDs: map[string]struct{}{ + shares[0].Id.OpaqueId: {}, + shares[0].Id.OpaqueId: {}, + }, + }, + }, + } + bytes, err := json.Marshal(cache) + Expect(err).ToNot(HaveOccurred()) + storage.On("SimpleDownload", mock.Anything, mock.Anything).Return(bytes, nil) + cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) // Trigger reload + + shares, err = m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(2)) }) }) diff --git a/pkg/share/manager/jsoncs3/providercache.go b/pkg/share/manager/jsoncs3/providercache.go new file mode 100644 index 0000000000..856a61883e --- /dev/null +++ b/pkg/share/manager/jsoncs3/providercache.go @@ -0,0 +1,76 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package jsoncs3 + +import collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + +type ProviderCache struct { + Providers map[string]*ProviderSpaces +} + +type ProviderSpaces struct { + Spaces map[string]*ProviderShares +} + +type ProviderShares struct { + Shares map[string]*collaboration.Share +} + +func NewProviderCache() ProviderCache { + return ProviderCache{ + Providers: map[string]*ProviderSpaces{}, + } +} + +func (pc *ProviderCache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { + if pc.Providers[storageID] == nil { + pc.Providers[storageID] = &ProviderSpaces{ + Spaces: map[string]*ProviderShares{}, + } + } + if pc.Providers[storageID].Spaces[spaceID] == nil { + pc.Providers[storageID].Spaces[spaceID] = &ProviderShares{ + Shares: map[string]*collaboration.Share{}, + } + } + pc.Providers[storageID].Spaces[spaceID].Shares[shareID] = share +} + +func (pc *ProviderCache) Remove(storageID, spaceID, shareID string) { + if pc.Providers[storageID] == nil || + pc.Providers[storageID].Spaces[spaceID] == nil { + return + } + delete(pc.Providers[storageID].Spaces[spaceID].Shares, shareID) +} + +func (pc *ProviderCache) Get(storageID, spaceID, shareID string) *collaboration.Share { + if pc.Providers[storageID] == nil || + pc.Providers[storageID].Spaces[spaceID] == nil { + return nil + } + return pc.Providers[storageID].Spaces[spaceID].Shares[shareID] +} + +func (pc *ProviderCache) ListSpace(storageID, spaceID string) *ProviderShares { + if pc.Providers[storageID] == nil { + return &ProviderShares{} + } + return pc.Providers[storageID].Spaces[spaceID] +} diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache.go index 0ce35b9e0a..ae5301ba89 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache.go @@ -25,38 +25,38 @@ import ( "github.com/cs3org/reva/v2/pkg/storagespace" ) -type shareCache struct { - userShares map[string]*userShareCache +type ShareCache struct { + UserShares map[string]*UserShareCache } -type userShareCache struct { - mtime time.Time - userShares map[string]*spaceShareIDs +type UserShareCache struct { + Mtime time.Time + UserShares map[string]*SpaceShareIDs } -type spaceShareIDs struct { - mtime time.Time +type SpaceShareIDs struct { + Mtime time.Time IDs map[string]struct{} } -func NewShareCache() shareCache { - return shareCache{ - userShares: map[string]*userShareCache{}, +func NewShareCache() ShareCache { + return ShareCache{ + UserShares: map[string]*UserShareCache{}, } } -func (c *shareCache) Has(userid string) bool { - return c.userShares[userid] != nil +func (c *ShareCache) Has(userid string) bool { + return c.UserShares[userid] != nil } -func (c *shareCache) GetShareCache(userid string) *userShareCache { - return c.userShares[userid] +func (c *ShareCache) GetShareCache(userid string) *UserShareCache { + return c.UserShares[userid] } -func (c *shareCache) SetShareCache(userid string, shareCache *userShareCache) { - c.userShares[userid] = shareCache +func (c *ShareCache) SetShareCache(userid string, shareCache *UserShareCache) { + c.UserShares[userid] = shareCache } -func (c *shareCache) Add(userid, shareID string) error { +func (c *ShareCache) Add(userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -67,24 +67,24 @@ func (c *shareCache) Add(userid, shareID string) error { }) now := time.Now() - if c.userShares[userid] == nil { - c.userShares[userid] = &userShareCache{ - userShares: map[string]*spaceShareIDs{}, + if c.UserShares[userid] == nil { + c.UserShares[userid] = &UserShareCache{ + UserShares: map[string]*SpaceShareIDs{}, } } - if c.userShares[userid].userShares[ssid] == nil { - c.userShares[userid].userShares[ssid] = &spaceShareIDs{ + if c.UserShares[userid].UserShares[ssid] == nil { + c.UserShares[userid].UserShares[ssid] = &SpaceShareIDs{ IDs: map[string]struct{}{}, } } // add share id - c.userShares[userid].mtime = now - c.userShares[userid].userShares[ssid].mtime = now - c.userShares[userid].userShares[ssid].IDs[shareID] = struct{}{} + c.UserShares[userid].Mtime = now + c.UserShares[userid].UserShares[ssid].Mtime = now + c.UserShares[userid].UserShares[ssid].IDs[shareID] = struct{}{} return nil } -func (c *shareCache) Remove(userid, shareID string) error { +func (c *ShareCache) Remove(userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -94,27 +94,27 @@ func (c *shareCache) Remove(userid, shareID string) error { SpaceId: spaceid, }) - if c.userShares[userid] != nil { - if c.userShares[userid].userShares[ssid] != nil { + if c.UserShares[userid] != nil { + if c.UserShares[userid].UserShares[ssid] != nil { // remove share id now := time.Now() - c.userShares[userid].mtime = now - c.userShares[userid].userShares[ssid].mtime = now - delete(c.userShares[userid].userShares[ssid].IDs, shareID) + c.UserShares[userid].Mtime = now + c.UserShares[userid].UserShares[ssid].Mtime = now + delete(c.UserShares[userid].UserShares[ssid].IDs, shareID) } } return nil } -func (c *shareCache) List(userid string) map[string]spaceShareIDs { - r := map[string]spaceShareIDs{} - if c.userShares[userid] == nil { +func (c *ShareCache) List(userid string) map[string]SpaceShareIDs { + r := map[string]SpaceShareIDs{} + if c.UserShares[userid] == nil { return r } - for ssid, cached := range c.userShares[userid].userShares { - r[ssid] = spaceShareIDs{ - mtime: cached.mtime, + for ssid, cached := range c.UserShares[userid].UserShares { + r[ssid] = SpaceShareIDs{ + Mtime: cached.Mtime, IDs: cached.IDs, } } From a9e4edb6fe05f0c8322919e94fd0645fee634571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 09:07:53 +0200 Subject: [PATCH 35/94] Test that expired UserShareCaches are reloaded properly --- pkg/share/manager/jsoncs3/jsoncs3.go | 54 +++++++++++------------ pkg/share/manager/jsoncs3/jsoncs3_test.go | 18 ++++---- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 9617d1cb3c..2b1ec22cde 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -78,11 +78,11 @@ type receivedShareState struct { mountPoint *provider.Reference } -type manager struct { +type Manager struct { sync.RWMutex - // cache holds the all shares, sharded by provider id and space id - cache ProviderCache + // Cache holds the all shares, sharded by provider id and space id + Cache ProviderCache // createdCache holds the list of shares a user has created, sharded by user id and space id createdCache ShareCache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id @@ -113,9 +113,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { } // New returns a new manager instance. -func New(s metadata.Storage) (share.Manager, error) { - return &manager{ - cache: NewProviderCache(), +func New(s metadata.Storage) (*Manager, error) { + return &Manager{ + Cache: NewProviderCache(), createdCache: NewShareCache(), userReceivedStates: receivedCache{}, groupReceivedCache: NewShareCache(), @@ -166,7 +166,7 @@ func New(s metadata.Storage) (share.Manager, error) { // /{userid}/received/{storageid}/{spaceid} // /{userid}/created/{storageid}/{spaceid} -func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { +func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() @@ -220,7 +220,7 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Mtime: ts, } - m.cache.Add(md.Id.StorageId, md.Id.SpaceId, shareID, s) + m.Cache.Add(md.Id.StorageId, md.Id.SpaceId, shareID, s) err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -262,13 +262,13 @@ func (m *manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } // getByID must be called in a lock-controlled block. -func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, error) { +func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, error) { shareid, err := storagespace.ParseID(id.OpaqueId) if err != nil { // invalid share id, does not exist return nil, errtypes.NotFound(id.String()) } - share := m.cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) + share := m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) if share == nil { return nil, errtypes.NotFound(id.String()) } @@ -276,8 +276,8 @@ func (m *manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } // getByKey must be called in a lock-controlled block. -func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { - spaceShares := m.cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) +func (m *Manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { + spaceShares := m.Cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) for _, share := range spaceShares.Shares { if utils.GranteeEqual(key.Grantee, share.Grantee) { return share, nil @@ -287,7 +287,7 @@ func (m *manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, e } // get must be called in a lock-controlled block. -func (m *manager) get(ref *collaboration.ShareReference) (s *collaboration.Share, err error) { +func (m *Manager) get(ref *collaboration.ShareReference) (s *collaboration.Share, err error) { switch { case ref.GetId() != nil: s, err = m.getByID(ref.GetId()) @@ -299,7 +299,7 @@ func (m *manager) get(ref *collaboration.ShareReference) (s *collaboration.Share return } -func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { +func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { m.Lock() defer m.Unlock() s, err := m.get(ref) @@ -316,7 +316,7 @@ func (m *manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc return nil, errtypes.NotFound(ref.String()) } -func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { +func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() user := ctxpkg.ContextMustGetUser(ctx) @@ -332,7 +332,7 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference } shareid, err := storagespace.ParseID(s.Id.OpaqueId) - m.cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) + m.Cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) // remove from created cache err = m.removeFromCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) @@ -345,7 +345,7 @@ func (m *manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return nil } -func (m *manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid string) error { +func (m *Manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid string) error { if err := m.createdCache.Remove(creatorid, shareid); err != nil { return err } @@ -360,7 +360,7 @@ func (m *manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid return nil } -func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { +func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() s, err := m.get(ref) @@ -388,7 +388,7 @@ func (m *manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer return s, nil } -func (m *manager) setCreatedCache(ctx context.Context, creatorid, shareid string) error { +func (m *Manager) setCreatedCache(ctx context.Context, creatorid, shareid string) error { if err := m.createdCache.Add(creatorid, shareid); err != nil { return err } @@ -404,7 +404,7 @@ func (m *manager) setCreatedCache(ctx context.Context, creatorid, shareid string } // ListShares -func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { +func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { m.Lock() defer m.Unlock() @@ -468,7 +468,7 @@ func (m *manager) ListShares(ctx context.Context, filters []*collaboration.Filte if err != nil { continue } - spaceShares := m.cache.ListSpace(providerid, spaceid) + spaceShares := m.Cache.ListSpace(providerid, spaceid) for shareid, _ := range spaceShareIDs.IDs { s := spaceShares.Shares[shareid] if s == nil { @@ -491,7 +491,7 @@ func userCreatedPath(userid string) string { } // we list the shares that are targeted to the user in context or to the user groups. -func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { +func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() @@ -548,7 +548,7 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } - spaceShares := m.cache.ListSpace(providerid, spaceid) + spaceShares := m.Cache.ListSpace(providerid, spaceid) for shareId, state := range rspace.receivedShareStates { s := spaceShares.Shares[shareId] if s == nil { @@ -571,7 +571,7 @@ func (m *manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // convert must be called in a lock-controlled block. -func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { +func (m *Manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { rs := &collaboration.ReceivedShare{ Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, @@ -594,11 +594,11 @@ func (m *manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Shar return rs } -func (m *manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { +func (m *Manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { return m.getReceived(ctx, ref) } -func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { +func (m *Manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() s, err := m.get(ref) @@ -612,7 +612,7 @@ func (m *manager) getReceived(ctx context.Context, ref *collaboration.ShareRefer return m.convert(user.Id, s), nil } -func (m *manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { +func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}}) if err != nil { return nil, err diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index fbdef97fa2..5794385085 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -30,10 +30,8 @@ import ( providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" - "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/stretchr/testify/mock" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -114,7 +112,7 @@ var _ = Describe("Jsoncs3", func() { } cacheStatInfo *provider.ResourceInfo storage *storagemocks.Storage - m share.Manager + m *jsoncs3.Manager ctx = ctxpkg.ContextSetUser(context.Background(), user1) granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) @@ -386,17 +384,19 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(1)) - providerid, spaceid, _, err := storagespace.SplitID(shares[0].Id.OpaqueId) - Expect(err).ToNot(HaveOccurred()) + // Add a second cache to the provider cache so it can be referenced + m.Cache.Add("storageid", "spaceid", "storageid$spaceid!secondshare", &collaboration.Share{ + Creator: user1.Id, + }) cache := jsoncs3.UserShareCache{ Mtime: time.Now(), UserShares: map[string]*jsoncs3.SpaceShareIDs{ - providerid + "$" + spaceid: { + "storageid$spaceid": { Mtime: time.Now(), IDs: map[string]struct{}{ - shares[0].Id.OpaqueId: {}, - shares[0].Id.OpaqueId: {}, + shares[0].Id.OpaqueId: {}, + "storageid$spaceid!secondshare": {}, }, }, }, @@ -404,8 +404,8 @@ var _ = Describe("Jsoncs3", func() { bytes, err := json.Marshal(cache) Expect(err).ToNot(HaveOccurred()) storage.On("SimpleDownload", mock.Anything, mock.Anything).Return(bytes, nil) - cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) // Trigger reload + cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) // Trigger reload shares, err = m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(2)) From 16347bf6e2136b7da0695160ce0d3b256d501e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 09:36:33 +0200 Subject: [PATCH 36/94] Extract ShareCache into a subpackage --- pkg/share/manager/jsoncs3/jsoncs3.go | 23 ++++++++++--------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 8 +++++-- .../jsoncs3/{ => sharecache}/sharecache.go | 4 ++-- 3 files changed, 20 insertions(+), 15 deletions(-) rename pkg/share/manager/jsoncs3/{ => sharecache}/sharecache.go (98%) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 2b1ec22cde..64bae70c94 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -25,6 +25,11 @@ import ( "sync" "time" + "github.com/google/uuid" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "google.golang.org/genproto/protobuf/field_mask" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -32,14 +37,10 @@ import ( ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" + "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages "github.com/cs3org/reva/v2/pkg/storagespace" - "github.com/google/uuid" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "google.golang.org/genproto/protobuf/field_mask" - - "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -84,9 +85,9 @@ type Manager struct { // Cache holds the all shares, sharded by provider id and space id Cache ProviderCache // createdCache holds the list of shares a user has created, sharded by user id and space id - createdCache ShareCache + createdCache sharecache.ShareCache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id - groupReceivedCache ShareCache + groupReceivedCache sharecache.ShareCache // userReceivedStates holds the state of shares a user has received, sharded by user id and space id userReceivedStates receivedCache @@ -116,9 +117,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: NewProviderCache(), - createdCache: NewShareCache(), + createdCache: sharecache.New(), userReceivedStates: receivedCache{}, - groupReceivedCache: NewShareCache(), + groupReceivedCache: sharecache.New(), storage: s, }, nil } @@ -447,7 +448,7 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte // - update cached list of created shares for the user in memory if changed createdBlob, err := m.storage.SimpleDownload(ctx, userCreatedPath) if err == nil { - newShareCache := UserShareCache{} + newShareCache := sharecache.UserShareCache{} err := json.Unmarshal(createdBlob, &newShareCache) if err != nil { // TODO log error but continue? diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 5794385085..bd1fde4355 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -31,6 +31,7 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" "github.com/stretchr/testify/mock" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -246,6 +247,9 @@ var _ = Describe("Jsoncs3", func() { Expect(err).To(HaveOccurred()) Expect(s).To(BeNil()) }) + + PIt("reloads the provider cache when it is outdated") + PIt("uses the new data after reload") }) Describe("UnShare", func() { @@ -389,9 +393,9 @@ var _ = Describe("Jsoncs3", func() { Creator: user1.Id, }) - cache := jsoncs3.UserShareCache{ + cache := sharecache.UserShareCache{ Mtime: time.Now(), - UserShares: map[string]*jsoncs3.SpaceShareIDs{ + UserShares: map[string]*sharecache.SpaceShareIDs{ "storageid$spaceid": { Mtime: time.Now(), IDs: map[string]struct{}{ diff --git a/pkg/share/manager/jsoncs3/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go similarity index 98% rename from pkg/share/manager/jsoncs3/sharecache.go rename to pkg/share/manager/jsoncs3/sharecache/sharecache.go index ae5301ba89..437e42dde2 100644 --- a/pkg/share/manager/jsoncs3/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package jsoncs3 +package sharecache import ( "time" @@ -39,7 +39,7 @@ type SpaceShareIDs struct { IDs map[string]struct{} } -func NewShareCache() ShareCache { +func New() ShareCache { return ShareCache{ UserShares: map[string]*UserShareCache{}, } From d7e9d5a7fe89dcecec9c29eeb2bf031cbc225e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 09:51:21 +0200 Subject: [PATCH 37/94] Extract ShareCache persistence into the sharecache subpackage --- pkg/share/manager/jsoncs3/jsoncs3.go | 59 ++--------------- .../manager/jsoncs3/sharecache/sharecache.go | 66 ++++++++++++++++--- 2 files changed, 63 insertions(+), 62 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 64bae70c94..fc8adba227 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -20,8 +20,6 @@ package jsoncs3 import ( "context" - "encoding/json" - "path/filepath" "sync" "time" @@ -117,9 +115,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: NewProviderCache(), - createdCache: sharecache.New(), + createdCache: sharecache.New(s), userReceivedStates: receivedCache{}, - groupReceivedCache: sharecache.New(), + groupReceivedCache: sharecache.New(s), storage: s, }, nil } @@ -350,12 +348,7 @@ func (m *Manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid if err := m.createdCache.Remove(creatorid, shareid); err != nil { return err } - createdBytes, err := json.Marshal(m.createdCache.GetShareCache(creatorid)) - if err != nil { - return err - } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := m.storage.SimpleUpload(ctx, userCreatedPath(creatorid), createdBytes); err != nil { + if err := m.createdCache.Persist(ctx, creatorid); err != nil { return err } return nil @@ -393,12 +386,7 @@ func (m *Manager) setCreatedCache(ctx context.Context, creatorid, shareid string if err := m.createdCache.Add(creatorid, shareid); err != nil { return err } - createdBytes, err := json.Marshal(m.createdCache.GetShareCache(creatorid)) - if err != nil { - return err - } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := m.storage.SimpleUpload(ctx, userCreatedPath(creatorid), createdBytes); err != nil { + if err := m.createdCache.Persist(ctx, creatorid); err != nil { return err } return nil @@ -426,40 +414,8 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte // Decision: use touch for now as it works withe plain posix and is easier to test // TODO check if a created or owned filter is set - - var mtime time.Time - // - do we have a cached list of created shares for the user in memory? - if usc := m.createdCache.GetShareCache(userid); usc != nil { - mtime = usc.Mtime - // - y: set If-Modified-Since header to only download if it changed - } else { - mtime = time.Time{} // Set zero time so that data from storage always takes precedence - } - // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed - userCreatedPath := userCreatedPath(userid) - info, err := m.storage.Stat(ctx, userCreatedPath) - if err != nil { - // TODO check other cases, we currently only assume it does not exist - return ss, nil - } - // check mtime of /users/{userid}/created.json - if utils.TSToTime(info.Mtime).After(mtime) { - // - update cached list of created shares for the user in memory if changed - createdBlob, err := m.storage.SimpleDownload(ctx, userCreatedPath) - if err == nil { - newShareCache := sharecache.UserShareCache{} - err := json.Unmarshal(createdBlob, &newShareCache) - if err != nil { - // TODO log error but continue? - // data corrupted, admin needs to take action - // the service still has data. dump it before ding? - } - m.createdCache.SetShareCache(userid, &newShareCache) - } else { - // TODO log error but continue with current cached data - } - } + m.createdCache.Sync(ctx, userid) for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { @@ -486,11 +442,6 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte return ss, nil } -func userCreatedPath(userid string) string { - userCreatedPath := filepath.Join("/users", userid, "created.json") - return userCreatedPath -} - // we list the shares that are targeted to the user in context or to the user groups. func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { m.Lock() diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 437e42dde2..3af502f284 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -19,14 +19,21 @@ package sharecache import ( + "context" + "encoding/json" + "path/filepath" "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/utils" ) type ShareCache struct { UserShares map[string]*UserShareCache + + storage metadata.Storage } type UserShareCache struct { @@ -39,22 +46,16 @@ type SpaceShareIDs struct { IDs map[string]struct{} } -func New() ShareCache { +func New(s metadata.Storage) ShareCache { return ShareCache{ UserShares: map[string]*UserShareCache{}, + storage: s, } } func (c *ShareCache) Has(userid string) bool { return c.UserShares[userid] != nil } -func (c *ShareCache) GetShareCache(userid string) *UserShareCache { - return c.UserShares[userid] -} - -func (c *ShareCache) SetShareCache(userid string, shareCache *UserShareCache) { - c.UserShares[userid] = shareCache -} func (c *ShareCache) Add(userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) @@ -120,3 +121,52 @@ func (c *ShareCache) List(userid string) map[string]SpaceShareIDs { } return r } + +func (c *ShareCache) Sync(ctx context.Context, userid string) error { + var mtime time.Time + // - do we have a cached list of created shares for the user in memory? + if usc := c.UserShares[userid]; usc != nil { + mtime = usc.Mtime + // - y: set If-Modified-Since header to only download if it changed + } else { + mtime = time.Time{} // Set zero time so that data from storage always takes precedence + } + + userCreatedPath := userCreatedPath(userid) + info, err := c.storage.Stat(ctx, userCreatedPath) + if err != nil { + return err + } + // check mtime of /users/{userid}/created.json + if utils.TSToTime(info.Mtime).After(mtime) { + // - update cached list of created shares for the user in memory if changed + createdBlob, err := c.storage.SimpleDownload(ctx, userCreatedPath) + if err != nil { + return err + } + newShareCache := &UserShareCache{} + err = json.Unmarshal(createdBlob, newShareCache) + if err != nil { + return err + } + c.UserShares[userid] = newShareCache + } + return nil +} + +func (c *ShareCache) Persist(ctx context.Context, userid string) error { + createdBytes, err := json.Marshal(c.UserShares[userid]) + if err != nil { + return err + } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := c.storage.SimpleUpload(ctx, userCreatedPath(userid), createdBytes); err != nil { + return err + } + return nil +} + +func userCreatedPath(userid string) string { + userCreatedPath := filepath.Join("/users", userid, "created.json") + return userCreatedPath +} From 45bfff5b718b316f9e4fd5d4e194d5485984edc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 10:13:49 +0200 Subject: [PATCH 38/94] Refactor --- pkg/share/manager/jsoncs3/jsoncs3.go | 28 ++++++------ .../{ => providercache}/providercache.go | 42 +++++++++-------- .../providercache/providercache_test.go | 45 +++++++++++++++++++ .../manager/jsoncs3/sharecache/sharecache.go | 18 ++++---- 4 files changed, 91 insertions(+), 42 deletions(-) rename pkg/share/manager/jsoncs3/{ => providercache}/providercache.go (63%) create mode 100644 pkg/share/manager/jsoncs3/providercache/providercache_test.go diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index fc8adba227..dcc894b7d8 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -35,6 +35,7 @@ import ( ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages @@ -81,11 +82,11 @@ type Manager struct { sync.RWMutex // Cache holds the all shares, sharded by provider id and space id - Cache ProviderCache + Cache providercache.Cache // createdCache holds the list of shares a user has created, sharded by user id and space id - createdCache sharecache.ShareCache + createdCache sharecache.Cache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id - groupReceivedCache sharecache.ShareCache + groupReceivedCache sharecache.Cache // userReceivedStates holds the state of shares a user has received, sharded by user id and space id userReceivedStates receivedCache @@ -114,7 +115,7 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { // New returns a new manager instance. func New(s metadata.Storage) (*Manager, error) { return &Manager{ - Cache: NewProviderCache(), + Cache: providercache.New(s), createdCache: sharecache.New(s), userReceivedStates: receivedCache{}, groupReceivedCache: sharecache.New(s), @@ -331,10 +332,17 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference } shareid, err := storagespace.ParseID(s.Id.OpaqueId) + if err != nil { + return err + } m.Cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) // remove from created cache - err = m.removeFromCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + err = m.createdCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + if err != nil { + return err + } + err = m.createdCache.Persist(ctx, s.GetCreator().GetOpaqueId()) if err != nil { return err } @@ -344,16 +352,6 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return nil } -func (m *Manager) removeFromCreatedCache(ctx context.Context, creatorid, shareid string) error { - if err := m.createdCache.Remove(creatorid, shareid); err != nil { - return err - } - if err := m.createdCache.Persist(ctx, creatorid); err != nil { - return err - } - return nil -} - func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() diff --git a/pkg/share/manager/jsoncs3/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go similarity index 63% rename from pkg/share/manager/jsoncs3/providercache.go rename to pkg/share/manager/jsoncs3/providercache/providercache.go index 856a61883e..f6998fd498 100644 --- a/pkg/share/manager/jsoncs3/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -16,43 +16,49 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package jsoncs3 +package providercache -import collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" +import ( + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" +) -type ProviderCache struct { - Providers map[string]*ProviderSpaces +type Cache struct { + Providers map[string]*Spaces + + storage metadata.Storage } -type ProviderSpaces struct { - Spaces map[string]*ProviderShares +type Spaces struct { + Spaces map[string]*Shares } -type ProviderShares struct { +type Shares struct { Shares map[string]*collaboration.Share } -func NewProviderCache() ProviderCache { - return ProviderCache{ - Providers: map[string]*ProviderSpaces{}, +func New(s metadata.Storage) Cache { + return Cache{ + Providers: map[string]*Spaces{}, + storage: s, } } -func (pc *ProviderCache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { +func (pc *Cache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { if pc.Providers[storageID] == nil { - pc.Providers[storageID] = &ProviderSpaces{ - Spaces: map[string]*ProviderShares{}, + pc.Providers[storageID] = &Spaces{ + Spaces: map[string]*Shares{}, } } if pc.Providers[storageID].Spaces[spaceID] == nil { - pc.Providers[storageID].Spaces[spaceID] = &ProviderShares{ + pc.Providers[storageID].Spaces[spaceID] = &Shares{ Shares: map[string]*collaboration.Share{}, } } pc.Providers[storageID].Spaces[spaceID].Shares[shareID] = share } -func (pc *ProviderCache) Remove(storageID, spaceID, shareID string) { +func (pc *Cache) Remove(storageID, spaceID, shareID string) { if pc.Providers[storageID] == nil || pc.Providers[storageID].Spaces[spaceID] == nil { return @@ -60,7 +66,7 @@ func (pc *ProviderCache) Remove(storageID, spaceID, shareID string) { delete(pc.Providers[storageID].Spaces[spaceID].Shares, shareID) } -func (pc *ProviderCache) Get(storageID, spaceID, shareID string) *collaboration.Share { +func (pc *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { if pc.Providers[storageID] == nil || pc.Providers[storageID].Spaces[spaceID] == nil { return nil @@ -68,9 +74,9 @@ func (pc *ProviderCache) Get(storageID, spaceID, shareID string) *collaboration. return pc.Providers[storageID].Spaces[spaceID].Shares[shareID] } -func (pc *ProviderCache) ListSpace(storageID, spaceID string) *ProviderShares { +func (pc *Cache) ListSpace(storageID, spaceID string) *Shares { if pc.Providers[storageID] == nil { - return &ProviderShares{} + return &Shares{} } return pc.Providers[storageID].Spaces[spaceID] } diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go new file mode 100644 index 0000000000..5898a0a800 --- /dev/null +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -0,0 +1,45 @@ +package providercache_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache" + storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" +) + +var _ = Describe("Cache", func() { + var ( + c providercache.Cache + storage *storagemocks.Storage + + // storageId = "storageid" + // spaceid = "spaceid" + // share1 = &collaboration.Share{ + // Id: &collaboration.ShareId{ + // OpaqueId: "share1", + // }, + // } + ) + + BeforeEach(func() { + storage = &storagemocks.Storage{} + + c = providercache.New(storage) + Expect(c).ToNot(BeNil()) + }) + + Describe("Add", func() { + It("adds a share", func() { + // oldMtime := c.Providers. + // s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + // Expect(s).To(BeNil()) + + // c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + + // s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + // Expect(s).ToNot(BeNil()) + // Expect(s).To(Equal(share1)) + }) + }) +}) diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 3af502f284..cf310c2c72 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" ) -type ShareCache struct { +type Cache struct { UserShares map[string]*UserShareCache storage metadata.Storage @@ -46,18 +46,18 @@ type SpaceShareIDs struct { IDs map[string]struct{} } -func New(s metadata.Storage) ShareCache { - return ShareCache{ +func New(s metadata.Storage) Cache { + return Cache{ UserShares: map[string]*UserShareCache{}, storage: s, } } -func (c *ShareCache) Has(userid string) bool { +func (c *Cache) Has(userid string) bool { return c.UserShares[userid] != nil } -func (c *ShareCache) Add(userid, shareID string) error { +func (c *Cache) Add(userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -85,7 +85,7 @@ func (c *ShareCache) Add(userid, shareID string) error { return nil } -func (c *ShareCache) Remove(userid, shareID string) error { +func (c *Cache) Remove(userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -107,7 +107,7 @@ func (c *ShareCache) Remove(userid, shareID string) error { return nil } -func (c *ShareCache) List(userid string) map[string]SpaceShareIDs { +func (c *Cache) List(userid string) map[string]SpaceShareIDs { r := map[string]SpaceShareIDs{} if c.UserShares[userid] == nil { return r @@ -122,7 +122,7 @@ func (c *ShareCache) List(userid string) map[string]SpaceShareIDs { return r } -func (c *ShareCache) Sync(ctx context.Context, userid string) error { +func (c *Cache) Sync(ctx context.Context, userid string) error { var mtime time.Time // - do we have a cached list of created shares for the user in memory? if usc := c.UserShares[userid]; usc != nil { @@ -154,7 +154,7 @@ func (c *ShareCache) Sync(ctx context.Context, userid string) error { return nil } -func (c *ShareCache) Persist(ctx context.Context, userid string) error { +func (c *Cache) Persist(ctx context.Context, userid string) error { createdBytes, err := json.Marshal(c.UserShares[userid]) if err != nil { return err From 89e4b1d2cd93bdb0c75b0303f608e2bb8874c5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 13:13:31 +0200 Subject: [PATCH 39/94] Persist the provider cache --- pkg/share/manager/jsoncs3/jsoncs3.go | 58 +------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 27 ++-- .../jsoncs3/providercache/providercache.go | 114 ++++++++++++---- .../providercache/providercache_suite_test.go | 13 ++ .../providercache/providercache_test.go | 126 +++++++++++++++--- .../manager/jsoncs3/sharecache/sharecache.go | 3 +- 6 files changed, 226 insertions(+), 115 deletions(-) create mode 100644 pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index dcc894b7d8..e626060a09 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -397,7 +397,6 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte //log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) - userid := user.Id.OpaqueId var ss []*collaboration.Share // Q: how do we detect that a created list changed? @@ -413,7 +412,7 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte // TODO check if a created or owned filter is set // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed - m.createdCache.Sync(ctx, userid) + m.createdCache.Sync(ctx, user.Id.OpaqueId) for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { @@ -607,58 +606,3 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab return rs, nil } - -// // Dump exports shares and received shares to channels (e.g. during migration) -// func (m *manager) Dump(ctx context.Context, shareChan chan<- *collaboration.Share, receivedShareChan chan<- share.ReceivedShareWithUser) error { -// log := appctx.GetLogger(ctx) -// for _, s := range m.model.Shares { -// shareChan <- s -// } - -// for userIDString, states := range m.model.State { -// userMountPoints := m.model.MountPoint[userIDString] -// id := &userv1beta1.UserId{} -// mV2 := proto.MessageV2(id) -// if err := prototext.Unmarshal([]byte(userIDString), mV2); err != nil { -// log.Error().Err(err).Msg("error unmarshalling the user id") -// continue -// } - -// for shareIDString, state := range states { -// sid := &collaboration.ShareId{} -// mV2 := proto.MessageV2(sid) -// if err := prototext.Unmarshal([]byte(shareIDString), mV2); err != nil { -// log.Error().Err(err).Msg("error unmarshalling the user id") -// continue -// } - -// var s *collaboration.Share -// for _, is := range m.model.Shares { -// if is.Id.OpaqueId == sid.OpaqueId { -// s = is -// break -// } -// } -// if s == nil { -// log.Warn().Str("share id", sid.OpaqueId).Msg("Share not found") -// continue -// } - -// var mp *provider.Reference -// if userMountPoints != nil { -// mp = userMountPoints[shareIDString] -// } - -// receivedShareChan <- share.ReceivedShareWithUser{ -// UserID: id, -// ReceivedShare: &collaboration.ReceivedShare{ -// Share: s, -// State: state, -// MountPoint: mp, -// }, -// } -// } -// } - -// return nil -// } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index bd1fde4355..f9063e194c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -177,13 +177,22 @@ var _ = Describe("Jsoncs3", func() { Context("with an existing share", func() { var ( - share *collaboration.Share + share *collaboration.Share + shareRef *collaboration.ShareReference ) BeforeEach(func() { var err error share, err = m.Share(ctx, sharedResource, grant) Expect(err).ToNot(HaveOccurred()) + + shareRef = &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + } }) Describe("GetShare", func() { @@ -215,13 +224,7 @@ var _ = Describe("Jsoncs3", func() { }) It("retrieves an existing share by id", func() { - s, err := m.GetShare(ctx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: share.Id.OpaqueId, - }, - }, - }) + s, err := m.GetShare(ctx, shareRef) Expect(err).ToNot(HaveOccurred()) Expect(s).ToNot(BeNil()) Expect(share.ResourceId).To(Equal(sharedResource.Id)) @@ -237,13 +240,7 @@ var _ = Describe("Jsoncs3", func() { }) It("does not return other users' shares", func() { - s, err := m.GetShare(otherCtx, &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: share.Id.OpaqueId, - }, - }, - }) + s, err := m.GetShare(otherCtx, shareRef) Expect(err).To(HaveOccurred()) Expect(s).To(BeNil()) }) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index f6998fd498..4a403ac1a0 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -19,8 +19,15 @@ package providercache import ( + "context" + "encoding/json" + "path" + "path/filepath" + "time" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" + "github.com/cs3org/reva/v2/pkg/utils" ) type Cache struct { @@ -35,6 +42,7 @@ type Spaces struct { type Shares struct { Shares map[string]*collaboration.Share + Mtime time.Time } func New(s metadata.Storage) Cache { @@ -44,39 +52,99 @@ func New(s metadata.Storage) Cache { } } -func (pc *Cache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { - if pc.Providers[storageID] == nil { - pc.Providers[storageID] = &Spaces{ - Spaces: map[string]*Shares{}, - } - } - if pc.Providers[storageID].Spaces[spaceID] == nil { - pc.Providers[storageID].Spaces[spaceID] = &Shares{ - Shares: map[string]*collaboration.Share{}, - } - } - pc.Providers[storageID].Spaces[spaceID].Shares[shareID] = share +func (c *Cache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { + c.initializeIfNeeded(storageID, spaceID) + c.Providers[storageID].Spaces[spaceID].Shares[shareID] = share } -func (pc *Cache) Remove(storageID, spaceID, shareID string) { - if pc.Providers[storageID] == nil || - pc.Providers[storageID].Spaces[spaceID] == nil { +func (c *Cache) Remove(storageID, spaceID, shareID string) { + if c.Providers[storageID] == nil || + c.Providers[storageID].Spaces[spaceID] == nil { return } - delete(pc.Providers[storageID].Spaces[spaceID].Shares, shareID) + delete(c.Providers[storageID].Spaces[spaceID].Shares, shareID) } -func (pc *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { - if pc.Providers[storageID] == nil || - pc.Providers[storageID].Spaces[spaceID] == nil { +func (c *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { + if c.Providers[storageID] == nil || + c.Providers[storageID].Spaces[spaceID] == nil { return nil } - return pc.Providers[storageID].Spaces[spaceID].Shares[shareID] + return c.Providers[storageID].Spaces[spaceID].Shares[shareID] } -func (pc *Cache) ListSpace(storageID, spaceID string) *Shares { - if pc.Providers[storageID] == nil { +func (c *Cache) ListSpace(storageID, spaceID string) *Shares { + if c.Providers[storageID] == nil { return &Shares{} } - return pc.Providers[storageID].Spaces[spaceID] + return c.Providers[storageID].Spaces[spaceID] +} + +func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { + if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { + return nil + } + + createdBytes, err := json.Marshal(c.Providers[storageID].Spaces[spaceID]) + if err != nil { + return err + } + jsonPath := spaceJSONPath(storageID, spaceID) + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + return err + } + if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { + return err + } + return nil +} + +func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { + var mtime time.Time + if c.Providers[storageID] != nil && c.Providers[storageID].Spaces[spaceID] != nil { + mtime = c.Providers[storageID].Spaces[spaceID].Mtime + // - y: set If-Modified-Since header to only download if it changed + } else { + mtime = time.Time{} // Set zero time so that data from storage always takes precedence + } + + jsonPath := spaceJSONPath(storageID, spaceID) + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + // check mtime of /users/{userid}/created.json + if utils.TSToTime(info.Mtime).After(mtime) { + // - update cached list of created shares for the user in memory if changed + createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) + if err != nil { + return err + } + newShares := &Shares{} + err = json.Unmarshal(createdBlob, newShares) + if err != nil { + return err + } + c.initializeIfNeeded(storageID, spaceID) + c.Providers[storageID].Spaces[spaceID] = newShares + } + return nil +} + +func (c *Cache) initializeIfNeeded(storageID, spaceID string) { + if c.Providers[storageID] == nil { + c.Providers[storageID] = &Spaces{ + Spaces: map[string]*Shares{}, + } + } + if c.Providers[storageID].Spaces[spaceID] == nil { + c.Providers[storageID].Spaces[spaceID] = &Shares{ + Shares: map[string]*collaboration.Share{}, + } + } +} + +func spaceJSONPath(storageID, spaceID string) string { + return filepath.Join("/storages", storageID, spaceID+".json") } diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go new file mode 100644 index 0000000000..7098cbb6af --- /dev/null +++ b/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go @@ -0,0 +1,13 @@ +package providercache_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestProvidercache(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Providercache Suite") +} diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 5898a0a800..2851490565 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -1,45 +1,135 @@ package providercache_test import ( + "context" + "io/ioutil" + "os" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache" - storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" ) var _ = Describe("Cache", func() { var ( c providercache.Cache - storage *storagemocks.Storage - - // storageId = "storageid" - // spaceid = "spaceid" - // share1 = &collaboration.Share{ - // Id: &collaboration.ShareId{ - // OpaqueId: "share1", - // }, - // } + storage metadata.Storage + + storageId = "storageid" + spaceid = "spaceid" + share1 = &collaboration.Share{ + Id: &collaboration.ShareId{ + OpaqueId: "share1", + }, + } + ctx context.Context + tmpdir string ) BeforeEach(func() { - storage = &storagemocks.Storage{} + ctx = context.Background() + + var err error + tmpdir, err = ioutil.TempDir("", "providercache-test") + Expect(err).ToNot(HaveOccurred()) + + err = os.MkdirAll(tmpdir, 0755) + Expect(err).ToNot(HaveOccurred()) + + storage, err = metadata.NewDiskStorage(tmpdir) + Expect(err).ToNot(HaveOccurred()) c = providercache.New(storage) Expect(c).ToNot(BeNil()) }) + AfterEach(func() { + if tmpdir != "" { + os.RemoveAll(tmpdir) + } + }) + Describe("Add", func() { It("adds a share", func() { - // oldMtime := c.Providers. - // s := c.Get(storageId, spaceid, share1.Id.OpaqueId) - // Expect(s).To(BeNil()) + s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).To(BeNil()) + + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + + s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).ToNot(BeNil()) + Expect(s).To(Equal(share1)) + }) + }) + + Context("with an existing entry", func() { + BeforeEach(func() { + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + }) + + Describe("Remove", func() { + It("removes the entry", func() { + s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).ToNot(BeNil()) + Expect(s).To(Equal(share1)) + + c.Remove(storageId, spaceid, share1.Id.OpaqueId) + + s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).To(BeNil()) + }) + }) + + Describe("Persist", func() { + It("handles non-existent storages", func() { + Expect(c.Persist(ctx, "foo", "bar")).To(Succeed()) + }) + It("handles non-existent spaces", func() { + Expect(c.Persist(ctx, storageId, "bar")).To(Succeed()) + }) + + It("persists", func() { + Expect(c.Persist(ctx, storageId, spaceid)).To(Succeed()) + }) + }) + + Describe("Sync", func() { + BeforeEach(func() { + Expect(c.Persist(ctx, storageId, spaceid)).To(Succeed()) + // reset in-memory cache + c = providercache.New(storage) + }) + + It("downloads if needed", func() { + s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).To(BeNil()) + + c.Sync(ctx, storageId, spaceid) + + s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).ToNot(BeNil()) + }) + + It("does not download if not needed", func() { + s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).To(BeNil()) - // c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + c.Providers[storageId] = &providercache.Spaces{ + Spaces: map[string]*providercache.Shares{ + spaceid: { + Mtime: time.Now(), + }, + }, + } + c.Sync(ctx, storageId, spaceid) - // s = c.Get(storageId, spaceid, share1.Id.OpaqueId) - // Expect(s).ToNot(BeNil()) - // Expect(s).To(Equal(share1)) + s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + Expect(s).To(BeNil()) // Sync from disk didn't happen because in-memory mtime is later than on disk + }) }) }) }) diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index cf310c2c72..bc5825f0dd 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -167,6 +167,5 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { } func userCreatedPath(userid string) string { - userCreatedPath := filepath.Join("/users", userid, "created.json") - return userCreatedPath + return filepath.Join("/users", userid, "created.json") } From 02a5585b23faf778e24ba27874acaee538266059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 13:25:52 +0200 Subject: [PATCH 40/94] Update the space mtime when adding/removing shares --- .../jsoncs3/providercache/providercache.go | 2 ++ .../providercache/providercache_test.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 4a403ac1a0..b3da02e1d5 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -55,6 +55,7 @@ func New(s metadata.Storage) Cache { func (c *Cache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID].Shares[shareID] = share + c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() } func (c *Cache) Remove(storageID, spaceID, shareID string) { @@ -63,6 +64,7 @@ func (c *Cache) Remove(storageID, spaceID, shareID string) { return } delete(c.Providers[storageID].Spaces[spaceID].Shares, shareID) + c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() } func (c *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 2851490565..1e3ac1abb1 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -64,6 +64,18 @@ var _ = Describe("Cache", func() { Expect(s).ToNot(BeNil()) Expect(s).To(Equal(share1)) }) + + It("sets the mtime", func() { + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(time.Time{})) + }) + + It("updates the mtime", func() { + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + old := c.Providers[storageId].Spaces[spaceid].Mtime + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(old)) + }) }) Context("with an existing entry", func() { @@ -82,6 +94,13 @@ var _ = Describe("Cache", func() { s = c.Get(storageId, spaceid, share1.Id.OpaqueId) Expect(s).To(BeNil()) }) + + It("updates the mtime", func() { + c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + old := c.Providers[storageId].Spaces[spaceid].Mtime + c.Remove(storageId, spaceid, share1.Id.OpaqueId) + Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(old)) + }) }) Describe("Persist", func() { From 55ef7aeb442502297eccd06d64d6bcdc2b94cce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 13:55:47 +0200 Subject: [PATCH 41/94] Fix persisting user shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 18 ++--- pkg/share/manager/jsoncs3/jsoncs3_test.go | 74 ++++++++----------- .../providercache/providercache_test.go | 71 ++++++++++-------- .../manager/jsoncs3/sharecache/sharecache.go | 7 +- 4 files changed, 84 insertions(+), 86 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index e626060a09..e2f2e3464c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -83,8 +83,8 @@ type Manager struct { // Cache holds the all shares, sharded by provider id and space id Cache providercache.Cache - // createdCache holds the list of shares a user has created, sharded by user id and space id - createdCache sharecache.Cache + // CreatedCache holds the list of shares a user has created, sharded by user id and space id + CreatedCache sharecache.Cache // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id groupReceivedCache sharecache.Cache // userReceivedStates holds the state of shares a user has received, sharded by user id and space id @@ -116,7 +116,7 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: providercache.New(s), - createdCache: sharecache.New(s), + CreatedCache: sharecache.New(s), userReceivedStates: receivedCache{}, groupReceivedCache: sharecache.New(s), storage: s, @@ -338,11 +338,11 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference m.Cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) // remove from created cache - err = m.createdCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + err = m.CreatedCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) if err != nil { return err } - err = m.createdCache.Persist(ctx, s.GetCreator().GetOpaqueId()) + err = m.CreatedCache.Persist(ctx, s.GetCreator().GetOpaqueId()) if err != nil { return err } @@ -381,10 +381,10 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer } func (m *Manager) setCreatedCache(ctx context.Context, creatorid, shareid string) error { - if err := m.createdCache.Add(creatorid, shareid); err != nil { + if err := m.CreatedCache.Add(creatorid, shareid); err != nil { return err } - if err := m.createdCache.Persist(ctx, creatorid); err != nil { + if err := m.CreatedCache.Persist(ctx, creatorid); err != nil { return err } return nil @@ -412,9 +412,9 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte // TODO check if a created or owned filter is set // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed - m.createdCache.Sync(ctx, user.Id.OpaqueId) + m.CreatedCache.Sync(ctx, user.Id.OpaqueId) - for ssid, spaceShareIDs := range m.createdCache.List(user.Id.OpaqueId) { + for ssid, spaceShareIDs := range m.CreatedCache.List(user.Id.OpaqueId) { if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index f9063e194c..9636b162bc 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -21,6 +21,9 @@ package jsoncs3_test import ( "context" "encoding/json" + "io/ioutil" + "os" + "path/filepath" "time" groupv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" @@ -28,12 +31,10 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" - storagemocks "github.com/cs3org/reva/v2/pkg/storage/utils/metadata/mocks" - "github.com/stretchr/testify/mock" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "google.golang.org/protobuf/types/known/fieldmaskpb" . "github.com/onsi/ginkgo/v2" @@ -111,9 +112,9 @@ var _ = Describe("Jsoncs3", func() { Permissions: readPermissions, }, } - cacheStatInfo *provider.ResourceInfo - storage *storagemocks.Storage - m *jsoncs3.Manager + storage metadata.Storage + tmpdir string + m *jsoncs3.Manager ctx = ctxpkg.ContextSetUser(context.Background(), user1) granteeCtx = ctxpkg.ContextSetUser(context.Background(), user2) @@ -133,23 +134,26 @@ var _ = Describe("Jsoncs3", func() { ) BeforeEach(func() { - cacheStatInfo = &provider.ResourceInfo{ - Name: "created.json", - Size: 10, - Mtime: &typesv1beta1.Timestamp{Seconds: 100}, - } + var err error + tmpdir, err = ioutil.TempDir("", "jsoncs3-test") + Expect(err).ToNot(HaveOccurred()) - storage = &storagemocks.Storage{} - storage.On("Init", mock.Anything, mock.Anything).Return(nil) - storage.On("MakeDirIfNotExist", mock.Anything, mock.Anything).Return(nil) - storage.On("SimpleUpload", mock.Anything, mock.Anything, mock.Anything).Return(nil) - storage.On("Stat", mock.Anything, mock.Anything).Return(cacheStatInfo, nil) + err = os.MkdirAll(tmpdir, 0755) + Expect(err).ToNot(HaveOccurred()) + + storage, err = metadata.NewDiskStorage(tmpdir) + Expect(err).ToNot(HaveOccurred()) - var err error m, err = jsoncs3.New(storage) Expect(err).ToNot(HaveOccurred()) }) + AfterEach(func() { + if tmpdir != "" { + os.RemoveAll(tmpdir) + } + }) + Describe("Share", func() { It("fails if the share already exists", func() { _, err := m.Share(ctx, sharedResource, grant) @@ -245,7 +249,10 @@ var _ = Describe("Jsoncs3", func() { Expect(s).To(BeNil()) }) - PIt("reloads the provider cache when it is outdated") + It("reloads the provider cache when it is outdated", func() { + + }) + PIt("uses the new data after reload") }) @@ -356,31 +363,7 @@ var _ = Describe("Jsoncs3", func() { Expect(shares[0].Id).To(Equal(share.Id)) }) - It("loads the list of created shares if it hasn't been cashed yet", func() { - storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte("{}"), nil) - - emptyCtx := ctxpkg.ContextSetUser(context.Background(), &userpb.User{Id: &userpb.UserId{OpaqueId: "emptyuser"}}) - shares, err := m.ListShares(emptyCtx, nil) - Expect(err).ToNot(HaveOccurred()) - Expect(shares).To(HaveLen(0)) - storage.AssertCalled(GinkgoT(), "SimpleDownload", mock.Anything, "/users/emptyuser/created.json") - }) - - It("reloads the shares only if the cache was invalidated", func() { - storage.On("SimpleDownload", mock.Anything, mock.Anything).Return([]byte("{}"), nil) - - _, err := m.ListShares(ctx, nil) // data in storage is older -> no download - Expect(err).ToNot(HaveOccurred()) - - cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) - - _, err = m.ListShares(ctx, nil) // data in storage is younger -> download - Expect(err).ToNot(HaveOccurred()) - - storage.AssertNumberOfCalls(GinkgoT(), "SimpleDownload", 1) - }) - - It("uses the data from the storage after reload", func() { + FIt("uses the data from the storage after reload", func() { shares, err := m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(1)) @@ -404,9 +387,10 @@ var _ = Describe("Jsoncs3", func() { } bytes, err := json.Marshal(cache) Expect(err).ToNot(HaveOccurred()) - storage.On("SimpleDownload", mock.Anything, mock.Anything).Return(bytes, nil) + err = os.WriteFile(filepath.Join(tmpdir, "users/admin/created.json"), bytes, 0x755) + Expect(err).ToNot(HaveOccurred()) - cacheStatInfo.Mtime.Seconds = uint64(time.Now().UnixNano()) // Trigger reload + m.CreatedCache.UserShares["admin"].Mtime = time.Time{} // trigger reload shares, err = m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(2)) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 1e3ac1abb1..abbbfdb573 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -19,8 +19,9 @@ var _ = Describe("Cache", func() { c providercache.Cache storage metadata.Storage - storageId = "storageid" - spaceid = "spaceid" + storageID = "storageid" + spaceID = "spaceid" + shareID = "storageid$spaceid!share1" share1 = &collaboration.Share{ Id: &collaboration.ShareId{ OpaqueId: "share1", @@ -55,51 +56,59 @@ var _ = Describe("Cache", func() { Describe("Add", func() { It("adds a share", func() { - s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + c.Add(storageID, spaceID, shareID, share1) - s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + s = c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) Expect(s).To(Equal(share1)) }) It("sets the mtime", func() { - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) - Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(time.Time{})) + c.Add(storageID, spaceID, shareID, share1) + Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(time.Time{})) }) It("updates the mtime", func() { - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) - old := c.Providers[storageId].Spaces[spaceid].Mtime - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) - Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(old)) + c.Add(storageID, spaceID, shareID, share1) + old := c.Providers[storageID].Spaces[spaceID].Mtime + c.Add(storageID, spaceID, shareID, share1) + Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) Context("with an existing entry", func() { BeforeEach(func() { - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) + c.Add(storageID, spaceID, shareID, share1) + Expect(c.Persist(context.Background(), storageID, spaceID)).To(Succeed()) + }) + + Describe("Get", func() { + It("returns the entry", func() { + s := c.Get(storageID, spaceID, shareID) + Expect(s).ToNot(BeNil()) + }) }) Describe("Remove", func() { It("removes the entry", func() { - s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + s := c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) Expect(s).To(Equal(share1)) - c.Remove(storageId, spaceid, share1.Id.OpaqueId) + c.Remove(storageID, spaceID, shareID) - s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + s = c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) }) It("updates the mtime", func() { - c.Add(storageId, spaceid, share1.Id.OpaqueId, share1) - old := c.Providers[storageId].Spaces[spaceid].Mtime - c.Remove(storageId, spaceid, share1.Id.OpaqueId) - Expect(c.Providers[storageId].Spaces[spaceid].Mtime).ToNot(Equal(old)) + c.Add(storageID, spaceID, shareID, share1) + old := c.Providers[storageID].Spaces[spaceID].Mtime + c.Remove(storageID, spaceID, shareID) + Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) @@ -108,46 +117,46 @@ var _ = Describe("Cache", func() { Expect(c.Persist(ctx, "foo", "bar")).To(Succeed()) }) It("handles non-existent spaces", func() { - Expect(c.Persist(ctx, storageId, "bar")).To(Succeed()) + Expect(c.Persist(ctx, storageID, "bar")).To(Succeed()) }) It("persists", func() { - Expect(c.Persist(ctx, storageId, spaceid)).To(Succeed()) + Expect(c.Persist(ctx, storageID, spaceID)).To(Succeed()) }) }) Describe("Sync", func() { BeforeEach(func() { - Expect(c.Persist(ctx, storageId, spaceid)).To(Succeed()) + Expect(c.Persist(ctx, storageID, spaceID)).To(Succeed()) // reset in-memory cache c = providercache.New(storage) }) It("downloads if needed", func() { - s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Sync(ctx, storageId, spaceid) + c.Sync(ctx, storageID, spaceID) - s = c.Get(storageId, spaceid, share1.Id.OpaqueId) + s = c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) }) It("does not download if not needed", func() { - s := c.Get(storageId, spaceid, share1.Id.OpaqueId) + s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Providers[storageId] = &providercache.Spaces{ + c.Providers[storageID] = &providercache.Spaces{ Spaces: map[string]*providercache.Shares{ - spaceid: { + spaceID: { Mtime: time.Now(), }, }, } - c.Sync(ctx, storageId, spaceid) + c.Sync(ctx, storageID, spaceID) // Sync from disk won't happen because in-memory mtime is later than on disk - s = c.Get(storageId, spaceid, share1.Id.OpaqueId) - Expect(s).To(BeNil()) // Sync from disk didn't happen because in-memory mtime is later than on disk + s = c.Get(storageID, spaceID, shareID) + Expect(s).To(BeNil()) }) }) }) diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index bc5825f0dd..545bc560ec 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -21,6 +21,7 @@ package sharecache import ( "context" "encoding/json" + "path" "path/filepath" "time" @@ -159,8 +160,12 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { if err != nil { return err } + jsonPath := userCreatedPath(userid) + if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + return err + } // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := c.storage.SimpleUpload(ctx, userCreatedPath(userid), createdBytes); err != nil { + if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { return err } return nil From 9871e20cc8cd3b96a6f4efbe8be24cbcdfcceda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 3 Aug 2022 14:18:02 +0200 Subject: [PATCH 42/94] Test that GetShare syncs the cache --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 28 +++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 9636b162bc..f34319ddef 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -243,17 +243,31 @@ var _ = Describe("Jsoncs3", func() { Expect(s.Id.OpaqueId).To(Equal(share.Id.OpaqueId)) }) + It("reloads the provider cache when it is outdated", func() { + s, err := m.GetShare(ctx, shareRef) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + + cache := m.Cache.Providers["storageid"].Spaces["spaceid"] + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true + bytes, err := json.Marshal(cache) + Expect(err).ToNot(HaveOccurred()) + storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(err).ToNot(HaveOccurred()) + + m.CreatedCache.UserShares["admin"].Mtime = time.Time{} // trigger reload + s, err = m.GetShare(ctx, shareRef) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.Permissions.Permissions.InitiateFileUpload).To(BeTrue()) + }) + It("does not return other users' shares", func() { s, err := m.GetShare(otherCtx, shareRef) Expect(err).To(HaveOccurred()) Expect(s).To(BeNil()) }) - - It("reloads the provider cache when it is outdated", func() { - - }) - - PIt("uses the new data after reload") }) Describe("UnShare", func() { @@ -363,7 +377,7 @@ var _ = Describe("Jsoncs3", func() { Expect(shares[0].Id).To(Equal(share.Id)) }) - FIt("uses the data from the storage after reload", func() { + It("uses the data from the storage after reload", func() { shares, err := m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(1)) From b5642c92463e23802a6cec6971f909328496c1a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 10:59:06 +0200 Subject: [PATCH 43/94] Reload cache if a Share wasn't found. Fix unmarshaling shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 11 ++++- pkg/share/manager/jsoncs3/jsoncs3_test.go | 12 ++++++ .../jsoncs3/providercache/providercache.go | 40 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index e2f2e3464c..15f7c60a7a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -270,7 +270,16 @@ func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } share := m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) if share == nil { - return nil, errtypes.NotFound(id.String()) + // reload cache, maybe out data is outdated + err = m.Cache.Sync(context.Background(), shareid.StorageId, shareid.SpaceId) + if err != nil { + return nil, err + } + + share = m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) + if share == nil { + return nil, errtypes.NotFound(id.String()) + } } return share, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index f34319ddef..144b591534 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -263,6 +263,18 @@ var _ = Describe("Jsoncs3", func() { Expect(s.Permissions.Permissions.InitiateFileUpload).To(BeTrue()) }) + It("loads the cache when it doesn't have an entry", func() { + err := m.Cache.Persist(context.Background(), "storageid", "spaceid") + Expect(err).ToNot(HaveOccurred()) + + m, err = jsoncs3.New(storage) + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, shareRef) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + }) + It("does not return other users' shares", func() { s, err := m.GetShare(otherCtx, shareRef) Expect(err).To(HaveOccurred()) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index b3da02e1d5..b58866d1b4 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -26,6 +26,7 @@ import ( "time" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -45,6 +46,45 @@ type Shares struct { Mtime time.Time } +func (s *Shares) UnmarshalJSON(data []byte) error { + // Shares are tricky to unmarshal because the contain an interface (Grantee) which makes the json Unmarshal bail out + // To work around that problem we unmarshal into json.RawMessage in a first step and then try to manually unmarshal + // into the specific types in a second step. + tmp := struct { + Shares map[string]json.RawMessage + Mtime time.Time + }{} + + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + + s.Mtime = tmp.Mtime + s.Shares = make(map[string]*collaboration.Share, len(tmp.Shares)) + for id, genericShare := range tmp.Shares { + userShare := &collaboration.Share{ + Grantee: &provider.Grantee{Id: &provider.Grantee_UserId{}}, + } + err = json.Unmarshal(genericShare, userShare) // is this a user share? + if err == nil { + s.Shares[id] = userShare + continue + } + + groupShare := &collaboration.Share{ + Grantee: &provider.Grantee{Id: &provider.Grantee_GroupId{}}, + } + err = json.Unmarshal(genericShare, groupShare) // try to unmarshal to a group share if the user share unmarshalling failed + if err != nil { + return err + } + s.Shares[id] = groupShare + } + + return nil +} + func New(s metadata.Storage) Cache { return Cache{ Providers: map[string]*Spaces{}, From 824590d5c284d9c17563a7c69c136f26b817bc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 11:24:58 +0200 Subject: [PATCH 44/94] Persist the cache when adding/removing shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 30 ++++++++++----- pkg/share/manager/jsoncs3/jsoncs3_test.go | 47 ++++++++++++++++++++++- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 15f7c60a7a..3174c98d0b 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -191,7 +191,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.Lock() defer m.Unlock() - _, err := m.getByKey(key) + _, err := m.getByKey(ctx, key) if err == nil { // share already exists return nil, errtypes.AlreadyExists(key.String()) @@ -221,6 +221,10 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } m.Cache.Add(md.Id.StorageId, md.Id.SpaceId, shareID, s) + err = m.Cache.Persist(ctx, md.Id.StorageId, md.Id.SpaceId) + if err != nil { + return nil, err + } err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -270,7 +274,7 @@ func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } share := m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) if share == nil { - // reload cache, maybe out data is outdated + // reload cache, maybe our data is outdated err = m.Cache.Sync(context.Background(), shareid.StorageId, shareid.SpaceId) if err != nil { return nil, err @@ -285,7 +289,11 @@ func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro } // getByKey must be called in a lock-controlled block. -func (m *Manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, error) { +func (m *Manager) getByKey(ctx context.Context, key *collaboration.ShareKey) (*collaboration.Share, error) { + err := m.Cache.Sync(ctx, key.ResourceId.StorageId, key.ResourceId.SpaceId) + if err != nil { + return nil, err + } spaceShares := m.Cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) for _, share := range spaceShares.Shares { if utils.GranteeEqual(key.Grantee, share.Grantee) { @@ -296,12 +304,12 @@ func (m *Manager) getByKey(key *collaboration.ShareKey) (*collaboration.Share, e } // get must be called in a lock-controlled block. -func (m *Manager) get(ref *collaboration.ShareReference) (s *collaboration.Share, err error) { +func (m *Manager) get(ctx context.Context, ref *collaboration.ShareReference) (s *collaboration.Share, err error) { switch { case ref.GetId() != nil: s, err = m.getByID(ref.GetId()) case ref.GetKey() != nil: - s, err = m.getByKey(ref.GetKey()) + s, err = m.getByKey(ctx, ref.GetKey()) default: err = errtypes.NotFound(ref.String()) } @@ -311,7 +319,7 @@ func (m *Manager) get(ref *collaboration.ShareReference) (s *collaboration.Share func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - s, err := m.get(ref) + s, err := m.get(ctx, ref) if err != nil { return nil, err } @@ -330,7 +338,7 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference defer m.Unlock() user := ctxpkg.ContextMustGetUser(ctx) - s, err := m.get(ref) + s, err := m.get(ctx, ref) if err != nil { return err } @@ -345,6 +353,10 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return err } m.Cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) + err = m.Cache.Persist(ctx, shareid.StorageId, shareid.SpaceId) + if err != nil { + return err + } // remove from created cache err = m.CreatedCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) @@ -364,7 +376,7 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() - s, err := m.get(ref) + s, err := m.get(ctx, ref) if err != nil { return nil, err } @@ -559,7 +571,7 @@ func (m *Manager) GetReceivedShare(ctx context.Context, ref *collaboration.Share func (m *Manager) getReceived(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() - s, err := m.get(ref) + s, err := m.get(ctx, ref) if err != nil { return nil, err } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 144b591534..9343a848d5 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -177,6 +177,26 @@ var _ = Describe("Jsoncs3", func() { Expect(share).ToNot(BeNil()) Expect(share.ResourceId).To(Equal(sharedResource.Id)) }) + + It("persists the share", func() { + _, err := m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s).ToNot(BeNil()) + + m, err = jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + s = shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s).ToNot(BeNil()) + }) }) Context("with an existing share", func() { @@ -267,7 +287,7 @@ var _ = Describe("Jsoncs3", func() { err := m.Cache.Persist(context.Background(), "storageid", "spaceid") Expect(err).ToNot(HaveOccurred()) - m, err = jsoncs3.New(storage) + m, err = jsoncs3.New(storage) // Reset in-memory cache Expect(err).ToNot(HaveOccurred()) s, err := m.GetShare(ctx, shareRef) @@ -316,6 +336,31 @@ var _ = Describe("Jsoncs3", func() { Expect(err).To(HaveOccurred()) Expect(s).To(BeNil()) }) + + It("removes an existing share from the storage", func() { + err := m.Unshare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + m, err = jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }, + }, + }) + Expect(err).To(HaveOccurred()) + Expect(s).To(BeNil()) + }) }) Describe("UpdateShare", func() { From 1d7a73f4eb0cfccb70fc4d71e32885ab7d6121a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 11:34:08 +0200 Subject: [PATCH 45/94] Implicitly persist the cache when adding/removing entries --- pkg/share/manager/jsoncs3/jsoncs3.go | 12 ++---------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 2 +- .../jsoncs3/providercache/providercache.go | 10 +++++++--- .../jsoncs3/providercache/providercache_test.go | 17 ++++++++--------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 3174c98d0b..2bb99c1499 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -220,11 +220,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Mtime: ts, } - m.Cache.Add(md.Id.StorageId, md.Id.SpaceId, shareID, s) - err = m.Cache.Persist(ctx, md.Id.StorageId, md.Id.SpaceId) - if err != nil { - return nil, err - } + m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -352,11 +348,7 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference if err != nil { return err } - m.Cache.Remove(shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) - err = m.Cache.Persist(ctx, shareid.StorageId, shareid.SpaceId) - if err != nil { - return err - } + m.Cache.Remove(ctx, shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) // remove from created cache err = m.CreatedCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 9343a848d5..e54956e801 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -440,7 +440,7 @@ var _ = Describe("Jsoncs3", func() { Expect(len(shares)).To(Equal(1)) // Add a second cache to the provider cache so it can be referenced - m.Cache.Add("storageid", "spaceid", "storageid$spaceid!secondshare", &collaboration.Share{ + m.Cache.Add(ctx, "storageid", "spaceid", "storageid$spaceid!secondshare", &collaboration.Share{ Creator: user1.Id, }) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index b58866d1b4..2d4b657e6f 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -92,19 +92,23 @@ func New(s metadata.Storage) Cache { } } -func (c *Cache) Add(storageID, spaceID, shareID string, share *collaboration.Share) { +func (c *Cache) Add(ctx context.Context, storageID, spaceID, shareID string, share *collaboration.Share) error { c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID].Shares[shareID] = share c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() + + return c.Persist(ctx, storageID, spaceID) } -func (c *Cache) Remove(storageID, spaceID, shareID string) { +func (c *Cache) Remove(ctx context.Context, storageID, spaceID, shareID string) error { if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { - return + return nil } delete(c.Providers[storageID].Spaces[spaceID].Shares, shareID) c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() + + return c.Persist(ctx, storageID, spaceID) } func (c *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index abbbfdb573..490c693c26 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -59,7 +59,7 @@ var _ = Describe("Cache", func() { s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Add(storageID, spaceID, shareID, share1) + c.Add(ctx, storageID, spaceID, shareID, share1) s = c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) @@ -67,22 +67,21 @@ var _ = Describe("Cache", func() { }) It("sets the mtime", func() { - c.Add(storageID, spaceID, shareID, share1) + c.Add(ctx, storageID, spaceID, shareID, share1) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(time.Time{})) }) It("updates the mtime", func() { - c.Add(storageID, spaceID, shareID, share1) + c.Add(ctx, storageID, spaceID, shareID, share1) old := c.Providers[storageID].Spaces[spaceID].Mtime - c.Add(storageID, spaceID, shareID, share1) + c.Add(ctx, storageID, spaceID, shareID, share1) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) Context("with an existing entry", func() { BeforeEach(func() { - c.Add(storageID, spaceID, shareID, share1) - Expect(c.Persist(context.Background(), storageID, spaceID)).To(Succeed()) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) }) Describe("Get", func() { @@ -98,16 +97,16 @@ var _ = Describe("Cache", func() { Expect(s).ToNot(BeNil()) Expect(s).To(Equal(share1)) - c.Remove(storageID, spaceID, shareID) + c.Remove(ctx, storageID, spaceID, shareID) s = c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) }) It("updates the mtime", func() { - c.Add(storageID, spaceID, shareID, share1) + c.Add(ctx, storageID, spaceID, shareID, share1) old := c.Providers[storageID].Spaces[spaceID].Mtime - c.Remove(storageID, spaceID, shareID) + c.Remove(ctx, storageID, spaceID, shareID) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) From 305d1dd4e905ca5bb19a52c3264501261da704d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 11:52:00 +0200 Subject: [PATCH 46/94] Also implicitly persist the ShareCache when adding/removing items --- pkg/share/manager/jsoncs3/jsoncs3.go | 25 ++++++------------- .../manager/jsoncs3/sharecache/sharecache.go | 9 ++++--- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 2bb99c1499..48025c9ab8 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -222,7 +222,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) - err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), shareID) + err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { return nil, err } @@ -253,7 +253,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() - if err := m.groupReceivedCache.Add(groupid, shareID); err != nil { + if err := m.groupReceivedCache.Add(ctx, groupid, shareID); err != nil { return nil, err } } @@ -348,14 +348,13 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference if err != nil { return err } - m.Cache.Remove(ctx, shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) - - // remove from created cache - err = m.CreatedCache.Remove(s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + err = m.Cache.Remove(ctx, shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) if err != nil { return err } - err = m.CreatedCache.Persist(ctx, s.GetCreator().GetOpaqueId()) + + // remove from created cache + err = m.CreatedCache.Remove(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) if err != nil { return err } @@ -385,7 +384,7 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer Nanos: uint32(now % int64(time.Second)), } - err = m.setCreatedCache(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) if err != nil { return nil, err } @@ -393,16 +392,6 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer return s, nil } -func (m *Manager) setCreatedCache(ctx context.Context, creatorid, shareid string) error { - if err := m.CreatedCache.Add(creatorid, shareid); err != nil { - return err - } - if err := m.CreatedCache.Persist(ctx, creatorid); err != nil { - return err - } - return nil -} - // ListShares func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { m.Lock() diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 545bc560ec..947227e12d 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -58,7 +58,7 @@ func (c *Cache) Has(userid string) bool { return c.UserShares[userid] != nil } -func (c *Cache) Add(userid, shareID string) error { +func (c *Cache) Add(ctx context.Context, userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -83,10 +83,10 @@ func (c *Cache) Add(userid, shareID string) error { c.UserShares[userid].Mtime = now c.UserShares[userid].UserShares[ssid].Mtime = now c.UserShares[userid].UserShares[ssid].IDs[shareID] = struct{}{} - return nil + return c.Persist(ctx, userid) } -func (c *Cache) Remove(userid, shareID string) error { +func (c *Cache) Remove(ctx context.Context, userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { return err @@ -105,7 +105,8 @@ func (c *Cache) Remove(userid, shareID string) error { delete(c.UserShares[userid].UserShares[ssid].IDs, shareID) } } - return nil + + return c.Persist(ctx, userid) } func (c *Cache) List(userid string) map[string]SpaceShareIDs { From c4af393506f1a3313eb1a4a6899893ef3b91df43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 12:05:28 +0200 Subject: [PATCH 47/94] Consider the resource id when looking for shares by key --- pkg/share/manager/jsoncs3/jsoncs3.go | 2 +- pkg/share/manager/jsoncs3/jsoncs3_test.go | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 48025c9ab8..b2c47e7cc8 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -292,7 +292,7 @@ func (m *Manager) getByKey(ctx context.Context, key *collaboration.ShareKey) (*c } spaceShares := m.Cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) for _, share := range spaceShares.Shares { - if utils.GranteeEqual(key.Grantee, share.Grantee) { + if utils.GranteeEqual(key.Grantee, share.Grantee) && utils.ResourceIDEqual(share.ResourceId, key.ResourceId) { return share, nil } } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index e54956e801..a421a47dfb 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -247,6 +247,23 @@ var _ = Describe("Jsoncs3", func() { Expect(err).To(HaveOccurred()) }) + It("considers the resource id part of the key", func() { + s, err := m.GetShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Key{ + Key: &collaboration.ShareKey{ + ResourceId: &providerv1beta1.ResourceId{ + StorageId: "storageid", + SpaceId: "spaceid", + OpaqueId: "unknown", + }, + Grantee: grant.Grantee, + }, + }, + }) + Expect(s).To(BeNil()) + Expect(err).To(HaveOccurred()) + }) + It("retrieves an existing share by id", func() { s, err := m.GetShare(ctx, shareRef) Expect(err).ToNot(HaveOccurred()) From 2d83a26800396a03772037cb87b32c077d909a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 13:22:12 +0200 Subject: [PATCH 48/94] Add more tests --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 38 ++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index a421a47dfb..52f4273d55 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -301,10 +301,7 @@ var _ = Describe("Jsoncs3", func() { }) It("loads the cache when it doesn't have an entry", func() { - err := m.Cache.Persist(context.Background(), "storageid", "spaceid") - Expect(err).ToNot(HaveOccurred()) - - m, err = jsoncs3.New(storage) // Reset in-memory cache + m, err := jsoncs3.New(storage) // Reset in-memory cache Expect(err).ToNot(HaveOccurred()) s, err := m.GetShare(ctx, shareRef) @@ -441,7 +438,40 @@ var _ = Describe("Jsoncs3", func() { }) Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) }) + + It("persists the change", func() { + s := shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + + // enhance privileges + us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: share.Id.OpaqueId, + }, + }, + }, &collaboration.SharePermissions{ + Permissions: writePermissions, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(us).ToNot(BeNil()) + Expect(us.GetPermissions().GetPermissions()).To(Equal(writePermissions)) + + m, err = jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + s = shareBykey(&collaboration.ShareKey{ + ResourceId: sharedResource.Id, + Grantee: grant.Grantee, + }) + Expect(s.GetPermissions().GetPermissions()).To(Equal(writePermissions)) + + }) }) + Describe("ListShares", func() { It("lists an existing share", func() { shares, err := m.ListShares(ctx, nil) From 7efe451486bfc7d2980200bed78829a267bb82bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 15:59:00 +0200 Subject: [PATCH 49/94] Persist updates in the provider cache --- pkg/share/manager/jsoncs3/jsoncs3.go | 6 +- pkg/share/manager/jsoncs3/jsoncs3_test.go | 98 ++++++++++++++--------- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index b2c47e7cc8..c0fffeb62f 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -384,10 +384,8 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer Nanos: uint32(now % int64(time.Second)), } - err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) - if err != nil { - return nil, err - } + // Update provider cache + m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) return s, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 52f4273d55..ae0a73062a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -76,30 +76,7 @@ var _ = Describe("Jsoncs3", func() { Id: user2.Id, Groups: []string{"users"}, } - readPermissions = &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - } - writePermissions = &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - InitiateFileUpload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - } - grant = &collaboration.ShareGrant{ - Grantee: &provider.Grantee{ - Type: provider.GranteeType_GRANTEE_TYPE_USER, - Id: &provider.Grantee_UserId{UserId: grantee.GetId()}, - }, - Permissions: &collaboration.SharePermissions{ - Permissions: readPermissions, - }, - } + grant *collaboration.ShareGrant groupGrant = &collaboration.ShareGrant{ Grantee: &provider.Grantee{ @@ -109,7 +86,9 @@ var _ = Describe("Jsoncs3", func() { }}, }, Permissions: &collaboration.SharePermissions{ - Permissions: readPermissions, + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: false, + }, }, } storage metadata.Storage @@ -134,6 +113,18 @@ var _ = Describe("Jsoncs3", func() { ) BeforeEach(func() { + grant = &collaboration.ShareGrant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{UserId: grantee.GetId()}, + }, + Permissions: &collaboration.SharePermissions{ + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: false, + }, + }, + } + var err error tmpdir, err = ioutil.TempDir("", "jsoncs3-test") Expect(err).ToNot(HaveOccurred()) @@ -386,7 +377,9 @@ var _ = Describe("Jsoncs3", func() { }, }, }, &collaboration.SharePermissions{ - Permissions: writePermissions, + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: true, + }, }) Expect(err).To(HaveOccurred()) }) @@ -396,7 +389,7 @@ var _ = Describe("Jsoncs3", func() { ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) // enhance privileges us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ @@ -406,17 +399,19 @@ var _ = Describe("Jsoncs3", func() { }, }, }, &collaboration.SharePermissions{ - Permissions: writePermissions, + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: true, + }, }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions()).To(Equal(writePermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions()).To(Equal(writePermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) // reduce privileges us, err = m.UpdateShare(ctx, &collaboration.ShareReference{ @@ -426,17 +421,19 @@ var _ = Describe("Jsoncs3", func() { }, }, }, &collaboration.SharePermissions{ - Permissions: readPermissions, + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: false, + }, }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) }) It("persists the change", func() { @@ -444,7 +441,7 @@ var _ = Describe("Jsoncs3", func() { ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions()).To(Equal(readPermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) // enhance privileges us, err := m.UpdateShare(ctx, &collaboration.ShareReference{ @@ -454,11 +451,13 @@ var _ = Describe("Jsoncs3", func() { }, }, }, &collaboration.SharePermissions{ - Permissions: writePermissions, + Permissions: &providerv1beta1.ResourcePermissions{ + InitiateFileUpload: true, + }, }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(us.GetPermissions().GetPermissions()).To(Equal(writePermissions)) + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) m, err = jsoncs3.New(storage) // Reset in-memory cache Expect(err).ToNot(HaveOccurred()) @@ -467,8 +466,7 @@ var _ = Describe("Jsoncs3", func() { ResourceId: sharedResource.Id, Grantee: grant.Grantee, }) - Expect(s.GetPermissions().GetPermissions()).To(Equal(writePermissions)) - + Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) }) }) @@ -481,7 +479,29 @@ var _ = Describe("Jsoncs3", func() { Expect(shares[0].Id).To(Equal(share.Id)) }) - It("uses the data from the storage after reload", func() { + It("syncronizes the provider cache before listing", func() { + shares, err := m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].Id.OpaqueId).To(Equal(share.Id.OpaqueId)) + Expect(shares[0].Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + + cache := m.Cache.Providers["storageid"].Spaces["spaceid"] + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true + bytes, err := json.Marshal(cache) + Expect(err).ToNot(HaveOccurred()) + storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(err).ToNot(HaveOccurred()) + + m.CreatedCache.UserShares["admin"].Mtime = time.Time{} // trigger reload + shares, err = m.ListShares(ctx, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(len(shares)).To(Equal(1)) + Expect(shares[0].Id.OpaqueId).To(Equal(share.Id.OpaqueId)) + Expect(shares[0].Permissions.Permissions.InitiateFileUpload).To(BeTrue()) + }) + + It("syncronizes the share cache before listing", func() { shares, err := m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(1)) From 83cdc028b73c1b66839d6690bbb2e7ef52de5487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Aug 2022 16:04:40 +0200 Subject: [PATCH 50/94] Update the MTime when persisting the caches --- pkg/share/manager/jsoncs3/providercache/providercache.go | 5 ++--- .../manager/jsoncs3/providercache/providercache_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 2d4b657e6f..4fd36cf9ae 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -95,7 +95,6 @@ func New(s metadata.Storage) Cache { func (c *Cache) Add(ctx context.Context, storageID, spaceID, shareID string, share *collaboration.Share) error { c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID].Shares[shareID] = share - c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() return c.Persist(ctx, storageID, spaceID) } @@ -106,7 +105,6 @@ func (c *Cache) Remove(ctx context.Context, storageID, spaceID, shareID string) return nil } delete(c.Providers[storageID].Spaces[spaceID].Shares, shareID) - c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() return c.Persist(ctx, storageID, spaceID) } @@ -131,15 +129,16 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return nil } + c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() createdBytes, err := json.Marshal(c.Providers[storageID].Spaces[spaceID]) if err != nil { return err } jsonPath := spaceJSONPath(storageID, spaceID) - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { return err } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { return err } diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 490c693c26..cd087e4706 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -122,6 +122,13 @@ var _ = Describe("Cache", func() { It("persists", func() { Expect(c.Persist(ctx, storageID, spaceID)).To(Succeed()) }) + + It("updates the mtime", func() { + oldMtime := c.Providers[storageID].Spaces[spaceID].Mtime + + Expect(c.Persist(ctx, storageID, spaceID)).To(Succeed()) + Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(oldMtime)) + }) }) Describe("Sync", func() { From 9c16653fc3ae50f779eea4a86d0b82bf758c33e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 09:31:06 +0200 Subject: [PATCH 51/94] Set the MTime when persisting a UserCache --- .../manager/jsoncs3/sharecache/sharecache.go | 2 + .../sharecache/sharecache_suite_test.go | 13 ++++ .../jsoncs3/sharecache/sharecache_test.go | 65 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go create mode 100644 pkg/share/manager/jsoncs3/sharecache/sharecache_test.go diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 947227e12d..ff4b2289b9 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -157,6 +157,8 @@ func (c *Cache) Sync(ctx context.Context, userid string) error { } func (c *Cache) Persist(ctx context.Context, userid string) error { + c.UserShares[userid].Mtime = time.Now() + createdBytes, err := json.Marshal(c.UserShares[userid]) if err != nil { return err diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go b/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go new file mode 100644 index 0000000000..6c6f050884 --- /dev/null +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go @@ -0,0 +1,13 @@ +package sharecache_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSharecache(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Sharecache Suite") +} diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go new file mode 100644 index 0000000000..63bb2e139f --- /dev/null +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go @@ -0,0 +1,65 @@ +package sharecache_test + +import ( + "context" + "io/ioutil" + "os" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" +) + +var _ = Describe("Sharecache", func() { + var ( + c sharecache.Cache + storage metadata.Storage + + userid = "user" + shareID = "storageid$spaceid!share1" + ctx context.Context + tmpdir string + ) + + BeforeEach(func() { + ctx = context.Background() + + var err error + tmpdir, err = ioutil.TempDir("", "providercache-test") + Expect(err).ToNot(HaveOccurred()) + + err = os.MkdirAll(tmpdir, 0755) + Expect(err).ToNot(HaveOccurred()) + + storage, err = metadata.NewDiskStorage(tmpdir) + Expect(err).ToNot(HaveOccurred()) + + c = sharecache.New(storage) + Expect(c).ToNot(BeNil()) + }) + + AfterEach(func() { + if tmpdir != "" { + os.RemoveAll(tmpdir) + } + }) + + Describe("Persist", func() { + Context("with an existing entry", func() { + BeforeEach(func() { + Expect(c.Add(ctx, userid, shareID)).To(Succeed()) + }) + + It("updates the mtime", func() { + oldMtime := c.UserShares[userid].Mtime + Expect(oldMtime).ToNot(Equal(time.Time{})) + + Expect(c.Persist(ctx, userid)).To(Succeed()) + Expect(c.UserShares[userid]).ToNot(Equal(oldMtime)) + }) + }) + }) +}) From 38b24c8c764b8e881944f4ea912a0e6245da4217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 10:24:41 +0200 Subject: [PATCH 52/94] Fix reloading the provider cache from disk when outdated --- pkg/share/manager/jsoncs3/jsoncs3.go | 25 ++++++++++++++--------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 14 ++++++++++++- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index c0fffeb62f..06fd4ac21a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -268,18 +268,16 @@ func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, erro // invalid share id, does not exist return nil, errtypes.NotFound(id.String()) } + + // sync cache, maybe our data is outdated + err = m.Cache.Sync(context.Background(), shareid.StorageId, shareid.SpaceId) + if err != nil { + return nil, err + } + share := m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) if share == nil { - // reload cache, maybe our data is outdated - err = m.Cache.Sync(context.Background(), shareid.StorageId, shareid.SpaceId) - if err != nil { - return nil, err - } - - share = m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) - if share == nil { - return nil, errtypes.NotFound(id.String()) - } + return nil, errtypes.NotFound(id.String()) } return share, nil } @@ -290,6 +288,13 @@ func (m *Manager) getByKey(ctx context.Context, key *collaboration.ShareKey) (*c if err != nil { return nil, err } + + // sync cache, maybe our data is outdated + err = m.Cache.Sync(context.Background(), key.ResourceId.StorageId, key.ResourceId.SpaceId) + if err != nil { + return nil, err + } + spaceShares := m.Cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) for _, share := range spaceShares.Shares { if utils.GranteeEqual(key.Grantee, share.Grantee) && utils.ResourceIDEqual(share.ResourceId, key.ResourceId) { diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index ae0a73062a..29f8517ba3 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -277,6 +277,7 @@ var _ = Describe("Jsoncs3", func() { Expect(s).ToNot(BeNil()) Expect(s.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + // Change providercache on disk cache := m.Cache.Providers["storageid"].Spaces["spaceid"] cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true bytes, err := json.Marshal(cache) @@ -284,7 +285,18 @@ var _ = Describe("Jsoncs3", func() { storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) Expect(err).ToNot(HaveOccurred()) - m.CreatedCache.UserShares["admin"].Mtime = time.Time{} // trigger reload + // Reset providercache in memory + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = false + + // Set local cache mtime to something later then on disk + m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Now().Add(time.Hour) + s, err = m.GetShare(ctx, shareRef) + Expect(err).ToNot(HaveOccurred()) + Expect(s).ToNot(BeNil()) + Expect(s.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + + // Set local cache mtime to something earlier then on disk + m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Now().Add(-time.Hour) s, err = m.GetShare(ctx, shareRef) Expect(err).ToNot(HaveOccurred()) Expect(s).ToNot(BeNil()) From fff4359a8320cd3ea08440637f9e0f0c22273ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 10:27:03 +0200 Subject: [PATCH 53/94] Cleanup --- pkg/share/manager/jsoncs3/jsoncs3.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 06fd4ac21a..249de9c129 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -90,10 +90,8 @@ type Manager struct { // userReceivedStates holds the state of shares a user has received, sharded by user id and space id userReceivedStates receivedCache - storage metadata.Storage - serviceUser *userv1beta1.User - SpaceRoot *provider.ResourceId - initialized bool + storage metadata.Storage + SpaceRoot *provider.ResourceId } // NewDefault returns a new manager instance with default dependencies @@ -125,7 +123,6 @@ func New(s metadata.Storage) (*Manager, error) { // File structure in the jsoncs3 space: // -// /shares/{shareid.json} // points to {storageid}/{spaceid} for looking up the share information // /storages/{storageid}/{spaceid.json} // contains all share information of all shares in that space // /users/{userid}/created/{storageid}/{spaceid} // points to a space the user created shares in // /users/{userid}/received/{storageid}/{spaceid}.json // holds the states of received shares of the users in the according space From e9a0eae883f998d9fdf729068005941a1b86d911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 10:31:01 +0200 Subject: [PATCH 54/94] Fix syncing the provider cache when listing shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 5 ++++- pkg/share/manager/jsoncs3/jsoncs3_test.go | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 249de9c129..cfe1c2647c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -164,7 +164,6 @@ func New(s metadata.Storage) (*Manager, error) { // /{userid}/created/{storageid}/{spaceid} func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { - user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() ts := &typespb.Timestamp{ @@ -424,6 +423,10 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte if err != nil { continue } + err = m.Cache.Sync(ctx, providerid, spaceid) + if err != nil { + continue + } spaceShares := m.Cache.ListSpace(providerid, spaceid) for shareid, _ := range spaceShareIDs.IDs { s := spaceShares.Shares[shareid] diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 29f8517ba3..92a9f16e97 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -498,6 +498,7 @@ var _ = Describe("Jsoncs3", func() { Expect(shares[0].Id.OpaqueId).To(Equal(share.Id.OpaqueId)) Expect(shares[0].Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + // Change providercache on disk cache := m.Cache.Providers["storageid"].Spaces["spaceid"] cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true bytes, err := json.Marshal(cache) @@ -505,7 +506,10 @@ var _ = Describe("Jsoncs3", func() { storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) Expect(err).ToNot(HaveOccurred()) - m.CreatedCache.UserShares["admin"].Mtime = time.Time{} // trigger reload + // Reset providercache in memory + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = false + + m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Time{} // trigger reload shares, err = m.ListShares(ctx, nil) Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(1)) From 1c30339a77dfcbbe5212c3e3177b1876e8ae4e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 10:38:09 +0200 Subject: [PATCH 55/94] Sync the providercache before listing received shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 4 ++++ pkg/share/manager/jsoncs3/jsoncs3_test.go | 24 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index cfe1c2647c..b26872c98c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -502,6 +502,10 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } + err = m.Cache.Sync(ctx, providerid, spaceid) + if err != nil { + continue + } spaceShares := m.Cache.ListSpace(providerid, spaceid) for shareId, state := range rspace.receivedShareStates { s := spaceShares.Shares[shareId] diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 92a9f16e97..bec006300b 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -601,6 +601,30 @@ var _ = Describe("Jsoncs3", func() { Expect(received[0].Share.ResourceId).To(Equal(sharedResource.Id)) Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) }) + + It("syncronizes the provider cache before listing", func() { + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + + // Change providercache on disk + cache := m.Cache.Providers["storageid"].Spaces["spaceid"] + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true + bytes, err := json.Marshal(cache) + Expect(err).ToNot(HaveOccurred()) + storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(err).ToNot(HaveOccurred()) + + // Reset providercache in memory + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = false + + m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Time{} // trigger reload + received, err = m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeTrue()) + }) }) Describe("GetReceivedShare", func() { From b721d5d1500878ade82932a29702405cfa93ba1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 16:22:31 +0200 Subject: [PATCH 56/94] Add a cache for the received shares states --- .../receivedShareCache_suite_test.go | 13 ++ .../receivedsharecache/receivedsharecache.go | 144 ++++++++++++++++++ .../receivedsharecache_test.go | 134 ++++++++++++++++ 3 files changed, 291 insertions(+) create mode 100644 pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go create mode 100644 pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go create mode 100644 pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache_test.go diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go new file mode 100644 index 0000000000..f93838acd0 --- /dev/null +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go @@ -0,0 +1,13 @@ +package receivedsharecache_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestReceivedShareCache(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ReceivedShareCache Suite") +} diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go new file mode 100644 index 0000000000..d2f4bedea2 --- /dev/null +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -0,0 +1,144 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package receivedsharecache + +import ( + "context" + "encoding/json" + "path" + "path/filepath" + "time" + + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" + "github.com/cs3org/reva/v2/pkg/utils" +) + +type Cache struct { + ReceivedSpaces map[string]*Spaces + + storage metadata.Storage +} + +type Spaces struct { + Mtime time.Time + Spaces map[string]*Space +} + +type Space struct { + Mtime time.Time + States map[string]*State +} +type State struct { + State collaboration.ShareState + MountPoint *provider.Reference +} + +func New(s metadata.Storage) Cache { + return Cache{ + ReceivedSpaces: map[string]*Spaces{}, + storage: s, + } +} + +func (c *Cache) Add(ctx context.Context, userID, spaceID string, rs *collaboration.ReceivedShare) error { + if c.ReceivedSpaces[userID] == nil { + c.ReceivedSpaces[userID] = &Spaces{ + Spaces: map[string]*Space{}, + } + } + if c.ReceivedSpaces[userID].Spaces[spaceID] == nil { + c.ReceivedSpaces[userID].Spaces[spaceID] = &Space{} + } + + receivedSpace := c.ReceivedSpaces[userID].Spaces[spaceID] + receivedSpace.Mtime = time.Now() + if receivedSpace.States == nil { + receivedSpace.States = map[string]*State{} + } + receivedSpace.States[rs.Share.Id.GetOpaqueId()] = &State{ + State: rs.State, + MountPoint: rs.MountPoint, + } + + return c.Persist(ctx, userID) +} + +func (c *Cache) Get(userID, spaceID, shareID string) *State { + if c.ReceivedSpaces[userID] == nil || c.ReceivedSpaces[userID].Spaces[spaceID] == nil { + return nil + } + return c.ReceivedSpaces[userID].Spaces[spaceID].States[shareID] +} + +func (c *Cache) Sync(ctx context.Context, userID string) error { + var mtime time.Time + if c.ReceivedSpaces[userID] != nil { + mtime = c.ReceivedSpaces[userID].Mtime + } else { + mtime = time.Time{} // Set zero time so that data from storage always takes precedence + } + + jsonPath := userJSONPath(userID) + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + // check mtime of /users/{userid}/created.json + if utils.TSToTime(info.Mtime).After(mtime) { + // - update cached list of created shares for the user in memory if changed + createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) + if err != nil { + return err + } + newSpaces := &Spaces{} + err = json.Unmarshal(createdBlob, newSpaces) + if err != nil { + return err + } + c.ReceivedSpaces[userID] = newSpaces + } + return nil +} + +func (c *Cache) Persist(ctx context.Context, userID string) error { + if c.ReceivedSpaces[userID] == nil { + return nil + } + + c.ReceivedSpaces[userID].Mtime = time.Now() + createdBytes, err := json.Marshal(c.ReceivedSpaces[userID]) + if err != nil { + return err + } + jsonPath := userJSONPath(userID) + if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + return err + } + // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { + return err + } + return nil +} + +func userJSONPath(userID string) string { + return filepath.Join("/users", userID, "received.json") +} diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache_test.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache_test.go new file mode 100644 index 0000000000..bd3fcee34a --- /dev/null +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache_test.go @@ -0,0 +1,134 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package receivedsharecache_test + +import ( + "context" + "io/ioutil" + "os" + + collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + collaborationv1beta1 "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache" + "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cache", func() { + var ( + c receivedsharecache.Cache + storage metadata.Storage + + userID = "user" + spaceID = "spaceid" + shareID = "storageid$spaceid!share1" + share = &collaboration.Share{ + Id: &collaborationv1beta1.ShareId{ + OpaqueId: shareID, + }, + } + ctx context.Context + tmpdir string + ) + + BeforeEach(func() { + ctx = context.Background() + + var err error + tmpdir, err = ioutil.TempDir("", "providercache-test") + Expect(err).ToNot(HaveOccurred()) + + err = os.MkdirAll(tmpdir, 0755) + Expect(err).ToNot(HaveOccurred()) + + storage, err = metadata.NewDiskStorage(tmpdir) + Expect(err).ToNot(HaveOccurred()) + + c = receivedsharecache.New(storage) + Expect(c).ToNot(BeNil()) + }) + + AfterEach(func() { + if tmpdir != "" { + os.RemoveAll(tmpdir) + } + }) + + Describe("Add", func() { + It("adds an entry", func() { + rs := &collaboration.ReceivedShare{ + Share: share, + State: collaborationv1beta1.ShareState_SHARE_STATE_PENDING, + } + err := c.Add(ctx, userID, spaceID, rs) + Expect(err).ToNot(HaveOccurred()) + + s := c.Get(userID, spaceID, shareID) + Expect(s).ToNot(BeNil()) + }) + + It("persists the new entry", func() { + rs := &collaboration.ReceivedShare{ + Share: share, + State: collaborationv1beta1.ShareState_SHARE_STATE_PENDING, + } + err := c.Add(ctx, userID, spaceID, rs) + Expect(err).ToNot(HaveOccurred()) + + c = receivedsharecache.New(storage) + Expect(c.Sync(ctx, userID)).To(Succeed()) + s := c.Get(userID, spaceID, shareID) + Expect(s).ToNot(BeNil()) + }) + }) + + Describe("with an existing entry", func() { + BeforeEach(func() { + rs := &collaboration.ReceivedShare{ + Share: share, + State: collaborationv1beta1.ShareState_SHARE_STATE_PENDING, + } + Expect(c.Add(ctx, userID, spaceID, rs)).To(Succeed()) + }) + + Describe("Get", func() { + It("handles unknown users", func() { + s := c.Get("something", spaceID, shareID) + Expect(s).To(BeNil()) + }) + + It("handles unknown spaces", func() { + s := c.Get(userID, "something", shareID) + Expect(s).To(BeNil()) + }) + + It("handles unknown shares", func() { + s := c.Get(userID, spaceID, "something") + Expect(s).To(BeNil()) + }) + + It("gets the entry", func() { + s := c.Get(userID, spaceID, shareID) + Expect(s).ToNot(BeNil()) + }) + }) + }) +}) From bfd860075332856286ad270eebd39a276ed82e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 16:22:49 +0200 Subject: [PATCH 57/94] Use the new userreceivedshares cache --- pkg/share/manager/jsoncs3/jsoncs3.go | 103 ++++++++-------------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 83 ++++++++++------- 2 files changed, 88 insertions(+), 98 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index b26872c98c..a69ba9311a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -28,7 +28,6 @@ import ( "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" @@ -36,6 +35,7 @@ import ( "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages @@ -67,17 +67,6 @@ type config struct { MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` } -type receivedCache map[string]receivedSpaces -type receivedSpaces map[string]*receivedSpace -type receivedSpace struct { - mtime int64 - receivedShareStates map[string]receivedShareState -} -type receivedShareState struct { - state collaboration.ShareState - mountPoint *provider.Reference -} - type Manager struct { sync.RWMutex @@ -88,7 +77,7 @@ type Manager struct { // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id groupReceivedCache sharecache.Cache // userReceivedStates holds the state of shares a user has received, sharded by user id and space id - userReceivedStates receivedCache + userReceivedStates receivedsharecache.Cache storage metadata.Storage SpaceRoot *provider.ResourceId @@ -115,7 +104,7 @@ func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: providercache.New(s), CreatedCache: sharecache.New(s), - userReceivedStates: receivedCache{}, + userReceivedStates: receivedsharecache.New(s), groupReceivedCache: sharecache.New(s), storage: s, }, nil @@ -231,22 +220,12 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla switch g.Grantee.Type { case provider.GranteeType_GRANTEE_TYPE_USER: userid := g.Grantee.GetUserId().GetOpaqueId() - if m.userReceivedStates[userid] == nil { - m.userReceivedStates[userid] = receivedSpaces{} // receivedSpaces - } - if m.userReceivedStates[userid][spaceID] == nil { - m.userReceivedStates[userid][spaceID] = &receivedSpace{} - } - receivedSpace := m.userReceivedStates[userid][spaceID] - receivedSpace.mtime = now - if receivedSpace.receivedShareStates == nil { - receivedSpace.receivedShareStates = map[string]receivedShareState{} - } - receivedSpace.receivedShareStates[shareID] = receivedShareState{ - state: collaboration.ShareState_SHARE_STATE_PENDING, - // mountpoint stays empty until user accepts the share + rs := &collaboration.ReceivedShare{ + Share: s, + State: collaboration.ShareState_SHARE_STATE_PENDING, } + m.userReceivedStates.Add(ctx, userid, spaceID, rs) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() if err := m.groupReceivedCache.Add(ctx, groupid, shareID); err != nil { @@ -457,7 +436,7 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati // con: when adding a received share we MUST have if-match for the initiate-file-upload request // to ensure consistency / prevent lost updates - ssids := map[string]*receivedSpace{} + ssids := map[string]*receivedsharecache.Space{} // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { @@ -467,14 +446,14 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // add a pending entry, the state will be updated // when reading the received shares below if they have already been accepted or denied - rs := receivedSpace{ - mtime: spaceShareIDs.Mtime.UnixNano(), - receivedShareStates: make(map[string]receivedShareState, len(spaceShareIDs.IDs)), + rs := receivedsharecache.Space{ + Mtime: spaceShareIDs.Mtime, + States: make(map[string]*receivedsharecache.State, len(spaceShareIDs.IDs)), } for shareid, _ := range spaceShareIDs.IDs { - rs.receivedShareStates[shareid] = receivedShareState{ - state: collaboration.ShareState_SHARE_STATE_PENDING, + rs.States[shareid] = &receivedsharecache.State{ + State: collaboration.ShareState_SHARE_STATE_PENDING, } } ssids[ssid] = &rs @@ -482,15 +461,16 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // add all spaces the user has receved shares for, this includes mount points and share state for groups - for ssid, rspace := range m.userReceivedStates[user.Id.OpaqueId] { - if rspace.mtime < time.Now().Add(-30*time.Second).UnixNano() { + m.userReceivedStates.Sync(ctx, user.Id.OpaqueId) + for ssid, rspace := range m.userReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { + if time.Now().Sub(rspace.Mtime) > time.Second*30 { // TODO reread from disk } // TODO use younger mtime to determine if if rs, ok := ssids[ssid]; ok { - for shareid, state := range rspace.receivedShareStates { + for shareid, state := range rspace.States { // overwrite state - rs.receivedShareStates[shareid] = state + rs.States[shareid] = state } } else { ssids[ssid] = rspace @@ -507,7 +487,7 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati continue } spaceShares := m.Cache.ListSpace(providerid, spaceid) - for shareId, state := range rspace.receivedShareStates { + for shareId, state := range rspace.States { s := spaceShares.Shares[shareId] if s == nil { continue @@ -516,8 +496,8 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati if share.MatchesFilters(s, filters) { rs := &collaboration.ReceivedShare{ Share: s, - State: state.state, - MountPoint: state.mountPoint, + State: state.State, + MountPoint: state.MountPoint, } rss = append(rss, rs) } @@ -529,25 +509,25 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // convert must be called in a lock-controlled block. -func (m *Manager) convert(currentUser *userv1beta1.UserId, s *collaboration.Share) *collaboration.ReceivedShare { +func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.Share) *collaboration.ReceivedShare { rs := &collaboration.ReceivedShare{ Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - providerid, spaceid, _, err := storagespace.SplitID(s.Id.OpaqueId) + providerid, sid, _, err := storagespace.SplitID(s.Id.OpaqueId) if err != nil { return rs } - spaceId := storagespace.FormatResourceID(provider.ResourceId{ + spaceID := storagespace.FormatResourceID(provider.ResourceId{ StorageId: providerid, - SpaceId: spaceid, + SpaceId: sid, }) - if rspace := m.userReceivedStates[currentUser.OpaqueId][spaceId]; rspace != nil { - if state, ok := rspace.receivedShareStates[s.Id.OpaqueId]; ok { - rs.State = state.state - rs.MountPoint = state.mountPoint - } + m.userReceivedStates.Sync(ctx, userID) + state := m.userReceivedStates.Get(userID, spaceID, s.Id.GetOpaqueId()) + if state != nil { + rs.State = state.State + rs.MountPoint = state.MountPoint } return rs } @@ -567,7 +547,7 @@ func (m *Manager) getReceived(ctx context.Context, ref *collaboration.ShareRefer if !share.IsGrantedToUser(s, user) { return nil, errtypes.NotFound(ref.String()) } - return m.convert(user.Id, s), nil + return m.convert(ctx, user.Id.GetOpaqueId(), s), nil } func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { @@ -590,28 +570,15 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab } } - newState := receivedShareState{ - state: rs.State, - mountPoint: rs.MountPoint, - } - // write back - currentUser := ctxpkg.ContextMustGetUser(ctx) - spaceId := storagespace.FormatResourceID(provider.ResourceId{ + + userID := ctxpkg.ContextMustGetUser(ctx) + spaceID := storagespace.FormatResourceID(provider.ResourceId{ StorageId: rs.Share.ResourceId.StorageId, SpaceId: rs.Share.ResourceId.SpaceId, }) - rspace := m.userReceivedStates[currentUser.Id.OpaqueId][spaceId] - if rspace == nil { - m.userReceivedStates[currentUser.Id.OpaqueId][spaceId] = - &receivedSpace{ - receivedShareStates: map[string]receivedShareState{}, - } - } - // update entry - rspace.receivedShareStates[rs.Share.Id.OpaqueId] = newState - rspace.mtime = time.Now().UnixNano() + m.userReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), spaceID, rs) return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index bec006300b..b756664ca2 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -555,6 +555,48 @@ var _ = Describe("Jsoncs3", func() { PIt("filters by owner") PIt("filters by creator") PIt("filters by grantee type") + PIt("syncronizes the group received cache before listing") + + It("lists the received shares", func() { + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.ResourceId).To(Equal(sharedResource.Id)) + Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + }) + + It("syncronizes the provider cache before listing", func() { + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) + + // Change providercache on disk + cache := m.Cache.Providers["storageid"].Spaces["spaceid"] + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true + bytes, err := json.Marshal(cache) + Expect(err).ToNot(HaveOccurred()) + storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(err).ToNot(HaveOccurred()) + + // Reset providercache in memory + cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = false + + m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Time{} // trigger reload + received, err = m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeTrue()) + }) + + It("syncronizes the user received cache before listing", func() { + m, err := jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(1)) + }) It("filters by resource id", func() { share2, err := m.Share(ctx, sharedResource2, grant) @@ -593,42 +635,23 @@ var _ = Describe("Jsoncs3", func() { Expect(received[0].Share.Id).To(Equal(share2.Id)) }) + }) - It("lists the received shares", func() { - received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Describe("GetReceivedShare", func() { + It("gets the state", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) Expect(err).ToNot(HaveOccurred()) - Expect(len(received)).To(Equal(1)) - Expect(received[0].Share.ResourceId).To(Equal(sharedResource.Id)) - Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) }) - It("syncronizes the provider cache before listing", func() { - received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(received)).To(Equal(1)) - Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeFalse()) - - // Change providercache on disk - cache := m.Cache.Providers["storageid"].Spaces["spaceid"] - cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true - bytes, err := json.Marshal(cache) - Expect(err).ToNot(HaveOccurred()) - storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) - Expect(err).ToNot(HaveOccurred()) - - // Reset providercache in memory - cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = false - - m.Cache.Providers["storageid"].Spaces["spaceid"].Mtime = time.Time{} // trigger reload - received, err = m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + It("syncs the cache", func() { + m, err := jsoncs3.New(storage) // Reset in-memory cache Expect(err).ToNot(HaveOccurred()) - Expect(len(received)).To(Equal(1)) - Expect(received[0].Share.Permissions.Permissions.InitiateFileUpload).To(BeTrue()) - }) - }) - Describe("GetReceivedShare", func() { - It("gets the state", func() { rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ Id: share.Id, From e65d31bb88f7440dd26949bf79fc4378c2090dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 8 Aug 2022 16:33:10 +0200 Subject: [PATCH 58/94] Increase test coverage of UpdateReceivedShare --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index b756664ca2..a4bbff54c9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -685,6 +685,43 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) }) + + It("updates the mountpoint", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.MountPoint).To(BeNil()) + + rs.MountPoint = &providerv1beta1.Reference{ + Path: "newMP", + } + rs, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"mount_point"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.MountPoint.Path).To(Equal("newMP")) + + rs, err = m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.MountPoint.Path).To(Equal("newMP")) + }) + + It("handles invalid field masks", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: share.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + _, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"invalid"}}) + Expect(err).To(HaveOccurred()) + }) }) }) }) From 61e96bbbbb3412dcdea39082e1ab69283b368336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 09:18:59 +0200 Subject: [PATCH 59/94] Fix listing group shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 2 +- pkg/share/manager/jsoncs3/jsoncs3_test.go | 50 +++++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index a69ba9311a..14f519d927 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -492,7 +492,7 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati if s == nil { continue } - if utils.UserEqual(user.GetId(), s.GetGrantee().GetUserId()) { + if share.IsGrantedToUser(s, user) { if share.MatchesFilters(s, filters) { rs := &collaboration.ReceivedShare{ Share: s, diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index a4bbff54c9..2de960f644 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -54,6 +54,7 @@ var _ = Describe("Jsoncs3", func() { Idp: "https://localhost:9200", OpaqueId: "einstein", }, + Groups: []string{"group1"}, } sharedResource = &providerv1beta1.ResourceInfo{ @@ -552,9 +553,6 @@ var _ = Describe("Jsoncs3", func() { }) Describe("ListReceivedShares", func() { - PIt("filters by owner") - PIt("filters by creator") - PIt("filters by grantee type") PIt("syncronizes the group received cache before listing") It("lists the received shares", func() { @@ -633,7 +631,29 @@ var _ = Describe("Jsoncs3", func() { Expect(received[0].Share.ResourceId).To(Equal(sharedResource2.Id)) Expect(received[0].State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) Expect(received[0].Share.Id).To(Equal(share2.Id)) + }) + + Context("with a group share", func() { + var ( + gshare *collaboration.Share + ) + + BeforeEach(func() { + var err error + gshare, err = m.Share(ctx, sharedResource, groupGrant) + Expect(err).ToNot(HaveOccurred()) + }) + It("lists the group share", func() { + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(2)) + ids := []string{} + for _, s := range received { + ids = append(ids, s.Share.Id.OpaqueId) + } + Expect(ids).To(ConsistOf(share.Id.OpaqueId, gshare.Id.OpaqueId)) + }) }) }) @@ -660,6 +680,30 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) }) + + Context("with a group share", func() { + var ( + gshare *collaboration.Share + ) + + BeforeEach(func() { + var err error + gshare, err = m.Share(ctx, sharedResource, groupGrant) + Expect(err).ToNot(HaveOccurred()) + }) + + PIt("syncs the group cache") + + It("gets the group share", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs).ToNot(BeNil()) + }) + }) }) Describe("UpdateReceivedShare", func() { From 678387ccc814324b0c350916e4d473694da09694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 09:20:27 +0200 Subject: [PATCH 60/94] Get the individual shares from the cache instead of listing all of them first --- pkg/share/manager/jsoncs3/jsoncs3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 14f519d927..826f3932e2 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -486,12 +486,12 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } - spaceShares := m.Cache.ListSpace(providerid, spaceid) for shareId, state := range rspace.States { - s := spaceShares.Shares[shareId] + s := m.Cache.Get(providerid, spaceid, shareId) if s == nil { continue } + if share.IsGrantedToUser(s, user) { if share.MatchesFilters(s, filters) { rs := &collaboration.ReceivedShare{ From ea6c5eca4217ebf23f00a8317d03a79e5c8b051c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 09:32:31 +0200 Subject: [PATCH 61/94] Fix unmarshaling group shares --- pkg/share/manager/jsoncs3/providercache/providercache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 4fd36cf9ae..2cccc5fed3 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -67,7 +67,7 @@ func (s *Shares) UnmarshalJSON(data []byte) error { Grantee: &provider.Grantee{Id: &provider.Grantee_UserId{}}, } err = json.Unmarshal(genericShare, userShare) // is this a user share? - if err == nil { + if err == nil && userShare.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER { s.Shares[id] = userShare continue } From b78e9ae9626864d07d969484dc990c74db7f8add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 09:32:42 +0200 Subject: [PATCH 62/94] sync the GroupReceivedCache before listing received shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 29 ++++++++++++----------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 16 +++++++++++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 826f3932e2..41f1059013 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -74,10 +74,10 @@ type Manager struct { Cache providercache.Cache // CreatedCache holds the list of shares a user has created, sharded by user id and space id CreatedCache sharecache.Cache - // groupReceivedCache holds the list of shares a group has access to, sharded by group id and space id - groupReceivedCache sharecache.Cache - // userReceivedStates holds the state of shares a user has received, sharded by user id and space id - userReceivedStates receivedsharecache.Cache + // GroupReceivedCache holds the list of shares a group has access to, sharded by group id and space id + GroupReceivedCache sharecache.Cache + // UserReceivedStates holds the state of shares a user has received, sharded by user id and space id + UserReceivedStates receivedsharecache.Cache storage metadata.Storage SpaceRoot *provider.ResourceId @@ -104,8 +104,8 @@ func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: providercache.New(s), CreatedCache: sharecache.New(s), - userReceivedStates: receivedsharecache.New(s), - groupReceivedCache: sharecache.New(s), + UserReceivedStates: receivedsharecache.New(s), + GroupReceivedCache: sharecache.New(s), storage: s, }, nil } @@ -225,10 +225,10 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - m.userReceivedStates.Add(ctx, userid, spaceID, rs) + m.UserReceivedStates.Add(ctx, userid, spaceID, rs) case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() - if err := m.groupReceivedCache.Add(ctx, groupid, shareID); err != nil { + if err := m.GroupReceivedCache.Add(ctx, groupid, shareID); err != nil { return nil, err } } @@ -440,7 +440,8 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { - for ssid, spaceShareIDs := range m.groupReceivedCache.List(group) { + m.GroupReceivedCache.Sync(ctx, group) + for ssid, spaceShareIDs := range m.GroupReceivedCache.List(group) { if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } @@ -461,8 +462,8 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // add all spaces the user has receved shares for, this includes mount points and share state for groups - m.userReceivedStates.Sync(ctx, user.Id.OpaqueId) - for ssid, rspace := range m.userReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { + m.UserReceivedStates.Sync(ctx, user.Id.OpaqueId) + for ssid, rspace := range m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { if time.Now().Sub(rspace.Mtime) > time.Second*30 { // TODO reread from disk } @@ -523,8 +524,8 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S StorageId: providerid, SpaceId: sid, }) - m.userReceivedStates.Sync(ctx, userID) - state := m.userReceivedStates.Get(userID, spaceID, s.Id.GetOpaqueId()) + m.UserReceivedStates.Sync(ctx, userID) + state := m.UserReceivedStates.Get(userID, spaceID, s.Id.GetOpaqueId()) if state != nil { rs.State = state.State rs.MountPoint = state.MountPoint @@ -578,7 +579,7 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab SpaceId: rs.Share.ResourceId.SpaceId, }) - m.userReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), spaceID, rs) + m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), spaceID, rs) return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 2de960f644..238f08ca17 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -553,8 +553,6 @@ var _ = Describe("Jsoncs3", func() { }) Describe("ListReceivedShares", func() { - PIt("syncronizes the group received cache before listing") - It("lists the received shares", func() { received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) Expect(err).ToNot(HaveOccurred()) @@ -654,6 +652,20 @@ var _ = Describe("Jsoncs3", func() { } Expect(ids).To(ConsistOf(share.Id.OpaqueId, gshare.Id.OpaqueId)) }) + + It("syncronizes the group received cache before listing", func() { + m, err := jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(2)) + ids := []string{} + for _, s := range received { + ids = append(ids, s.Share.Id.OpaqueId) + } + Expect(ids).To(ConsistOf(share.Id.OpaqueId, gshare.Id.OpaqueId)) + }) }) }) From 1f7182599c266c452f79f27c167896d7f7e05d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 09:49:12 +0200 Subject: [PATCH 63/94] Increase test coverage regarding group shares --- pkg/share/manager/jsoncs3/jsoncs3.go | 1 + pkg/share/manager/jsoncs3/jsoncs3_test.go | 76 ++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 41f1059013..bf578672a1 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -524,6 +524,7 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S StorageId: providerid, SpaceId: sid, }) + m.UserReceivedStates.Sync(ctx, userID) state := m.UserReceivedStates.Get(userID, spaceID, s.Id.GetOpaqueId()) if state != nil { diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 238f08ca17..ddd505bd7c 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -704,8 +704,6 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) }) - PIt("syncs the group cache") - It("gets the group share", func() { rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ Spec: &collaboration.ShareReference_Id{ @@ -715,6 +713,19 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) Expect(rs).ToNot(BeNil()) }) + + It("syncs the cache", func() { + m, err := jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs).ToNot(BeNil()) + }) }) }) @@ -778,6 +789,67 @@ var _ = Describe("Jsoncs3", func() { _, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"invalid"}}) Expect(err).To(HaveOccurred()) }) + + Context("with a group share", func() { + var ( + gshare *collaboration.Share + ) + + BeforeEach(func() { + var err error + gshare, err = m.Share(ctx, sharedResource, groupGrant) + Expect(err).ToNot(HaveOccurred()) + }) + + It("updates the received group share", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + + rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED + rs, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"state"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + + rs, err = m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + }) + + It("persists the change", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_PENDING)) + + rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED + rs, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"state"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + + m, err := jsoncs3.New(storage) // Reset in-memory cache + Expect(err).ToNot(HaveOccurred()) + + rs, err = m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(rs.State).To(Equal(collaboration.ShareState_SHARE_STATE_ACCEPTED)) + }) + }) }) }) }) From 3f58d483b8d1a91eb22a92fe9f971e1408875978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 10:01:55 +0200 Subject: [PATCH 64/94] Fix the path/filename of the groups received.json file in the storage --- pkg/share/manager/jsoncs3/jsoncs3.go | 4 ++-- .../manager/jsoncs3/sharecache/sharecache.go | 16 ++++++++++------ .../jsoncs3/sharecache/sharecache_test.go | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index bf578672a1..f7a120530d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -103,9 +103,9 @@ func NewDefault(m map[string]interface{}) (share.Manager, error) { func New(s metadata.Storage) (*Manager, error) { return &Manager{ Cache: providercache.New(s), - CreatedCache: sharecache.New(s), + CreatedCache: sharecache.New(s, "users", "created.json"), UserReceivedStates: receivedsharecache.New(s), - GroupReceivedCache: sharecache.New(s), + GroupReceivedCache: sharecache.New(s, "groups", "received.json"), storage: s, }, nil } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index ff4b2289b9..c88b57d552 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -34,7 +34,9 @@ import ( type Cache struct { UserShares map[string]*UserShareCache - storage metadata.Storage + storage metadata.Storage + namespace string + filename string } type UserShareCache struct { @@ -47,10 +49,12 @@ type SpaceShareIDs struct { IDs map[string]struct{} } -func New(s metadata.Storage) Cache { +func New(s metadata.Storage, namespace, filename string) Cache { return Cache{ UserShares: map[string]*UserShareCache{}, storage: s, + namespace: namespace, + filename: filename, } } @@ -134,7 +138,7 @@ func (c *Cache) Sync(ctx context.Context, userid string) error { mtime = time.Time{} // Set zero time so that data from storage always takes precedence } - userCreatedPath := userCreatedPath(userid) + userCreatedPath := c.userCreatedPath(userid) info, err := c.storage.Stat(ctx, userCreatedPath) if err != nil { return err @@ -163,7 +167,7 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { if err != nil { return err } - jsonPath := userCreatedPath(userid) + jsonPath := c.userCreatedPath(userid) if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { return err } @@ -174,6 +178,6 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { return nil } -func userCreatedPath(userid string) string { - return filepath.Join("/users", userid, "created.json") +func (c *Cache) userCreatedPath(userid string) string { + return filepath.Join("/", c.namespace, userid, c.filename) } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go index 63bb2e139f..36e8a647bc 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go @@ -37,7 +37,7 @@ var _ = Describe("Sharecache", func() { storage, err = metadata.NewDiskStorage(tmpdir) Expect(err).ToNot(HaveOccurred()) - c = sharecache.New(storage) + c = sharecache.New(storage, "users", "created.json") Expect(c).ToNot(BeNil()) }) From d1debf261f0702f9e4b3d4d24c1ab010225fa409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 10 Aug 2022 10:14:41 +0200 Subject: [PATCH 65/94] Update docs --- pkg/share/manager/jsoncs3/jsoncs3.go | 117 +++++++----------- .../jsoncs3/providercache/providercache.go | 17 ++- .../receivedsharecache/receivedsharecache.go | 12 ++ .../manager/jsoncs3/sharecache/sharecache.go | 15 ++- 4 files changed, 81 insertions(+), 80 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index f7a120530d..ddc332d73d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -44,7 +44,9 @@ import ( ) /* - The sharded json driver splits the json file per storage space. Similar to fileids shareids are prefixed with the spaceid. + The sharded json driver splits the json file per storage space. Similar to fileids shareids are prefixed with the spaceid for easier lookup. + In addition to the space json the share manager keeps lists for users and groups to cache their lists of created and received shares + and to hold the state of received shares. FAQ Q: Why not split shares by user and have a list per user? @@ -53,6 +55,35 @@ import ( in project spaces could not be managed collaboratively. By splitting by space, we are in fact not only splitting by user, but more granular, per space. + + File structure in the jsoncs3 space: + + /storages/{storageid}/{spaceid.json} // contains the share information of all shares in that space + /users/{userid}/created.json // points to the spaces the user created shares in including the list of shares + /users/{userid}/received.json // holds the states of received shares of the users + /groups/{groupid}/received.json // points to the spaces the group has received shares in including the list of shares + + Example: + ├── groups + │ └── group1 + │ └── received.json + ├── storages + │ └── storageid + │ └── spaceid.json + └── users + ├── admin + │ └── created.json + └── einstein + └── received.json + + Whenever a share is created, the share manager has to + 1. update the /storages/{storageid}/{spaceid}.json file, + 2. create /users/{userid}/created.json if it doesn't exist yet and add the space/share + 3. create /users/{userid}/received.json or /groups/{groupid}/received.json if it doesn exist yet and add the space/share + + When updating shares /storages/{storageid}/{spaceid}.json is updated accordingly. The mtime is used to invalidate in-memory caches. + + When updating received shares the mountpoint and state are updated in /users/{userid}/received.json (for both user and group shares). */ func init() { @@ -70,14 +101,10 @@ type config struct { type Manager struct { sync.RWMutex - // Cache holds the all shares, sharded by provider id and space id - Cache providercache.Cache - // CreatedCache holds the list of shares a user has created, sharded by user id and space id - CreatedCache sharecache.Cache - // GroupReceivedCache holds the list of shares a group has access to, sharded by group id and space id - GroupReceivedCache sharecache.Cache - // UserReceivedStates holds the state of shares a user has received, sharded by user id and space id - UserReceivedStates receivedsharecache.Cache + Cache providercache.Cache // holds all shares, sharded by provider id and space id + CreatedCache sharecache.Cache // holds the list of shares a user has created, sharded by user id + GroupReceivedCache sharecache.Cache // holds the list of shares a group has access to, sharded by group id + UserReceivedStates receivedsharecache.Cache // holds the state of shares a user has received, sharded by user id storage metadata.Storage SpaceRoot *provider.ResourceId @@ -110,48 +137,7 @@ func New(s metadata.Storage) (*Manager, error) { }, nil } -// File structure in the jsoncs3 space: -// -// /storages/{storageid}/{spaceid.json} // contains all share information of all shares in that space -// /users/{userid}/created/{storageid}/{spaceid} // points to a space the user created shares in -// /users/{userid}/received/{storageid}/{spaceid}.json // holds the states of received shares of the users in the according space -// /groups/{groupid}/received/{storageid}/{spaceid} // points to a space the group has received shares in - -// We store the shares in the metadata storage under /{storageid}/{spaceid}.json - -// To persist the mountpoints of group shares the /{userid}/received/{storageid}/{spaceid}.json file is used. -// - it allows every user to update his own mountpoint without having to update&reread the /{storageid}/{spaceid}.json file - -// To persist the accepted / pending state of shares the /{userid}/received/{storageid}/{spaceid}.json file is used. -// - it allows every user to update his own mountpoint without having to update&reread the /{storageid}/{spaceid}.json file - -// To determine access to group shares a /{groupid}/received/{storageid}/{spaceid} file is used. - -// Whenever a share is created, the share manager has to -// 1. update the /{storageid}/{spaceid}.json file, -// 2. touch /{userid}/created/{storageid}/{spaceid} and -// 3. touch /{userid}/received/{storageid}/{spaceid}.json or /{groupid}/received/{storageid}/{spaceid} -// - The /{userid}/received/{storageid}/{spaceid}.json file persists mountpoints and accepted / rejected state -// - (optional) to wrap these three steps in a transaction the share manager can create a transaction file befor the first step and clean it up when all steps succeded - -// To determine the spaces a user has access to we maintain an empty /{userid}/(received|created)/{storageid}/{spaceid} folder -// that we persist when initially traversing all shares in the metadata /{storageid}/{spaceid}.json files -// when a user creates a new share the jsoncs3 manager touches a new /{userid}/(received|created)/{storageid}/{spaceid} folder -// - the changed mtime can be used to determine when a space needs to be reread for redundant setups - -// when initializing we only initialize per user: -// - we list /{userid}/created/*, for every space we fetch /{storageid}/{spaceid}.json if we -// have not cached it yet, or if the /{userid}/created/{storageid}${spaceid} etag changed -// - if it does not exist we query the registry for every storage provider id, then -// we traverse /{storageid}/ in the metadata storage to -// 1. create /{userid}/created -// 2. touch /{userid}/created/{storageid}${spaceid} -// TODO 3. split storageid from spaceid touch /{userid}/created/{storageid} && /{userid}/created/{storageid}/{spaceid} (not needed when mtime propagation is enabled) - -// we need to split the two lists: -// /{userid}/received/{storageid}/{spaceid} -// /{userid}/created/{storageid}/{spaceid} - +// Share creates a new share func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() @@ -292,6 +278,7 @@ func (m *Manager) get(ctx context.Context, ref *collaboration.ShareReference) (s return } +// GetShare gets the information for a share by the given ref. func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { m.Lock() defer m.Unlock() @@ -309,6 +296,7 @@ func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc return nil, errtypes.NotFound(ref.String()) } +// Unshare deletes a share func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { m.Lock() defer m.Unlock() @@ -344,6 +332,7 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return nil } +// UpdateShare updates the mode of the given share. func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { m.Lock() defer m.Unlock() @@ -370,30 +359,15 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer return s, nil } -// ListShares +// ListShares returns the shares created by the user func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { m.Lock() defer m.Unlock() - //log := appctx.GetLogger(ctx) user := ctxpkg.ContextMustGetUser(ctx) var ss []*collaboration.Share - // Q: how do we detect that a created list changed? - // Option 1: we rely on etag propagation on the storage to bubble up changes in any space to a single created list - // - drawback should stop etag propagation at /{userid}/ to prevent further propagation to the root of the share provider space - // - we could use the user.ocis.propagation xattr in decomposedfs or the eos equivalent to optimize the storage - // - pro: more efficient, more elegant - // - con: more complex, does not work on plain posix - // Option 2: we stat users/{userid}/created.json - // - pro: easier to implement, works on plain posix, no folders - // Can this be hidden behind the metadata storage interface? - // Decision: use touch for now as it works withe plain posix and is easier to test - - // TODO check if a created or owned filter is set - // - read /users/{userid}/created.json (with If-Modified-Since header) aka read if changed m.CreatedCache.Sync(ctx, user.Id.OpaqueId) - for ssid, spaceShareIDs := range m.CreatedCache.List(user.Id.OpaqueId) { if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk @@ -423,7 +397,7 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte return ss, nil } -// we list the shares that are targeted to the user in context or to the user groups. +// ListReceivedShares returns the list of shares the user has access to. func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { m.Lock() defer m.Unlock() @@ -431,11 +405,6 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati var rss []*collaboration.ReceivedShare user := ctxpkg.ContextMustGetUser(ctx) - // Q: how do we detect that a received list changed? - // - similar to the created.json we stat and download a received.json - // con: when adding a received share we MUST have if-match for the initiate-file-upload request - // to ensure consistency / prevent lost updates - ssids := map[string]*receivedsharecache.Space{} // first collect all spaceids the user has access to as a group member @@ -534,6 +503,7 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S return rs } +// GetReceivedShare returns the information for a received share. func (m *Manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { return m.getReceived(ctx, ref) } @@ -552,6 +522,7 @@ func (m *Manager) getReceived(ctx context.Context, ref *collaboration.ShareRefer return m.convert(ctx, user.Id.GetOpaqueId(), s), nil } +// UpdateReceivedShare updates the received share with share state. func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}}) if err != nil { diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 2cccc5fed3..857fde6f13 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -31,25 +31,29 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" ) +// Cache holds share information structured by provider and space type Cache struct { Providers map[string]*Spaces storage metadata.Storage } +// Spaces holds the share information for provider type Spaces struct { Spaces map[string]*Shares } +// Shares hols the share information of one space type Shares struct { Shares map[string]*collaboration.Share Mtime time.Time } +// UnmarshalJSON overrides the default unmarshaling +// Shares are tricky to unmarshal because they contain an interface (Grantee) which makes the json Unmarshal bail out +// To work around that problem we unmarshal into json.RawMessage in a first step and then try to manually unmarshal +// into the specific types in a second step. func (s *Shares) UnmarshalJSON(data []byte) error { - // Shares are tricky to unmarshal because the contain an interface (Grantee) which makes the json Unmarshal bail out - // To work around that problem we unmarshal into json.RawMessage in a first step and then try to manually unmarshal - // into the specific types in a second step. tmp := struct { Shares map[string]json.RawMessage Mtime time.Time @@ -85,6 +89,7 @@ func (s *Shares) UnmarshalJSON(data []byte) error { return nil } +// New returns a new Cache instance func New(s metadata.Storage) Cache { return Cache{ Providers: map[string]*Spaces{}, @@ -92,6 +97,7 @@ func New(s metadata.Storage) Cache { } } +// Add adds a share to the cache func (c *Cache) Add(ctx context.Context, storageID, spaceID, shareID string, share *collaboration.Share) error { c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID].Shares[shareID] = share @@ -99,6 +105,7 @@ func (c *Cache) Add(ctx context.Context, storageID, spaceID, shareID string, sha return c.Persist(ctx, storageID, spaceID) } +// Remove removes a share from the cache func (c *Cache) Remove(ctx context.Context, storageID, spaceID, shareID string) error { if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { @@ -109,6 +116,7 @@ func (c *Cache) Remove(ctx context.Context, storageID, spaceID, shareID string) return c.Persist(ctx, storageID, spaceID) } +// Get returns one entry from the cache func (c *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { @@ -117,6 +125,7 @@ func (c *Cache) Get(storageID, spaceID, shareID string) *collaboration.Share { return c.Providers[storageID].Spaces[spaceID].Shares[shareID] } +// ListSpace returns the list of shares in a given space func (c *Cache) ListSpace(storageID, spaceID string) *Shares { if c.Providers[storageID] == nil { return &Shares{} @@ -124,6 +133,7 @@ func (c *Cache) ListSpace(storageID, spaceID string) *Shares { return c.Providers[storageID].Spaces[spaceID] } +// Persist persists the data of one space func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { return nil @@ -145,6 +155,7 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return nil } +// Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { var mtime time.Time if c.Providers[storageID] != nil && c.Providers[storageID].Spaces[spaceID] != nil { diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index d2f4bedea2..a0cf41fe02 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -31,26 +31,34 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" ) +// Cache stores the list of received shares and their states +// It functions as an in-memory cache with a persistence layer +// The storage is sharded by user type Cache struct { ReceivedSpaces map[string]*Spaces storage metadata.Storage } +// Spaces holds the received shares of one user per space type Spaces struct { Mtime time.Time Spaces map[string]*Space } +// Spaces holds the received shares in one space of one user type Space struct { Mtime time.Time States map[string]*State } + +// State holds the state information of a received share type State struct { State collaboration.ShareState MountPoint *provider.Reference } +// New returns a new Cache instance func New(s metadata.Storage) Cache { return Cache{ ReceivedSpaces: map[string]*Spaces{}, @@ -58,6 +66,7 @@ func New(s metadata.Storage) Cache { } } +// Add adds a new entry to the cache func (c *Cache) Add(ctx context.Context, userID, spaceID string, rs *collaboration.ReceivedShare) error { if c.ReceivedSpaces[userID] == nil { c.ReceivedSpaces[userID] = &Spaces{ @@ -81,6 +90,7 @@ func (c *Cache) Add(ctx context.Context, userID, spaceID string, rs *collaborati return c.Persist(ctx, userID) } +// Get returns one entry from the cache func (c *Cache) Get(userID, spaceID, shareID string) *State { if c.ReceivedSpaces[userID] == nil || c.ReceivedSpaces[userID].Spaces[spaceID] == nil { return nil @@ -88,6 +98,7 @@ func (c *Cache) Get(userID, spaceID, shareID string) *State { return c.ReceivedSpaces[userID].Spaces[spaceID].States[shareID] } +// Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, userID string) error { var mtime time.Time if c.ReceivedSpaces[userID] != nil { @@ -118,6 +129,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { return nil } +// Persist persists the data for one user to the storage func (c *Cache) Persist(ctx context.Context, userID string) error { if c.ReceivedSpaces[userID] == nil { return nil diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index c88b57d552..9f50b488c1 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -31,6 +31,9 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" ) +// Cache caches the list of share ids for users/groups +// It functions as an in-memory cache with a persistence layer +// The storage is sharded by user/group type Cache struct { UserShares map[string]*UserShareCache @@ -39,16 +42,19 @@ type Cache struct { filename string } +// UserShareCache holds the space/share map for one user type UserShareCache struct { Mtime time.Time UserShares map[string]*SpaceShareIDs } +// SpaceShareIDs holds the unique list of share ids for a space type SpaceShareIDs struct { Mtime time.Time IDs map[string]struct{} } +// New returns a new Cache instance func New(s metadata.Storage, namespace, filename string) Cache { return Cache{ UserShares: map[string]*UserShareCache{}, @@ -58,10 +64,7 @@ func New(s metadata.Storage, namespace, filename string) Cache { } } -func (c *Cache) Has(userid string) bool { - return c.UserShares[userid] != nil -} - +// Add adds a share to the cache func (c *Cache) Add(ctx context.Context, userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { @@ -90,6 +93,7 @@ func (c *Cache) Add(ctx context.Context, userid, shareID string) error { return c.Persist(ctx, userid) } +// Remove removes a share for the given user func (c *Cache) Remove(ctx context.Context, userid, shareID string) error { storageid, spaceid, _, err := storagespace.SplitID(shareID) if err != nil { @@ -113,6 +117,7 @@ func (c *Cache) Remove(ctx context.Context, userid, shareID string) error { return c.Persist(ctx, userid) } +// List return the list of spaces/shares for the given user/group func (c *Cache) List(userid string) map[string]SpaceShareIDs { r := map[string]SpaceShareIDs{} if c.UserShares[userid] == nil { @@ -128,6 +133,7 @@ func (c *Cache) List(userid string) map[string]SpaceShareIDs { return r } +// Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, userid string) error { var mtime time.Time // - do we have a cached list of created shares for the user in memory? @@ -160,6 +166,7 @@ func (c *Cache) Sync(ctx context.Context, userid string) error { return nil } +// Persist persists the data for one user/group to the storage func (c *Cache) Persist(ctx context.Context, userid string) error { c.UserShares[userid].Mtime = time.Now() From ffffe87523880cf480e2f14a2cd16e8933d22938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 11 Aug 2022 09:48:04 +0200 Subject: [PATCH 66/94] Initialize storage --- pkg/share/manager/jsoncs3/jsoncs3.go | 98 ++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index ddc332d73d..63a53857a2 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -108,6 +108,8 @@ type Manager struct { storage metadata.Storage SpaceRoot *provider.ResourceId + + initialized bool } // NewDefault returns a new manager instance with default dependencies @@ -137,8 +139,47 @@ func New(s metadata.Storage) (*Manager, error) { }, nil } +func (m *Manager) initialize() error { + if m.initialized { + return nil + } + + m.Lock() + defer m.Unlock() + + if m.initialized { // check if initialization happened while grabbing the lock + return nil + } + + ctx := context.Background() + err := m.storage.Init(ctx, "jsoncs3-share-manager-metadata") + if err != nil { + return err + } + + err = m.storage.MakeDirIfNotExist(ctx, "storages") + if err != nil { + return err + } + err = m.storage.MakeDirIfNotExist(ctx, "users") + if err != nil { + return err + } + err = m.storage.MakeDirIfNotExist(ctx, "groups") + if err != nil { + return err + } + + m.initialized = true + return nil +} + // Share creates a new share func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { + if err := m.initialize(); err != nil { + return nil, err + } + user := ctxpkg.ContextMustGetUser(ctx) now := time.Now().UnixNano() ts := &typespb.Timestamp{ @@ -191,7 +232,10 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Mtime: ts, } - m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) + err = m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) + if err != nil { + return nil, err + } err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { @@ -280,6 +324,10 @@ func (m *Manager) get(ctx context.Context, ref *collaboration.ShareReference) (s // GetShare gets the information for a share by the given ref. func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.Share, error) { + if err := m.initialize(); err != nil { + return nil, err + } + m.Lock() defer m.Unlock() s, err := m.get(ctx, ref) @@ -298,6 +346,10 @@ func (m *Manager) GetShare(ctx context.Context, ref *collaboration.ShareReferenc // Unshare deletes a share func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference) error { + if err := m.initialize(); err != nil { + return err + } + m.Lock() defer m.Unlock() user := ctxpkg.ContextMustGetUser(ctx) @@ -334,6 +386,10 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference // UpdateShare updates the mode of the given share. func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareReference, p *collaboration.SharePermissions) (*collaboration.Share, error) { + if err := m.initialize(); err != nil { + return nil, err + } + m.Lock() defer m.Unlock() s, err := m.get(ctx, ref) @@ -361,6 +417,10 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer // ListShares returns the shares created by the user func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.Share, error) { + if err := m.initialize(); err != nil { + return nil, err + } + m.Lock() defer m.Unlock() @@ -399,6 +459,10 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte // ListReceivedShares returns the list of shares the user has access to. func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaboration.Filter) ([]*collaboration.ReceivedShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + m.Lock() defer m.Unlock() @@ -432,18 +496,20 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati // add all spaces the user has receved shares for, this includes mount points and share state for groups m.UserReceivedStates.Sync(ctx, user.Id.OpaqueId) - for ssid, rspace := range m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { - if time.Now().Sub(rspace.Mtime) > time.Second*30 { - // TODO reread from disk - } - // TODO use younger mtime to determine if - if rs, ok := ssids[ssid]; ok { - for shareid, state := range rspace.States { - // overwrite state - rs.States[shareid] = state + if m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId] != nil { + for ssid, rspace := range m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { + if time.Now().Sub(rspace.Mtime) > time.Second*30 { + // TODO reread from disk + } + // TODO use younger mtime to determine if + if rs, ok := ssids[ssid]; ok { + for shareid, state := range rspace.States { + // overwrite state + rs.States[shareid] = state + } + } else { + ssids[ssid] = rspace } - } else { - ssids[ssid] = rspace } } @@ -505,6 +571,10 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S // GetReceivedShare returns the information for a received share. func (m *Manager) GetReceivedShare(ctx context.Context, ref *collaboration.ShareReference) (*collaboration.ReceivedShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + return m.getReceived(ctx, ref) } @@ -524,6 +594,10 @@ func (m *Manager) getReceived(ctx context.Context, ref *collaboration.ShareRefer // UpdateReceivedShare updates the received share with share state. func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collaboration.ReceivedShare, fieldMask *field_mask.FieldMask) (*collaboration.ReceivedShare, error) { + if err := m.initialize(); err != nil { + return nil, err + } + rs, err := m.getReceived(ctx, &collaboration.ShareReference{Spec: &collaboration.ShareReference_Id{Id: receivedShare.Share.Id}}) if err != nil { return nil, err From 2bb02bb6c73549e466e7a2724895de9c1b948f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 12 Aug 2022 09:32:44 +0200 Subject: [PATCH 67/94] Use a different shareid format to not clash with resource ids --- pkg/share/manager/jsoncs3/jsoncs3.go | 93 +++++++++-------------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 6 +- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 63a53857a2..c5b7177a60 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -20,6 +20,7 @@ package jsoncs3 import ( "context" + "strings" "sync" "time" @@ -39,7 +40,6 @@ import ( "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages - "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -174,6 +174,26 @@ func (m *Manager) initialize() error { return nil } +func encodeShareID(providerID, spaceID, shareID string) string { + return providerID + "^" + spaceID + "°" + shareID +} + +// share ids are of the format ^° +func decodeShareID(id string) (string, string, string) { + parts := strings.SplitN(id, "^", 2) + if len(parts) == 1 { + return "", "", parts[0] + } + + storageid := parts[0] + parts = strings.SplitN(parts[1], "°", 2) + if len(parts) == 1 { + return storageid, parts[0], "" + } + + return storageid, parts[0], parts[1] +} + // Share creates a new share func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { if err := m.initialize(); err != nil { @@ -208,17 +228,8 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // share already exists return nil, errtypes.AlreadyExists(key.String()) } - shareReference := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: md.GetId().StorageId, - SpaceId: md.GetId().SpaceId, - OpaqueId: uuid.NewString(), - }, - } - shareID, err := storagespace.FormatReference(shareReference) - if err != nil { - return nil, err - } + + shareID := encodeShareID(md.GetId().GetStorageId(), md.GetId().GetSpaceId(), uuid.NewString()) s := &collaboration.Share{ Id: &collaboration.ShareId{ OpaqueId: shareID, @@ -242,10 +253,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla return nil, err } - spaceID := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: md.Id.StorageId, - SpaceId: md.Id.SpaceId, - }) + spaceID := md.Id.StorageId + "^" + md.Id.SpaceId // set flag for grantee to have access to share switch g.Grantee.Type { case provider.GranteeType_GRANTEE_TYPE_USER: @@ -268,19 +276,14 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // getByID must be called in a lock-controlled block. func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, error) { - shareid, err := storagespace.ParseID(id.OpaqueId) - if err != nil { - // invalid share id, does not exist - return nil, errtypes.NotFound(id.String()) - } - + storageID, spaceID, _ := decodeShareID(id.OpaqueId) // sync cache, maybe our data is outdated - err = m.Cache.Sync(context.Background(), shareid.StorageId, shareid.SpaceId) + err := m.Cache.Sync(context.Background(), storageID, spaceID) if err != nil { return nil, err } - share := m.Cache.Get(shareid.StorageId, shareid.SpaceId, id.OpaqueId) + share := m.Cache.Get(storageID, spaceID, id.OpaqueId) if share == nil { return nil, errtypes.NotFound(id.String()) } @@ -364,11 +367,8 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return errtypes.NotFound(ref.String()) } - shareid, err := storagespace.ParseID(s.Id.OpaqueId) - if err != nil { - return err - } - err = m.Cache.Remove(ctx, shareid.StorageId, shareid.SpaceId, s.Id.OpaqueId) + storageID, spaceID, _ := decodeShareID(s.Id.OpaqueId) + err = m.Cache.Remove(ctx, storageID, spaceID, s.Id.OpaqueId) if err != nil { return err } @@ -432,15 +432,12 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } - providerid, spaceid, _, err := storagespace.SplitID(ssid) - if err != nil { - continue - } - err = m.Cache.Sync(ctx, providerid, spaceid) + storageID, spaceID, _ := decodeShareID(ssid) + err := m.Cache.Sync(ctx, storageID, spaceID) if err != nil { continue } - spaceShares := m.Cache.ListSpace(providerid, spaceid) + spaceShares := m.Cache.ListSpace(storageID, spaceID) for shareid, _ := range spaceShareIDs.IDs { s := spaceShares.Shares[shareid] if s == nil { @@ -514,16 +511,13 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } for ssid, rspace := range ssids { - providerid, spaceid, _, err := storagespace.SplitID(ssid) - if err != nil { - continue - } - err = m.Cache.Sync(ctx, providerid, spaceid) + storageID, spaceID, _ := decodeShareID(ssid) + err := m.Cache.Sync(ctx, storageID, spaceID) if err != nil { continue } for shareId, state := range rspace.States { - s := m.Cache.Get(providerid, spaceid, shareId) + s := m.Cache.Get(storageID, spaceID, shareId) if s == nil { continue } @@ -551,17 +545,10 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S State: collaboration.ShareState_SHARE_STATE_PENDING, } - providerid, sid, _, err := storagespace.SplitID(s.Id.OpaqueId) - if err != nil { - return rs - } - spaceID := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: providerid, - SpaceId: sid, - }) + storageID, spaceID, _ := decodeShareID(s.Id.OpaqueId) m.UserReceivedStates.Sync(ctx, userID) - state := m.UserReceivedStates.Get(userID, spaceID, s.Id.GetOpaqueId()) + state := m.UserReceivedStates.Get(userID, storageID+"^"+spaceID, s.Id.GetOpaqueId()) if state != nil { rs.State = state.State rs.MountPoint = state.MountPoint @@ -620,12 +607,8 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab // write back userID := ctxpkg.ContextMustGetUser(ctx) - spaceID := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: rs.Share.ResourceId.StorageId, - SpaceId: rs.Share.ResourceId.SpaceId, - }) - m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), spaceID, rs) + m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index ddd505bd7c..60a1c1cd02 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -524,18 +524,18 @@ var _ = Describe("Jsoncs3", func() { Expect(len(shares)).To(Equal(1)) // Add a second cache to the provider cache so it can be referenced - m.Cache.Add(ctx, "storageid", "spaceid", "storageid$spaceid!secondshare", &collaboration.Share{ + m.Cache.Add(ctx, "storageid", "spaceid", "storageid^spaceid°secondshare", &collaboration.Share{ Creator: user1.Id, }) cache := sharecache.UserShareCache{ Mtime: time.Now(), UserShares: map[string]*sharecache.SpaceShareIDs{ - "storageid$spaceid": { + "storageid^spaceid": { Mtime: time.Now(), IDs: map[string]struct{}{ shares[0].Id.OpaqueId: {}, - "storageid$spaceid!secondshare": {}, + "storageid^spaceid°secondshare": {}, }, }, }, From 2ba16dce1260dc57d1c450c3c97a8ef327fe63d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 12 Aug 2022 10:26:09 +0200 Subject: [PATCH 68/94] Fix group shares not being merged with the user states --- pkg/share/manager/jsoncs3/jsoncs3.go | 34 ++++----------- pkg/share/manager/jsoncs3/jsoncs3_test.go | 17 ++++++++ .../manager/jsoncs3/sharecache/sharecache.go | 23 +++-------- pkg/share/manager/jsoncs3/shareid/shareid.go | 41 +++++++++++++++++++ 4 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 pkg/share/manager/jsoncs3/shareid/shareid.go diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index c5b7177a60..28ea87c1a9 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -20,7 +20,6 @@ package jsoncs3 import ( "context" - "strings" "sync" "time" @@ -38,6 +37,7 @@ import ( "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/providercache" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/shareid" "github.com/cs3org/reva/v2/pkg/share/manager/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" // nolint:staticcheck // we need the legacy package to convert V1 to V2 messages "github.com/cs3org/reva/v2/pkg/utils" @@ -174,26 +174,6 @@ func (m *Manager) initialize() error { return nil } -func encodeShareID(providerID, spaceID, shareID string) string { - return providerID + "^" + spaceID + "°" + shareID -} - -// share ids are of the format ^° -func decodeShareID(id string) (string, string, string) { - parts := strings.SplitN(id, "^", 2) - if len(parts) == 1 { - return "", "", parts[0] - } - - storageid := parts[0] - parts = strings.SplitN(parts[1], "°", 2) - if len(parts) == 1 { - return storageid, parts[0], "" - } - - return storageid, parts[0], parts[1] -} - // Share creates a new share func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { if err := m.initialize(); err != nil { @@ -229,7 +209,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla return nil, errtypes.AlreadyExists(key.String()) } - shareID := encodeShareID(md.GetId().GetStorageId(), md.GetId().GetSpaceId(), uuid.NewString()) + shareID := shareid.Encode(md.GetId().GetStorageId(), md.GetId().GetSpaceId(), uuid.NewString()) s := &collaboration.Share{ Id: &collaboration.ShareId{ OpaqueId: shareID, @@ -276,7 +256,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla // getByID must be called in a lock-controlled block. func (m *Manager) getByID(id *collaboration.ShareId) (*collaboration.Share, error) { - storageID, spaceID, _ := decodeShareID(id.OpaqueId) + storageID, spaceID, _ := shareid.Decode(id.OpaqueId) // sync cache, maybe our data is outdated err := m.Cache.Sync(context.Background(), storageID, spaceID) if err != nil { @@ -367,7 +347,7 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference return errtypes.NotFound(ref.String()) } - storageID, spaceID, _ := decodeShareID(s.Id.OpaqueId) + storageID, spaceID, _ := shareid.Decode(s.Id.OpaqueId) err = m.Cache.Remove(ctx, storageID, spaceID, s.Id.OpaqueId) if err != nil { return err @@ -432,7 +412,7 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { // TODO reread from disk } - storageID, spaceID, _ := decodeShareID(ssid) + storageID, spaceID, _ := shareid.Decode(ssid) err := m.Cache.Sync(ctx, storageID, spaceID) if err != nil { continue @@ -511,7 +491,7 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } for ssid, rspace := range ssids { - storageID, spaceID, _ := decodeShareID(ssid) + storageID, spaceID, _ := shareid.Decode(ssid) err := m.Cache.Sync(ctx, storageID, spaceID) if err != nil { continue @@ -545,7 +525,7 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S State: collaboration.ShareState_SHARE_STATE_PENDING, } - storageID, spaceID, _ := decodeShareID(s.Id.OpaqueId) + storageID, spaceID, _ := shareid.Decode(s.Id.OpaqueId) m.UserReceivedStates.Sync(ctx, userID) state := m.UserReceivedStates.Get(userID, storageID+"^"+spaceID, s.Id.GetOpaqueId()) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 60a1c1cd02..215a741672 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -666,6 +666,23 @@ var _ = Describe("Jsoncs3", func() { } Expect(ids).To(ConsistOf(share.Id.OpaqueId, gshare.Id.OpaqueId)) }) + + It("merges the user state with the group share", func() { + rs, err := m.GetReceivedShare(granteeCtx, &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: gshare.Id, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + rs.State = collaboration.ShareState_SHARE_STATE_ACCEPTED + _, err = m.UpdateReceivedShare(granteeCtx, rs, &fieldmaskpb.FieldMask{Paths: []string{"state"}}) + Expect(err).ToNot(HaveOccurred()) + + received, err := m.ListReceivedShares(granteeCtx, []*collaboration.Filter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(received)).To(Equal(2)) + }) }) }) diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 9f50b488c1..1c6bf36a8c 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -25,9 +25,8 @@ import ( "path/filepath" "time" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/shareid" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" - "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -66,14 +65,8 @@ func New(s metadata.Storage, namespace, filename string) Cache { // Add adds a share to the cache func (c *Cache) Add(ctx context.Context, userid, shareID string) error { - storageid, spaceid, _, err := storagespace.SplitID(shareID) - if err != nil { - return err - } - ssid := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: storageid, - SpaceId: spaceid, - }) + storageid, spaceid, _ := shareid.Decode(shareID) + ssid := storageid + "^" + spaceid now := time.Now() if c.UserShares[userid] == nil { @@ -95,14 +88,8 @@ func (c *Cache) Add(ctx context.Context, userid, shareID string) error { // Remove removes a share for the given user func (c *Cache) Remove(ctx context.Context, userid, shareID string) error { - storageid, spaceid, _, err := storagespace.SplitID(shareID) - if err != nil { - return err - } - ssid := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: storageid, - SpaceId: spaceid, - }) + storageid, spaceid, _ := shareid.Decode(shareID) + ssid := storageid + "^" + spaceid if c.UserShares[userid] != nil { if c.UserShares[userid].UserShares[ssid] != nil { diff --git a/pkg/share/manager/jsoncs3/shareid/shareid.go b/pkg/share/manager/jsoncs3/shareid/shareid.go new file mode 100644 index 0000000000..4366b45a3e --- /dev/null +++ b/pkg/share/manager/jsoncs3/shareid/shareid.go @@ -0,0 +1,41 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package shareid + +import "strings" + +func Encode(providerID, spaceID, shareID string) string { + return providerID + "^" + spaceID + "°" + shareID +} + +// share ids are of the format ^° +func Decode(id string) (string, string, string) { + parts := strings.SplitN(id, "^", 2) + if len(parts) == 1 { + return "", "", parts[0] + } + + storageid := parts[0] + parts = strings.SplitN(parts[1], "°", 2) + if len(parts) == 1 { + return storageid, parts[0], "" + } + + return storageid, parts[0], parts[1] +} From 79882b4ffc3858ed40afd34f913cab9b092c2d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 16 Aug 2022 10:10:30 +0200 Subject: [PATCH 69/94] Fix hound issues --- pkg/share/manager/jsoncs3/jsoncs3.go | 9 +++++---- .../jsoncs3/receivedsharecache/receivedsharecache.go | 2 +- pkg/share/manager/jsoncs3/shareid/shareid.go | 2 ++ pkg/storage/utils/metadata/cs3.go | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 28ea87c1a9..a3363deb9d 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -98,6 +98,7 @@ type config struct { MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` } +// Manager implements a share manager using a cs3 storage backend with local caching type Manager struct { sync.RWMutex @@ -418,7 +419,7 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte continue } spaceShares := m.Cache.ListSpace(storageID, spaceID) - for shareid, _ := range spaceShareIDs.IDs { + for shareid := range spaceShareIDs.IDs { s := spaceShares.Shares[shareid] if s == nil { continue @@ -462,7 +463,7 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati States: make(map[string]*receivedsharecache.State, len(spaceShareIDs.IDs)), } - for shareid, _ := range spaceShareIDs.IDs { + for shareid := range spaceShareIDs.IDs { rs.States[shareid] = &receivedsharecache.State{ State: collaboration.ShareState_SHARE_STATE_PENDING, } @@ -496,8 +497,8 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati if err != nil { continue } - for shareId, state := range rspace.States { - s := m.Cache.Get(storageID, spaceID, shareId) + for shareID, state := range rspace.States { + s := m.Cache.Get(storageID, spaceID, shareID) if s == nil { continue } diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index a0cf41fe02..fa133c2db2 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -46,7 +46,7 @@ type Spaces struct { Spaces map[string]*Space } -// Spaces holds the received shares in one space of one user +// Space holds the received shares of one user in one space type Space struct { Mtime time.Time States map[string]*State diff --git a/pkg/share/manager/jsoncs3/shareid/shareid.go b/pkg/share/manager/jsoncs3/shareid/shareid.go index 4366b45a3e..65d774d198 100644 --- a/pkg/share/manager/jsoncs3/shareid/shareid.go +++ b/pkg/share/manager/jsoncs3/shareid/shareid.go @@ -20,10 +20,12 @@ package shareid import "strings" +// Encode encodes a share id func Encode(providerID, spaceID, shareID string) string { return providerID + "^" + spaceID + "°" + shareID } +// Decode decodes an encoded shareid // share ids are of the format ^° func Decode(id string) (string, string, string) { parts := strings.SplitN(id, "^", 2) diff --git a/pkg/storage/utils/metadata/cs3.go b/pkg/storage/utils/metadata/cs3.go index 63fb0276d7..9185ce878a 100644 --- a/pkg/storage/utils/metadata/cs3.go +++ b/pkg/storage/utils/metadata/cs3.go @@ -163,6 +163,7 @@ func (cs3 *CS3) SimpleUpload(ctx context.Context, uploadpath string, content []b return resp.Body.Close() } +// Stat returns the metadata for the given path func (cs3 *CS3) Stat(ctx context.Context, path string) (*provider.ResourceInfo, error) { client, err := cs3.providerClient() if err != nil { From 425d0b8fdddbf708f2680fcc7d7813a6417f6ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 16 Aug 2022 10:45:27 +0200 Subject: [PATCH 70/94] Remove TODO comments that aren't relevant anymore/yet --- pkg/share/manager/jsoncs3/jsoncs3.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index a3363deb9d..0542d12656 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -410,9 +410,6 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte m.CreatedCache.Sync(ctx, user.Id.OpaqueId) for ssid, spaceShareIDs := range m.CreatedCache.List(user.Id.OpaqueId) { - if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { - // TODO reread from disk - } storageID, spaceID, _ := shareid.Decode(ssid) err := m.Cache.Sync(ctx, storageID, spaceID) if err != nil { @@ -453,9 +450,6 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati for _, group := range user.Groups { m.GroupReceivedCache.Sync(ctx, group) for ssid, spaceShareIDs := range m.GroupReceivedCache.List(group) { - if time.Now().Sub(spaceShareIDs.Mtime) > time.Second*30 { - // TODO reread from disk - } // add a pending entry, the state will be updated // when reading the received shares below if they have already been accepted or denied rs := receivedsharecache.Space{ @@ -476,10 +470,6 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati m.UserReceivedStates.Sync(ctx, user.Id.OpaqueId) if m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId] != nil { for ssid, rspace := range m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { - if time.Now().Sub(rspace.Mtime) > time.Second*30 { - // TODO reread from disk - } - // TODO use younger mtime to determine if if rs, ok := ssids[ssid]; ok { for shareid, state := range rspace.States { // overwrite state From 5a3735a465dd8f3ab932c48e1d8e1436ce340ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 16 Aug 2022 16:32:02 +0200 Subject: [PATCH 71/94] Add support for the IfUnmodifiedSince upload condition --- .../storageprovider/storageprovider.go | 29 ++++++++------ pkg/storage/utils/metadata/cs3.go | 32 ++++++++++++--- pkg/storage/utils/metadata/disk.go | 39 ++++++++++++++++++- pkg/storage/utils/metadata/storage.go | 9 +++++ 4 files changed, 90 insertions(+), 19 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index db2c091cb3..6f4d1dc7f2 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -302,26 +302,33 @@ func (s *service) InitiateFileUpload(ctx context.Context, req *provider.Initiate if err != nil { return nil, err } + switch sRes.Status.Code { + case rpc.Code_CODE_OK, rpc.Code_CODE_NOT_FOUND: + // Just continue with a normal upload + default: + return &provider.InitiateFileUploadResponse{ + Status: sRes.Status, + }, nil + } metadata := map[string]string{} ifMatch := req.GetIfMatch() if ifMatch != "" { - switch sRes.Status.Code { - case rpc.Code_CODE_OK: - if sRes.Info.Etag != ifMatch { - return &provider.InitiateFileUploadResponse{ - Status: status.NewAborted(ctx, errors.New("etag mismatch"), "etag mismatch"), - }, nil - } - case rpc.Code_CODE_NOT_FOUND: - // Just continue with a normal upload - default: + if sRes.Info.Etag != ifMatch { return &provider.InitiateFileUploadResponse{ - Status: sRes.Status, + Status: status.NewFailedPrecondition(ctx, errors.New("etag mismatch"), "etag mismatch"), }, nil } metadata["if-match"] = ifMatch } + ifUnmodifiedSince := req.GetIfUnmodifiedSince() + if ifUnmodifiedSince != nil { + if utils.LaterTS(sRes.Info.Mtime, ifUnmodifiedSince) == sRes.Info.Mtime { + return &provider.InitiateFileUploadResponse{ + Status: status.NewFailedPrecondition(ctx, errors.New("resource has been modified"), "resource has been modified"), + }, nil + } + } ctx = ctxpkg.ContextSetLockID(ctx, req.LockId) diff --git a/pkg/storage/utils/metadata/cs3.go b/pkg/storage/utils/metadata/cs3.go index 9185ce878a..cbd679d24b 100644 --- a/pkg/storage/utils/metadata/cs3.go +++ b/pkg/storage/utils/metadata/cs3.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "net/http" "os" + "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -113,6 +114,14 @@ func (cs3 *CS3) Init(ctx context.Context, spaceid string) (err error) { // SimpleUpload uploads a file to the metadata storage func (cs3 *CS3) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error { + return cs3.Upload(ctx, UploadRequest{ + Path: uploadpath, + Content: content, + }) +} + +// Upload uploads a file to the metadata storage +func (cs3 *CS3) Upload(ctx context.Context, req UploadRequest) error { client, err := cs3.providerClient() if err != nil { return err @@ -122,14 +131,25 @@ func (cs3 *CS3) SimpleUpload(ctx context.Context, uploadpath string, content []b return err } - ref := provider.InitiateFileUploadRequest{ + ifuReq := &provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ ResourceId: cs3.SpaceRoot, - Path: utils.MakeRelativePath(uploadpath), + Path: utils.MakeRelativePath(req.Path), }, } - res, err := client.InitiateFileUpload(ctx, &ref) + if req.IfMatchEtag != "" { + ifuReq.Options = &provider.InitiateFileUploadRequest_IfMatch{ + IfMatch: req.IfMatchEtag, + } + } + if req.IfUnmodifiedSince != (time.Time{}) { + ifuReq.Options = &provider.InitiateFileUploadRequest_IfUnmodifiedSince{ + IfUnmodifiedSince: &types.Timestamp{Seconds: uint64(req.IfUnmodifiedSince.Second())}, + } + } + + res, err := client.InitiateFileUpload(ctx, ifuReq) if err != nil { return err } @@ -149,14 +169,14 @@ func (cs3 *CS3) SimpleUpload(ctx context.Context, uploadpath string, content []b return errors.New("metadata storage doesn't support the simple upload protocol") } - req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(content)) + httpReq, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(req.Content)) if err != nil { return err } md, _ := metadata.FromOutgoingContext(ctx) - req.Header.Add(ctxpkg.TokenHeader, md.Get(ctxpkg.TokenHeader)[0]) - resp, err := cs3.dataGatewayClient.Do(req) + httpReq.Header.Add(ctxpkg.TokenHeader, md.Get(ctxpkg.TokenHeader)[0]) + resp, err := cs3.dataGatewayClient.Do(httpReq) if err != nil { return err } diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index 261e29e5b8..5a65780854 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -20,13 +20,16 @@ package metadata import ( "context" + "errors" "io/fs" "io/ioutil" "os" "path" + "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/errtypes" ) // Disk represents a disk metadata storage @@ -73,8 +76,40 @@ func (disk *Disk) Stat(ctx context.Context, path string) (*provider.ResourceInfo } // SimpleUpload stores a file on disk -func (disk *Disk) SimpleUpload(_ context.Context, uploadpath string, content []byte) error { - return os.WriteFile(disk.targetPath(uploadpath), content, 0644) +func (disk *Disk) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error { + return disk.Upload(ctx, UploadRequest{ + Path: uploadpath, + Content: content, + }) +} + +func (disk *Disk) Upload(_ context.Context, req UploadRequest) error { + p := disk.targetPath(req.Path) + if req.IfMatchEtag != "" { + info, err := os.Stat(p) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } else if err == nil { + etag, err := calcEtag(info.ModTime(), info.Size()) + if err != nil { + return err + } + if etag != req.IfMatchEtag { + return errtypes.PreconditionFailed("etag mismatch") + } + } + } + if req.IfUnmodifiedSince != (time.Time{}) { + info, err := os.Stat(p) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } else if err == nil { + if info.ModTime().After(req.IfUnmodifiedSince) { + return errtypes.PreconditionFailed("resource has been modified") + } + } + } + return os.WriteFile(p, req.Content, 0644) } // SimpleDownload reads a file from disk diff --git a/pkg/storage/utils/metadata/storage.go b/pkg/storage/utils/metadata/storage.go index 45064e5116..bf17145663 100644 --- a/pkg/storage/utils/metadata/storage.go +++ b/pkg/storage/utils/metadata/storage.go @@ -30,11 +30,20 @@ import ( //go:generate make --no-print-directory -C ../../../.. mockery NAME=Storage +type UploadRequest struct { + Path string + Content []byte + + IfMatchEtag string + IfUnmodifiedSince time.Time +} + // Storage is the interface to maintain metadata in a storage type Storage interface { Backend() string Init(ctx context.Context, name string) (err error) + Upload(ctx context.Context, req UploadRequest) error SimpleUpload(ctx context.Context, uploadpath string, content []byte) error SimpleDownload(ctx context.Context, path string) ([]byte, error) Delete(ctx context.Context, path string) error From 2d7cb27130ffd4a1e75f0ff52cc52b1a58b69879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 16 Aug 2022 16:32:54 +0200 Subject: [PATCH 72/94] Do not overwrite newer data on the storage --- pkg/share/manager/jsoncs3/jsoncs3_test.go | 17 +++++++++++++++++ .../jsoncs3/providercache/providercache.go | 8 +++++++- .../jsoncs3/providercache/providercache_test.go | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 215a741672..7a66269542 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -550,6 +550,23 @@ var _ = Describe("Jsoncs3", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(shares)).To(Equal(2)) }) + + It("filters by resource id", func() { + shares, err := m.ListShares(ctx, []*collaboration.Filter{ + { + Type: collaboration.Filter_TYPE_RESOURCE_ID, + Term: &collaboration.Filter_ResourceId{ + ResourceId: &providerv1beta1.ResourceId{ + StorageId: "storageid", + SpaceId: "spaceid", + OpaqueId: "somethingelse", + }, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(0)) + }) }) Describe("ListReceivedShares", func() { diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 857fde6f13..75a024f0c8 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -139,6 +139,8 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return nil } + oldMtime := c.Providers[storageID].Spaces[spaceID].Mtime + c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() createdBytes, err := json.Marshal(c.Providers[storageID].Spaces[spaceID]) if err != nil { @@ -149,7 +151,11 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return err } // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { + if err := c.storage.Upload(ctx, metadata.UploadRequest{ + Path: jsonPath, + Content: createdBytes, + IfUnmodifiedSince: oldMtime, + }); err != nil { return err } return nil diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index cd087e4706..106c72e796 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -129,6 +129,12 @@ var _ = Describe("Cache", func() { Expect(c.Persist(ctx, storageID, spaceID)).To(Succeed()) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(oldMtime)) }) + + It("does not persist if the etag changed on disk", func() { + c.Providers[storageID].Spaces[spaceID].Mtime = time.Now().Add(-3 * time.Hour) + + Expect(c.Persist(ctx, storageID, spaceID)).ToNot(Succeed()) + }) }) Describe("Sync", func() { From 029581d2a2e538d86870b505c5e7a98caebba2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 18 Aug 2022 15:54:03 +0200 Subject: [PATCH 73/94] Allow for listing shares of other users as space manager --- pkg/share/manager/jsoncs3/jsoncs3.go | 43 ++++++++++++++++++ pkg/share/manager/jsoncs3/jsoncs3_test.go | 55 +++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 0542d12656..8501f3d670 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" @@ -406,6 +407,48 @@ func (m *Manager) ListShares(ctx context.Context, filters []*collaboration.Filte defer m.Unlock() user := ctxpkg.ContextMustGetUser(ctx) + grouped := share.GroupFiltersByType(filters) + + if len(grouped[collaboration.Filter_TYPE_RESOURCE_ID]) > 0 { + return m.listSharesByIDs(ctx, user, filters) + } + + return m.listCreatedShares(ctx, user, filters) +} + +func (m *Manager) listSharesByIDs(ctx context.Context, user *userv1beta1.User, filters []*collaboration.Filter) ([]*collaboration.Share, error) { + var ss []*collaboration.Share + + grouped := share.GroupFiltersByType(filters) + providerSpaces := map[string]map[string]bool{} + for _, f := range grouped[collaboration.Filter_TYPE_RESOURCE_ID] { + storageID := f.GetResourceId().GetStorageId() + spaceID := f.GetResourceId().GetSpaceId() + if providerSpaces[storageID] == nil { + providerSpaces[storageID] = map[string]bool{} + } + providerSpaces[storageID][spaceID] = true + } + + for providerID, spaces := range providerSpaces { + for spaceID, _ := range spaces { + err := m.Cache.Sync(ctx, providerID, spaceID) + if err != nil { + return nil, err + } + + shares := m.Cache.ListSpace(providerID, spaceID) + for _, s := range shares.Shares { + if share.MatchesFilters(s, filters) { + ss = append(ss, s) + } + } + } + } + return ss, nil +} + +func (m *Manager) listCreatedShares(ctx context.Context, user *userv1beta1.User, filters []*collaboration.Filter) ([]*collaboration.Share, error) { var ss []*collaboration.Share m.CreatedCache.Sync(ctx, user.Id.OpaqueId) diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 7a66269542..707214ef98 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -31,6 +31,7 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/conversions" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/sharecache" @@ -191,6 +192,60 @@ var _ = Describe("Jsoncs3", func() { }) }) + Context("with a space manager", func() { + var ( + share *collaboration.Share + + manager = &userpb.User{ + Id: &userpb.UserId{ + Idp: "https://localhost:9200", + OpaqueId: "spacemanager", + }, + } + managerCtx context.Context + ) + + BeforeEach(func() { + managerCtx = ctxpkg.ContextSetUser(context.Background(), manager) + + var err error + share, err = m.Share(ctx, sharedResource, grant) + Expect(err).ToNot(HaveOccurred()) + + _, err = m.Share(ctx, &providerv1beta1.ResourceInfo{ + Id: &providerv1beta1.ResourceId{ + StorageId: "storageid", + SpaceId: "spaceid", + OpaqueId: "spaceid", + }, + }, &collaboration.ShareGrant{ + Grantee: &providerv1beta1.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &providerv1beta1.Grantee_UserId{UserId: manager.Id}, + }, + Permissions: &collaboration.SharePermissions{ + Permissions: conversions.NewManagerRole().CS3ResourcePermissions(), + }, + }) + }) + + Describe("ListShares", func() { + It("returns the share requested by id even though it's not owned or created by the manager", func() { + shares, err := m.ListShares(managerCtx, []*collaboration.Filter{ + { + Type: collaboration.Filter_TYPE_RESOURCE_ID, + Term: &collaboration.Filter_ResourceId{ + ResourceId: sharedResource.Id, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(shares).To(HaveLen(1)) + Expect(shares[0].Id).To(Equal(share.Id)) + }) + }) + }) + Context("with an existing share", func() { var ( share *collaboration.Share From 77ea20830910735ca1852ad0727fc87f916f20ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 09:08:20 +0200 Subject: [PATCH 74/94] Bump go-cs3apis --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8d6638dabd..aa3da95c40 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20220719130120-361e9f987d64 + github.com/cs3org/go-cs3apis v0.0.0-20220818202316-e92afdddac6d github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/dgraph-io/ristretto v0.1.0 github.com/emvi/iso-639-1 v1.0.1 diff --git a/go.sum b/go.sum index 4e5a200695..28dc9bdb01 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20220719130120-361e9f987d64 h1:cFnankJOCWndnOns4sKRG7yzH61ammK2Am6rEGWCK40= -github.com/cs3org/go-cs3apis v0.0.0-20220719130120-361e9f987d64/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20220818202316-e92afdddac6d h1:toyZ7IsXlUdEPZ/IG8fg7hbM8HcLPY0bkX4FKBmgLVI= +github.com/cs3org/go-cs3apis v0.0.0-20220818202316-e92afdddac6d/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 5b668269b1e7fcb1df16b6a29d7983a8aed5045c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 09:16:19 +0200 Subject: [PATCH 75/94] Fix hound issues --- pkg/share/manager/jsoncs3/jsoncs3.go | 2 +- pkg/storage/utils/metadata/disk.go | 2 ++ pkg/storage/utils/metadata/storage.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 8501f3d670..13f1b0e05a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -431,7 +431,7 @@ func (m *Manager) listSharesByIDs(ctx context.Context, user *userv1beta1.User, f } for providerID, spaces := range providerSpaces { - for spaceID, _ := range spaces { + for spaceID := range spaces { err := m.Cache.Sync(ctx, providerID, spaceID) if err != nil { return nil, err diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index 5a65780854..beba6df0c3 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -54,6 +54,7 @@ func (disk *Disk) Backend() string { return "disk" } +// Stat returns the metadata for the given path func (disk *Disk) Stat(ctx context.Context, path string) (*provider.ResourceInfo, error) { info, err := os.Stat(disk.targetPath(path)) if err != nil { @@ -83,6 +84,7 @@ func (disk *Disk) SimpleUpload(ctx context.Context, uploadpath string, content [ }) } +// Upload stores a file on disk func (disk *Disk) Upload(_ context.Context, req UploadRequest) error { p := disk.targetPath(req.Path) if req.IfMatchEtag != "" { diff --git a/pkg/storage/utils/metadata/storage.go b/pkg/storage/utils/metadata/storage.go index bf17145663..be695374b3 100644 --- a/pkg/storage/utils/metadata/storage.go +++ b/pkg/storage/utils/metadata/storage.go @@ -30,6 +30,7 @@ import ( //go:generate make --no-print-directory -C ../../../.. mockery NAME=Storage +// UploadRequest represents an upload request and its options type UploadRequest struct { Path string Content []byte From 5de8377a12516c27a4004626574b32c66f7f9b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 09:57:14 +0200 Subject: [PATCH 76/94] Log the sync processes --- .../jsoncs3/providercache/providercache.go | 16 ++++++++++++- .../receivedsharecache/receivedsharecache.go | 16 ++++++++++++- .../manager/jsoncs3/sharecache/sharecache.go | 24 +++++++++++++++---- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 75a024f0c8..d4b8baa2e0 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -27,6 +27,8 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -163,6 +165,9 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { // Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { + log := appctx.GetLogger(ctx).With().Str("storageID", storageID).Str("spaceID", spaceID).Logger() + log.Debug().Msg("Syncing provider cache..") + var mtime time.Time if c.Providers[storageID] != nil && c.Providers[storageID].Spaces[spaceID] != nil { mtime = c.Providers[storageID].Spaces[spaceID].Mtime @@ -174,23 +179,32 @@ func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { jsonPath := spaceJSONPath(storageID, spaceID) info, err := c.storage.Stat(ctx, jsonPath) if err != nil { - return err + if _, ok := err.(errtypes.NotFound); ok { + return nil // Nothing to sync against + } else { + log.Error().Err(err).Msg("Failed to stat the provider cache") + return err + } } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { + log.Error().Err(err).Msg("Updating...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) if err != nil { + log.Error().Err(err).Msg("Failed to download the provider cache") return err } newShares := &Shares{} err = json.Unmarshal(createdBlob, newShares) if err != nil { + log.Error().Err(err).Msg("Failed to unmarshal the provider cache") return err } c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID] = newShares } + log.Error().Err(err).Msg("Provider cache ist up to date") return nil } diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index fa133c2db2..f8f674e368 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -27,6 +27,8 @@ import ( collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/appctx" + "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/cs3org/reva/v2/pkg/utils" ) @@ -100,6 +102,9 @@ func (c *Cache) Get(userID, spaceID, shareID string) *State { // Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, userID string) error { + log := appctx.GetLogger(ctx).With().Str("userID", userID).Logger() + log.Debug().Msg("Syncing received share cache...") + var mtime time.Time if c.ReceivedSpaces[userID] != nil { mtime = c.ReceivedSpaces[userID].Mtime @@ -110,22 +115,31 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { jsonPath := userJSONPath(userID) info, err := c.storage.Stat(ctx, jsonPath) if err != nil { - return err + if _, ok := err.(errtypes.NotFound); ok { + return nil // Nothing to sync against + } else { + log.Error().Err(err).Msg("Failed to stat the received share") + return err + } } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { + log.Debug().Msg("Updating...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) if err != nil { + log.Error().Err(err).Msg("Failed to download the received share") return err } newSpaces := &Spaces{} err = json.Unmarshal(createdBlob, newSpaces) if err != nil { + log.Error().Err(err).Msg("Failed to unmarshal the received share") return err } c.ReceivedSpaces[userID] = newSpaces } + log.Debug().Msg("Received share ist up to date") return nil } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 1c6bf36a8c..023160bb20 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -25,6 +25,8 @@ import ( "path/filepath" "time" + "github.com/cs3org/reva/v2/pkg/appctx" + "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/shareid" "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" "github.com/cs3org/reva/v2/pkg/utils" @@ -121,35 +123,47 @@ func (c *Cache) List(userid string) map[string]SpaceShareIDs { } // Sync updates the in-memory data with the data from the storage if it is outdated -func (c *Cache) Sync(ctx context.Context, userid string) error { +func (c *Cache) Sync(ctx context.Context, userID string) error { + log := appctx.GetLogger(ctx).With().Str("userID", userID).Logger() + log.Debug().Msg("Syncing share cache...") + var mtime time.Time // - do we have a cached list of created shares for the user in memory? - if usc := c.UserShares[userid]; usc != nil { + if usc := c.UserShares[userID]; usc != nil { mtime = usc.Mtime // - y: set If-Modified-Since header to only download if it changed } else { mtime = time.Time{} // Set zero time so that data from storage always takes precedence } - userCreatedPath := c.userCreatedPath(userid) + userCreatedPath := c.userCreatedPath(userID) info, err := c.storage.Stat(ctx, userCreatedPath) if err != nil { - return err + if _, ok := err.(errtypes.NotFound); ok { + return nil // Nothing to sync against + } else { + log.Error().Err(err).Msg("Failed to stat the share cache") + return err + } } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { + log.Debug().Msg("Updating...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, userCreatedPath) if err != nil { + log.Error().Err(err).Msg("Failed to download the share cache") return err } newShareCache := &UserShareCache{} err = json.Unmarshal(createdBlob, newShareCache) if err != nil { + log.Error().Err(err).Msg("Failed to unmarshal the share cache") return err } - c.UserShares[userid] = newShareCache + c.UserShares[userID] = newShareCache } + log.Debug().Msg("Share cache ist up to date") return nil } From 81a431d06de757ecc383dff4e27327bdd8569e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 10:51:51 +0200 Subject: [PATCH 77/94] Add changelog --- changelog/unreleased/jsoncs3-share-manager.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/jsoncs3-share-manager.md diff --git a/changelog/unreleased/jsoncs3-share-manager.md b/changelog/unreleased/jsoncs3-share-manager.md new file mode 100644 index 0000000000..b725babbe5 --- /dev/null +++ b/changelog/unreleased/jsoncs3-share-manager.md @@ -0,0 +1,6 @@ +Enhancement: Add new jsoncs3 share manager + +We've added a new jsoncs3 share manager which splits the json file per storage +space and caches data locally. + +https://github.com/cs3org/reva/pull/3148 From 4fa5bac81a236e3e556aa0957e755c76d62e95bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 10:53:58 +0200 Subject: [PATCH 78/94] Fix even more hound issues --- pkg/share/manager/jsoncs3/providercache/providercache.go | 5 ++--- .../manager/jsoncs3/receivedsharecache/receivedsharecache.go | 5 ++--- pkg/share/manager/jsoncs3/sharecache/sharecache.go | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index d4b8baa2e0..c82520574b 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -181,10 +181,9 @@ func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { if err != nil { if _, ok := err.(errtypes.NotFound); ok { return nil // Nothing to sync against - } else { - log.Error().Err(err).Msg("Failed to stat the provider cache") - return err } + log.Error().Err(err).Msg("Failed to stat the provider cache") + return err } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index f8f674e368..bb048794c4 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -117,10 +117,9 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { if err != nil { if _, ok := err.(errtypes.NotFound); ok { return nil // Nothing to sync against - } else { - log.Error().Err(err).Msg("Failed to stat the received share") - return err } + log.Error().Err(err).Msg("Failed to stat the received share") + return err } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 023160bb20..44659a7286 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -141,10 +141,9 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { if err != nil { if _, ok := err.(errtypes.NotFound); ok { return nil // Nothing to sync against - } else { - log.Error().Err(err).Msg("Failed to stat the share cache") - return err } + log.Error().Err(err).Msg("Failed to stat the share cache") + return err } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { From b5141d9c93dd7c9da5a811fbf5d6129393c40583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 10:57:39 +0200 Subject: [PATCH 79/94] Remove superfluous sync --- pkg/share/manager/jsoncs3/jsoncs3.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 13f1b0e05a..05c4464d09 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -279,12 +279,6 @@ func (m *Manager) getByKey(ctx context.Context, key *collaboration.ShareKey) (*c return nil, err } - // sync cache, maybe our data is outdated - err = m.Cache.Sync(context.Background(), key.ResourceId.StorageId, key.ResourceId.SpaceId) - if err != nil { - return nil, err - } - spaceShares := m.Cache.ListSpace(key.ResourceId.StorageId, key.ResourceId.SpaceId) for _, share := range spaceShares.Shares { if utils.GranteeEqual(key.Grantee, share.Grantee) && utils.ResourceIDEqual(share.ResourceId, key.ResourceId) { From 6183f486f61ec7969a2ac578c27c84fac833875c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 11:02:47 +0200 Subject: [PATCH 80/94] Regenerate mocks --- pkg/storage/utils/metadata/mocks/Storage.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/storage/utils/metadata/mocks/Storage.go b/pkg/storage/utils/metadata/mocks/Storage.go index 37aba4a3b7..e4ad15e87b 100644 --- a/pkg/storage/utils/metadata/mocks/Storage.go +++ b/pkg/storage/utils/metadata/mocks/Storage.go @@ -23,6 +23,7 @@ package mocks import ( context "context" + metadata "github.com/cs3org/reva/v2/pkg/storage/utils/metadata" mock "github.com/stretchr/testify/mock" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -232,6 +233,20 @@ func (_m *Storage) Stat(ctx context.Context, path string) (*providerv1beta1.Reso return r0, r1 } +// Upload provides a mock function with given fields: ctx, req +func (_m *Storage) Upload(ctx context.Context, req metadata.UploadRequest) error { + ret := _m.Called(ctx, req) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, metadata.UploadRequest) error); ok { + r0 = rf(ctx, req) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // NewStorage creates a new instance of Storage. It also registers a cleanup function to assert the mocks expectations. func NewStorage(t testing.TB) *Storage { mock := &Storage{} From 4148cddc124e9de41530121ac2e62c0864ce872b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 19 Aug 2022 11:37:31 +0000 Subject: [PATCH 81/94] use utils.NowTS() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 05c4464d09..db8af5f1a7 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -21,7 +21,6 @@ package jsoncs3 import ( "context" "sync" - "time" "github.com/google/uuid" "github.com/mitchellh/mapstructure" @@ -31,7 +30,6 @@ import ( userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/share" @@ -183,11 +181,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } user := ctxpkg.ContextMustGetUser(ctx) - now := time.Now().UnixNano() - ts := &typespb.Timestamp{ - Seconds: uint64(now / int64(time.Second)), - Nanos: uint32(now % int64(time.Second)), - } + ts := utils.TSNow() // do not allow share to myself or the owner if share is for a user // TODO(labkode): should not this be caught already at the gw level? @@ -378,12 +372,8 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer return nil, errtypes.NotFound(ref.String()) } - now := time.Now().UnixNano() s.Permissions = p - s.Mtime = &typespb.Timestamp{ - Seconds: uint64(now / int64(time.Second)), - Nanos: uint32(now % int64(time.Second)), - } + s.Mtime = utils.TSNow() // Update provider cache m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) From a2fade2281c1023c75c8880cc366410ca77af6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 19 Aug 2022 13:23:19 +0000 Subject: [PATCH 82/94] cleanup cache sync logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/providercache/providercache.go | 6 +++--- .../jsoncs3/receivedsharecache/receivedsharecache.go | 4 ++-- pkg/share/manager/jsoncs3/sharecache/sharecache.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index c82520574b..f828a40123 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -166,7 +166,7 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { // Sync updates the in-memory data with the data from the storage if it is outdated func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { log := appctx.GetLogger(ctx).With().Str("storageID", storageID).Str("spaceID", spaceID).Logger() - log.Debug().Msg("Syncing provider cache..") + log.Debug().Msg("Syncing provider cache...") var mtime time.Time if c.Providers[storageID] != nil && c.Providers[storageID].Spaces[spaceID] != nil { @@ -187,7 +187,7 @@ func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { - log.Error().Err(err).Msg("Updating...") + log.Debug().Msg("Updating provider cache...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) if err != nil { @@ -203,7 +203,7 @@ func (c *Cache) Sync(ctx context.Context, storageID, spaceID string) error { c.initializeIfNeeded(storageID, spaceID) c.Providers[storageID].Spaces[spaceID] = newShares } - log.Error().Err(err).Msg("Provider cache ist up to date") + log.Debug().Msg("Provider cache is up to date") return nil } diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index bb048794c4..29eed83bf3 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -123,7 +123,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { - log.Debug().Msg("Updating...") + log.Debug().Msg("Updating received share cache...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, jsonPath) if err != nil { @@ -138,7 +138,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { } c.ReceivedSpaces[userID] = newSpaces } - log.Debug().Msg("Received share ist up to date") + log.Debug().Msg("Received share cache is up to date") return nil } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 44659a7286..0e4d00fb0e 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -147,7 +147,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { } // check mtime of /users/{userid}/created.json if utils.TSToTime(info.Mtime).After(mtime) { - log.Debug().Msg("Updating...") + log.Debug().Msg("Updating share cache...") // - update cached list of created shares for the user in memory if changed createdBlob, err := c.storage.SimpleDownload(ctx, userCreatedPath) if err != nil { @@ -162,7 +162,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { } c.UserShares[userID] = newShareCache } - log.Debug().Msg("Share cache ist up to date") + log.Debug().Msg("Share cache is up to date") return nil } From 2bb66dbc209c4c87e10e091f9b02df64836708ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 19 Aug 2022 13:47:36 +0000 Subject: [PATCH 83/94] introduce utils.TimeToTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/utils/utils.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index f8f1cde42d..3e53660d6d 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -143,6 +143,14 @@ func TSToTime(ts *types.Timestamp) time.Time { return time.Unix(int64(ts.Seconds), int64(ts.Nanos)) } +// TimeToTS converts Go's time.Time to a protobuf Timestamp. +func TimeToTS(t time.Time) *types.Timestamp { + return &types.Timestamp{ + Seconds: uint64(t.Unix()), // implicity returns UTC + Nanos: uint32(t.Nanosecond()), + } +} + // LaterTS returns the timestamp which occurs later. func LaterTS(t1 *types.Timestamp, t2 *types.Timestamp) *types.Timestamp { if TSToUnixNano(t1) > TSToUnixNano(t2) { From 851376cc72e6568155ed2c5ff3eee505317e965d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 19 Aug 2022 13:48:09 +0000 Subject: [PATCH 84/94] convert to timestamp instead of taking time seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/metadata/cs3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storage/utils/metadata/cs3.go b/pkg/storage/utils/metadata/cs3.go index cbd679d24b..45f1401bb2 100644 --- a/pkg/storage/utils/metadata/cs3.go +++ b/pkg/storage/utils/metadata/cs3.go @@ -145,7 +145,7 @@ func (cs3 *CS3) Upload(ctx context.Context, req UploadRequest) error { } if req.IfUnmodifiedSince != (time.Time{}) { ifuReq.Options = &provider.InitiateFileUploadRequest_IfUnmodifiedSince{ - IfUnmodifiedSince: &types.Timestamp{Seconds: uint64(req.IfUnmodifiedSince.Second())}, + IfUnmodifiedSince: utils.TimeToTS(req.IfUnmodifiedSince), } } From ba1a2124fd24a5c421f46b9ec6d9522716547712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 19 Aug 2022 13:49:41 +0000 Subject: [PATCH 85/94] prevent nil when trying to remove not existing grant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/decomposedfs/grants.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index c9268989bd..cca3996d20 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -139,6 +139,10 @@ func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference return err } + if grant == nil { + return errtypes.NotFound("grant not found") + } + // you are allowed to remove grants if you created them yourself or have the proper permission if !utils.UserEqual(grant.Creator, ctxpkg.ContextMustGetUser(ctx).GetId()) { ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool { From a033534b51a90e56d8fd68236b91117cc3bcac54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 16:16:43 +0200 Subject: [PATCH 86/94] Update local mtime after writing to the cache --- pkg/share/manager/jsoncs3/providercache/providercache.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index f828a40123..f3e744663e 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -152,7 +152,7 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { return err } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments + if err := c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, @@ -160,6 +160,12 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { }); err != nil { return err } + + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + c.Providers[storageID].Spaces[spaceID].Mtime = utils.TSToTime(info.Mtime) return nil } From 8bfcfa082039fe6b59ff8850475709db050b6fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Aug 2022 16:43:54 +0200 Subject: [PATCH 87/94] WIP: use IfUnmodifiedSince flag when writing caches --- .../receivedsharecache/receivedsharecache.go | 16 ++++++++++++++-- .../manager/jsoncs3/sharecache/sharecache.go | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index 29eed83bf3..27f6c5dcd1 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -148,6 +148,7 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { return nil } + oldMtime := c.ReceivedSpaces[userID].Mtime c.ReceivedSpaces[userID].Mtime = time.Now() createdBytes, err := json.Marshal(c.ReceivedSpaces[userID]) if err != nil { @@ -157,10 +158,21 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { return err } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { + + if err := c.storage.Upload(ctx, metadata.UploadRequest{ + Path: jsonPath, + Content: createdBytes, + IfUnmodifiedSince: oldMtime, + }); err != nil { return err } + + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + c.ReceivedSpaces[userID].Mtime = utils.TSToTime(info.Mtime) + return nil } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 0e4d00fb0e..118950227f 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -168,6 +168,7 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { // Persist persists the data for one user/group to the storage func (c *Cache) Persist(ctx context.Context, userid string) error { + oldMtime := c.UserShares[userid].Mtime c.UserShares[userid].Mtime = time.Now() createdBytes, err := json.Marshal(c.UserShares[userid]) @@ -178,10 +179,21 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { return err } - // FIXME needs stat & upload if match combo to prevent lost update in redundant deployments - if err := c.storage.SimpleUpload(ctx, jsonPath, createdBytes); err != nil { + + if err := c.storage.Upload(ctx, metadata.UploadRequest{ + Path: jsonPath, + Content: createdBytes, + IfUnmodifiedSince: oldMtime, + }); err != nil { + return err + } + + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { return err } + c.UserShares[userid].Mtime = utils.TSToTime(info.Mtime) + return nil } From 776b30bda424a6ec52adcfc6dcfc8ef734ba9554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 09:21:34 +0000 Subject: [PATCH 88/94] comment stat after upload, update docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 21 +++++++++++++++---- pkg/share/manager/jsoncs3/jsoncs3_test.go | 6 +++--- .../jsoncs3/providercache/providercache.go | 16 ++++++++------ .../receivedsharecache/receivedsharecache.go | 15 +++++++------ .../manager/jsoncs3/sharecache/sharecache.go | 15 +++++++------ 5 files changed, 48 insertions(+), 25 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index db8af5f1a7..6b7aea6b30 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -58,8 +58,8 @@ import ( File structure in the jsoncs3 space: /storages/{storageid}/{spaceid.json} // contains the share information of all shares in that space - /users/{userid}/created.json // points to the spaces the user created shares in including the list of shares - /users/{userid}/received.json // holds the states of received shares of the users + /users/{userid}/created.json // points to the spaces the user created shares in, including the list of shares + /users/{userid}/received.json // holds the accepted/pending state and mount point of received shares for users /groups/{groupid}/received.json // points to the spaces the group has received shares in including the list of shares Example: @@ -80,9 +80,17 @@ import ( 2. create /users/{userid}/created.json if it doesn't exist yet and add the space/share 3. create /users/{userid}/received.json or /groups/{groupid}/received.json if it doesn exist yet and add the space/share - When updating shares /storages/{storageid}/{spaceid}.json is updated accordingly. The mtime is used to invalidate in-memory caches. + When updating shares /storages/{storageid}/{spaceid}.json is updated accordingly. The mtime is used to invalidate in-memory caches: + - TODO the upload is tried with an if-unmodified-since header + - TODO when if fails, the {spaceid}.json file is downloaded, the changes are reapplied and the upload is retried with the new mtime When updating received shares the mountpoint and state are updated in /users/{userid}/received.json (for both user and group shares). + + When reading the list of received shares the /users/{userid}/received.json file and the /groups/{groupid}/received.json files are statted. + - if the mtime changed we download the file to update the local cache + + When reading the list of created shares the /users/{userid}/created.json file is statted + - if the mtime changed we download the file to update the local cache */ func init() { @@ -184,7 +192,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla ts := utils.TSNow() // do not allow share to myself or the owner if share is for a user - // TODO(labkode): should not this be caught already at the gw level? + // TODO: should this not already be caught at the gw level? if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER && (utils.UserEqual(g.Grantee.GetUserId(), user.Id) || utils.UserEqual(g.Grantee.GetUserId(), md.Owner)) { return nil, errors.New("json: owner/creator and grantee are the same") @@ -226,6 +234,7 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), shareID) if err != nil { + // TODO when persisting fails, download, readd and persist again return nil, err } @@ -240,6 +249,8 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla State: collaboration.ShareState_SHARE_STATE_PENDING, } m.UserReceivedStates.Add(ctx, userid, spaceID, rs) + // TODO check error + // TODO when persisting fails, download, readd and persist again case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() if err := m.GroupReceivedCache.Add(ctx, groupid, shareID); err != nil { @@ -377,6 +388,7 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer // Update provider cache m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) + // TODO when persisting fails, download, readd and persist again return s, nil } @@ -607,6 +619,7 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab userID := ctxpkg.ContextMustGetUser(ctx) m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) + // TODO when persisting fails, download, readd and persist again return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index 707214ef98..c48892d109 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -473,7 +473,7 @@ var _ = Describe("Jsoncs3", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, @@ -495,7 +495,7 @@ var _ = Describe("Jsoncs3", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) + Expect(us.GetPermissions().GetPermissions().InitiateFileUpload).To(BeFalse()) s = shareBykey(&collaboration.ShareKey{ ResourceId: sharedResource.Id, @@ -525,7 +525,7 @@ var _ = Describe("Jsoncs3", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(us).ToNot(BeNil()) - Expect(s.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) + Expect(us.GetPermissions().GetPermissions().InitiateFileUpload).To(BeTrue()) m, err = jsoncs3.New(storage) // Reset in-memory cache Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index f3e744663e..bd6d3f71cc 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -45,7 +45,7 @@ type Spaces struct { Spaces map[string]*Shares } -// Shares hols the share information of one space +// Shares holds the share information of one space type Shares struct { Shares map[string]*collaboration.Share Mtime time.Time @@ -161,11 +161,15 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return err } - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.Providers[storageID].Spaces[spaceID].Mtime = utils.TSToTime(info.Mtime) + /* + FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat + the local cache is updated with Sync during reads + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + c.Providers[storageID].Spaces[spaceID].Mtime = utils.TSToTime(info.Mtime) + */ return nil } diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index 27f6c5dcd1..4b2d8a756d 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -167,12 +167,15 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { return err } - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.ReceivedSpaces[userID].Mtime = utils.TSToTime(info.Mtime) - + /* + FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat + the local cache is updated with Sync during reads + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + c.ReceivedSpaces[userID].Mtime = utils.TSToTime(info.Mtime) + */ return nil } diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 118950227f..7683b217da 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -188,12 +188,15 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { return err } - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.UserShares[userid].Mtime = utils.TSToTime(info.Mtime) - + /* + FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat + the local cache is updated with Sync during reads + info, err := c.storage.Stat(ctx, jsonPath) + if err != nil { + return err + } + c.UserShares[userid].Mtime = utils.TSToTime(info.Mtime) + */ return nil } From 9dc366d42070eb19304faa8324903f234ac024d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 10:59:37 +0000 Subject: [PATCH 89/94] use new mtime for if-unmodified-since MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 84 +++++++++++++++++-- .../jsoncs3/providercache/providercache.go | 21 +---- .../receivedsharecache/receivedsharecache.go | 20 +---- .../manager/jsoncs3/sharecache/sharecache.go | 20 +---- 4 files changed, 84 insertions(+), 61 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 6b7aea6b30..821b7baf7b 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -228,13 +228,26 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla } err = m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.Cache.Sync(ctx, md.Id.StorageId, md.Id.SpaceId); err != nil { + return nil, err + } + err = m.Cache.Add(ctx, md.Id.StorageId, md.Id.SpaceId, shareID, s) + // TODO try more often? + } if err != nil { return nil, err } err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), shareID) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.CreatedCache.Sync(ctx, s.GetCreator().GetOpaqueId()); err != nil { + return nil, err + } + err = m.CreatedCache.Add(ctx, s.GetCreator().GetOpaqueId(), shareID) + // TODO try more often? + } if err != nil { - // TODO when persisting fails, download, readd and persist again return nil, err } @@ -248,12 +261,28 @@ func (m *Manager) Share(ctx context.Context, md *provider.ResourceInfo, g *colla Share: s, State: collaboration.ShareState_SHARE_STATE_PENDING, } - m.UserReceivedStates.Add(ctx, userid, spaceID, rs) - // TODO check error - // TODO when persisting fails, download, readd and persist again + err = m.UserReceivedStates.Add(ctx, userid, spaceID, rs) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.UserReceivedStates.Sync(ctx, s.GetCreator().GetOpaqueId()); err != nil { + return nil, err + } + err = m.UserReceivedStates.Add(ctx, userid, spaceID, rs) + // TODO try more often? + } + if err != nil { + return nil, err + } case provider.GranteeType_GRANTEE_TYPE_GROUP: groupid := g.Grantee.GetGroupId().GetOpaqueId() - if err := m.GroupReceivedCache.Add(ctx, groupid, shareID); err != nil { + err := m.GroupReceivedCache.Add(ctx, groupid, shareID) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.GroupReceivedCache.Sync(ctx, groupid); err != nil { + return nil, err + } + err = m.GroupReceivedCache.Add(ctx, groupid, shareID) + // TODO try more often? + } + if err != nil { return nil, err } } @@ -350,12 +379,26 @@ func (m *Manager) Unshare(ctx context.Context, ref *collaboration.ShareReference storageID, spaceID, _ := shareid.Decode(s.Id.OpaqueId) err = m.Cache.Remove(ctx, storageID, spaceID, s.Id.OpaqueId) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.Cache.Sync(ctx, storageID, spaceID); err != nil { + return err + } + err = m.Cache.Remove(ctx, storageID, spaceID, s.Id.OpaqueId) + // TODO try more often? + } if err != nil { return err } // remove from created cache err = m.CreatedCache.Remove(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + if err := m.CreatedCache.Sync(ctx, s.GetCreator().GetOpaqueId()); err != nil { + return err + } + err = m.CreatedCache.Remove(ctx, s.GetCreator().GetOpaqueId(), s.Id.OpaqueId) + // TODO try more often? + } if err != nil { return err } @@ -387,8 +430,24 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer s.Mtime = utils.TSNow() // Update provider cache - m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) - // TODO when persisting fails, download, readd and persist again + err = m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) + // when persisting fails + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + // reupdate + s, err := m.get(ctx, ref) // does an implicit sync + if err != nil { + return nil, err + } + s.Permissions = p + s.Mtime = utils.TSNow() + + // persist again + err = m.Cache.Persist(ctx, s.ResourceId.StorageId, s.ResourceId.SpaceId) + // TODO try more often? + } + if err != nil { + return nil, err + } return s, nil } @@ -618,8 +677,15 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab userID := ctxpkg.ContextMustGetUser(ctx) - m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) - // TODO when persisting fails, download, readd and persist again + err = m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) + if _, ok := err.(errtypes.IsPreconditionFailed); ok { + // when persisting fails, download, readd and persist again + if err := m.UserReceivedStates.Sync(ctx, userID.GetId().GetOpaqueId()); err != nil { + return nil, err + } + err = m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) + // TODO try more often? + } return rs, nil } diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index bd6d3f71cc..13879271e9 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -141,8 +141,6 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return nil } - oldMtime := c.Providers[storageID].Spaces[spaceID].Mtime - c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() createdBytes, err := json.Marshal(c.Providers[storageID].Spaces[spaceID]) if err != nil { @@ -153,24 +151,11 @@ func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { return err } - if err := c.storage.Upload(ctx, metadata.UploadRequest{ + return c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, - IfUnmodifiedSince: oldMtime, - }); err != nil { - return err - } - - /* - FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat - the local cache is updated with Sync during reads - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.Providers[storageID].Spaces[spaceID].Mtime = utils.TSToTime(info.Mtime) - */ - return nil + IfUnmodifiedSince: c.Providers[storageID].Spaces[spaceID].Mtime, + }) } // Sync updates the in-memory data with the data from the storage if it is outdated diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index 4b2d8a756d..8eac0df917 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -148,7 +148,6 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { return nil } - oldMtime := c.ReceivedSpaces[userID].Mtime c.ReceivedSpaces[userID].Mtime = time.Now() createdBytes, err := json.Marshal(c.ReceivedSpaces[userID]) if err != nil { @@ -159,24 +158,11 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { return err } - if err := c.storage.Upload(ctx, metadata.UploadRequest{ + return c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, - IfUnmodifiedSince: oldMtime, - }); err != nil { - return err - } - - /* - FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat - the local cache is updated with Sync during reads - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.ReceivedSpaces[userID].Mtime = utils.TSToTime(info.Mtime) - */ - return nil + IfUnmodifiedSince: c.ReceivedSpaces[userID].Mtime, + }) } func userJSONPath(userID string) string { diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 7683b217da..40470d3d90 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -168,7 +168,6 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { // Persist persists the data for one user/group to the storage func (c *Cache) Persist(ctx context.Context, userid string) error { - oldMtime := c.UserShares[userid].Mtime c.UserShares[userid].Mtime = time.Now() createdBytes, err := json.Marshal(c.UserShares[userid]) @@ -180,24 +179,11 @@ func (c *Cache) Persist(ctx context.Context, userid string) error { return err } - if err := c.storage.Upload(ctx, metadata.UploadRequest{ + return c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, - IfUnmodifiedSince: oldMtime, - }); err != nil { - return err - } - - /* - FIXME stating here introduces a lost read because the file might have been overwritten written between the above upload and this stat - the local cache is updated with Sync during reads - info, err := c.storage.Stat(ctx, jsonPath) - if err != nil { - return err - } - c.UserShares[userid].Mtime = utils.TSToTime(info.Mtime) - */ - return nil + IfUnmodifiedSince: c.UserShares[userid].Mtime, + }) } func (c *Cache) userCreatedPath(userid string) string { From 285e35cc16816ad49073599ca49a5ab0464150bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 11:00:33 +0000 Subject: [PATCH 90/94] add timestamps to precondition failed error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/utils/metadata/disk.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/storage/utils/metadata/disk.go b/pkg/storage/utils/metadata/disk.go index beba6df0c3..816177a774 100644 --- a/pkg/storage/utils/metadata/disk.go +++ b/pkg/storage/utils/metadata/disk.go @@ -21,6 +21,7 @@ package metadata import ( "context" "errors" + "fmt" "io/fs" "io/ioutil" "os" @@ -107,7 +108,7 @@ func (disk *Disk) Upload(_ context.Context, req UploadRequest) error { return err } else if err == nil { if info.ModTime().After(req.IfUnmodifiedSince) { - return errtypes.PreconditionFailed("resource has been modified") + return errtypes.PreconditionFailed(fmt.Sprintf("resource has been modified, mtime: %s > since %s", info.ModTime(), req.IfUnmodifiedSince)) } } } From 18f2d669d470ac9df7035601f0b1dc2851476be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 12:05:09 +0000 Subject: [PATCH 91/94] revert mtime change on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../jsoncs3/providercache/providercache.go | 24 +++++++++++++++---- .../providercache/providercache_test.go | 7 +++--- .../receivedsharecache/receivedsharecache.go | 12 ++++++++-- .../manager/jsoncs3/sharecache/sharecache.go | 11 +++++++-- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache.go b/pkg/share/manager/jsoncs3/providercache/providercache.go index 13879271e9..83210d270b 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -135,27 +135,41 @@ func (c *Cache) ListSpace(storageID, spaceID string) *Shares { return c.Providers[storageID].Spaces[spaceID] } -// Persist persists the data of one space -func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { +// PersistWithTime persists the data of one space if it has not been modified since the given mtime +func (c *Cache) PersistWithTime(ctx context.Context, storageID, spaceID string, mtime time.Time) error { if c.Providers[storageID] == nil || c.Providers[storageID].Spaces[spaceID] == nil { return nil } - c.Providers[storageID].Spaces[spaceID].Mtime = time.Now() + oldMtime := c.Providers[storageID].Spaces[spaceID].Mtime + c.Providers[storageID].Spaces[spaceID].Mtime = mtime + + // FIXME there is a race when between this time now and the below Uploed another process also updates the file -> we need a lock createdBytes, err := json.Marshal(c.Providers[storageID].Spaces[spaceID]) if err != nil { + c.Providers[storageID].Spaces[spaceID].Mtime = oldMtime return err } jsonPath := spaceJSONPath(storageID, spaceID) if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + c.Providers[storageID].Spaces[spaceID].Mtime = oldMtime return err } - return c.storage.Upload(ctx, metadata.UploadRequest{ + if err = c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, IfUnmodifiedSince: c.Providers[storageID].Spaces[spaceID].Mtime, - }) + }); err != nil { + c.Providers[storageID].Spaces[spaceID].Mtime = oldMtime + return err + } + return nil +} + +// Persist persists the data of one space +func (c *Cache) Persist(ctx context.Context, storageID, spaceID string) error { + return c.PersistWithTime(ctx, storageID, spaceID, time.Now()) } // Sync updates the in-memory data with the data from the storage if it is outdated diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 106c72e796..7f31b6efd8 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -130,10 +130,11 @@ var _ = Describe("Cache", func() { Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(oldMtime)) }) - It("does not persist if the etag changed on disk", func() { - c.Providers[storageID].Spaces[spaceID].Mtime = time.Now().Add(-3 * time.Hour) + }) - Expect(c.Persist(ctx, storageID, spaceID)).ToNot(Succeed()) + Describe("PersistWithTime", func() { + It("does not persist if the mtime on disk is more recent", func() { + Expect(c.PersistWithTime(ctx, storageID, spaceID, time.Now().Add(-3*time.Hour))).ToNot(Succeed()) }) }) diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index 8eac0df917..d2302f8ae1 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -148,21 +148,29 @@ func (c *Cache) Persist(ctx context.Context, userID string) error { return nil } + oldMtime := c.ReceivedSpaces[userID].Mtime c.ReceivedSpaces[userID].Mtime = time.Now() + createdBytes, err := json.Marshal(c.ReceivedSpaces[userID]) if err != nil { + c.ReceivedSpaces[userID].Mtime = oldMtime return err } jsonPath := userJSONPath(userID) if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + c.ReceivedSpaces[userID].Mtime = oldMtime return err } - return c.storage.Upload(ctx, metadata.UploadRequest{ + if err = c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, IfUnmodifiedSince: c.ReceivedSpaces[userID].Mtime, - }) + }); err != nil { + c.ReceivedSpaces[userID].Mtime = oldMtime + return err + } + return nil } func userJSONPath(userID string) string { diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/pkg/share/manager/jsoncs3/sharecache/sharecache.go index 40470d3d90..82cb19b27e 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -168,22 +168,29 @@ func (c *Cache) Sync(ctx context.Context, userID string) error { // Persist persists the data for one user/group to the storage func (c *Cache) Persist(ctx context.Context, userid string) error { + oldMtime := c.UserShares[userid].Mtime c.UserShares[userid].Mtime = time.Now() createdBytes, err := json.Marshal(c.UserShares[userid]) if err != nil { + c.UserShares[userid].Mtime = oldMtime return err } jsonPath := c.userCreatedPath(userid) if err := c.storage.MakeDirIfNotExist(ctx, path.Dir(jsonPath)); err != nil { + c.UserShares[userid].Mtime = oldMtime return err } - return c.storage.Upload(ctx, metadata.UploadRequest{ + if err = c.storage.Upload(ctx, metadata.UploadRequest{ Path: jsonPath, Content: createdBytes, IfUnmodifiedSince: c.UserShares[userid].Mtime, - }) + }); err != nil { + c.UserShares[userid].Mtime = oldMtime + return err + } + return nil } func (c *Cache) userCreatedPath(userid string) string { From b41b20fa3c08fd20969733154c38e285e45de55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 12:27:33 +0000 Subject: [PATCH 92/94] add missing headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../providercache/providercache_suite_test.go | 18 ++++++++++++++++++ .../providercache/providercache_test.go | 18 ++++++++++++++++++ .../receivedShareCache_suite_test.go | 18 ++++++++++++++++++ .../sharecache/sharecache_suite_test.go | 18 ++++++++++++++++++ .../jsoncs3/sharecache/sharecache_test.go | 18 ++++++++++++++++++ 5 files changed, 90 insertions(+) diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go index 7098cbb6af..eb4a39b90e 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_suite_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package providercache_test import ( diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index 7f31b6efd8..d12a2e865d 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package providercache_test import ( diff --git a/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go b/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go index f93838acd0..5877755c37 100644 --- a/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go +++ b/pkg/share/manager/jsoncs3/receivedsharecache/receivedShareCache_suite_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package receivedsharecache_test import ( diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go b/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go index 6c6f050884..44026265b3 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache_suite_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package sharecache_test import ( diff --git a/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go index 36e8a647bc..f6bd3b91f9 100644 --- a/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go +++ b/pkg/share/manager/jsoncs3/sharecache/sharecache_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package sharecache_test import ( From 2e77af49d70d41ea038eabf5aade9bcd9833409d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 22 Aug 2022 14:18:21 +0000 Subject: [PATCH 93/94] lint fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/share/manager/jsoncs3/jsoncs3.go | 18 +++++++++++++----- pkg/share/manager/jsoncs3/jsoncs3_test.go | 11 ++++++----- .../providercache/providercache_test.go | 18 +++++++++--------- pkg/utils/utils.go | 2 +- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/pkg/share/manager/jsoncs3/jsoncs3.go b/pkg/share/manager/jsoncs3/jsoncs3.go index 821b7baf7b..0d93272f3a 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3.go +++ b/pkg/share/manager/jsoncs3/jsoncs3.go @@ -434,7 +434,7 @@ func (m *Manager) UpdateShare(ctx context.Context, ref *collaboration.ShareRefer // when persisting fails if _, ok := err.(errtypes.IsPreconditionFailed); ok { // reupdate - s, err := m.get(ctx, ref) // does an implicit sync + s, err = m.get(ctx, ref) // does an implicit sync if err != nil { return nil, err } @@ -506,7 +506,9 @@ func (m *Manager) listSharesByIDs(ctx context.Context, user *userv1beta1.User, f func (m *Manager) listCreatedShares(ctx context.Context, user *userv1beta1.User, filters []*collaboration.Filter) ([]*collaboration.Share, error) { var ss []*collaboration.Share - m.CreatedCache.Sync(ctx, user.Id.OpaqueId) + if err := m.CreatedCache.Sync(ctx, user.Id.OpaqueId); err != nil { + return ss, err + } for ssid, spaceShareIDs := range m.CreatedCache.List(user.Id.OpaqueId) { storageID, spaceID, _ := shareid.Decode(ssid) err := m.Cache.Sync(ctx, storageID, spaceID) @@ -546,7 +548,9 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati // first collect all spaceids the user has access to as a group member for _, group := range user.Groups { - m.GroupReceivedCache.Sync(ctx, group) + if err := m.GroupReceivedCache.Sync(ctx, group); err != nil { + continue // ignore error, cache will be updated on next read + } for ssid, spaceShareIDs := range m.GroupReceivedCache.List(group) { // add a pending entry, the state will be updated // when reading the received shares below if they have already been accepted or denied @@ -565,7 +569,8 @@ func (m *Manager) ListReceivedShares(ctx context.Context, filters []*collaborati } // add all spaces the user has receved shares for, this includes mount points and share state for groups - m.UserReceivedStates.Sync(ctx, user.Id.OpaqueId) + _ = m.UserReceivedStates.Sync(ctx, user.Id.OpaqueId) // ignore error, cache will be updated on next read + if m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId] != nil { for ssid, rspace := range m.UserReceivedStates.ReceivedSpaces[user.Id.OpaqueId].Spaces { if rs, ok := ssids[ssid]; ok { @@ -616,7 +621,7 @@ func (m *Manager) convert(ctx context.Context, userID string, s *collaboration.S storageID, spaceID, _ := shareid.Decode(s.Id.OpaqueId) - m.UserReceivedStates.Sync(ctx, userID) + _ = m.UserReceivedStates.Sync(ctx, userID) // ignore error, cache will be updated on next read state := m.UserReceivedStates.Get(userID, storageID+"^"+spaceID, s.Id.GetOpaqueId()) if state != nil { rs.State = state.State @@ -686,6 +691,9 @@ func (m *Manager) UpdateReceivedShare(ctx context.Context, receivedShare *collab err = m.UserReceivedStates.Add(ctx, userID.GetId().GetOpaqueId(), rs.Share.ResourceId.StorageId+"^"+rs.Share.ResourceId.SpaceId, rs) // TODO try more often? } + if err != nil { + return nil, err + } return rs, nil } diff --git a/pkg/share/manager/jsoncs3/jsoncs3_test.go b/pkg/share/manager/jsoncs3/jsoncs3_test.go index c48892d109..ba65b96037 100644 --- a/pkg/share/manager/jsoncs3/jsoncs3_test.go +++ b/pkg/share/manager/jsoncs3/jsoncs3_test.go @@ -227,6 +227,7 @@ var _ = Describe("Jsoncs3", func() { Permissions: conversions.NewManagerRole().CS3ResourcePermissions(), }, }) + Expect(err).ToNot(HaveOccurred()) }) Describe("ListShares", func() { @@ -338,7 +339,7 @@ var _ = Describe("Jsoncs3", func() { cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true bytes, err := json.Marshal(cache) Expect(err).ToNot(HaveOccurred()) - storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes)).To(Succeed()) Expect(err).ToNot(HaveOccurred()) // Reset providercache in memory @@ -559,7 +560,7 @@ var _ = Describe("Jsoncs3", func() { cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true bytes, err := json.Marshal(cache) Expect(err).ToNot(HaveOccurred()) - storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes)).To(Succeed()) Expect(err).ToNot(HaveOccurred()) // Reset providercache in memory @@ -579,9 +580,9 @@ var _ = Describe("Jsoncs3", func() { Expect(len(shares)).To(Equal(1)) // Add a second cache to the provider cache so it can be referenced - m.Cache.Add(ctx, "storageid", "spaceid", "storageid^spaceid°secondshare", &collaboration.Share{ + Expect(m.Cache.Add(ctx, "storageid", "spaceid", "storageid^spaceid°secondshare", &collaboration.Share{ Creator: user1.Id, - }) + })).To(Succeed()) cache := sharecache.UserShareCache{ Mtime: time.Now(), @@ -644,7 +645,7 @@ var _ = Describe("Jsoncs3", func() { cache.Shares[share.Id.OpaqueId].Permissions.Permissions.InitiateFileUpload = true bytes, err := json.Marshal(cache) Expect(err).ToNot(HaveOccurred()) - storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes) + Expect(storage.SimpleUpload(context.Background(), "storages/storageid/spaceid.json", bytes)).To(Succeed()) Expect(err).ToNot(HaveOccurred()) // Reset providercache in memory diff --git a/pkg/share/manager/jsoncs3/providercache/providercache_test.go b/pkg/share/manager/jsoncs3/providercache/providercache_test.go index d12a2e865d..9fdfdbc8f9 100644 --- a/pkg/share/manager/jsoncs3/providercache/providercache_test.go +++ b/pkg/share/manager/jsoncs3/providercache/providercache_test.go @@ -77,7 +77,7 @@ var _ = Describe("Cache", func() { s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Add(ctx, storageID, spaceID, shareID, share1) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) s = c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) @@ -85,14 +85,14 @@ var _ = Describe("Cache", func() { }) It("sets the mtime", func() { - c.Add(ctx, storageID, spaceID, shareID, share1) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(time.Time{})) }) It("updates the mtime", func() { - c.Add(ctx, storageID, spaceID, shareID, share1) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) old := c.Providers[storageID].Spaces[spaceID].Mtime - c.Add(ctx, storageID, spaceID, shareID, share1) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) @@ -115,16 +115,16 @@ var _ = Describe("Cache", func() { Expect(s).ToNot(BeNil()) Expect(s).To(Equal(share1)) - c.Remove(ctx, storageID, spaceID, shareID) + Expect(c.Remove(ctx, storageID, spaceID, shareID)).To(Succeed()) s = c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) }) It("updates the mtime", func() { - c.Add(ctx, storageID, spaceID, shareID, share1) + Expect(c.Add(ctx, storageID, spaceID, shareID, share1)).To(Succeed()) old := c.Providers[storageID].Spaces[spaceID].Mtime - c.Remove(ctx, storageID, spaceID, shareID) + Expect(c.Remove(ctx, storageID, spaceID, shareID)).To(Succeed()) Expect(c.Providers[storageID].Spaces[spaceID].Mtime).ToNot(Equal(old)) }) }) @@ -167,7 +167,7 @@ var _ = Describe("Cache", func() { s := c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) - c.Sync(ctx, storageID, spaceID) + Expect(c.Sync(ctx, storageID, spaceID)).To(Succeed()) s = c.Get(storageID, spaceID, shareID) Expect(s).ToNot(BeNil()) @@ -184,7 +184,7 @@ var _ = Describe("Cache", func() { }, }, } - c.Sync(ctx, storageID, spaceID) // Sync from disk won't happen because in-memory mtime is later than on disk + Expect(c.Sync(ctx, storageID, spaceID)).To(Succeed()) // Sync from disk won't happen because in-memory mtime is later than on disk s = c.Get(storageID, spaceID, shareID) Expect(s).To(BeNil()) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 3e53660d6d..0d99af4939 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -146,7 +146,7 @@ func TSToTime(ts *types.Timestamp) time.Time { // TimeToTS converts Go's time.Time to a protobuf Timestamp. func TimeToTS(t time.Time) *types.Timestamp { return &types.Timestamp{ - Seconds: uint64(t.Unix()), // implicity returns UTC + Seconds: uint64(t.Unix()), // implicitly returns UTC Nanos: uint32(t.Nanosecond()), } } From 033ae61a1f8afb7b2277b5faac20b4fc94a8bbb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 23 Aug 2022 08:11:24 +0000 Subject: [PATCH 94/94] fix nextcloud mock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- pkg/storage/fs/nextcloud/nextcloud_server_mock.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/storage/fs/nextcloud/nextcloud_server_mock.go b/pkg/storage/fs/nextcloud/nextcloud_server_mock.go index dd083b899c..a7a4d2b1cc 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_server_mock.go +++ b/pkg/storage/fs/nextcloud/nextcloud_server_mock.go @@ -111,6 +111,7 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetPathByID {"storage_id":"00000000-0000-0000-0000-000000000000","opaque_id":"fileid-/some/path"} EMPTY`: {200, "/subdir", serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"path":"/file"},"uploadLength":0,"metadata":{"providerID":""}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":0,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty},