Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

graph: reuse user- and group cache in graph/sharedbyme #7683

Merged
merged 3 commits into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions services/graph/pkg/identity/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import (
"net/url"

"github.com/CiscoM31/godata"
cs3 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
cs3group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
cs3user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/v2/services/graph/pkg/service/v0/errorcode"
)
Expand Down Expand Up @@ -103,9 +104,10 @@ type EducationBackend interface {
RemoveTeacherFromEducationClass(ctx context.Context, classID string, teacherID string) error
}

func CreateUserModelFromCS3(u *cs3.User) *libregraph.User {
// CreateUserModelFromCS3 converts a cs3 User object into a libregraph.User
func CreateUserModelFromCS3(u *cs3user.User) *libregraph.User {
if u.Id == nil {
u.Id = &cs3.UserId{}
u.Id = &cs3user.UserId{}
}
return &libregraph.User{
DisplayName: &u.DisplayName,
Expand All @@ -114,3 +116,14 @@ func CreateUserModelFromCS3(u *cs3.User) *libregraph.User {
Id: &u.Id.OpaqueId,
}
}

// CreateGroupModelFromCS3 converts a cs3 Group object into a libregraph.Group
func CreateGroupModelFromCS3(g *cs3group.Group) *libregraph.Group {
if g.Id == nil {
g.Id = &cs3group.GroupId{}
}
return &libregraph.Group{
Id: &g.Id.OpaqueId,
DisplayName: &g.GroupName,
}
}
139 changes: 139 additions & 0 deletions services/graph/pkg/identity/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package identity

import (
"context"
"time"

gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
cs3Group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
cs3User "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
revautils "github.com/cs3org/reva/v2/pkg/utils"
"github.com/jellydator/ttlcache/v3"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/v2/services/graph/pkg/service/v0/errorcode"
)

// IdentityCache implements a simple ttl based cache for looking up users and groups by ID
type IdentityCache struct {
users *ttlcache.Cache[string, libregraph.User]
groups *ttlcache.Cache[string, libregraph.Group]
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
}

type identityCacheOptions struct {
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
usersTTL time.Duration
groupsTTL time.Duration
}

// IdentityCacheOptiondefines a single option function.
type IdentityCacheOption func(o *identityCacheOptions)

// IdentityCacheWithGatewaySelector set the gatewaySelector for the Identity Cache
func IdentityCacheWithGatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) IdentityCacheOption {
return func(o *identityCacheOptions) {
o.gatewaySelector = gatewaySelector
}
}

// IdentityCacheWithUsersTTL sets the TTL for the users cache
func IdentityCacheWithUsersTTL(ttl time.Duration) IdentityCacheOption {
return func(o *identityCacheOptions) {
o.usersTTL = ttl
}
}

// IdentityCacheWithGroupsTTL sets the TTL for the groups cache
func IdentityCacheWithGroupsTTL(ttl time.Duration) IdentityCacheOption {
return func(o *identityCacheOptions) {
o.groupsTTL = ttl
}
}

func newOptions(opts ...IdentityCacheOption) identityCacheOptions {
opt := identityCacheOptions{}
for _, o := range opts {
o(&opt)
}
return opt
}

// NewIdentityCache instanciates a new IdentityCache and sets the supplied options
func NewIdentityCache(opts ...IdentityCacheOption) IdentityCache {
opt := newOptions(opts...)

var cache IdentityCache

cache.users = ttlcache.New(
ttlcache.WithTTL[string, libregraph.User](opt.usersTTL),
ttlcache.WithDisableTouchOnHit[string, libregraph.User](),
)
go cache.users.Start()

cache.groups = ttlcache.New(
ttlcache.WithTTL[string, libregraph.Group](opt.groupsTTL),
ttlcache.WithDisableTouchOnHit[string, libregraph.Group](),
)
go cache.groups.Start()

cache.gatewaySelector = opt.gatewaySelector

return cache
}

// GetUser looks up a user by id, if the user is not cached yet it will do a lookup via the CS3 API
func (cache IdentityCache) GetUser(ctx context.Context, userid string) (libregraph.User, error) {
var user libregraph.User
if item := cache.users.Get(userid); item == nil {
gatewayClient, err := cache.gatewaySelector.Next()
cs3UserID := &cs3User.UserId{
OpaqueId: userid,
}
cs3User, err := revautils.GetUser(cs3UserID, gatewayClient)
if err != nil {
if revautils.IsErrNotFound(err) {
return libregraph.User{}, ErrNotFound
}
return libregraph.User{}, errorcode.New(errorcode.GeneralException, err.Error())
}
user = *CreateUserModelFromCS3(cs3User)
cache.users.Set(userid, user, ttlcache.DefaultTTL)

} else {
user = item.Value()
}
return user, nil
}

// GetUser looks up a group by id, if the group is not cached yet it will do a lookup via the CS3 API
func (cache IdentityCache) GetGroup(ctx context.Context, groupid string) (libregraph.Group, error) {
var group libregraph.Group
if item := cache.groups.Get(groupid); item == nil {
gatewayClient, err := cache.gatewaySelector.Next()
cs3GroupID := &cs3Group.GroupId{
OpaqueId: groupid,
}
req := cs3Group.GetGroupRequest{
GroupId: cs3GroupID,
}
res, err := gatewayClient.GetGroup(ctx, &req)
if err != nil {
return group, errorcode.New(errorcode.GeneralException, err.Error())
}
switch res.Status.Code {
case rpc.Code_CODE_OK:
cs3Group := res.GetGroup()
group = *CreateGroupModelFromCS3(cs3Group)
cache.groups.Set(groupid, group, ttlcache.DefaultTTL)
case rpc.Code_CODE_NOT_FOUND:
return group, ErrNotFound
default:
return group, errorcode.New(errorcode.GeneralException, res.Status.Message)
}
} else {
group = item.Value()
}
return group, nil
}
15 changes: 2 additions & 13 deletions services/graph/pkg/identity/cs3.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (i *CS3) GetGroups(ctx context.Context, queryParam url.Values) ([]*libregra
groups := make([]*libregraph.Group, 0, len(res.Groups))

for _, group := range res.Groups {
groups = append(groups, createGroupModelFromCS3(group))
groups = append(groups, CreateGroupModelFromCS3(group))
}

return groups, nil
Expand Down Expand Up @@ -185,7 +185,7 @@ func (i *CS3) GetGroup(ctx context.Context, groupID string, queryParam url.Value
return nil, errorcode.New(errorcode.GeneralException, res.Status.Message)
}

return createGroupModelFromCS3(res.Group), nil
return CreateGroupModelFromCS3(res.Group), nil
}

// DeleteGroup implements the Backend Interface. It's currently not supported for the CS3 backend
Expand All @@ -212,14 +212,3 @@ func (i *CS3) AddMembersToGroup(ctx context.Context, groupID string, memberID []
func (i *CS3) RemoveMemberFromGroup(ctx context.Context, groupID string, memberID string) error {
return errorcode.New(errorcode.NotSupported, "not implemented")
}

func createGroupModelFromCS3(g *cs3group.Group) *libregraph.Group {
if g.Id == nil {
g.Id = &cs3group.GroupId{}
}
return &libregraph.Group{
Id: &g.Id.OpaqueId,
DisplayName: &g.GroupName,
// TODO when to fetch and expand memberof, usernames or ids?
}
}
25 changes: 6 additions & 19 deletions services/graph/pkg/service/v0/drives.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/go-chi/render"
"github.com/jellydator/ttlcache/v3"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/v2/ocis-pkg/service/grpc"
v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
Expand Down Expand Up @@ -780,28 +779,16 @@ func (g Graph) cs3PermissionsToLibreGraph(ctx context.Context, space *storagepro
tmp := id
var identitySet libregraph.IdentitySet
if _, ok := groupsMap[id]; ok {
var group libregraph.Group
if item := g.groupsCache.Get(id); item == nil {
if requestedGroup, err := g.identityBackend.GetGroup(ctx, id, url.Values{}); err == nil {
group = *requestedGroup
g.groupsCache.Set(id, group, ttlcache.DefaultTTL)
}
} else {
group = item.Value()
group, err := g.identityCache.GetGroup(ctx, tmp)
if err != nil {
g.logger.Warn().Str("groupid", tmp).Msg("Group not found by id")
}

identitySet = libregraph.IdentitySet{Group: &libregraph.Identity{Id: &tmp, DisplayName: group.GetDisplayName()}}
} else {
var user libregraph.User
if item := g.usersCache.Get(id); item == nil {
if requestedUser, err := g.identityBackend.GetUser(ctx, id, &godata.GoDataRequest{}); err == nil {
user = *requestedUser
g.usersCache.Set(id, user, ttlcache.DefaultTTL)
}
} else {
user = item.Value()
user, err := g.identityCache.GetUser(ctx, tmp)
if err != nil {
g.logger.Warn().Str("userid", tmp).Msg("User not found by id")
}

identitySet = libregraph.IdentitySet{User: &libregraph.Identity{Id: &tmp, DisplayName: user.GetDisplayName()}}
}

Expand Down
4 changes: 1 addition & 3 deletions services/graph/pkg/service/v0/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/go-chi/chi/v5"
"github.com/jellydator/ttlcache/v3"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/v2/ocis-pkg/keycloak"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
ehsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/eventhistory/v0"
Expand Down Expand Up @@ -71,8 +70,7 @@ type Graph struct {
roleService RoleService
permissionsService Permissions
specialDriveItemsCache *ttlcache.Cache[string, interface{}]
usersCache *ttlcache.Cache[string, libregraph.User]
groupsCache *ttlcache.Cache[string, libregraph.Group]
identityCache identity.IdentityCache
eventsPublisher events.Publisher
searchService searchsvc.SearchProviderService
keycloakClient keycloak.Client
Expand Down
22 changes: 5 additions & 17 deletions services/graph/pkg/service/v0/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/go-chi/chi/v5/middleware"
ldapv3 "github.com/go-ldap/ldap/v3"
"github.com/jellydator/ttlcache/v3"
libregraph "github.com/owncloud/libre-graph-api-go"
ocisldap "github.com/owncloud/ocis/v2/ocis-pkg/ldap"
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/roles"
Expand Down Expand Up @@ -125,29 +124,18 @@ func NewService(opts ...Option) (Graph, error) {
)
go spacePropertiesCache.Start()

usersCache := ttlcache.New(
ttlcache.WithTTL[string, libregraph.User](
time.Duration(options.Config.Spaces.UsersCacheTTL),
),
ttlcache.WithDisableTouchOnHit[string, libregraph.User](),
)
go usersCache.Start()

groupsCache := ttlcache.New(
ttlcache.WithTTL[string, libregraph.Group](
time.Duration(options.Config.Spaces.GroupsCacheTTL),
),
ttlcache.WithDisableTouchOnHit[string, libregraph.Group](),
identityCache := identity.NewIdentityCache(
identity.IdentityCacheWithGatewaySelector(options.GatewaySelector),
identity.IdentityCacheWithUsersTTL(time.Duration(options.Config.Spaces.UsersCacheTTL)),
identity.IdentityCacheWithGroupsTTL(time.Duration(options.Config.Spaces.GroupsCacheTTL)),
)
go groupsCache.Start()

svc := Graph{
config: options.Config,
mux: m,
logger: &options.Logger,
specialDriveItemsCache: spacePropertiesCache,
usersCache: usersCache,
groupsCache: groupsCache,
identityCache: identityCache,
eventsPublisher: options.EventsPublisher,
gatewaySelector: options.GatewaySelector,
searchService: options.SearchService,
Expand Down
Loading