diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index c8d8d6b68d8..19bdb913519 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -12,10 +12,13 @@ import ( "strings" "github.com/cs3org/reva/pkg/auth/scope" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/utils" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" @@ -32,6 +35,7 @@ type CS3Repo struct { tm token.Manager storageProvider provider.ProviderAPIClient dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + space *provider.StorageSpace } // NewCS3Repo creates a new cs3 repo @@ -49,7 +53,7 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { return nil, err } - return CS3Repo{ + repo := CS3Repo{ cfg: cfg, tm: tokenManager, storageProvider: client, @@ -58,7 +62,58 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { Transport: http.DefaultTransport, }, }, - }, nil + } + + if err := repo.Init(); err != nil { + return nil, err + } + + return &repo, nil +} + +// init creates the metadata space +func (r *CS3Repo) Init() (err error) { + ctx := context.Background() + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) + su := &user.User{ + Id: &user.UserId{ + OpaqueId: r.cfg.ServiceUser.UUID, + }, + Groups: []string{}, + UidNumber: r.cfg.ServiceUser.UID, + GidNumber: r.cfg.ServiceUser.GID, + } + // FIXME change CS3 api to allow sending a space id + cssr, err := r.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ + Opaque: &typesv1beta1.Opaque{ + Map: map[string]*typesv1beta1.OpaqueEntry{ + "spaceid": { + Decoder: "plain", + Value: []byte(r.cfg.ServiceUser.UUID), + }, + }, + }, + Owner: su, + Name: "Metadata", + Type: "metadata", + }) + switch { + case err != nil: + return err + case cssr.Status.Code == v1beta11.Code_CODE_OK: + // continue + case cssr.Status.Code != v1beta11.Code_CODE_ALREADY_EXISTS: + // continue + default: + return errtypes.NewErrtypeFromStatus(cssr.Status) + } + r.space = cssr.StorageSpace + return nil } // WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id). @@ -108,7 +163,8 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", accountsFolder), + ResourceId: r.space.Root, + Path: utils.MakeRelativePath(accountsFolder), }, }) if err != nil { @@ -159,7 +215,8 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: path.Join("/", accountsFolder, id), + ResourceId: r.space.Root, + Path: utils.MakeRelativePath(filepath.Join("/", accountsFolder, id)), }, }) @@ -222,7 +279,8 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", groupsFolder), + ResourceId: r.space.Root, + Path: utils.MakeRelativePath(groupsFolder), }, }) if err != nil { @@ -273,7 +331,8 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: path.Join("/", groupsFolder, id), + ResourceId: r.space.Root, + Path: utils.MakeRelativePath(filepath.Join(groupsFolder, id)), }, }) @@ -311,21 +370,22 @@ func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manage } func (r CS3Repo) accountURL(id string) string { - return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, accountsFolder, id)) + return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, "spaces", r.space.Id.OpaqueId+"!"+r.space.Id.OpaqueId, accountsFolder, id)) } func (r CS3Repo) groupURL(id string) string { - return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, groupsFolder, id)) + return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, "spaces", r.space.Id.OpaqueId+"!"+r.space.Id.OpaqueId, groupsFolder, id)) } func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error { - return MakeDirIfNotExist(ctx, r.storageProvider, folder) + return MakeDirIfNotExist(ctx, r.storageProvider, r.space.Root, folder) } // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context. -func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folder string) error { +func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, root *provider.ResourceId, folder string) error { var rootPathRef = &provider.Reference{ - Path: path.Join("/", folder), + ResourceId: root, + Path: utils.MakeRelativePath(folder), } resp, err := sp.Stat(ctx, &provider.StatRequest{ diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index 81cc98cec1a..ee2703a4aba 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -22,6 +22,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" + "github.com/cs3org/reva/pkg/utils" "google.golang.org/grpc/metadata" "github.com/owncloud/ocis/ocis-pkg/indexer/index" @@ -72,6 +73,7 @@ func NewAutoincrementIndex(o ...option.Option) index.Index { }, dataProvider: dataProviderClient{ baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), + spaceid: opts.ServiceUser.UUID, client: http.Client{ Transport: http.DefaultTransport, }, @@ -177,7 +179,11 @@ func (idx *Autoincrement) Remove(id string, v string) error { deletePath := path.Join("/", idx.indexRootDir, v) resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: deletePath, + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(deletePath), }, }) @@ -215,7 +221,11 @@ func (idx *Autoincrement) Search(pattern string) ([]string, error) { res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", idx.indexRootDir), + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(idx.indexRootDir), }, }) @@ -313,7 +323,10 @@ func (idx *Autoincrement) resolveSymlink(name string) (string, error) { } func (idx *Autoincrement) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, folder) } func (idx *Autoincrement) authenticate(ctx context.Context) (token string, err error) { @@ -328,7 +341,11 @@ func (idx *Autoincrement) next() (int, error) { res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", idx.indexRootDir), + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(idx.indexRootDir), }, }) @@ -375,5 +392,5 @@ func (idx *Autoincrement) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) } diff --git a/ocis-pkg/indexer/index/cs3/data_provider_client.go b/ocis-pkg/indexer/index/cs3/data_provider_client.go index 3a2d42f7ed0..baec5ddcb91 100644 --- a/ocis-pkg/indexer/index/cs3/data_provider_client.go +++ b/ocis-pkg/indexer/index/cs3/data_provider_client.go @@ -3,16 +3,18 @@ package cs3 import ( "io" "net/http" + "path/filepath" "strings" ) type dataProviderClient struct { client http.Client + spaceid string baseURL string } func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, url), body) + req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), body) if err != nil { return nil, err } @@ -22,7 +24,7 @@ func (d dataProviderClient) put(url string, body io.Reader, token string) (*http } func (d dataProviderClient) get(url string, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, url), nil) + req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), nil) if err != nil { return nil, err } diff --git a/ocis-pkg/indexer/index/cs3/helper.go b/ocis-pkg/indexer/index/cs3/helper.go index 127b839c155..5024557e8cf 100644 --- a/ocis-pkg/indexer/index/cs3/helper.go +++ b/ocis-pkg/indexer/index/cs3/helper.go @@ -3,16 +3,20 @@ package cs3 import ( "context" "fmt" - "path" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/utils" ) -func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, indexRootDir string) error { +func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, spaceid, indexRootDir string) error { res, err := storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: path.Join("/", indexRootDir), + ResourceId: &provider.ResourceId{ + StorageId: spaceid, + OpaqueId: spaceid, + }, + Path: utils.MakeRelativePath(indexRootDir), }, }) if err != nil { diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index 60d65e75e11..73034016505 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -18,6 +18,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" + "github.com/cs3org/reva/pkg/utils" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" "github.com/owncloud/ocis/ocis-pkg/indexer/option" @@ -76,6 +77,7 @@ func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index { }, dataProvider: dataProviderClient{ baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), + spaceid: opts.ServiceUser.UUID, client: http.Client{ Transport: http.DefaultTransport, }, @@ -133,7 +135,11 @@ func (idx *NonUnique) Lookup(v string) ([]string, error) { res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", idx.indexRootDir, v), + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(path.Join("/", idx.indexRootDir, v)), }, }) @@ -193,7 +199,11 @@ func (idx *NonUnique) Remove(id string, v string) error { deletePath := path.Join("/", idx.indexRootDir, v, id) resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: deletePath, + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(deletePath), }, }) @@ -208,7 +218,11 @@ func (idx *NonUnique) Remove(id string, v string) error { toStat := path.Join("/", idx.indexRootDir, v) lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: toStat, + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(toStat), }, }) if err != nil { @@ -219,7 +233,11 @@ func (idx *NonUnique) Remove(id string, v string) error { deletePath = path.Join("/", idx.indexRootDir, v) _, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: deletePath, + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(deletePath), }, }) if err != nil { @@ -263,7 +281,11 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { matches := make([]string, 0) res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", idx.indexRootDir), + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(idx.indexRootDir), }, }) @@ -284,7 +306,11 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { for i := range foldersMatched { res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: foldersMatched[i], + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(foldersMatched[i]), }, }) @@ -317,7 +343,10 @@ func (idx *NonUnique) FilesDir() string { } func (idx *NonUnique) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, folder) } func (idx *NonUnique) createSymlink(oldname, newname string) error { @@ -386,7 +415,7 @@ func (idx *NonUnique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) } func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) { diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index 75e96389743..c0573ce5060 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -20,6 +20,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" + "github.com/cs3org/reva/pkg/utils" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" "github.com/owncloud/ocis/ocis-pkg/indexer/option" @@ -80,6 +81,7 @@ func NewUniqueIndexWithOptions(o ...option.Option) index.Index { }, dataProvider: dataProviderClient{ baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), + spaceid: opts.ServiceUser.UUID, client: http.Client{ Transport: http.DefaultTransport, }, @@ -192,7 +194,11 @@ func (idx *Unique) Remove(id string, v string) error { ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: deletePath, + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(deletePath), }, }) @@ -241,7 +247,11 @@ func (idx *Unique) Search(pattern string) ([]string, error) { ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/", idx.indexRootDir), + ResourceId: &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, + Path: utils.MakeRelativePath(idx.indexRootDir), }, }) @@ -339,7 +349,10 @@ func (idx *Unique) resolveSymlink(name string) (string, error) { } func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, folder) } func (idx *Unique) authenticate(ctx context.Context) (token string, err error) { @@ -362,5 +375,5 @@ func (idx *Unique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) }