From 70d083076a49c5533627a47d2c0a32bf6b258f46 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 01:49:26 +0000 Subject: [PATCH 1/8] first attempt at templates in go --- go.mod | 4 ++ pkg/storage/pw/context/context.go | 63 +++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d21944c86f..0765b057df 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/fatih/color v1.7.0 // indirect github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.2.0+incompatible + github.com/gogo/protobuf v1.2.0 // indirect github.com/golang/protobuf v1.3.2 github.com/gomodule/redigo v2.0.0+incompatible github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 @@ -24,10 +25,12 @@ require ( github.com/pkg/errors v0.8.1 github.com/pkg/xattr v0.4.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect + github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect github.com/rs/cors v1.7.0 github.com/rs/zerolog v1.17.2 go.opencensus.io v0.22.1 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a + golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 google.golang.org/grpc v1.25.1 @@ -35,6 +38,7 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.2.2 // indirect + honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect ) go 1.13 diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go index 02a09978f4..eafd632f6d 100644 --- a/pkg/storage/pw/context/context.go +++ b/pkg/storage/pw/context/context.go @@ -19,9 +19,12 @@ package context import ( + "bytes" "context" + "fmt" "path" "strings" + "text/template" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" @@ -37,10 +40,11 @@ func init() { type config struct { Prefix string `mapstructure:"prefix"` + Layout string `mapstructure:"layout"` } func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} + c := &config{Layout: "{{.Username}}"} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "error decoding conf") return nil, err @@ -55,23 +59,50 @@ func New(m map[string]interface{}) (storage.PathWrapper, error) { if err != nil { return nil, err } - return &pw{prefix: c.Prefix}, nil + return &pw{prefix: c.Prefix, layout: c.Layout}, nil } type pw struct { prefix string + layout string +} + +type layoutTemplate struct { + Username string + FirstLetter string + Provider string +} + +func (pw *pw) getUserHomePath(username string) (string, error) { + usernameSplit := strings.Split(username, "@") + if len(usernameSplit) == 1 { + usernameSplit = append(usernameSplit, "_Unknown") + } + if usernameSplit[1] == "" { + usernameSplit[1] = "_Unknown" + } + + pathTemplate := layoutTemplate{ + Username: username, + FirstLetter: string([]rune(usernameSplit[0])[1]), + Provider: usernameSplit[1], + } + tmpl, err := template.New("userhomepath").Parse(pw.layout) + if err != nil { + return "", err + } + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, pathTemplate) + if err != nil { + return "", err + } + + return buf.String(), nil } // Only works when a user is in context func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - // TODO how do we get the users home path? - // - construct based on homedir prefix + username/userid? - // - look into context? - // - query preferences? - // - do nothing - // -> screams for a wrapper/unwrapper 'strategy' - u, ok := user.ContextGetUser(ctx) if !ok { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") @@ -79,9 +110,13 @@ func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { if u.Username == "" { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") } - - return path.Join("/", pw.prefix, u.Username, rp), nil + userHomePath, err := pw.getUserHomePath(u.Username) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) + } + return path.Join("/", pw.prefix, userHomePath, rp), nil } + func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { u, ok := user.ContextGetUser(ctx) if !ok { @@ -90,5 +125,9 @@ func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { if u.Username == "" { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") } - return strings.TrimPrefix(rp, path.Join("/", u.Username)), nil + userHomePath, err := pw.getUserHomePath(u.Username) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) + } + return strings.TrimPrefix(rp, path.Join("/", userHomePath)), nil } From 5d78398259ed73588c270fcb0ec22231d185e293 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 02:46:51 +0000 Subject: [PATCH 2/8] simplify code --- pkg/storage/pw/context/context.go | 41 +++++++++++++------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go index eafd632f6d..bbc7ea1559 100644 --- a/pkg/storage/pw/context/context.go +++ b/pkg/storage/pw/context/context.go @@ -73,8 +73,16 @@ type layoutTemplate struct { Provider string } -func (pw *pw) getUserHomePath(username string) (string, error) { - usernameSplit := strings.Split(username, "@") +func (pw *pw) getUserHomePath(ctx context.Context) (string, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + } + if u.Username == "" { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") + } + + usernameSplit := strings.Split(u.Username, "@") if len(usernameSplit) == 1 { usernameSplit = append(usernameSplit, "_Unknown") } @@ -83,18 +91,18 @@ func (pw *pw) getUserHomePath(username string) (string, error) { } pathTemplate := layoutTemplate{ - Username: username, + Username: u.Username, FirstLetter: string([]rune(usernameSplit[0])[1]), Provider: usernameSplit[1], } tmpl, err := template.New("userhomepath").Parse(pw.layout) if err != nil { - return "", err + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error())) } buf := new(bytes.Buffer) err = tmpl.Execute(buf, pathTemplate) if err != nil { - return "", err + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error())) } return buf.String(), nil @@ -102,32 +110,17 @@ func (pw *pw) getUserHomePath(username string) (string, error) { // Only works when a user is in context func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userHomePath, err := pw.getUserHomePath(u.Username) + userHomePath, err := pw.getUserHomePath(ctx) if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) + return "", err } return path.Join("/", pw.prefix, userHomePath, rp), nil } func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userHomePath, err := pw.getUserHomePath(u.Username) + userHomePath, err := pw.getUserHomePath(ctx) if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) + return "", err } return strings.TrimPrefix(rp, path.Join("/", userHomePath)), nil } From cfa192385127795368f8a93667b462b79de51164 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 04:50:46 +0000 Subject: [PATCH 3/8] created a storage helper so the template work lives in one spot for all storage providers, easy to expand --- pkg/storage/fs/eos/eos.go | 237 ++++++++++++++++++++++------ pkg/storage/fs/owncloud/owncloud.go | 96 +++++++---- pkg/storage/helper/helper.go | 68 ++++++++ pkg/storage/pw/context/context.go | 59 ++----- 4 files changed, 338 insertions(+), 122 deletions(-) create mode 100644 pkg/storage/helper/helper.go diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index bad21e9016..c715d9caf5 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -32,6 +32,7 @@ import ( "strings" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" @@ -108,6 +109,9 @@ type config struct { // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. UseKeytab bool `mapstrucuture:"use_keytab"` + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` + // SecProtocol specifies the xrootd security protocol to use between the server and EOS. SecProtocol string `mapstructure:"sec_protocol"` @@ -116,12 +120,15 @@ type config struct { // SingleUsername is the username to use when SingleUserMode is enabled SingleUsername string `mapstructure:"single_username"` + + // Layout + Layout string `mapstructure:"layout"` } func getUser(ctx context.Context) (*userpb.User, error) { u, ok := user.ContextGetUser(ctx) if !ok { - err := errors.Wrap(errtypes.UserRequired(""), "storage_eos: error getting user from ctx") + err := errors.Wrap(errtypes.UserRequired(""), "eos: error getting user from ctx") return nil, err } return u, nil @@ -152,6 +159,10 @@ func (c *config) init() { if c.CacheDirectory == "" { c.CacheDirectory = os.TempDir() } + + if c.Layout == "" { + c.Layout = "{{.Username}}" + } } // New returns a new implementation of the storage.FS interface that connects to EOS. @@ -194,6 +205,16 @@ func New(m map[string]interface{}) (storage.FS, error) { return eosStorage, nil } +func (fs *eosStorage) getHomeForUser(u *userpb.User) (string, error) { + userhome, err := helper.GetUserHomePath(u, fs.conf.Layout) + if err != nil { + return "", err + } + + home := path.Join(fs.mountpoint, userhome) + return home, nil +} + func (fs *eosStorage) Shutdown(ctx context.Context) error { // TODO(labkode): in a grpc implementation we can close connections. return nil @@ -216,19 +237,19 @@ func (fs *eosStorage) removeNamespace(ctx context.Context, np string) string { func (fs *eosStorage) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { u, err := getUser(ctx) if err != nil { - return "", errors.Wrap(err, "storage_eos: no user in ctx") + return "", errors.Wrap(err, "eos: no user in ctx") } // parts[0] = 868317, parts[1] = photos, ... parts := strings.Split(id.OpaqueId, "/") fileID, err := strconv.ParseUint(parts[0], 10, 64) if err != nil { - return "", errors.Wrap(err, "storage_eos: error parsing fileid string") + return "", errors.Wrap(err, "eos: error parsing fileid string") } eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fileID) if err != nil { - return "", errors.Wrap(err, "storage_eos: error getting file info by inode") + return "", errors.Wrap(err, "eos: error getting file info by inode") } fi := fs.convertToResourceInfo(ctx, eosFileInfo) @@ -260,7 +281,7 @@ func (fs *eosStorage) getPath(ctx context.Context, u *userpb.User, id *provider. } eosFileInfo, err := fs.c.GetFileInfoByInode(ctx, u.Username, fid) if err != nil { - return "", errors.Wrap(err, "storage_eos: error getting file info by inode") + return "", errors.Wrap(err, "eos: error getting file info by inode") } return eosFileInfo.File, nil } @@ -268,12 +289,12 @@ func (fs *eosStorage) getPath(ctx context.Context, u *userpb.User, id *provider. func (fs *eosStorage) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } eosACL, err := fs.getEosACL(g) @@ -283,7 +304,7 @@ func (fs *eosStorage) AddGrant(ctx context.Context, ref *provider.Reference, g * err = fs.c.AddACL(ctx, u.Username, fn, eosACL) if err != nil { - return errors.Wrap(err, "storage_eos: error adding acl") + return errors.Wrap(err, "eos: error adding acl") } return nil @@ -354,7 +375,7 @@ func (fs *eosStorage) UnsetArbitraryMetadata(ctx context.Context, ref *provider. func (fs *eosStorage) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } eosACLType, err := getEosACLType(g.Grantee.Type) @@ -364,12 +385,12 @@ func (fs *eosStorage) RemoveGrant(ctx context.Context, ref *provider.Reference, fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } err = fs.c.RemoveACL(ctx, u.Username, fn, eosACLType, g.Grantee.Id.OpaqueId) if err != nil { - return errors.Wrap(err, "storage_eos: error removing acl") + return errors.Wrap(err, "eos: error removing acl") } return nil } @@ -377,22 +398,22 @@ func (fs *eosStorage) RemoveGrant(ctx context.Context, ref *provider.Reference, func (fs *eosStorage) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } eosACL, err := fs.getEosACL(g) if err != nil { - return errors.Wrap(err, "storage_eos: error mapping acl") + return errors.Wrap(err, "eos: error mapping acl") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } err = fs.c.AddACL(ctx, u.Username, fn, eosACL) if err != nil { - return errors.Wrap(err, "storage_eos: error updating acl") + return errors.Wrap(err, "eos: error updating acl") } return nil } @@ -405,7 +426,7 @@ func (fs *eosStorage) ListGrants(ctx context.Context, ref *provider.Reference) ( fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } acls, err := fs.c.ListACLs(ctx, u.Username, fn) @@ -505,7 +526,7 @@ func (fs *eosStorage) GetMD(ctx context.Context, ref *provider.Reference) (*prov } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, u.Username, fn) @@ -519,17 +540,17 @@ func (fs *eosStorage) GetMD(ctx context.Context, ref *provider.Reference) (*prov func (fs *eosStorage) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) { u, err := getUser(ctx) if err != nil { - return nil, errors.Wrap(err, "storage_eos: no user in ctx") + return nil, errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } eosFileInfos, err := fs.c.List(ctx, u.Username, fn) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error listing") + return nil, errors.Wrap(err, "eos: error listing") } finfos := []*provider.ResourceInfo{} @@ -550,35 +571,161 @@ func (fs *eosStorage) ListFolder(ctx context.Context, ref *provider.Reference) ( func (fs *eosStorage) GetQuota(ctx context.Context) (int, int, error) { u, err := getUser(ctx) if err != nil { - return 0, 0, errors.Wrap(err, "storage_eos: no user in ctx") + return 0, 0, errors.Wrap(err, "eos: no user in ctx") } return fs.c.GetQuota(ctx, u.Username, fs.conf.Namespace) } +func (fs *eosStorage) GetHome(ctx context.Context) (string, error) { + u, err := getUser(ctx) + if err != nil { + return "", errors.Wrap(err, "eos: no user in ctx") + } + + home, err := fs.getHomeForUser(u) + + if err != nil { + return "", err + } + + return home, nil +} + +func (fs *eosStorage) CreateHome(ctx context.Context) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + home, err := fs.getHomeForUser(u) + if err != nil { + return err + } + + _, err = fs.c.GetFileInfoByPath(ctx, "root", home) + if err == nil { // home already exists + return nil + } + + // TODO(labkode): abort on any error that is not found + if _, ok := err.(errtypes.IsNotFound); !ok { + return errors.Wrap(err, "eos: error verifying if user home directory exists") + } + + // TODO(labkode): only trigger creation on not found, copy from CERNBox logic. + err = fs.c.CreateDir(ctx, "root", home) + if err != nil { + // EOS will return success on mkdir over an existing directory. + return errors.Wrap(err, "eos: error creating dir") + } + err = fs.c.Chown(ctx, "root", u.Username, home) + if err != nil { + return errors.Wrap(err, "eos: error chowning directory") + } + err = fs.c.Chmod(ctx, "root", "2770", home) + if err != nil { + return errors.Wrap(err, "eos: error chmoding directory") + } + + attrs := []*eosclient.Attribute{ + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "mask", + Val: "700", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "allow.oc.sync", + Val: "1", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "mtime.propagation", + Val: "1", + }, + &eosclient.Attribute{ + Type: eosclient.SystemAttr, + Key: "forced.atomic", + Val: "1", + }, + } + + for _, attr := range attrs { + err = fs.c.SetAttr(ctx, "root", attr, home) + if err != nil { + return errors.Wrap(err, "eos: error setting attribute") + } + + } + return nil +} + func (fs *eosStorage) CreateDir(ctx context.Context, fn string) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } fn = fs.getInternalPath(ctx, fn) return fs.c.CreateDir(ctx, u.Username, fn) } -func (fs *eosStorage) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { - // TODO(labkode):implement - return errtypes.NotSupported("eos: operation not supported") +func (fs *eosStorage) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + ref := &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: p, + }, + } + + fn, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + // TODO(labkode): with grpc we can touch with xattrs. + // Current mechanism is: touch to hidden file, set xattr, rename. + dir, base := path.Split(fn) + tmp := path.Join(dir, fmt.Sprintf(".sys.r#.%s", base)) + if err := fs.c.Touch(ctx, u.Username, tmp); err != nil { + err = errors.Wrapf(err, "eos: error creating temporary ref file") + return err + } + + // set xattr on ref + attr := &eosclient.Attribute{ + Type: eosclient.UserAttr, + Key: "reva.ref", + Val: targetURI.String(), + } + + if err := fs.c.SetAttr(ctx, u.Username, attr, tmp); err != nil { + err = errors.Wrapf(err, "eos: error setting reva.ref attr on file: %q", tmp) + return err + } + + // rename to have the file visible in user space. + if err := fs.c.Rename(ctx, u.Username, tmp, fn); err != nil { + err = errors.Wrapf(err, "eos: error renaming from: %q to %q", tmp, fn) + return err + } + + return nil } func (fs *eosStorage) Delete(ctx context.Context, ref *provider.Reference) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } return fs.c.Remove(ctx, u.Username, fn) @@ -587,16 +734,16 @@ func (fs *eosStorage) Delete(ctx context.Context, ref *provider.Reference) error func (fs *eosStorage) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } oldPath, err := fs.resolve(ctx, u, oldRef) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } newPath, err := fs.resolve(ctx, u, newRef) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } return fs.c.Rename(ctx, u.Username, oldPath, newPath) @@ -605,12 +752,12 @@ func (fs *eosStorage) Move(ctx context.Context, oldRef, newRef *provider.Referen func (fs *eosStorage) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { u, err := getUser(ctx) if err != nil { - return nil, errors.Wrap(err, "storage_eos: no user in ctx") + return nil, errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } return fs.c.Read(ctx, u.Username, fn) @@ -619,12 +766,12 @@ func (fs *eosStorage) Download(ctx context.Context, ref *provider.Reference) (io func (fs *eosStorage) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } return fs.c.Write(ctx, u.Username, fn, r) @@ -633,17 +780,17 @@ func (fs *eosStorage) Upload(ctx context.Context, ref *provider.Reference, r io. func (fs *eosStorage) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { u, err := getUser(ctx) if err != nil { - return nil, errors.Wrap(err, "storage_eos: no user in ctx") + return nil, errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } eosRevisions, err := fs.c.ListVersions(ctx, u.Username, fn) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error listing versions") + return nil, errors.Wrap(err, "eos: error listing versions") } revisions := []*provider.FileVersion{} for _, eosRev := range eosRevisions { @@ -656,12 +803,12 @@ func (fs *eosStorage) ListRevisions(ctx context.Context, ref *provider.Reference func (fs *eosStorage) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { u, err := getUser(ctx) if err != nil { - return nil, errors.Wrap(err, "storage_eos: no user in ctx") + return nil, errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error resolving reference") + return nil, errors.Wrap(err, "eos: error resolving reference") } fn = fs.getInternalPath(ctx, fn) @@ -671,12 +818,12 @@ func (fs *eosStorage) DownloadRevision(ctx context.Context, ref *provider.Refere func (fs *eosStorage) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } fn, err := fs.resolve(ctx, u, ref) if err != nil { - return errors.Wrap(err, "storage_eos: error resolving reference") + return errors.Wrap(err, "eos: error resolving reference") } return fs.c.RollbackToVersion(ctx, u.Username, fn, revisionKey) @@ -693,7 +840,7 @@ func (fs *eosStorage) PurgeRecycleItem(ctx context.Context, key string) error { func (fs *eosStorage) EmptyRecycle(ctx context.Context) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } return fs.c.PurgeDeletedEntries(ctx, u.Username) } @@ -701,11 +848,11 @@ func (fs *eosStorage) EmptyRecycle(ctx context.Context) error { func (fs *eosStorage) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, error) { u, err := getUser(ctx) if err != nil { - return nil, errors.Wrap(err, "storage_eos: no user in ctx") + return nil, errors.Wrap(err, "eos: no user in ctx") } eosDeletedEntries, err := fs.c.ListDeletedEntries(ctx, u.Username) if err != nil { - return nil, errors.Wrap(err, "storage_eos: error listing deleted entries") + return nil, errors.Wrap(err, "eos: error listing deleted entries") } recycleEntries := []*provider.RecycleItem{} for _, entry := range eosDeletedEntries { @@ -725,7 +872,7 @@ func (fs *eosStorage) ListRecycle(ctx context.Context) ([]*provider.RecycleItem, func (fs *eosStorage) RestoreRecycleItem(ctx context.Context, key string) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "storage_eos: no user in ctx") + return errors.Wrap(err, "eos: no user in ctx") } return fs.c.RestoreDeletedEntry(ctx, u.Username, key) } diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index fd8378355a..cefcd786cd 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -41,6 +41,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/gomodule/redigo/redis" @@ -152,8 +153,8 @@ func init() { type config struct { DataDirectory string `mapstructure:"datadirectory"` Scan bool `mapstructure:"scan"` - Autocreate bool `mapstructure:"autocreate"` Redis string `mapstructure:"redis"` + Layout string `mapstructure:"layout"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -169,14 +170,13 @@ func (c *config) init(m map[string]interface{}) { if c.Redis == "" { c.Redis = ":6379" } + if c.Layout == "" { + c.Layout = "{{.Username}}" + } // default to scanning if not configured if _, ok := m["scan"]; !ok { c.Scan = true } - // default to autocreate if not configured - if _, ok := m["autocreate"]; !ok { - c.Autocreate = true - } } // New returns an implementation to of the storage.FS interface that talk to @@ -230,6 +230,15 @@ func (fs *ocFS) Shutdown(ctx context.Context) error { return fs.pool.Close() } +func getUser(ctx context.Context) (*userpb.User, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired(""), "owncloud: error getting user from ctx") + return nil, err + } + return u, nil +} + // scan files and add uuid to path mapping to kv store func (fs *ocFS) scanFiles(ctx context.Context, conn redis.Conn) { if fs.c.Scan { @@ -462,29 +471,6 @@ func readOrCreateID(ctx context.Context, np string, conn redis.Conn) string { return uid.String() } -func (fs *ocFS) autocreate(ctx context.Context, fsfn string) { - if fs.c.Autocreate { - parts := strings.SplitN(fsfn, "/files", 2) - switch len(parts) { - case 1: - return // error? there is no files in here ... - case 2: - if parts[1] == "" { - // nothing to do, fsfn is the home - } else { - // only create home - fsfn = path.Join(parts[0], "files") - } - err := os.MkdirAll(fsfn, 0700) - if err != nil { - appctx.GetLogger(ctx).Debug().Err(err). - Str("fsfn", fsfn). - Msg("could not autocreate dir") - } - } - } -} - func (fs *ocFS) getPath(ctx context.Context, id *provider.ResourceId) (string, error) { c := fs.pool.Get() defer c.Close() @@ -868,6 +854,54 @@ func (fs *ocFS) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, nil } +func (fs *ocFS) getHomeForUser(u *userpb.User) (string, error) { + userhome, err := helper.GetUserHomePath(u, fs.c.Layout) + if err != nil { + return "", err + } + + return path.Join("/", userhome) +} + +func (fs *ocFS) CreateHome(ctx context.Context) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "ocFS: no user in ctx") + } + + home, err := fs.getHomeForUser(u) + if err != nil { + return err + } + + homePaths := []string{ + path.Join(fs.c.DataDirectory, home, "files"), + path.Join(fs.c.DataDirectory, home, "files_trashbin"), + path.Join(fs.c.DataDirectory, home, "files_versions"), + } + + for _, v := range homePaths { + if err = os.MkdirAll(v, 0700); err != nil { + return errors.Wrap(err, "ocFS: error creating home path: "+v) + } + } + + return nil +} + +func (fs *ocFS) GetHome(ctx context.Context) (string, error) { + u, err := getUser(ctx) + if err != nil { + return "", errors.Wrap(err, "ocFS: no user in ctx") + } + + home, err := fs.getHomeForUser(u) + if err != nil { + return "", err + } + return home, nil +} + func (fs *ocFS) CreateDir(ctx context.Context, fn string) (err error) { np := fs.getInternalPath(ctx, fn) if err = os.Mkdir(np, 0700); err != nil { @@ -1164,8 +1198,6 @@ func (fs *ocFS) GetMD(ctx context.Context, ref *provider.Reference) (*provider.R return nil, errors.Wrap(err, "ocFS: error resolving reference") } - fs.autocreate(ctx, np) - md, err := os.Stat(np) if err != nil { if os.IsNotExist(err) { @@ -1186,8 +1218,6 @@ func (fs *ocFS) ListFolder(ctx context.Context, ref *provider.Reference) ([]*pro return nil, errors.Wrap(err, "ocFS: error resolving reference") } - fs.autocreate(ctx, np) - mds, err := ioutil.ReadDir(np) if err != nil { if os.IsNotExist(err) { @@ -1305,8 +1335,6 @@ func (fs *ocFS) ListRevisions(ctx context.Context, ref *provider.Reference) ([]* } vp := fs.getVersionsPath(ctx, np) - fs.autocreate(ctx, vp) - bn := path.Base(np) revisions := []*provider.FileVersion{} diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go new file mode 100644 index 0000000000..5a895bf8eb --- /dev/null +++ b/pkg/storage/helper/helper.go @@ -0,0 +1,68 @@ +// Copyright 2018-2019 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package helper + +import ( + "bytes" + "fmt" + "strings" + "text/template" + + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/errors" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" +) + +type layoutTemplate struct { + Username string + FirstLetter string + Provider string +} + +func GetUserHomePath(u *userpb.User, layout string) (string, error) { + if u.Username == "" { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") + } + + usernameSplit := strings.Split(u.Username, "@") + if len(usernameSplit) == 1 { + usernameSplit = append(usernameSplit, "_Unknown") + } + if usernameSplit[1] == "" { + usernameSplit[1] = "_Unknown" + } + + pathTemplate := layoutTemplate{ + Username: u.Username, + FirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), + Provider: usernameSplit[1], + } + tmpl, err := template.New("userhomepath").Parse(layout) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error())) + } + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, pathTemplate) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error())) + } + + return buf.String(), nil +} diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go index bbc7ea1559..0bf2045e92 100644 --- a/pkg/storage/pw/context/context.go +++ b/pkg/storage/pw/context/context.go @@ -19,15 +19,14 @@ package context import ( - "bytes" "context" "fmt" "path" "strings" - "text/template" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/storage/pw/registry" "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" @@ -67,13 +66,9 @@ type pw struct { layout string } -type layoutTemplate struct { - Username string - FirstLetter string - Provider string -} +// Only works when a user is in context +func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { -func (pw *pw) getUserHomePath(ctx context.Context) (string, error) { u, ok := user.ContextGetUser(ctx) if !ok { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") @@ -81,46 +76,24 @@ func (pw *pw) getUserHomePath(ctx context.Context) (string, error) { if u.Username == "" { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") } - - usernameSplit := strings.Split(u.Username, "@") - if len(usernameSplit) == 1 { - usernameSplit = append(usernameSplit, "_Unknown") - } - if usernameSplit[1] == "" { - usernameSplit[1] = "_Unknown" - } - - pathTemplate := layoutTemplate{ - Username: u.Username, - FirstLetter: string([]rune(usernameSplit[0])[1]), - Provider: usernameSplit[1], - } - tmpl, err := template.New("userhomepath").Parse(pw.layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error())) - } - buf := new(bytes.Buffer) - err = tmpl.Execute(buf, pathTemplate) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error())) - } - - return buf.String(), nil -} - -// Only works when a user is in context -func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - userHomePath, err := pw.getUserHomePath(ctx) + userhome, err := helper.GetUserHomePath(u, pw.layout) if err != nil { - return "", err + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) } - return path.Join("/", pw.prefix, userHomePath, rp), nil + return path.Join("/", pw.prefix, userhome, rp), nil } func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { - userHomePath, err := pw.getUserHomePath(ctx) + u, ok := user.ContextGetUser(ctx) + if !ok { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + } + if u.Username == "" { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") + } + userhome, err := helper.GetUserHomePath(u, pw.layout) if err != nil { - return "", err + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) } - return strings.TrimPrefix(rp, path.Join("/", userHomePath)), nil + return strings.TrimPrefix(rp, path.Join("/", userhome)), nil } From 53a686a6a8b075fe8161f8457aede7ce48f9db61 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 04:55:14 +0000 Subject: [PATCH 4/8] added layout to storage-home.toml --- examples/separate/storage-home.toml | 53 +++++++---------------------- 1 file changed, 12 insertions(+), 41 deletions(-) diff --git a/examples/separate/storage-home.toml b/examples/separate/storage-home.toml index 8a26517d48..c434c1f594 100644 --- a/examples/separate/storage-home.toml +++ b/examples/separate/storage-home.toml @@ -1,4 +1,8 @@ # This storage-home.toml config file will start a reva service that: +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + # - authenticates grpc storage provider requests using the internal jwt token # - authenticates http upload and download requests requests using basic auth # - serves the home storage provider on grpc port 12000 @@ -8,66 +12,33 @@ # The home storage will inject the username into the path and jail users into # their home directory -[core] -max_cpus = "2" - -[log] -level = "debug" - [grpc] -network = "tcp" address = "0.0.0.0:12000" -enabled_services = ["storageprovider"] -enabled_interceptors = ["auth"] # This is a storage proider that grants direct acces to the wrapped storage +# TODO same storage id as the /oc/ storage provider +# if we have an id, we can directly go to that storage, no need to wrap paths +# we have a locally running dataprovider +# this is where clients can find it +# the context path wrapper reads tho username from the context and prefixes the relative storage path with it [grpc.services.storageprovider] driver = "owncloud" -# the context path wrapper reads tho username from the context and prefixes the relative storage path with it -path_wrapper = "context" mount_path = "/home" -# TODO same storage id as the /oc/ storage provider -# if we have an id, we can directly go to that storage, no need to wrap paths mount_id = "123e4567-e89b-12d3-a456-426655440000" -# we have a locally running dataprovider expose_data_server = true -# this is where clients can find it +path_wrapper = "context" data_server_url = "http://localhost:12001/data" - -[grpc.services.storageprovider.available_checksums] -md5 = 100 -unset = 1000 +enable_home_creation = true [grpc.services.storageprovider.drivers.owncloud] datadirectory = "/var/tmp/reva/data" - -[grpc.services.storageprovider.path_wrappers.context] -prefix = "" - -[grpc.interceptors.auth] -token_manager = "jwt" - -[grpc.interceptors.auth.token_managers.jwt] -secret = "Pive-Fumkiu4" +layout = "{{.FirstLetter}}/{{.UsernameLower}}" [http] address = "0.0.0.0:12001" -enabled_services = ["dataprovider"] -enabled_middlewares = ["auth"] - -[http.middlewares.auth] -gatewaysvc = "localhost:19000" -credential_chain = ["basic", "bearer"] -token_strategy = "header" -token_writer = "header" -token_manager = "jwt" - -[http.middlewares.auth.token_managers.jwt] -secret = "Pive-Fumkiu4" [http.services.dataprovider] driver = "owncloud" -prefix = "data" temp_folder = "/var/tmp/" [http.services.dataprovider.drivers.owncloud] From 2d7d7c5801fcc629be5e7414b4f2ff4a0c44489d Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 05:01:12 +0000 Subject: [PATCH 5/8] add UsernameLower --- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/helper/helper.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index cefcd786cd..b1d8f7ef3a 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -860,7 +860,7 @@ func (fs *ocFS) getHomeForUser(u *userpb.User) (string, error) { return "", err } - return path.Join("/", userhome) + return path.Join("/", userhome), nil } func (fs *ocFS) CreateHome(ctx context.Context) error { diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go index 5a895bf8eb..defcc0b4b0 100644 --- a/pkg/storage/helper/helper.go +++ b/pkg/storage/helper/helper.go @@ -31,11 +31,13 @@ import ( ) type layoutTemplate struct { - Username string - FirstLetter string - Provider string + Username string //the username + UsernameLower string //the username in lowercase + FirstLetter string //first letter of username in lowercase + Provider string //Provider/domain of user in lowercase } +// Converts username into user's home path according to layout func GetUserHomePath(u *userpb.User, layout string) (string, error) { if u.Username == "" { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") @@ -43,16 +45,17 @@ func GetUserHomePath(u *userpb.User, layout string) (string, error) { usernameSplit := strings.Split(u.Username, "@") if len(usernameSplit) == 1 { - usernameSplit = append(usernameSplit, "_Unknown") + usernameSplit = append(usernameSplit, "_unknown") } if usernameSplit[1] == "" { - usernameSplit[1] = "_Unknown" + usernameSplit[1] = "_unknown" } pathTemplate := layoutTemplate{ - Username: u.Username, - FirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), - Provider: usernameSplit[1], + Username: u.Username, + UsernameLower: strings.ToLower(u.Username), + FirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), + Provider: strings.ToLower(usernameSplit[1]), } tmpl, err := template.New("userhomepath").Parse(layout) if err != nil { From 9a6209f8d6a8911e1d03827c16324dd2e55f6487 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 16 Jan 2020 05:03:59 +0000 Subject: [PATCH 6/8] better comment for exported function --- pkg/storage/helper/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go index defcc0b4b0..0a7c7cf269 100644 --- a/pkg/storage/helper/helper.go +++ b/pkg/storage/helper/helper.go @@ -37,7 +37,7 @@ type layoutTemplate struct { Provider string //Provider/domain of user in lowercase } -// Converts username into user's home path according to layout +// GetUserHomePath converts username into user's home path according to layout func GetUserHomePath(u *userpb.User, layout string) (string, error) { if u.Username == "" { return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") From 70960ead0dcf06cd50ee60134008d0c69c8a8eb4 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Tue, 21 Jan 2020 03:10:36 +0000 Subject: [PATCH 7/8] add UsernamePrefixCount --- examples/separate/storage-home.toml | 2 +- go.sum | 1 + pkg/storage/helper/helper.go | 34 ++++++++++++++++++++++------- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/examples/separate/storage-home.toml b/examples/separate/storage-home.toml index c434c1f594..5c0428979d 100644 --- a/examples/separate/storage-home.toml +++ b/examples/separate/storage-home.toml @@ -32,7 +32,7 @@ enable_home_creation = true [grpc.services.storageprovider.drivers.owncloud] datadirectory = "/var/tmp/reva/data" -layout = "{{.FirstLetter}}/{{.UsernameLower}}" +layout = "{{.UsernamePrefixCount.1}}/{{.UsernameLower}}" [http] address = "0.0.0.0:12001" diff --git a/go.sum b/go.sum index c3af132227..5d924b4206 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go index 0a7c7cf269..8fabe13e6b 100644 --- a/pkg/storage/helper/helper.go +++ b/pkg/storage/helper/helper.go @@ -21,6 +21,8 @@ package helper import ( "bytes" "fmt" + "regexp" + "strconv" "strings" "text/template" @@ -31,10 +33,11 @@ import ( ) type layoutTemplate struct { - Username string //the username - UsernameLower string //the username in lowercase - FirstLetter string //first letter of username in lowercase - Provider string //Provider/domain of user in lowercase + Username string //the username + UsernameLower string //the username in lowercase + UsernamePrefixCount string //first letters of username in lowercase eg: {{.UsernamePrefixCount.3}} will take the first 3 chars and make them lowercase, defaults to 1 + UsernameFirstLetter string //first letter of username in lowercase, equivalent as {{.UsernamePrefixCount.1}} but easy to read + Provider string //Provider/domain of user in lowercase } // GetUserHomePath converts username into user's home path according to layout @@ -51,11 +54,26 @@ func GetUserHomePath(u *userpb.User, layout string) (string, error) { usernameSplit[1] = "_unknown" } + // handle {{.UsernamePrefixCount.x}} + // where x is an int, pull it out and remove it from the go template + letters := 1 + reg := regexp.MustCompile(`\{\{\.UsernamePrefixCount\.[0-9]+\}\}`) + rmatches := reg.FindAllString(layout, -1) + if rmatches != nil { + reg := regexp.MustCompile("[^0-9]+") + f, _ := strconv.ParseInt(reg.ReplaceAllString(rmatches[0], ""), 10, 64) + if f > 1 { + letters = int(f) + } + layout = strings.Replace(layout, "{{.UsernamePrefixCount."+strconv.Itoa(letters)+"}}", "{{.UsernamePrefixCount}}", -1) + } + pathTemplate := layoutTemplate{ - Username: u.Username, - UsernameLower: strings.ToLower(u.Username), - FirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), - Provider: strings.ToLower(usernameSplit[1]), + Username: u.Username, + UsernameLower: strings.ToLower(u.Username), + UsernamePrefixCount: strings.ToLower(string([]rune(usernameSplit[0])[0:letters])), + UsernameFirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), + Provider: strings.ToLower(usernameSplit[1]), } tmpl, err := template.New("userhomepath").Parse(layout) if err != nil { From 80a9519216804baa9a134a9e9d7d72dc728ed0ff Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Thu, 23 Jan 2020 02:38:40 +0000 Subject: [PATCH 8/8] remove pw as it is not needed anymore --- cmd/revad/runtime/loader.go | 1 - examples/separate/storage-home.toml | 1 - .../storageprovider/storageprovider.go | 33 ------- pkg/storage/pw/context/context.go | 99 ------------------- pkg/storage/pw/loader/loader.go | 25 ----- pkg/storage/pw/registry/registry.go | 34 ------- 6 files changed, 193 deletions(-) delete mode 100644 pkg/storage/pw/context/context.go delete mode 100644 pkg/storage/pw/loader/loader.go delete mode 100644 pkg/storage/pw/registry/registry.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index dfaed8b845..856c0ab84d 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -32,7 +32,6 @@ import ( _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/share/manager/loader" _ "github.com/cs3org/reva/pkg/storage/fs/loader" - _ "github.com/cs3org/reva/pkg/storage/pw/loader" _ "github.com/cs3org/reva/pkg/storage/registry/loader" _ "github.com/cs3org/reva/pkg/token/manager/loader" _ "github.com/cs3org/reva/pkg/user/manager/loader" diff --git a/examples/separate/storage-home.toml b/examples/separate/storage-home.toml index 5c0428979d..f32485b28a 100644 --- a/examples/separate/storage-home.toml +++ b/examples/separate/storage-home.toml @@ -26,7 +26,6 @@ driver = "owncloud" mount_path = "/home" mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true -path_wrapper = "context" data_server_url = "http://localhost:12001/data" enable_home_creation = true diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 16bd78b7b5..d50e6a5177 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - pwregistry "github.com/cs3org/reva/pkg/storage/pw/registry" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -62,7 +61,6 @@ type config struct { type service struct { conf *config storage storage.FS - pathWrapper storage.PathWrapper mountPath, mountID string tmpFolder string dataServerURL *url.URL @@ -134,10 +132,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { if err != nil { return nil, err } - pw, err := getPW(c) - if err != nil { - return nil, err - } // parse data server url u, err := url.Parse(c.DataServerURL) @@ -158,7 +152,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, storage: fs, - pathWrapper: pw, tmpFolder: tmpFolder, mountPath: mountPath, mountID: mountID, @@ -789,16 +782,6 @@ func getFS(c *config) (storage.FS, error) { return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getPW(c *config) (storage.PathWrapper, error) { - if c.PathWrapper == "" { - return nil, nil - } - if f, ok := pwregistry.NewFuncs[c.PathWrapper]; ok { - return f(c.PathWrappers[c.PathWrapper]) - } - return nil, fmt.Errorf("path wrapper not found: %s", c.Driver) -} - func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) { if ref.GetId() != nil { idRef := &provider.Reference{ @@ -824,13 +807,6 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provide return nil, err } - if s.pathWrapper != nil { - fsfn, err = s.pathWrapper.Unwrap(ctx, fsfn) - if err != nil { - return nil, err - } - } - pathRef := &provider.Reference{ Spec: &provider.Reference_Path{ Path: fsfn, @@ -849,15 +825,6 @@ func (s *service) trimMountPrefix(fn string) (string, error) { func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo) error { ri.Id.StorageId = s.mountID - - if s.pathWrapper != nil { - var err error - ri.Path, err = s.pathWrapper.Wrap(ctx, ri.Path) - if err != nil { - return err - } - } - ri.Path = path.Join(s.mountPath, ri.Path) return nil } diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go deleted file mode 100644 index 0bf2045e92..0000000000 --- a/pkg/storage/pw/context/context.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package context - -import ( - "context" - "fmt" - "path" - "strings" - - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/helper" - "github.com/cs3org/reva/pkg/storage/pw/registry" - "github.com/cs3org/reva/pkg/user" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -func init() { - registry.Register("context", New) -} - -type config struct { - Prefix string `mapstructure:"prefix"` - Layout string `mapstructure:"layout"` -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{Layout: "{{.Username}}"} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -// New returns an implementation to of the storage.PathWrapper interface that -// is used to wrap and unwrap storage paths -func New(m map[string]interface{}) (storage.PathWrapper, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - return &pw{prefix: c.Prefix, layout: c.Layout}, nil -} - -type pw struct { - prefix string - layout string -} - -// Only works when a user is in context -func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userhome, err := helper.GetUserHomePath(u, pw.layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) - } - return path.Join("/", pw.prefix, userhome, rp), nil -} - -func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - userhome, err := helper.GetUserHomePath(u, pw.layout) - if err != nil { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template error: %s", err.Error())) - } - return strings.TrimPrefix(rp, path.Join("/", userhome)), nil -} diff --git a/pkg/storage/pw/loader/loader.go b/pkg/storage/pw/loader/loader.go deleted file mode 100644 index 91a2d6b458..0000000000 --- a/pkg/storage/pw/loader/loader.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package loader - -import ( - // Load core storage path wrapper backends. - _ "github.com/cs3org/reva/pkg/storage/pw/context" - // Add your own here -) diff --git a/pkg/storage/pw/registry/registry.go b/pkg/storage/pw/registry/registry.go deleted file mode 100644 index ef181ada2d..0000000000 --- a/pkg/storage/pw/registry/registry.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package registry - -import "github.com/cs3org/reva/pkg/storage" - -// NewFunc is the function that storage implementations -// should register at init time. -type NewFunc func(map[string]interface{}) (storage.PathWrapper, error) - -// NewFuncs is a map containing all the registered storage backends. -var NewFuncs = map[string]NewFunc{} - -// Register registers a new storage backend new function. -// Not safe for concurrent use. Safe for use from package init. -func Register(name string, f NewFunc) { - NewFuncs[name] = f -}