From 9ac900448b079be67ffeeaae3fbde1bc740f5a58 Mon Sep 17 00:00:00 2001 From: Michael D'Silva Date: Fri, 7 Feb 2020 02:41:51 +1100 Subject: [PATCH] Use storage layout template for home directory definition (#476) --- cmd/revad/runtime/loader.go | 1 - examples/separate/storage-home.toml | 2 +- go.mod | 3 + go.sum | 1 + .../storageprovider/storageprovider.go | 33 ------- pkg/storage/fs/eos/eos.go | 32 +++++-- pkg/storage/fs/owncloud/owncloud.go | 28 ++++-- pkg/storage/helper/helper.go | 89 ++++++++++++++++++ pkg/storage/pw/context/context.go | 94 ------------------- pkg/storage/pw/loader/loader.go | 25 ----- pkg/storage/pw/registry/registry.go | 34 ------- 11 files changed, 141 insertions(+), 201 deletions(-) create mode 100644 pkg/storage/helper/helper.go delete mode 100644 pkg/storage/pw/context/context.go delete mode 100644 pkg/storage/pw/loader/loader.go delete mode 100644 pkg/storage/pw/registry/registry.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index dfaed8b845..856c0ab84d 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -32,7 +32,6 @@ import ( _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/share/manager/loader" _ "github.com/cs3org/reva/pkg/storage/fs/loader" - _ "github.com/cs3org/reva/pkg/storage/pw/loader" _ "github.com/cs3org/reva/pkg/storage/registry/loader" _ "github.com/cs3org/reva/pkg/token/manager/loader" _ "github.com/cs3org/reva/pkg/user/manager/loader" diff --git a/examples/separate/storage-home.toml b/examples/separate/storage-home.toml index 9c93ad1888..f32485b28a 100644 --- a/examples/separate/storage-home.toml +++ b/examples/separate/storage-home.toml @@ -26,12 +26,12 @@ driver = "owncloud" mount_path = "/home" mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true -path_wrapper = "context" data_server_url = "http://localhost:12001/data" enable_home_creation = true [grpc.services.storageprovider.drivers.owncloud] datadirectory = "/var/tmp/reva/data" +layout = "{{.UsernamePrefixCount.1}}/{{.UsernameLower}}" [http] address = "0.0.0.0:12001" diff --git a/go.mod b/go.mod index ddc7d3cd5a..1172528e9e 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/fatih/color v1.7.0 // indirect github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.2.0+incompatible + github.com/gogo/protobuf v1.2.0 // indirect github.com/golang/protobuf v1.3.2 github.com/gomodule/redigo v2.0.0+incompatible github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 @@ -29,6 +30,7 @@ require ( github.com/rs/zerolog v1.17.2 go.opencensus.io v0.22.1 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a + golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 google.golang.org/grpc v1.26.0 @@ -36,6 +38,7 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect gopkg.in/ldap.v2 v2.5.1 gopkg.in/square/go-jose.v2 v2.2.2 // indirect + honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect ) go 1.13 diff --git a/go.sum b/go.sum index c3af132227..5d924b4206 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 16bd78b7b5..d50e6a5177 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -34,7 +34,6 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - pwregistry "github.com/cs3org/reva/pkg/storage/pw/registry" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -62,7 +61,6 @@ type config struct { type service struct { conf *config storage storage.FS - pathWrapper storage.PathWrapper mountPath, mountID string tmpFolder string dataServerURL *url.URL @@ -134,10 +132,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { if err != nil { return nil, err } - pw, err := getPW(c) - if err != nil { - return nil, err - } // parse data server url u, err := url.Parse(c.DataServerURL) @@ -158,7 +152,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, storage: fs, - pathWrapper: pw, tmpFolder: tmpFolder, mountPath: mountPath, mountID: mountID, @@ -789,16 +782,6 @@ func getFS(c *config) (storage.FS, error) { return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getPW(c *config) (storage.PathWrapper, error) { - if c.PathWrapper == "" { - return nil, nil - } - if f, ok := pwregistry.NewFuncs[c.PathWrapper]; ok { - return f(c.PathWrappers[c.PathWrapper]) - } - return nil, fmt.Errorf("path wrapper not found: %s", c.Driver) -} - func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) { if ref.GetId() != nil { idRef := &provider.Reference{ @@ -824,13 +807,6 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provide return nil, err } - if s.pathWrapper != nil { - fsfn, err = s.pathWrapper.Unwrap(ctx, fsfn) - if err != nil { - return nil, err - } - } - pathRef := &provider.Reference{ Spec: &provider.Reference_Path{ Path: fsfn, @@ -849,15 +825,6 @@ func (s *service) trimMountPrefix(fn string) (string, error) { func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo) error { ri.Id.StorageId = s.mountID - - if s.pathWrapper != nil { - var err error - ri.Path, err = s.pathWrapper.Wrap(ctx, ri.Path) - if err != nil { - return err - } - } - ri.Path = path.Join(s.mountPath, ri.Path) return nil } diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 2c66444240..341a7f44ad 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -32,6 +32,7 @@ import ( "strings" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/eosclient" @@ -119,6 +120,9 @@ type config struct { // SingleUsername is the username to use when SingleUserMode is enabled SingleUsername string `mapstructure:"single_username"` + + // Layout + Layout string `mapstructure:"layout"` } func getUser(ctx context.Context) (*userpb.User, error) { @@ -155,6 +159,10 @@ func (c *config) init() { if c.CacheDirectory == "" { c.CacheDirectory = os.TempDir() } + + if c.Layout == "" { + c.Layout = "{{.Username}}" + } } // New returns a new implementation of the storage.FS interface that connects to EOS. @@ -197,11 +205,14 @@ func New(m map[string]interface{}) (storage.FS, error) { return eosStorage, nil } -func (fs *eosStorage) getHomeForUser(u *userpb.User) string { - // TODO(labkode): define home path layout in configuration - // like home: %letter%/%username% and then do string substitution. - home := path.Join(fs.mountpoint, u.Username) - return home +func (fs *eosStorage) getHomeForUser(u *userpb.User) (string, error) { + userhome, err := helper.GetUserHomePath(u, fs.conf.Layout) + if err != nil { + return "", err + } + + home := path.Join(fs.mountpoint, userhome) + return home, nil } func (fs *eosStorage) Shutdown(ctx context.Context) error { @@ -571,7 +582,11 @@ func (fs *eosStorage) GetHome(ctx context.Context) (string, error) { return "", errors.Wrap(err, "eos: no user in ctx") } - home := fs.getHomeForUser(u) + home, err := fs.getHomeForUser(u) + if err != nil { + return "", err + } + return home, nil } @@ -581,7 +596,10 @@ func (fs *eosStorage) CreateHome(ctx context.Context) error { return errors.Wrap(err, "eos: no user in ctx") } - home := fs.getHomeForUser(u) + home, err := fs.getHomeForUser(u) + if err != nil { + return err + } _, err = fs.c.GetFileInfoByPath(ctx, "root", home) if err == nil { // home already exists diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 5eeec1bcf7..b1d8f7ef3a 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -41,6 +41,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/helper" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" "github.com/gomodule/redigo/redis" @@ -153,6 +154,7 @@ type config struct { DataDirectory string `mapstructure:"datadirectory"` Scan bool `mapstructure:"scan"` Redis string `mapstructure:"redis"` + Layout string `mapstructure:"layout"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -168,6 +170,9 @@ func (c *config) init(m map[string]interface{}) { if c.Redis == "" { c.Redis = ":6379" } + if c.Layout == "" { + c.Layout = "{{.Username}}" + } // default to scanning if not configured if _, ok := m["scan"]; !ok { c.Scan = true @@ -849,17 +854,25 @@ func (fs *ocFS) GetQuota(ctx context.Context) (int, int, error) { return 0, 0, nil } -func (fs *ocFS) getHomeForUser(u *userpb.User) string { - return path.Join("/", u.Username) +func (fs *ocFS) getHomeForUser(u *userpb.User) (string, error) { + userhome, err := helper.GetUserHomePath(u, fs.c.Layout) + if err != nil { + return "", err + } + + return path.Join("/", userhome), nil } func (fs *ocFS) CreateHome(ctx context.Context) error { u, err := getUser(ctx) if err != nil { - return errors.Wrap(err, "eos: no user in ctx") + return errors.Wrap(err, "ocFS: no user in ctx") } - home := fs.getHomeForUser(u) + home, err := fs.getHomeForUser(u) + if err != nil { + return err + } homePaths := []string{ path.Join(fs.c.DataDirectory, home, "files"), @@ -879,10 +892,13 @@ func (fs *ocFS) CreateHome(ctx context.Context) error { func (fs *ocFS) GetHome(ctx context.Context) (string, error) { u, err := getUser(ctx) if err != nil { - return "", errors.Wrap(err, "eos: no user in ctx") + return "", errors.Wrap(err, "ocFS: no user in ctx") } - home := fs.getHomeForUser(u) + home, err := fs.getHomeForUser(u) + if err != nil { + return "", err + } return home, nil } diff --git a/pkg/storage/helper/helper.go b/pkg/storage/helper/helper.go new file mode 100644 index 0000000000..8fabe13e6b --- /dev/null +++ b/pkg/storage/helper/helper.go @@ -0,0 +1,89 @@ +// Copyright 2018-2019 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package helper + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + "strings" + "text/template" + + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/errors" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" +) + +type layoutTemplate struct { + Username string //the username + UsernameLower string //the username in lowercase + UsernamePrefixCount string //first letters of username in lowercase eg: {{.UsernamePrefixCount.3}} will take the first 3 chars and make them lowercase, defaults to 1 + UsernameFirstLetter string //first letter of username in lowercase, equivalent as {{.UsernamePrefixCount.1}} but easy to read + Provider string //Provider/domain of user in lowercase +} + +// GetUserHomePath converts username into user's home path according to layout +func GetUserHomePath(u *userpb.User, layout string) (string, error) { + if u.Username == "" { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") + } + + usernameSplit := strings.Split(u.Username, "@") + if len(usernameSplit) == 1 { + usernameSplit = append(usernameSplit, "_unknown") + } + if usernameSplit[1] == "" { + usernameSplit[1] = "_unknown" + } + + // handle {{.UsernamePrefixCount.x}} + // where x is an int, pull it out and remove it from the go template + letters := 1 + reg := regexp.MustCompile(`\{\{\.UsernamePrefixCount\.[0-9]+\}\}`) + rmatches := reg.FindAllString(layout, -1) + if rmatches != nil { + reg := regexp.MustCompile("[^0-9]+") + f, _ := strconv.ParseInt(reg.ReplaceAllString(rmatches[0], ""), 10, 64) + if f > 1 { + letters = int(f) + } + layout = strings.Replace(layout, "{{.UsernamePrefixCount."+strconv.Itoa(letters)+"}}", "{{.UsernamePrefixCount}}", -1) + } + + pathTemplate := layoutTemplate{ + Username: u.Username, + UsernameLower: strings.ToLower(u.Username), + UsernamePrefixCount: strings.ToLower(string([]rune(usernameSplit[0])[0:letters])), + UsernameFirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])), + Provider: strings.ToLower(usernameSplit[1]), + } + tmpl, err := template.New("userhomepath").Parse(layout) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error())) + } + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, pathTemplate) + if err != nil { + return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error())) + } + + return buf.String(), nil +} diff --git a/pkg/storage/pw/context/context.go b/pkg/storage/pw/context/context.go deleted file mode 100644 index 02a09978f4..0000000000 --- a/pkg/storage/pw/context/context.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package context - -import ( - "context" - "path" - "strings" - - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage" - "github.com/cs3org/reva/pkg/storage/pw/registry" - "github.com/cs3org/reva/pkg/user" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -func init() { - registry.Register("context", New) -} - -type config struct { - Prefix string `mapstructure:"prefix"` -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -// New returns an implementation to of the storage.PathWrapper interface that -// is used to wrap and unwrap storage paths -func New(m map[string]interface{}) (storage.PathWrapper, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - return &pw{prefix: c.Prefix}, nil -} - -type pw struct { - prefix string -} - -// Only works when a user is in context -func (pw *pw) Unwrap(ctx context.Context, rp string) (string, error) { - - // TODO how do we get the users home path? - // - construct based on homedir prefix + username/userid? - // - look into context? - // - query preferences? - // - do nothing - // -> screams for a wrapper/unwrapper 'strategy' - - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - - return path.Join("/", pw.prefix, u.Username, rp), nil -} -func (pw *pw) Wrap(ctx context.Context, rp string) (string, error) { - u, ok := user.ContextGetUser(ctx) - if !ok { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - } - if u.Username == "" { - return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username") - } - return strings.TrimPrefix(rp, path.Join("/", u.Username)), nil -} diff --git a/pkg/storage/pw/loader/loader.go b/pkg/storage/pw/loader/loader.go deleted file mode 100644 index 91a2d6b458..0000000000 --- a/pkg/storage/pw/loader/loader.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package loader - -import ( - // Load core storage path wrapper backends. - _ "github.com/cs3org/reva/pkg/storage/pw/context" - // Add your own here -) diff --git a/pkg/storage/pw/registry/registry.go b/pkg/storage/pw/registry/registry.go deleted file mode 100644 index ef181ada2d..0000000000 --- a/pkg/storage/pw/registry/registry.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2019 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package registry - -import "github.com/cs3org/reva/pkg/storage" - -// NewFunc is the function that storage implementations -// should register at init time. -type NewFunc func(map[string]interface{}) (storage.PathWrapper, error) - -// NewFuncs is a map containing all the registered storage backends. -var NewFuncs = map[string]NewFunc{} - -// Register registers a new storage backend new function. -// Not safe for concurrent use. Safe for use from package init. -func Register(name string, f NewFunc) { - NewFuncs[name] = f -}