Skip to content

Commit

Permalink
Use storage layout template for home directory definition (#476)
Browse files Browse the repository at this point in the history
  • Loading branch information
madsi1m authored Feb 6, 2020
1 parent e6c8246 commit 9ac9004
Show file tree
Hide file tree
Showing 11 changed files with 141 additions and 201 deletions.
1 change: 0 additions & 1 deletion cmd/revad/runtime/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
_ "github.com/cs3org/reva/pkg/publicshare/manager/loader"
_ "github.com/cs3org/reva/pkg/share/manager/loader"
_ "github.com/cs3org/reva/pkg/storage/fs/loader"
_ "github.com/cs3org/reva/pkg/storage/pw/loader"
_ "github.com/cs3org/reva/pkg/storage/registry/loader"
_ "github.com/cs3org/reva/pkg/token/manager/loader"
_ "github.com/cs3org/reva/pkg/user/manager/loader"
Expand Down
2 changes: 1 addition & 1 deletion examples/separate/storage-home.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ driver = "owncloud"
mount_path = "/home"
mount_id = "123e4567-e89b-12d3-a456-426655440000"
expose_data_server = true
path_wrapper = "context"
data_server_url = "http://localhost:12001/data"
enable_home_creation = true

[grpc.services.storageprovider.drivers.owncloud]
datadirectory = "/var/tmp/reva/data"
layout = "{{.UsernamePrefixCount.1}}/{{.UsernameLower}}"

[http]
address = "0.0.0.0:12001"
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/fatih/color v1.7.0 // indirect
github.com/go-openapi/strfmt v0.19.2 // indirect
github.com/gofrs/uuid v3.2.0+incompatible
github.com/gogo/protobuf v1.2.0 // indirect
github.com/golang/protobuf v1.3.2
github.com/gomodule/redigo v2.0.0+incompatible
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
Expand All @@ -29,13 +30,15 @@ require (
github.com/rs/zerolog v1.17.2
go.opencensus.io v0.22.1
golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
google.golang.org/grpc v1.26.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
gopkg.in/ldap.v2 v2.5.1
gopkg.in/square/go-jose.v2 v2.2.2 // indirect
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
)

go 1.13
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
Expand Down
33 changes: 0 additions & 33 deletions internal/grpc/services/storageprovider/storageprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (
"github.com/cs3org/reva/pkg/rgrpc/status"
"github.com/cs3org/reva/pkg/storage"
"github.com/cs3org/reva/pkg/storage/fs/registry"
pwregistry "github.com/cs3org/reva/pkg/storage/pw/registry"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"go.opencensus.io/trace"
Expand Down Expand Up @@ -62,7 +61,6 @@ type config struct {
type service struct {
conf *config
storage storage.FS
pathWrapper storage.PathWrapper
mountPath, mountID string
tmpFolder string
dataServerURL *url.URL
Expand Down Expand Up @@ -134,10 +132,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
if err != nil {
return nil, err
}
pw, err := getPW(c)
if err != nil {
return nil, err
}

// parse data server url
u, err := url.Parse(c.DataServerURL)
Expand All @@ -158,7 +152,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
service := &service{
conf: c,
storage: fs,
pathWrapper: pw,
tmpFolder: tmpFolder,
mountPath: mountPath,
mountID: mountID,
Expand Down Expand Up @@ -789,16 +782,6 @@ func getFS(c *config) (storage.FS, error) {
return nil, fmt.Errorf("driver not found: %s", c.Driver)
}

func getPW(c *config) (storage.PathWrapper, error) {
if c.PathWrapper == "" {
return nil, nil
}
if f, ok := pwregistry.NewFuncs[c.PathWrapper]; ok {
return f(c.PathWrappers[c.PathWrapper])
}
return nil, fmt.Errorf("path wrapper not found: %s", c.Driver)
}

func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) {
if ref.GetId() != nil {
idRef := &provider.Reference{
Expand All @@ -824,13 +807,6 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provide
return nil, err
}

if s.pathWrapper != nil {
fsfn, err = s.pathWrapper.Unwrap(ctx, fsfn)
if err != nil {
return nil, err
}
}

pathRef := &provider.Reference{
Spec: &provider.Reference_Path{
Path: fsfn,
Expand All @@ -849,15 +825,6 @@ func (s *service) trimMountPrefix(fn string) (string, error) {

func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo) error {
ri.Id.StorageId = s.mountID

if s.pathWrapper != nil {
var err error
ri.Path, err = s.pathWrapper.Wrap(ctx, ri.Path)
if err != nil {
return err
}
}

ri.Path = path.Join(s.mountPath, ri.Path)
return nil
}
32 changes: 25 additions & 7 deletions pkg/storage/fs/eos/eos.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"strings"

"github.com/cs3org/reva/pkg/storage/fs/registry"
"github.com/cs3org/reva/pkg/storage/helper"

"github.com/cs3org/reva/pkg/appctx"
"github.com/cs3org/reva/pkg/eosclient"
Expand Down Expand Up @@ -119,6 +120,9 @@ type config struct {

// SingleUsername is the username to use when SingleUserMode is enabled
SingleUsername string `mapstructure:"single_username"`

// Layout
Layout string `mapstructure:"layout"`
}

func getUser(ctx context.Context) (*userpb.User, error) {
Expand Down Expand Up @@ -155,6 +159,10 @@ func (c *config) init() {
if c.CacheDirectory == "" {
c.CacheDirectory = os.TempDir()
}

if c.Layout == "" {
c.Layout = "{{.Username}}"
}
}

// New returns a new implementation of the storage.FS interface that connects to EOS.
Expand Down Expand Up @@ -197,11 +205,14 @@ func New(m map[string]interface{}) (storage.FS, error) {
return eosStorage, nil
}

func (fs *eosStorage) getHomeForUser(u *userpb.User) string {
// TODO(labkode): define home path layout in configuration
// like home: %letter%/%username% and then do string substitution.
home := path.Join(fs.mountpoint, u.Username)
return home
func (fs *eosStorage) getHomeForUser(u *userpb.User) (string, error) {
userhome, err := helper.GetUserHomePath(u, fs.conf.Layout)
if err != nil {
return "", err
}

home := path.Join(fs.mountpoint, userhome)
return home, nil
}

func (fs *eosStorage) Shutdown(ctx context.Context) error {
Expand Down Expand Up @@ -571,7 +582,11 @@ func (fs *eosStorage) GetHome(ctx context.Context) (string, error) {
return "", errors.Wrap(err, "eos: no user in ctx")
}

home := fs.getHomeForUser(u)
home, err := fs.getHomeForUser(u)
if err != nil {
return "", err
}

return home, nil
}

Expand All @@ -581,7 +596,10 @@ func (fs *eosStorage) CreateHome(ctx context.Context) error {
return errors.Wrap(err, "eos: no user in ctx")
}

home := fs.getHomeForUser(u)
home, err := fs.getHomeForUser(u)
if err != nil {
return err
}

_, err = fs.c.GetFileInfoByPath(ctx, "root", home)
if err == nil { // home already exists
Expand Down
28 changes: 22 additions & 6 deletions pkg/storage/fs/owncloud/owncloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
"github.com/cs3org/reva/pkg/mime"
"github.com/cs3org/reva/pkg/storage"
"github.com/cs3org/reva/pkg/storage/fs/registry"
"github.com/cs3org/reva/pkg/storage/helper"
"github.com/cs3org/reva/pkg/user"
"github.com/gofrs/uuid"
"github.com/gomodule/redigo/redis"
Expand Down Expand Up @@ -153,6 +154,7 @@ type config struct {
DataDirectory string `mapstructure:"datadirectory"`
Scan bool `mapstructure:"scan"`
Redis string `mapstructure:"redis"`
Layout string `mapstructure:"layout"`
}

func parseConfig(m map[string]interface{}) (*config, error) {
Expand All @@ -168,6 +170,9 @@ func (c *config) init(m map[string]interface{}) {
if c.Redis == "" {
c.Redis = ":6379"
}
if c.Layout == "" {
c.Layout = "{{.Username}}"
}
// default to scanning if not configured
if _, ok := m["scan"]; !ok {
c.Scan = true
Expand Down Expand Up @@ -849,17 +854,25 @@ func (fs *ocFS) GetQuota(ctx context.Context) (int, int, error) {
return 0, 0, nil
}

func (fs *ocFS) getHomeForUser(u *userpb.User) string {
return path.Join("/", u.Username)
func (fs *ocFS) getHomeForUser(u *userpb.User) (string, error) {
userhome, err := helper.GetUserHomePath(u, fs.c.Layout)
if err != nil {
return "", err
}

return path.Join("/", userhome), nil
}

func (fs *ocFS) CreateHome(ctx context.Context) error {
u, err := getUser(ctx)
if err != nil {
return errors.Wrap(err, "eos: no user in ctx")
return errors.Wrap(err, "ocFS: no user in ctx")
}

home := fs.getHomeForUser(u)
home, err := fs.getHomeForUser(u)
if err != nil {
return err
}

homePaths := []string{
path.Join(fs.c.DataDirectory, home, "files"),
Expand All @@ -879,10 +892,13 @@ func (fs *ocFS) CreateHome(ctx context.Context) error {
func (fs *ocFS) GetHome(ctx context.Context) (string, error) {
u, err := getUser(ctx)
if err != nil {
return "", errors.Wrap(err, "eos: no user in ctx")
return "", errors.Wrap(err, "ocFS: no user in ctx")
}

home := fs.getHomeForUser(u)
home, err := fs.getHomeForUser(u)
if err != nil {
return "", err
}
return home, nil
}

Expand Down
89 changes: 89 additions & 0 deletions pkg/storage/helper/helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// Copyright 2018-2019 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

package helper

import (
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
"text/template"

"github.com/cs3org/reva/pkg/errtypes"
"github.com/pkg/errors"

userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
)

type layoutTemplate struct {
Username string //the username
UsernameLower string //the username in lowercase
UsernamePrefixCount string //first letters of username in lowercase eg: {{.UsernamePrefixCount.3}} will take the first 3 chars and make them lowercase, defaults to 1
UsernameFirstLetter string //first letter of username in lowercase, equivalent as {{.UsernamePrefixCount.1}} but easy to read
Provider string //Provider/domain of user in lowercase
}

// GetUserHomePath converts username into user's home path according to layout
func GetUserHomePath(u *userpb.User, layout string) (string, error) {
if u.Username == "" {
return "", errors.Wrap(errtypes.UserRequired("userrequired"), "user has no username")
}

usernameSplit := strings.Split(u.Username, "@")
if len(usernameSplit) == 1 {
usernameSplit = append(usernameSplit, "_unknown")
}
if usernameSplit[1] == "" {
usernameSplit[1] = "_unknown"
}

// handle {{.UsernamePrefixCount.x}}
// where x is an int, pull it out and remove it from the go template
letters := 1
reg := regexp.MustCompile(`\{\{\.UsernamePrefixCount\.[0-9]+\}\}`)
rmatches := reg.FindAllString(layout, -1)
if rmatches != nil {
reg := regexp.MustCompile("[^0-9]+")
f, _ := strconv.ParseInt(reg.ReplaceAllString(rmatches[0], ""), 10, 64)
if f > 1 {
letters = int(f)
}
layout = strings.Replace(layout, "{{.UsernamePrefixCount."+strconv.Itoa(letters)+"}}", "{{.UsernamePrefixCount}}", -1)
}

pathTemplate := layoutTemplate{
Username: u.Username,
UsernameLower: strings.ToLower(u.Username),
UsernamePrefixCount: strings.ToLower(string([]rune(usernameSplit[0])[0:letters])),
UsernameFirstLetter: strings.ToLower(string([]rune(usernameSplit[0])[0])),
Provider: strings.ToLower(usernameSplit[1]),
}
tmpl, err := template.New("userhomepath").Parse(layout)
if err != nil {
return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template parse error: %s", err.Error()))
}
buf := new(bytes.Buffer)
err = tmpl.Execute(buf, pathTemplate)
if err != nil {
return "", errors.Wrap(errtypes.UserRequired("userrequired"), fmt.Sprintf("template execute error: %s", err.Error()))
}

return buf.String(), nil
}
Loading

0 comments on commit 9ac9004

Please sign in to comment.