spaces: WIP CreateStorageSoace (#2041)

changelog/unreleased/spaces-creation.md

@@ -0,0 +1,5 @@
+Enhancement: Create operations for Spaces
+
+DecomposedFS is aware now of the concept of Spaces, and supports for creating them.
+
+https://github.com/cs3org/reva/pull/2041

internal/grpc/services/gateway/storageprovider.go

@@ -101,7 +101,7 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) (
 func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
 	log := appctx.GetLogger(ctx)
 	// TODO: needs to be fixed
-	c, err := s.findByPath(ctx, req.Type)
+	c, err := s.findByPath(ctx, "/users")
 	if err != nil {
 		return &provider.CreateStorageSpaceResponse{
 			Status: status.NewStatusFromErrType(ctx, "error finding path", err),

internal/grpc/services/storageprovider/storageprovider.go

@@ -431,10 +431,9 @@ func (s *service) CreateHome(ctx context.Context, req *provider.CreateHomeReques
 	return res, nil
 }
 
+// CreateStorageSpace creates a storage space
 func (s *service) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
-	return &provider.CreateStorageSpaceResponse{
-		Status: status.NewUnimplemented(ctx, errtypes.NotSupported("CreateStorageSpace not implemented"), "CreateStorageSpace not implemented"),
-	}, nil
+	return s.storage.CreateStorageSpace(ctx, req)
 }
 
 func hasNodeID(s *provider.StorageSpace) bool {

pkg/storage/fs/nextcloud/nextcloud.go

@@ -76,6 +76,11 @@ func New(m map[string]interface{}) (storage.FS, error) {
 	return NewStorageDriver(conf)
 }
 
+// CreateStorageSpace creates a storage space
+func (nc *StorageDriver) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
+	return nil, fmt.Errorf("unimplemented: CreateStorageSpace")
+}
+
 // NewStorageDriver returns a new NextcloudStorageDriver
 func NewStorageDriver(c *StorageDriverConfig) (*StorageDriver, error) {
 	var client *http.Client

pkg/storage/fs/owncloud/owncloud.go

@@ -705,6 +705,11 @@ func getResourceType(isDir bool) provider.ResourceType {
 	return provider.ResourceType_RESOURCE_TYPE_FILE
 }
 
+// CreateStorageSpace creates a storage space
+func (fs *ocfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
+	return nil, fmt.Errorf("unimplemented: CreateStorageSpace")
+}
+
 func readOrCreateID(ctx context.Context, ip string, conn redis.Conn) string {
 	log := appctx.GetLogger(ctx)
 

pkg/storage/fs/owncloudsql/owncloudsql.go

@@ -489,6 +489,11 @@ func (fs *owncloudsqlfs) getUserStorage(user string) (int, error) {
 	return id, err
 }
 
+// CreateStorageSpace creates a storage space
+func (fs *owncloudsqlfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
+	return nil, fmt.Errorf("unimplemented: CreateStorageSpace")
+}
+
 func (fs *owncloudsqlfs) convertToResourceInfo(ctx context.Context, entry *filecache.File, ip string, mdKeys []string) (*provider.ResourceInfo, error) {
 	mdKeysMap := make(map[string]struct{})
 	for _, k := range mdKeys {

pkg/storage/fs/s3/s3.go

@@ -383,6 +383,11 @@ func (fs *s3FS) Delete(ctx context.Context, ref *provider.Reference) error {
 	return nil
 }
 
+// CreateStorageSpace creates a storage space
+func (fs *s3FS) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) {
+	return nil, fmt.Errorf("unimplemented: CreateStorageSpace")
+}
+
 func (fs *s3FS) moveObject(ctx context.Context, oldKey string, newKey string) error {
 
 	// Copy

pkg/storage/storage.go

@@ -58,6 +58,7 @@ type FS interface {
 	SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error
 	UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error
 	ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error)
+	CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error)
 }
 
 // Registry is the interface that storage registries implement

pkg/storage/utils/decomposedfs/decomposedfs.go

@@ -24,7 +24,6 @@ package decomposedfs
 import (
 	"context"
 	"io"
-	"math"
 	"net/url"
 	"os"
 	"path"
@@ -33,9 +32,7 @@ import (
 	"strings"
 	"syscall"
 
-	userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
-	types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
 	"github.com/cs3org/reva/pkg/appctx"
 	ctxpkg "github.com/cs3org/reva/pkg/ctx"
 	"github.com/cs3org/reva/pkg/errtypes"
@@ -226,27 +223,6 @@ func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) {
 	return
 }
 
-func (fs *Decomposedfs) createStorageSpace(ctx context.Context, spaceType, nodeID string) error {
-
-	// create space type dir
-	if err := os.MkdirAll(filepath.Join(fs.o.Root, "spaces", spaceType), 0700); err != nil {
-		return err
-	}
-
-	// we can reuse the node id as the space id
-	err := os.Symlink("../../nodes/"+nodeID, filepath.Join(fs.o.Root, "spaces", spaceType, nodeID))
-	if err != nil {
-		if isAlreadyExists(err) {
-			appctx.GetLogger(ctx).Debug().Err(err).Str("node", nodeID).Str("spacetype", spaceType).Msg("symlink already exists")
-		} else {
-			// TODO how should we handle error cases here?
-			appctx.GetLogger(ctx).Error().Err(err).Str("node", nodeID).Str("spacetype", spaceType).Msg("could not create symlink")
-		}
-	}
-
-	return nil
-}
-
 // The os not exists error is buried inside the xattr error,
 // so we cannot just use os.IsNotExists().
 func isAlreadyExists(err error) bool {
@@ -522,158 +498,6 @@ func (fs *Decomposedfs) Download(ctx context.Context, ref *provider.Reference) (
 	return reader, nil
 }
 
-// ListStorageSpaces returns a list of StorageSpaces.
-// The list can be filtered by space type or space id.
-// Spaces are persisted with symlinks in /spaces/<type>/<spaceid> pointing to ../../nodes/<nodeid>, the root node of the space
-// The spaceid is a concatenation of storageid + "!" + nodeid
-func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) {
-	// TODO check filters
-
-	// TODO when a space symlink is broken delete the space for cleanup
-	// read permissions are deduced from the node?
-
-	// TODO for absolute references this actually requires us to move all user homes into a subfolder of /nodes/root,
-	// e.g. /nodes/root/<space type> otherwise storage space names might collide even though they are of different types
-	// /nodes/root/personal/foo and /nodes/root/shares/foo might be two very different spaces, a /nodes/root/foo is not expressive enough
-	// we would not need /nodes/root if access always happened via spaceid+relative path
-
-	var (
-		spaceType = "*"
-		spaceID   = "*"
-	)
-
-	for i := range filter {
-		switch filter[i].Type {
-		case provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE:
-			spaceType = filter[i].GetSpaceType()
-		case provider.ListStorageSpacesRequest_Filter_TYPE_ID:
-			parts := strings.SplitN(filter[i].GetId().OpaqueId, "!", 2)
-			if len(parts) == 2 {
-				spaceID = parts[1]
-			}
-		}
-	}
-
-	// build the glob path, eg.
-	// /path/to/root/spaces/personal/nodeid
-	// /path/to/root/spaces/shared/nodeid
-	matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceType, spaceID))
-	if err != nil {
-		return nil, err
-	}
-
-	spaces := make([]*provider.StorageSpace, 0, len(matches))
-
-	u, ok := ctxpkg.ContextGetUser(ctx)
-	if !ok {
-		appctx.GetLogger(ctx).Debug().Msg("expected user in context")
-		return spaces, nil
-	}
-
-	for i := range matches {
-		// always read link in case storage space id != node id
-		if target, err := os.Readlink(matches[i]); err != nil {
-			appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[i]).Msg("could not read link, skipping")
-			continue
-		} else {
-			n, err := node.ReadNode(ctx, fs.lu, filepath.Base(target))
-			if err != nil {
-				appctx.GetLogger(ctx).Error().Err(err).Str("id", filepath.Base(target)).Msg("could not read node, skipping")
-				continue
-			}
-			owner, err := n.Owner()
-			if err != nil {
-				appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not read owner, skipping")
-				continue
-			}
-
-			// TODO apply more filters
-
-			space := &provider.StorageSpace{
-				// FIXME the driver should know its id move setting the spaceid from the storage provider to the drivers
-				//Id: &provider.StorageSpaceId{OpaqueId: "1284d238-aa92-42ce-bdc4-0b0000009157!" + n.ID},
-				Root: &provider.ResourceId{
-					// FIXME the driver should know its id move setting the spaceid from the storage provider to the drivers
-					//StorageId: "1284d238-aa92-42ce-bdc4-0b0000009157",
-					OpaqueId: n.ID,
-				},
-				Name:      n.Name,
-				SpaceType: filepath.Base(filepath.Dir(matches[i])),
-				// Mtime is set either as node.tmtime or as fi.mtime below
-			}
-
-			if space.SpaceType == "share" {
-				if utils.UserEqual(u.Id, owner) {
-					// do not list shares as spaces for the owner
-					continue
-				}
-			} else {
-				space.Name = "root" // do not expose the id as name, this is the root of a space
-				// TODO read from extended attribute for project / group spaces
-			}
-
-			// filter out spaces user cannot access (currently based on stat permission)
-			p, err := n.ReadUserPermissions(ctx, u)
-			if err != nil {
-				appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not read permissions, skipping")
-				continue
-			}
-			if !p.Stat {
-				continue
-			}
-
-			// fill in user object if the current user is the owner
-			if utils.UserEqual(u.Id, owner) {
-				space.Owner = u
-			} else {
-				space.Owner = &userv1beta1.User{ // FIXME only return a UserID, not a full blown user object
-					Id: owner,
-				}
-			}
-
-			// we set the space mtime to the root item mtime
-			// override the stat mtime with a tmtime if it is present
-			if tmt, err := n.GetTMTime(); err == nil {
-				un := tmt.UnixNano()
-				space.Mtime = &types.Timestamp{
-					Seconds: uint64(un / 1000000000),
-					Nanos:   uint32(un % 1000000000),
-				}
-			} else if fi, err := os.Stat(matches[i]); err == nil {
-				// fall back to stat mtime
-				un := fi.ModTime().UnixNano()
-				space.Mtime = &types.Timestamp{
-					Seconds: uint64(un / 1000000000),
-					Nanos:   uint32(un % 1000000000),
-				}
-			}
-
-			// quota
-			v, err := xattr.Get(matches[i], xattrs.QuotaAttr)
-			if err == nil {
-				// make sure we have a proper signed int
-				// we use the same magic numbers to indicate:
-				// -1 = uncalculated
-				// -2 = unknown
-				// -3 = unlimited
-				if quota, err := strconv.ParseUint(string(v), 10, 64); err == nil {
-					space.Quota = &provider.Quota{
-						QuotaMaxBytes: quota,
-						QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited?
-					}
-				} else {
-					appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", matches[i]).Msg("could not read quota")
-				}
-			}
-
-			spaces = append(spaces, space)
-		}
-	}
-
-	return spaces, nil
-
-}
-
 func (fs *Decomposedfs) copyMD(s string, t string) (err error) {
 	var attrs []string
 	if attrs, err = xattr.List(s); err != nil {

pkg/storage/utils/decomposedfs/grants.go

@@ -32,6 +32,9 @@ import (
 	"github.com/pkg/xattr"
 )
 
+// SpaceGrant is the key used to signal not to create a new space when a grant is assigned to a storage space.
+var SpaceGrant struct{}
+
 // DenyGrant denies access to a resource.
 func (fs *Decomposedfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error {
 	return errtypes.NotSupported("decomposedfs: not supported")
@@ -68,8 +71,12 @@ func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g
 		return err
 	}
 
-	if err := fs.createStorageSpace(ctx, "share", node.ID); err != nil {
-		return err
+	// when a grant is added to a space, do not add a new space under "shares"
+	if spaceGrant := ctx.Value(SpaceGrant); spaceGrant == nil {
+		err := fs.createStorageSpace(ctx, "share", node.ID)
+		if err != nil {
+			return err
+		}
 	}
 
 	return fs.tp.Propagate(ctx, node)

pkg/storage/utils/decomposedfs/node/node.go

@@ -120,6 +120,16 @@ func (n *Node) ChangeOwner(new *userpb.UserId) (err error) {
 	return
 }
 
+// SetMetadata populates a given key with its value.
+// Note that consumers should be aware of the metadata options on xattrs.go.
+func (n *Node) SetMetadata(key string, val string) (err error) {
+	nodePath := n.InternalPath()
+	if err := xattr.Set(nodePath, key, []byte(val)); err != nil {
+		return errors.Wrap(err, "Decomposedfs: could not set parentid attribute")
+	}
+	return nil
+}
+
 // WriteMetadata writes the Node metadata to disk
 func (n *Node) WriteMetadata(owner *userpb.UserId) (err error) {
 	nodePath := n.InternalPath()