From c4b4e50384da16eacb6665cc9e3adcce1fc7c4b8 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 14 Aug 2020 17:03:26 +0200 Subject: [PATCH 1/6] OCM: Add logic for resolving storage references over webdav --- cmd/reva/ocm-share-create.go | 17 +-- examples/ocm-partners/providers.demo.json | 12 +- examples/ocmd/ocmd-server-1.toml | 2 + examples/ocmd/ocmd-server-2.toml | 2 + examples/ocmd/providers.demo.json | 8 +- examples/storage-references/gateway.toml | 1 + go.mod | 3 +- go.sum | 4 + .../grpc/services/gateway/ocmshareprovider.go | 28 +++-- .../grpc/services/gateway/storageprovider.go | 57 ++++----- .../services/gateway/webdavstorageprovider.go | 115 ++++++++++++++++++ internal/grpc/services/ocmcore/ocmcore.go | 39 ++++-- .../ocmshareprovider/ocmshareprovider.go | 18 +-- internal/http/services/ocmd/shares.go | 42 ++++--- .../handlers/apps/sharing/shares/shares.go | 13 +- pkg/ocm/share/manager/json/json.go | 14 ++- pkg/ocm/share/manager/memory/memory.go | 14 ++- pkg/ocm/share/share.go | 3 +- pkg/user/manager/demo/demo.go | 26 +++- 19 files changed, 294 insertions(+), 124 deletions(-) create mode 100644 internal/grpc/services/gateway/webdavstorageprovider.go diff --git a/cmd/reva/ocm-share-create.go b/cmd/reva/ocm-share-create.go index 1ea3d594c5..186391feea 100644 --- a/cmd/reva/ocm-share-create.go +++ b/cmd/reva/ocm-share-create.go @@ -19,11 +19,9 @@ package main import ( - "encoding/json" "fmt" "io" "os" - "path" "strconv" "time" @@ -119,20 +117,17 @@ func ocmShareCreateCommand() *command { }, } - permissionMap := map[string]string{"name": strconv.Itoa(pint)} - val, err := json.Marshal(permissionMap) - if err != nil { - return err - } + fmt.Println("res.Info.Path" + res.Info.Path) + opaqueObj := &types.Opaque{ Map: map[string]*types.OpaqueEntry{ "permissions": &types.OpaqueEntry{ - Decoder: "json", - Value: val, + Decoder: "plain", + Value: []byte(strconv.Itoa(pint)), }, "name": &types.OpaqueEntry{ Decoder: "plain", - Value: []byte(path.Base(res.Info.Path)), + Value: []byte(res.Info.Path), }, }, } @@ -148,12 +143,12 @@ func ocmShareCreateCommand() *command { if err != nil { return err } - fmt.Println("create share done") if shareRes.Status.Code != rpc.Code_CODE_OK { return formatError(shareRes.Status) } + fmt.Println("create share done") t := table.NewWriter() t.SetOutputMirror(os.Stdout) t.AppendHeader(table.Row{"#", "Owner.Idp", "Owner.OpaqueId", "ResourceId", "Permissions", "Type", "Grantee.Idp", "Grantee.OpaqueId", "Created", "Updated"}) diff --git a/examples/ocm-partners/providers.demo.json b/examples/ocm-partners/providers.demo.json index 567302e942..f0ee47a881 100644 --- a/examples/ocm-partners/providers.demo.json +++ b/examples/ocm-partners/providers.demo.json @@ -27,7 +27,7 @@ "description": "CERNBox Webdav API" }, "name": "CERNBox - Webdav API", - "path": "https://sciencemesh.cernbox.cern.ch/iop/webdav/", + "path": "https://sciencemesh.cernbox.cern.ch/iop/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -63,7 +63,7 @@ "description": "CESNET Webdav API" }, "name": "CESNET - Webdav API", - "path": "https://sciencemesh.cesnet.cz/iop/webdav/", + "path": "https://sciencemesh.cesnet.cz/iop/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -99,7 +99,7 @@ "description": "WWU Webdav API" }, "name": "WWU - Webdav API", - "path": "https://sciencemesh-test.uni-muenster.de/api/webdav/", + "path": "https://sciencemesh-test.uni-muenster.de/api/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -135,7 +135,7 @@ "description": "Cubbit Webdav API" }, "name": "Cubbit - Webdav API", - "path": "https://sciencemesh.cubbit.io/webdav/", + "path": "https://sciencemesh.cubbit.io/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -171,7 +171,7 @@ "description": "Ailleron Webdav API" }, "name": "Ailleron - Webdav API", - "path": "https://sciencemesh.softwaremind.com/iop/webdav/", + "path": "https://sciencemesh.softwaremind.com/iop/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -207,7 +207,7 @@ "description": "Surfsara Webdav API" }, "name": "Surfsara - Webdav API", - "path": "https://app.cs3mesh-iop.k8s.surfsara.nl/iop/webdav/", + "path": "https://app.cs3mesh-iop.k8s.surfsara.nl/iop/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index 5de60fd4da..7fbef64534 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -131,4 +131,6 @@ providers = "providers.demo.json" [http.services.ocs] prefix = "ocs" +[http.services.ocdav] + [http.middlewares.cors] diff --git a/examples/ocmd/ocmd-server-2.toml b/examples/ocmd/ocmd-server-2.toml index 1b306b8152..7637d53b65 100644 --- a/examples/ocmd/ocmd-server-2.toml +++ b/examples/ocmd/ocmd-server-2.toml @@ -114,4 +114,6 @@ providers = "providers.demo.json" [http.services.ocs] prefix = "ocs" +[http.services.ocdav] + [http.middlewares.cors] diff --git a/examples/ocmd/providers.demo.json b/examples/ocmd/providers.demo.json index 910006174b..79e82355e1 100644 --- a/examples/ocmd/providers.demo.json +++ b/examples/ocmd/providers.demo.json @@ -27,7 +27,7 @@ "description": "CERNBox Webdav API" }, "name": "CERNBox - Webdav API", - "path": "http://127.0.0.1:19001/webdav/", + "path": "http://127.0.0.1:19001/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -63,7 +63,7 @@ "description": "CESNET Webdav API" }, "name": "CESNET - Webdav API", - "path": "http://127.0.0.1:17001/webdav/", + "path": "http://127.0.0.1:17001/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -99,7 +99,7 @@ "description": "Example Webdav API" }, "name": "Example - Webdav API", - "path": "http://127.0.0.1:19001/webdav/", + "path": "http://127.0.0.1:19001/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", @@ -135,7 +135,7 @@ "description": "Test Webdav API" }, "name": "Test - Webdav API", - "path": "http://127.0.0.1:19001/webdav/", + "path": "http://127.0.0.1:19001/remote.php/webdav/", "is_monitored": true }, "api_version": "0.0.1", diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 8a59ea2929..35d7b429cf 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -25,3 +25,4 @@ home_provider = "/home" [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] +[http.services.ocdav] diff --git a/go.mod b/go.mod index ac5f6aa7c8..80bbe7a7ad 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200626150132-28a40e643719 - github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd + github.com/cs3org/go-cs3apis v0.0.0-20200810113633-b00aca449666 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/go-ldap/ldap/v3 v3.2.3 @@ -40,6 +40,7 @@ require ( github.com/rs/cors v1.7.0 github.com/rs/zerolog v1.19.0 github.com/stretchr/testify v1.6.1 + github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1 github.com/tus/tusd v1.1.1-0.20200416115059-9deabf9d80c2 go.opencensus.io v0.22.4 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 diff --git a/go.sum b/go.sum index 5e21dddb50..3876f55374 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/cs3org/go-cs3apis v0.0.0-20200728114537-4efa23660dbe h1:CQ/Grq7oVFqwi github.com/cs3org/go-cs3apis v0.0.0-20200728114537-4efa23660dbe/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd h1:uMaudkC7znaiIKT9rxIhoRYzrhTg1Nc78X7XEqhmjSk= github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20200810113633-b00aca449666 h1:E7VsSSN/2YZLSwrDMJJdAWU11lP7W1qkcXbrslb0PM0= +github.com/cs3org/go-cs3apis v0.0.0-20200810113633-b00aca449666/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -696,6 +698,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1 h1:TPyHV/OgChqNcnYqCoCvIFjR9TU60gFXXBKnhOBzVEI= +github.com/studio-b12/gowebdav v0.0.0-20200303150724-9380631c29a1/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s= github.com/subosito/gotenv v1.1.1/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index 1a2024ea65..5cdb7e70d2 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -302,30 +302,36 @@ func (s *svc) createWebdavReference(ctx context.Context, share *ocm.Share) (*rpc } } + var token string + tokenOpaque, ok := share.Grantee.Opaque.Map["token"] + if !ok { + return status.NewNotFound(ctx, "token not found"), nil + } + switch tokenOpaque.Decoder { + case "plain": + token = string(tokenOpaque.Value) + default: + err := errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder) + return status.NewInternal(ctx, err, "invalid opaque entry decoder"), nil + } + homeRes, err := s.GetHome(ctx, &provider.GetHomeRequest{}) if err != nil { err := errors.Wrap(err, "gateway: error calling GetHome") return status.NewInternal(ctx, err, "error updating received share"), nil } - parts := strings.Split(share.Id.OpaqueId, ":") - if len(parts) < 2 { - err := errors.New("resource ID does not follow the layout id:name ") - return status.NewInternal(ctx, err, "error decoding resource ID"), nil - } - // reference path is the home path + some name on the corresponding - // mesh provider (webdav:/home/MyShares/x@webdav_endpoint) + // mesh provider (/home/MyShares/x) // It is the responsibility of the gateway to resolve these references and merge the response back // from the main request. - // TODO(labkode): the name of the share should be the filename it points to by default. - refPath := path.Join(homeRes.Path, s.c.ShareFolder, path.Base(parts[1])) + refPath := path.Join(homeRes.Path, s.c.ShareFolder, path.Base(share.Name)) log.Info().Msg("mount path will be:" + refPath) createRefReq := &provider.CreateReferenceRequest{ Path: refPath, - // webdav is the Scheme and /@webdav_endpoint are the Opaque parts of the URL. - TargetUri: fmt.Sprintf("webdav:%s/%s@%s", share.ResourceId.GetStorageId(), share.ResourceId.GetOpaqueId(), webdavEndpoint), + // webdav is the scheme, token@webdav_endpoint the opaque part and the share name the query of the URL. + TargetUri: fmt.Sprintf("webdav:%s@%s?name=%s", token, webdavEndpoint, share.Name), } c, err := s.findByPath(ctx, refPath) diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index d83b75ca05..10f1b4ac4e 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -891,11 +891,20 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St panic("gateway: a share name must be of type reference: ref:" + res.Info.Path) } - ri, err := s.checkRef(ctx, res.Info) + ri, err := s.handleWebdavRefStat(ctx, res.Info) if err != nil { - return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "gateway: error resolving reference:"+p), - }, nil + if _, ok := err.(errtypes.IsNotSupported); !ok { + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+p), + }, nil + } else { + ri, err = s.checkRef(ctx, res.Info) + if err != nil { + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving reference: "+p), + }, nil + } + } } // we need to make sure we don't expose the reference target in the resource @@ -980,35 +989,21 @@ func (s *svc) checkRef(ctx context.Context, ri *provider.ResourceInfo) (*provide } // reference types MUST have a target resource id. - target := ri.Target - if target == "" { + if ri.Target == "" { err := errors.New("gateway: ref target is an empty uri") return nil, err } - newResourceInfo, err := s.handleRef(ctx, ri) - if err != nil { - err := errors.Wrapf(err, "gateway: error handling ref target:%s", target) - return nil, err - } - return newResourceInfo, nil -} - -func (s *svc) handleRef(ctx context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, error) { uri, err := url.Parse(ri.Target) if err != nil { - return nil, errors.Wrapf(err, "gateway: error parsing target uri:%s", ri.Target) + return nil, errors.Wrapf(err, "gateway: error parsing target uri: %s", ri.Target) } - scheme := uri.Scheme - - switch scheme { + switch uri.Scheme { case "cs3": return s.handleCS3Ref(ctx, uri.Opaque) - case "webdav": - return s.handleWebdavRef(ctx, ri) default: - err := errors.New("gateway: no reference handler for scheme:" + scheme) + err := errors.New("gateway: no reference handler for scheme: " + uri.Scheme) return nil, err } } @@ -1021,16 +1016,12 @@ func (s *svc) handleCS3Ref(ctx context.Context, opaque string) (*provider.Resour return nil, err } - storageid := parts[0] - opaqueid := parts[1] - id := &provider.ResourceId{ - StorageId: storageid, - OpaqueId: opaqueid, - } - ref := &provider.Reference{ Spec: &provider.Reference_Id{ - Id: id, + Id: &provider.ResourceId{ + StorageId: parts[0], + OpaqueId: parts[1], + }, }, } @@ -1056,12 +1047,6 @@ func (s *svc) handleCS3Ref(ctx context.Context, opaque string) (*provider.Resour return res.Info, nil } -func (s *svc) handleWebdavRef(_ context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, error) { - // A webdav ref has the following layout: /@webdav_endpoint - // TODO: Once file transfer functionalities have been added. - return ri, nil -} - func (s *svc) ListContainerStream(_ *provider.ListContainerStreamRequest, _ gateway.GatewayAPI_ListContainerStreamServer) error { return errors.New("Unimplemented") } diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go new file mode 100644 index 0000000000..b5e9f9b7c0 --- /dev/null +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -0,0 +1,115 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package gateway + +import ( + "context" + "net/url" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/token" + "github.com/pkg/errors" + "github.com/studio-b12/gowebdav" +) + +type webdavEndpoint struct { + filePath string + endpoint string + token string +} + +func extractEndpointInfo(ri *provider.ResourceInfo) (*webdavEndpoint, error) { + if ri.Type != provider.ResourceType_RESOURCE_TYPE_REFERENCE { + panic("gateway: calling handleWebdavRefStat on a non reference type:" + ri.String()) + } + // reference types MUST have a target resource id. + if ri.Target == "" { + err := errors.New("gateway: ref target is an empty uri") + return nil, err + } + + uri, err := url.Parse(ri.Target) + if err != nil { + return nil, errors.Wrap(err, "gateway: error parsing target uri: "+ri.Target) + } + if uri.Scheme != "webdav" { + return nil, errtypes.NotSupported("ref target does not have the webdav scheme") + } + + parts := strings.SplitN(uri.Opaque, "@", 2) + if len(parts) < 2 { + err := errors.New("gateway: webdav ref does not follow the layout token@webdav_endpoint?name " + ri.Target) + return nil, err + } + m, err := url.ParseQuery(uri.RawQuery) + if err != nil { + return nil, errors.Wrap(err, "gateway: error parsing target resource name") + } + + return &webdavEndpoint{ + filePath: m["name"][0], + endpoint: parts[1], + token: parts[0], + }, nil +} + +func (s *svc) handleWebdavRefStat(ctx context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, error) { + log := appctx.GetLogger(ctx) + ep, err := extractEndpointInfo(ri) + log.Info().Msgf("ep: %+v", ep) + if err != nil { + return nil, err + } + c := gowebdav.NewClient(ep.endpoint, "", "") + c.SetHeader(token.TokenHeader, ep.token) + + // We need to call PROPFIND ourselves as we need ownloud-specific fields + // to read the resource ID and permissions. + info, err := c.Stat(ep.filePath) + fileInfo := info.(*gowebdav.File) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) + } + + md := &provider.ResourceInfo{ + // Add Id, PermissionSet, Owner + Path: fileInfo.Path(), + Type: getResourceType(fileInfo.IsDir()), + Etag: fileInfo.ETag(), + MimeType: fileInfo.ContentType(), + Size: uint64(fileInfo.Size()), + Mtime: &types.Timestamp{ + Seconds: uint64(fileInfo.ModTime().Unix()), + }, + } + log.Info().Msgf("md: %+v", md) + + return md, nil +} + +func getResourceType(isDir bool) provider.ResourceType { + if isDir { + return provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + return provider.ResourceType_RESOURCE_TYPE_FILE +} diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index 8300efabb8..bb8ee28baa 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -120,19 +120,42 @@ func (s *service) CreateOCMCoreShare(ctx context.Context, req *ocmcore.CreateOCM OpaqueId: parts[1], } - opaqueObj := req.Protocol.Opaque.Map["permissions"] - if opaqueObj.Decoder != "json" { - err := errors.New("opaque entry decoder is not json") + var resourcePermissions *provider.ResourcePermissions + permOpaque, ok := req.Protocol.Opaque.Map["permissions"] + if !ok { + return &ocmcore.CreateOCMCoreShareResponse{ + Status: status.NewInternal(ctx, errors.New("resource permissions not set"), ""), + }, nil + } + switch permOpaque.Decoder { + case "json": + err := json.Unmarshal(permOpaque.Value, &resourcePermissions) + if err != nil { + return &ocmcore.CreateOCMCoreShareResponse{ + Status: status.NewInternal(ctx, err, "error decoding resource permissions"), + }, nil + } + default: + err := errors.New("opaque entry decoder not recognized") return &ocmcore.CreateOCMCoreShareResponse{ Status: status.NewInternal(ctx, err, "invalid opaque entry decoder"), }, nil } - var resourcePermissions *provider.ResourcePermissions - err := json.Unmarshal(opaqueObj.Value, &resourcePermissions) - if err != nil { + var token string + tokenOpaque, ok := req.Protocol.Opaque.Map["token"] + if !ok { return &ocmcore.CreateOCMCoreShareResponse{ - Status: status.NewInternal(ctx, err, "error decoding resource permissions"), + Status: status.NewInternal(ctx, errors.New("token not set"), ""), + }, nil + } + switch tokenOpaque.Decoder { + case "plain": + token = string(tokenOpaque.Value) + default: + err := errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder) + return &ocmcore.CreateOCMCoreShareResponse{ + Status: status.NewInternal(ctx, err, "invalid opaque entry decoder"), }, nil } @@ -146,7 +169,7 @@ func (s *service) CreateOCMCoreShare(ctx context.Context, req *ocmcore.CreateOCM }, } - share, err := s.sm.Share(ctx, resource, grant, req.Name, nil, "", req.Owner) + share, err := s.sm.Share(ctx, resource, grant, req.Name, nil, "", req.Owner, token) if err != nil { return &ocmcore.CreateOCMCoreShareResponse{ Status: status.NewInternal(ctx, err, "error creating ocm core share"), diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index f2be71782d..89e02de334 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -20,7 +20,6 @@ package ocmshareprovider import ( "context" - "encoding/json" "fmt" ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" @@ -111,35 +110,30 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq }, nil } + var permissions string permOpaque, ok := req.Opaque.Map["permissions"] if !ok { return &ocm.CreateOCMShareResponse{ Status: status.NewInternal(ctx, errors.New("resource permissions not set"), ""), }, nil } - var permissions map[string]string switch permOpaque.Decoder { - case "json": - err := json.Unmarshal(permOpaque.Value, &permissions) - if err != nil { - return &ocm.CreateOCMShareResponse{ - Status: status.NewInternal(ctx, err, "error decoding resource permissions"), - }, nil - } + case "plain": + permissions = string(permOpaque.Value) default: - err := errors.New("opaque entry decoder not recognized") + err := errors.New("opaque entry decoder not recognized: " + permOpaque.Decoder) return &ocm.CreateOCMShareResponse{ Status: status.NewInternal(ctx, err, "invalid opaque entry decoder"), }, nil } + var name string nameOpaque, ok := req.Opaque.Map["name"] if !ok { return &ocm.CreateOCMShareResponse{ Status: status.NewInternal(ctx, errors.New("resource name not set"), ""), }, nil } - var name string switch nameOpaque.Decoder { case "plain": name = string(nameOpaque.Value) @@ -150,7 +144,7 @@ func (s *service) CreateOCMShare(ctx context.Context, req *ocm.CreateOCMShareReq }, nil } - share, err := s.sm.Share(ctx, req.ResourceId, req.Grant, name, req.RecipientMeshProvider, permissions["name"], nil) + share, err := s.sm.Share(ctx, req.ResourceId, req.Grant, name, req.RecipientMeshProvider, permissions, nil, "") if err != nil { return &ocm.CreateOCMShareResponse{ Status: status.NewInternal(ctx, err, "error creating share"), diff --git a/internal/http/services/ocmd/shares.go b/internal/http/services/ocmd/shares.go index 8131284cbe..108886a4d9 100644 --- a/internal/http/services/ocmd/shares.go +++ b/internal/http/services/ocmd/shares.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "net/http" - "strconv" "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -125,28 +124,29 @@ func (h *sharesHandler) createShare(w http.ResponseWriter, r *http.Request) { } var permissions conversions.Permissions - var role string - options, ok := protocolDecoded["options"].(map[string]string) + var role, token string + options, ok := protocolDecoded["options"].(map[string]interface{}) + if !ok { + WriteError(w, r, APIErrorInvalidParameter, "protocol: webdav token not provided", nil) + return + } + + token, ok = options["token"].(string) + if !ok { + WriteError(w, r, APIErrorInvalidParameter, "protocol: webdav token not provided", nil) + return + } + + pval, ok := options["permissions"].(int) if !ok { - // by default only allow read permissions / assign viewer role role = conversions.RoleViewer } else { - pval, ok := options["permissions"] - if !ok { - role = conversions.RoleViewer - } else { - pint, err := strconv.Atoi(pval) - if err != nil { - WriteError(w, r, APIErrorInvalidParameter, "permissions must be an integer", err) - return - } - permissions, err = conversions.NewPermissions(pint) - if err != nil { - WriteError(w, r, APIErrorInvalidParameter, err.Error(), nil) - return - } - role = conversions.Permissions2Role(permissions) + permissions, err = conversions.NewPermissions(pval) + if err != nil { + WriteError(w, r, APIErrorInvalidParameter, err.Error(), nil) + return } + role = conversions.Permissions2Role(permissions) } var resourcePermissions *provider.ResourcePermissions @@ -177,6 +177,10 @@ func (h *sharesHandler) createShare(w http.ResponseWriter, r *http.Request) { Decoder: "json", Value: val, }, + "token": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte(token), + }, }, }, }, diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index ecefdb6ed8..a530ab292b 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -506,13 +506,6 @@ func (h *Handler) createFederatedCloudShare(w http.ResponseWriter, r *http.Reque resourcePermissions = asCS3Permissions(permissions, nil) } - permissionMap := map[string]string{"name": strconv.Itoa(int(permissions))} - val, err := json.Marshal(permissionMap) - if err != nil { - response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "could not encode role", err) - return - } - statReq := &provider.StatRequest{ Ref: &provider.Reference{ Spec: &provider.Reference_Path{ @@ -538,12 +531,12 @@ func (h *Handler) createFederatedCloudShare(w http.ResponseWriter, r *http.Reque Opaque: &types.Opaque{ Map: map[string]*types.OpaqueEntry{ "permissions": &types.OpaqueEntry{ - Decoder: "json", - Value: val, + Decoder: "plain", + Value: []byte(strconv.Itoa(int(permissions))), }, "name": &types.OpaqueEntry{ Decoder: "plain", - Value: []byte(path.Base(statRes.Info.Path)), + Value: []byte(statRes.Info.Path), }, }, }, diff --git a/pkg/ocm/share/manager/json/json.go b/pkg/ocm/share/manager/json/json.go index 376dc7acb4..b1a5768fad 100644 --- a/pkg/ocm/share/manager/json/json.go +++ b/pkg/ocm/share/manager/json/json.go @@ -41,6 +41,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/manager/registry" "github.com/cs3org/reva/pkg/rhttp" + tokenpkg "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/user" "github.com/google/uuid" "github.com/mitchellh/mapstructure" @@ -189,7 +190,7 @@ func getOCMEndpoint(originProvider *ocmprovider.ProviderInfo) (string, error) { } func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGrant, name string, - pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId) (*ocm.Share, error) { + pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId, token string) (*ocm.Share, error) { id := genID() now := time.Now().UnixNano() @@ -216,7 +217,14 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr return nil, errors.New("json: owner of resource not provided") } userID = owner - id += ":" + name + g.Grantee.Opaque = &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "token": &typespb.OpaqueEntry{ + Decoder: "plain", + Value: []byte(token), + }, + }, + } } else { userID = user.ContextMustGetUser(ctx).GetId() } @@ -244,6 +252,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr Id: &ocm.ShareId{ OpaqueId: id, }, + Name: name, ResourceId: md, Permissions: g.Permissions, Grantee: g.Grantee, @@ -261,6 +270,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr "name": "webdav", "options": map[string]string{ "permissions": pm, + "token": tokenpkg.ContextMustGetToken(ctx), }, }, ) diff --git a/pkg/ocm/share/manager/memory/memory.go b/pkg/ocm/share/manager/memory/memory.go index 3a91e271a7..83137e07e6 100644 --- a/pkg/ocm/share/manager/memory/memory.go +++ b/pkg/ocm/share/manager/memory/memory.go @@ -38,6 +38,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/rhttp" + tokenpkg "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/user" "github.com/google/uuid" "github.com/mitchellh/mapstructure" @@ -99,7 +100,7 @@ func getOCMEndpoint(originProvider *ocmprovider.ProviderInfo) (string, error) { } func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGrant, name string, - pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId) (*ocm.Share, error) { + pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId, token string) (*ocm.Share, error) { id := genID() now := time.Now().UnixNano() @@ -124,7 +125,14 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr return nil, errors.New("json: owner of resource not provided") } userID = owner - id += ":" + name + g.Grantee.Opaque = &typespb.Opaque{ + Map: map[string]*typespb.OpaqueEntry{ + "token": &typespb.OpaqueEntry{ + Decoder: "plain", + Value: []byte(token), + }, + }, + } } else { userID = user.ContextMustGetUser(ctx).GetId() } @@ -153,6 +161,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr Id: &ocm.ShareId{ OpaqueId: id, }, + Name: name, ResourceId: md, Permissions: g.Permissions, Grantee: g.Grantee, @@ -171,6 +180,7 @@ func (m *mgr) Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGr "name": "webdav", "options": map[string]string{ "permissions": pm, + "token": tokenpkg.ContextMustGetToken(ctx), }, }, ) diff --git a/pkg/ocm/share/share.go b/pkg/ocm/share/share.go index d6f1bbd8dd..cce1eb7517 100644 --- a/pkg/ocm/share/share.go +++ b/pkg/ocm/share/share.go @@ -30,7 +30,8 @@ import ( // Manager is the interface that manipulates the OCM shares. type Manager interface { // Create a new share in fn with the given acl. - Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGrant, name string, pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId) (*ocm.Share, error) + Share(ctx context.Context, md *provider.ResourceId, g *ocm.ShareGrant, name string, + pi *ocmprovider.ProviderInfo, pm string, owner *userpb.UserId, token string) (*ocm.Share, error) // GetShare gets the information for a share by the given ref. GetShare(ctx context.Context, ref *ocm.ShareReference) (*ocm.Share, error) diff --git a/pkg/user/manager/demo/demo.go b/pkg/user/manager/demo/demo.go index e4286192ba..7888edb736 100644 --- a/pkg/user/manager/demo/demo.go +++ b/pkg/user/manager/demo/demo.go @@ -20,7 +20,8 @@ package demo import ( "context" - "errors" + "encoding/json" + "io/ioutil" "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -28,6 +29,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" + "github.com/pkg/errors" ) func init() { @@ -41,6 +43,16 @@ type manager struct { // New returns a new user manager. func New(m map[string]interface{}) (user.Manager, error) { cat := getUsers() + data, err := json.Marshal(cat) + if err != nil { + err = errors.Wrap(err, "error encoding to json") + return nil, err + } + + if err := ioutil.WriteFile("/var/tmp/reva/users.json", data, 0644); err != nil { + err = errors.Wrap(err, "error writing to file") + return nil, err + } return &manager{catalog: cat}, nil } @@ -172,6 +184,18 @@ func getUsers() map[string]*userpb.User { Groups: []string{"quantum-lovers", "philosophy-haters", "physics-lovers"}, Mail: "richard@example.org", DisplayName: "Richard Feynman", + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "uid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("468"), + }, + "gid": &types.OpaqueEntry{ + Decoder: "plain", + Value: []byte("135"), + }, + }, + }, }, } } From 7bb84c221544fa395109b90c858f2fbc23cd0b71 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 14 Aug 2020 17:16:43 +0200 Subject: [PATCH 2/6] Add changelog --- changelog/unreleased/webdav-storage.md | 7 +++++++ internal/grpc/services/gateway/storageprovider.go | 13 ++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/webdav-storage.md diff --git a/changelog/unreleased/webdav-storage.md b/changelog/unreleased/webdav-storage.md new file mode 100644 index 0000000000..256f2af310 --- /dev/null +++ b/changelog/unreleased/webdav-storage.md @@ -0,0 +1,7 @@ +Enhancement: Add logic for resolving storage references over webdav + +This PR adds the functionality to resolve webdav references using the ocs +webdav service by passing the resource's owner's token. This would need to be +changed in production. + +https://github.com/cs3org/reva/pull/1094 diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 10f1b4ac4e..c501dc83f1 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -897,13 +897,12 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St return &provider.StatResponse{ Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+p), }, nil - } else { - ri, err = s.checkRef(ctx, res.Info) - if err != nil { - return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "gateway: error resolving reference: "+p), - }, nil - } + } + ri, err = s.checkRef(ctx, res.Info) + if err != nil { + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving reference: "+p), + }, nil } } From a3f74f5689360314f06e2cf7983cec7e87d662e6 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 14 Aug 2020 17:20:28 +0200 Subject: [PATCH 3/6] Remove stray changes --- pkg/user/manager/demo/demo.go | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/pkg/user/manager/demo/demo.go b/pkg/user/manager/demo/demo.go index 7888edb736..e4286192ba 100644 --- a/pkg/user/manager/demo/demo.go +++ b/pkg/user/manager/demo/demo.go @@ -20,8 +20,7 @@ package demo import ( "context" - "encoding/json" - "io/ioutil" + "errors" "strings" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -29,7 +28,6 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" - "github.com/pkg/errors" ) func init() { @@ -43,16 +41,6 @@ type manager struct { // New returns a new user manager. func New(m map[string]interface{}) (user.Manager, error) { cat := getUsers() - data, err := json.Marshal(cat) - if err != nil { - err = errors.Wrap(err, "error encoding to json") - return nil, err - } - - if err := ioutil.WriteFile("/var/tmp/reva/users.json", data, 0644); err != nil { - err = errors.Wrap(err, "error writing to file") - return nil, err - } return &manager{catalog: cat}, nil } @@ -184,18 +172,6 @@ func getUsers() map[string]*userpb.User { Groups: []string{"quantum-lovers", "philosophy-haters", "physics-lovers"}, Mail: "richard@example.org", DisplayName: "Richard Feynman", - Opaque: &types.Opaque{ - Map: map[string]*types.OpaqueEntry{ - "uid": &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte("468"), - }, - "gid": &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte("135"), - }, - }, - }, }, } } From e25b402c1312ee9c26e50b3cbf9e73154b47b56c Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 18 Aug 2020 13:59:12 +0200 Subject: [PATCH 4/6] Add logic for webdav operations --- cmd/reva/download.go | 106 ++++++--- cmd/reva/upload.go | 65 +++++- examples/storage-references/gateway.toml | 3 + .../grpc/services/gateway/storageprovider.go | 180 +++++++++++++-- .../services/gateway/webdavstorageprovider.go | 217 ++++++++++++++---- internal/http/services/dataprovider/put.go | 2 +- internal/http/services/owncloud/ocdav/put.go | 4 +- pkg/storage/utils/localfs/localfs.go | 4 +- 8 files changed, 480 insertions(+), 101 deletions(-) diff --git a/cmd/reva/download.go b/cmd/reva/download.go index 8a85390389..bdcf67be98 100644 --- a/cmd/reva/download.go +++ b/cmd/reva/download.go @@ -28,10 +28,14 @@ import ( "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rhttp" + tokenpkg "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" + "github.com/studio-b12/gowebdav" ) func downloadCommand() *command { @@ -85,30 +89,38 @@ func downloadCommand() *command { // TODO(labkode): upload to data server fmt.Printf("Downloading from: %s\n", res.DownloadEndpoint) - dataServerURL := res.DownloadEndpoint - // TODO(labkode): do a protocol switch - httpReq, err := rhttp.NewRequest(ctx, "GET", dataServerURL, nil) + content, err := checkDownloadWebdavRef(res.DownloadEndpoint, res.Opaque) if err != nil { - return err - } - - httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token) - httpClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - // TODO make insecure configurable - rhttp.Insecure(true), - // TODO make timeout configurable - rhttp.Timeout(time.Duration(24*int64(time.Hour))), - ) - - httpRes, err := httpClient.Do(httpReq) - if err != nil { - return err - } - defer httpRes.Body.Close() - - if httpRes.StatusCode != http.StatusOK { - return err + if _, ok := err.(errtypes.IsNotSupported); !ok { + return err + } + + dataServerURL := res.DownloadEndpoint + // TODO(labkode): do a protocol switch + httpReq, err := rhttp.NewRequest(ctx, "GET", dataServerURL, nil) + if err != nil { + return err + } + + httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token) + httpClient := rhttp.GetHTTPClient( + rhttp.Context(ctx), + // TODO make insecure configurable + rhttp.Insecure(true), + // TODO make timeout configurable + rhttp.Timeout(time.Duration(24*int64(time.Hour))), + ) + + httpRes, err := httpClient.Do(httpReq) + if err != nil { + return err + } + defer httpRes.Body.Close() + + if httpRes.StatusCode != http.StatusOK { + return err + } + content = httpRes.Body } absPath, err := utils.ResolvePath(local) @@ -116,20 +128,58 @@ func downloadCommand() *command { return err } + bar := pb.New(int(info.Size)).SetUnits(pb.U_BYTES) + bar.Start() + reader := bar.NewProxyReader(content) + fd, err := os.OpenFile(absPath, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err } - - bar := pb.New(int(info.Size)).SetUnits(pb.U_BYTES) - bar.Start() - reader := bar.NewProxyReader(httpRes.Body) if _, err := io.Copy(fd, reader); err != nil { return err } bar.Finish() return nil - } return cmd } + +func checkDownloadWebdavRef(endpoint string, opaque *typespb.Opaque) (io.Reader, error) { + if opaque == nil { + return nil, errtypes.NotSupported("opaque object not defined") + } + + var token string + tokenOpaque, ok := opaque.Map["webdav-token"] + if !ok { + return nil, errtypes.NotSupported("webdav token not defined") + } + switch tokenOpaque.Decoder { + case "plain": + token = string(tokenOpaque.Value) + default: + return nil, errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder) + } + + var filePath string + fileOpaque, ok := opaque.Map["webdav-file-path"] + if !ok { + return nil, errtypes.NotSupported("webdav file path not defined") + } + switch fileOpaque.Decoder { + case "plain": + filePath = string(fileOpaque.Value) + default: + return nil, errors.New("opaque entry decoder not recognized: " + fileOpaque.Decoder) + } + + c := gowebdav.NewClient(endpoint, "", "") + c.SetHeader(tokenpkg.TokenHeader, token) + + reader, err := c.ReadStream(filePath) + if err != nil { + return nil, err + } + return reader, nil +} diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index 9f3f8367de..ddcdbe9ee0 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -35,10 +35,11 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - + "github.com/cs3org/reva/pkg/errtypes" tokenpkg "github.com/cs3org/reva/pkg/token" "github.com/eventials/go-tus" "github.com/eventials/go-tus/memorystore" + "github.com/studio-b12/gowebdav" // TODO(labkode): this should not come from this package. "github.com/cs3org/reva/internal/grpc/services/storageprovider" @@ -51,11 +52,11 @@ func uploadCommand() *command { cmd := newCommand("upload") cmd.Description = func() string { return "upload a local file to the remote server" } cmd.Usage = func() string { return "Usage: upload [-flags] " } - disabletusFlag := cmd.Bool("disable-tus", false, "whether to disable tus protocol") + disableTusFlag := cmd.Bool("disable-tus", false, "whether to disable tus protocol") xsFlag := cmd.String("xs", "negotiate", "compute checksum") cmd.ResetFlags = func() { - *disabletusFlag, *xsFlag = false, "negotiate" + *disableTusFlag, *xsFlag = false, "negotiate" } cmd.Action = func(w ...io.Writer) error { @@ -121,6 +122,14 @@ func uploadCommand() *command { fmt.Printf("Data server: %s\n", res.UploadEndpoint) fmt.Printf("Allowed checksums: %+v\n", res.AvailableChecksums) + if err = checkUploadWebdavRef(res.UploadEndpoint, res.Opaque, md, fd); err != nil { + if _, ok := err.(errtypes.IsNotSupported); !ok { + return err + } + } else { + return nil + } + xsType, err := guessXS(*xsFlag, res.AvailableChecksums) if err != nil { return err @@ -144,7 +153,7 @@ func uploadCommand() *command { bar.Start() reader := bar.NewProxyReader(fd) - if *disabletusFlag { + if *disableTusFlag { httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader) if err != nil { bar.Finish() @@ -253,6 +262,54 @@ func uploadCommand() *command { return cmd } +func checkUploadWebdavRef(endpoint string, opaque *typespb.Opaque, md os.FileInfo, fd *os.File) error { + if opaque == nil { + return errtypes.NotSupported("opaque object not defined") + } + + var token string + tokenOpaque, ok := opaque.Map["webdav-token"] + if !ok { + return errtypes.NotSupported("webdav token not defined") + } + switch tokenOpaque.Decoder { + case "plain": + token = string(tokenOpaque.Value) + default: + return errors.New("opaque entry decoder not recognized: " + tokenOpaque.Decoder) + } + + var filePath string + fileOpaque, ok := opaque.Map["webdav-file-path"] + if !ok { + return errtypes.NotSupported("webdav file path not defined") + } + switch fileOpaque.Decoder { + case "plain": + filePath = string(fileOpaque.Value) + default: + return errors.New("opaque entry decoder not recognized: " + fileOpaque.Decoder) + } + + bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) + bar.Start() + reader := bar.NewProxyReader(fd) + + c := gowebdav.NewClient(endpoint, "", "") + c.SetHeader(tokenpkg.TokenHeader, token) + c.SetHeader("Upload-Length", strconv.FormatInt(md.Size(), 10)) + + err := c.WriteStream(filePath, reader, 0700) + if err != nil { + bar.Finish() + return err + } + + bar.Finish() + fmt.Println("File uploaded") + return nil +} + func computeXS(t provider.ResourceChecksumType, r io.Reader) (string, error) { switch t { case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32: diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 35d7b429cf..5b53a7ce76 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -3,6 +3,9 @@ commit_share_to_storage_grant = true commit_share_to_storage_ref = true +[grpc.services.gateway.token_managers.jwt] +expires = 86400 + [grpc.services.storageregistry] [grpc.services.storageregistry.drivers.static] home_provider = "/home" diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index c501dc83f1..6cacf6ce10 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -179,7 +179,7 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &gateway.InitiateFileDownloadResponse{ @@ -187,6 +187,22 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi }, nil } + if protocol == "webdav" { + // TODO(ishank011): pass this through the datagateway service + // for now, we just expose the file server to the user + ep, opaque, err := s.webdavRefTransferEndpoint(ctx, statRes.Info.Target, shareChild) + if err != nil { + return &gateway.InitiateFileDownloadResponse{ + Status: status.NewInternal(ctx, err, "gateway: error creating container on webdav host: "+p), + }, nil + } + return &gateway.InitiateFileDownloadResponse{ + Opaque: opaque, + Status: status.NewOK(ctx), + DownloadEndpoint: ep, + }, nil + } + // append child to target target := path.Join(ri.Path, shareChild) ref = &provider.Reference{ @@ -313,7 +329,7 @@ func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFile }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &gateway.InitiateFileUploadResponse{ @@ -321,6 +337,22 @@ func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFile }, nil } + if protocol == "webdav" { + // TODO(ishank011): pass this through the datagateway service + // for now, we just expose the file server to the user + ep, opaque, err := s.webdavRefTransferEndpoint(ctx, statRes.Info.Target, shareChild) + if err != nil { + return &gateway.InitiateFileUploadResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving webdav host: "+p), + }, nil + } + return &gateway.InitiateFileUploadResponse{ + Opaque: opaque, + Status: status.NewOK(ctx), + UploadEndpoint: ep, + }, nil + } + // append child to target target := path.Join(ri.Path, shareChild) ref = &provider.Reference{ @@ -483,7 +515,7 @@ func (s *svc) CreateContainer(ctx context.Context, req *provider.CreateContainer }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &provider.CreateContainerResponse{ @@ -491,6 +523,18 @@ func (s *svc) CreateContainer(ctx context.Context, req *provider.CreateContainer }, nil } + if protocol == "webdav" { + err = s.webdavRefMkdir(ctx, statRes.Info.Target, shareChild) + if err != nil { + return &provider.CreateContainerResponse{ + Status: status.NewInternal(ctx, err, "gateway: error creating container on webdav host: "+p), + }, nil + } + return &provider.CreateContainerResponse{ + Status: status.NewOK(ctx), + }, nil + } + // append child to target target := path.Join(ri.Path, shareChild) ref = &provider.Reference{ @@ -605,7 +649,7 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &provider.DeleteResponse{ @@ -613,6 +657,18 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide }, nil } + if protocol == "webdav" { + err = s.webdavRefDelete(ctx, statRes.Info.Target, shareChild) + if err != nil { + return &provider.DeleteResponse{ + Status: status.NewInternal(ctx, err, "gateway: error deleting resource on webdav host: "+p), + }, nil + } + return &provider.DeleteResponse{ + Status: status.NewOK(ctx), + }, nil + } + // append child to target target := path.Join(ri.Path, shareChild) ref = &provider.Reference{ @@ -721,7 +777,7 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &provider.MoveResponse{ @@ -729,6 +785,18 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo }, nil } + if protocol == "webdav" { + err = s.webdavRefMove(ctx, statRes.Info.Target, shareChild, dshareChild) + if err != nil { + return &provider.MoveResponse{ + Status: status.NewInternal(ctx, err, "gateway: error moving resource on webdav host: "+p), + }, nil + } + return &provider.MoveResponse{ + Status: status.NewOK(ctx), + }, nil + } + src := &provider.Reference{ Spec: &provider.Reference_Path{ Path: path.Join(ri.Path, shareChild), @@ -746,7 +814,9 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo return s.move(ctx, req) } - panic("gateway: move called on unknown path:" + p) + return &provider.MoveResponse{ + Status: status.NewInternal(ctx, errors.New("gateway: move called on unknown path: "+p), ""), + }, nil } func (s *svc) move(ctx context.Context, req *provider.MoveRequest) (*provider.MoveResponse, error) { @@ -891,17 +961,18 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St panic("gateway: a share name must be of type reference: ref:" + res.Info.Path) } - ri, err := s.handleWebdavRefStat(ctx, res.Info) + ri, protocol, err := s.checkRef(ctx, res.Info) if err != nil { - if _, ok := err.(errtypes.IsNotSupported); !ok { - return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+p), - }, nil - } - ri, err = s.checkRef(ctx, res.Info) + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving reference: "+p), + }, nil + } + + if protocol == "webdav" { + ri, err = s.webdavRefStat(ctx, res.Info.Target) if err != nil { return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "gateway: error resolving reference: "+p), + Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+p), }, nil } } @@ -942,7 +1013,7 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St }, nil } - ri, err := s.checkRef(ctx, statRes.Info) + ri, protocol, err := s.checkRef(ctx, statRes.Info) if err != nil { log.Err(err).Msg("gateway: error resolving reference") return &provider.StatResponse{ @@ -950,6 +1021,20 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St }, nil } + if protocol == "webdav" { + ri, err = s.webdavRefStat(ctx, statRes.Info.Target, shareChild) + if err != nil { + return &provider.StatResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+p), + }, nil + } + ri.Path = p + return &provider.StatResponse{ + Status: status.NewOK(ctx), + Info: ri, + }, nil + } + // append child to target target := path.Join(ri.Path, shareChild) ref = &provider.Reference{ @@ -982,7 +1067,7 @@ func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.St panic("gateway: stating an unknown path:" + p) } -func (s *svc) checkRef(ctx context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, error) { +func (s *svc) checkRef(ctx context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, string, error) { if ri.Type != provider.ResourceType_RESOURCE_TYPE_REFERENCE { panic("gateway: calling checkRef on a non reference type:" + ri.String()) } @@ -990,20 +1075,23 @@ func (s *svc) checkRef(ctx context.Context, ri *provider.ResourceInfo) (*provide // reference types MUST have a target resource id. if ri.Target == "" { err := errors.New("gateway: ref target is an empty uri") - return nil, err + return nil, "", err } uri, err := url.Parse(ri.Target) if err != nil { - return nil, errors.Wrapf(err, "gateway: error parsing target uri: %s", ri.Target) + return nil, "", errors.Wrapf(err, "gateway: error parsing target uri: %s", ri.Target) } switch uri.Scheme { case "cs3": - return s.handleCS3Ref(ctx, uri.Opaque) + ref, err := s.handleCS3Ref(ctx, uri.Opaque) + return ref, "cs3", err + case "webdav": + return nil, "webdav", nil default: err := errors.New("gateway: no reference handler for scheme: " + uri.Scheme) - return nil, err + return nil, "", err } } @@ -1097,12 +1185,22 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ } for i, ref := range lcr.Infos { - info, err := s.checkRef(ctx, ref) + info, protocol, err := s.checkRef(ctx, ref) if err != nil { return &provider.ListContainerResponse{ Status: status.NewInternal(ctx, err, "gateway: error resolving reference:"+ref.Path), }, nil } + + if protocol == "webdav" { + info, err = s.webdavRefStat(ctx, ref.Target) + if err != nil { + return &provider.ListContainerResponse{ + Status: status.NewInternal(ctx, err, "gateway: error resolving webdav reference: "+ref.Target), + }, nil + } + } + base := path.Base(ref.Path) info.Path = path.Join(p, base) lcr.Infos[i] = info @@ -1134,13 +1232,31 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ }, nil } - ri, err := s.checkRef(ctx, res.Info) + ri, protocol, err := s.checkRef(ctx, res.Info) if err != nil { return &provider.ListContainerResponse{ Status: status.NewInternal(ctx, err, "gateway: error resolving reference:"+p), }, nil } + if protocol == "webdav" { + infos, err := s.webdavRefLs(ctx, res.Info.Target) + if err != nil { + return &provider.ListContainerResponse{ + Status: status.NewInternal(ctx, err, "gateway: error listing webdav reference: "+p), + }, nil + } + + for _, info := range infos { + base := path.Base(info.Path) + info.Path = path.Join(p, base) + } + return &provider.ListContainerResponse{ + Status: status.NewOK(ctx), + Infos: infos, + }, nil + } + if ri.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER { err := errtypes.NotSupported("gateway: list container: cannot list non-container type:" + ri.Path) log.Err(err).Msg("gateway: error listing") @@ -1206,13 +1322,31 @@ func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequ }, nil } - ri, err := s.checkRef(ctx, res.Info) + ri, protocol, err := s.checkRef(ctx, res.Info) if err != nil { return &provider.ListContainerResponse{ Status: status.NewInternal(ctx, err, "gateway: error resolving reference:"+p), }, nil } + if protocol == "webdav" { + infos, err := s.webdavRefLs(ctx, res.Info.Target, shareChild) + if err != nil { + return &provider.ListContainerResponse{ + Status: status.NewInternal(ctx, err, "gateway: error listing webdav reference: "+p), + }, nil + } + + for _, info := range infos { + base := path.Base(info.Path) + info.Path = path.Join(shareName, shareChild, base) + } + return &provider.ListContainerResponse{ + Status: status.NewOK(ctx), + Infos: infos, + }, nil + } + if ri.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER { err := errtypes.NotSupported("gateway: list container: cannot list non-container type:" + ri.Path) log.Err(err).Msg("gateway: error listing") diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go index b5e9f9b7c0..3e25cc1629 100644 --- a/internal/grpc/services/gateway/webdavstorageprovider.go +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -21,11 +21,11 @@ package gateway import ( "context" "net/url" + "path" "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/token" "github.com/pkg/errors" @@ -38,73 +38,193 @@ type webdavEndpoint struct { token string } -func extractEndpointInfo(ri *provider.ResourceInfo) (*webdavEndpoint, error) { - if ri.Type != provider.ResourceType_RESOURCE_TYPE_REFERENCE { - panic("gateway: calling handleWebdavRefStat on a non reference type:" + ri.String()) +func (s *svc) webdavRefStat(ctx context.Context, targetURL string, nameQueries ...string) (*provider.ResourceInfo, error) { + targetURL, err := appendNameQuery(targetURL, nameQueries...) + if err != nil { + return nil, err } - // reference types MUST have a target resource id. - if ri.Target == "" { - err := errors.New("gateway: ref target is an empty uri") + + ep, err := extractEndpointInfo(targetURL) + if err != nil { return nil, err } + c := gowebdav.NewClient(ep.endpoint, "", "") + c.SetHeader(token.TokenHeader, ep.token) - uri, err := url.Parse(ri.Target) + // TODO(ishank011): We need to call PROPFIND ourselves as we need to retrieve + // ownloud-specific fields to get the resource ID and permissions. + info, err := c.Stat(ep.filePath) if err != nil { - return nil, errors.Wrap(err, "gateway: error parsing target uri: "+ri.Target) + return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) } - if uri.Scheme != "webdav" { - return nil, errtypes.NotSupported("ref target does not have the webdav scheme") + return normalize(info.(*gowebdav.File)), nil +} + +func (s *svc) webdavRefLs(ctx context.Context, targetURL string, nameQueries ...string) ([]*provider.ResourceInfo, error) { + targetURL, err := appendNameQuery(targetURL, nameQueries...) + if err != nil { + return nil, err } - parts := strings.SplitN(uri.Opaque, "@", 2) - if len(parts) < 2 { - err := errors.New("gateway: webdav ref does not follow the layout token@webdav_endpoint?name " + ri.Target) + ep, err := extractEndpointInfo(targetURL) + if err != nil { return nil, err } - m, err := url.ParseQuery(uri.RawQuery) + c := gowebdav.NewClient(ep.endpoint, "", "") + c.SetHeader(token.TokenHeader, ep.token) + + // TODO(ishank011): We need to call PROPFIND ourselves as we need to retrieve + // ownloud-specific fields to get the resource ID and permissions. + infos, err := c.ReadDir(ep.filePath) if err != nil { - return nil, errors.Wrap(err, "gateway: error parsing target resource name") + return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) } - return &webdavEndpoint{ - filePath: m["name"][0], - endpoint: parts[1], - token: parts[0], - }, nil + var mds []*provider.ResourceInfo + mds = make([]*provider.ResourceInfo, len(infos)) + for _, fi := range infos { + info := fi.(gowebdav.File) + mds = append(mds, normalize(&info)) + } + return mds, nil } -func (s *svc) handleWebdavRefStat(ctx context.Context, ri *provider.ResourceInfo) (*provider.ResourceInfo, error) { - log := appctx.GetLogger(ctx) - ep, err := extractEndpointInfo(ri) - log.Info().Msgf("ep: %+v", ep) +func (s *svc) webdavRefMkdir(ctx context.Context, targetURL string, nameQueries ...string) error { + targetURL, err := appendNameQuery(targetURL, nameQueries...) if err != nil { - return nil, err + return err + } + + ep, err := extractEndpointInfo(targetURL) + if err != nil { + return err } c := gowebdav.NewClient(ep.endpoint, "", "") c.SetHeader(token.TokenHeader, ep.token) - // We need to call PROPFIND ourselves as we need ownloud-specific fields - // to read the resource ID and permissions. - info, err := c.Stat(ep.filePath) - fileInfo := info.(*gowebdav.File) + err = c.Mkdir(ep.filePath, 0700) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) + return errors.Wrap(err, "gateway: error calling mkdir at the webdav endpoint: "+ep.endpoint) + } + return nil +} + +func (s *svc) webdavRefMove(ctx context.Context, targetURL, src, destination string) error { + srcURL, err := appendNameQuery(targetURL, src) + if err != nil { + return err + } + srcEP, err := extractEndpointInfo(srcURL) + if err != nil { + return err + } + + destURL, err := appendNameQuery(targetURL, destination) + if err != nil { + return err + } + destEP, err := extractEndpointInfo(destURL) + if err != nil { + return err + } + + c := gowebdav.NewClient(srcEP.endpoint, "", "") + c.SetHeader(token.TokenHeader, srcEP.token) + + err = c.Rename(srcEP.filePath, destEP.filePath, true) + if err != nil { + return errors.Wrap(err, "gateway: error calling rename at the webdav endpoint: "+srcEP.endpoint) + } + return nil +} + +func (s *svc) webdavRefDelete(ctx context.Context, targetURL string, nameQueries ...string) error { + targetURL, err := appendNameQuery(targetURL, nameQueries...) + if err != nil { + return err + } + + ep, err := extractEndpointInfo(targetURL) + if err != nil { + return err + } + c := gowebdav.NewClient(ep.endpoint, "", "") + c.SetHeader(token.TokenHeader, ep.token) + + err = c.Remove(ep.filePath) + if err != nil { + return errors.Wrap(err, "gateway: error calling remove at the webdav endpoint: "+ep.endpoint) + } + return nil +} + +func (s *svc) webdavRefTransferEndpoint(ctx context.Context, targetURL string, nameQueries ...string) (string, *types.Opaque, error) { + targetURL, err := appendNameQuery(targetURL, nameQueries...) + if err != nil { + return "", nil, err } - md := &provider.ResourceInfo{ - // Add Id, PermissionSet, Owner - Path: fileInfo.Path(), - Type: getResourceType(fileInfo.IsDir()), - Etag: fileInfo.ETag(), - MimeType: fileInfo.ContentType(), - Size: uint64(fileInfo.Size()), + ep, err := extractEndpointInfo(targetURL) + if err != nil { + return "", nil, err + } + + return ep.endpoint, &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "webdav-file-path": { + Decoder: "plain", + Value: []byte(ep.filePath), + }, + "webdav-token": { + Decoder: "plain", + Value: []byte(ep.token), + }, + }, + }, nil +} + +func normalize(info *gowebdav.File) *provider.ResourceInfo { + return &provider.ResourceInfo{ + // TODO(ishank011): Add Id, PermissionSet, Owner + Path: info.Path(), + Type: getResourceType(info.IsDir()), + Etag: info.ETag(), + MimeType: info.ContentType(), + Size: uint64(info.Size()), Mtime: &types.Timestamp{ - Seconds: uint64(fileInfo.ModTime().Unix()), + Seconds: uint64(info.ModTime().Unix()), }, } - log.Info().Msgf("md: %+v", md) +} + +func extractEndpointInfo(targetURL string) (*webdavEndpoint, error) { + if targetURL == "" { + return nil, errors.New("gateway: ref target is an empty uri") + } - return md, nil + uri, err := url.Parse(targetURL) + if err != nil { + return nil, errors.Wrap(err, "gateway: error parsing target uri: "+targetURL) + } + if uri.Scheme != "webdav" { + return nil, errtypes.NotSupported("ref target does not have the webdav scheme") + } + + parts := strings.SplitN(uri.Opaque, "@", 2) + if len(parts) < 2 { + err := errors.New("gateway: webdav ref does not follow the layout token@webdav_endpoint?name " + targetURL) + return nil, err + } + m, err := url.ParseQuery(uri.RawQuery) + if err != nil { + return nil, errors.Wrap(err, "gateway: error parsing target resource name") + } + + return &webdavEndpoint{ + filePath: m["name"][0], + endpoint: parts[1], + token: parts[0], + }, nil } func getResourceType(isDir bool) provider.ResourceType { @@ -113,3 +233,18 @@ func getResourceType(isDir bool) provider.ResourceType { } return provider.ResourceType_RESOURCE_TYPE_FILE } + +func appendNameQuery(targetURL string, nameQueries ...string) (string, error) { + uri, err := url.Parse(targetURL) + if err != nil { + return "", err + } + q, err := url.ParseQuery(uri.RawQuery) + if err != nil { + return "", err + } + name := append([]string{q["name"][0]}, nameQueries...) + q.Set("name", path.Join(name...)) + uri.RawQuery = q.Encode() + return uri.String(), nil +} diff --git a/internal/http/services/dataprovider/put.go b/internal/http/services/dataprovider/put.go index 5dc93e57c8..b38d153084 100644 --- a/internal/http/services/dataprovider/put.go +++ b/internal/http/services/dataprovider/put.go @@ -63,7 +63,7 @@ func (s *svc) doTusPut(w http.ResponseWriter, r *http.Request) { return } - length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) + length, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) if err != nil { w.WriteHeader(http.StatusBadRequest) return diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 24d8eb576e..e1f7c2e7a7 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -202,7 +202,7 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } } - length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) + length, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) if err != nil { w.WriteHeader(http.StatusBadRequest) return @@ -211,7 +211,7 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { opaqueMap := map[string]*typespb.OpaqueEntry{ "Upload-Length": { Decoder: "plain", - Value: []byte(r.Header.Get("Content-Length")), + Value: []byte(r.Header.Get("Upload-Length")), }, } diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index d275cb911e..054cd79bed 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -1147,14 +1147,14 @@ func (fs *localfs) RestoreRecycleItem(ctx context.Context, restoreKey string) er func (fs *localfs) propagate(ctx context.Context, leafPath string) error { var root string - if fs.isShareFolderChild(ctx, leafPath) { + if fs.isShareFolderChild(ctx, leafPath) || strings.HasSuffix(path.Clean(leafPath), fs.conf.ShareFolder) { root = fs.wrapReferences(ctx, "/") } else { root = fs.wrap(ctx, "/") } if !strings.HasPrefix(leafPath, root) { - return errors.New("internal path outside root") + return errors.New("internal path: " + leafPath + " outside root: " + root) } fi, err := os.Stat(leafPath) From cd678474744b8e5704954f24a2fac51b8ffa110c Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 18 Aug 2020 14:18:25 +0200 Subject: [PATCH 5/6] Fix linting --- internal/grpc/services/gateway/webdavstorageprovider.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go index 3e25cc1629..ba12c9fc80 100644 --- a/internal/grpc/services/gateway/webdavstorageprovider.go +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -80,8 +80,7 @@ func (s *svc) webdavRefLs(ctx context.Context, targetURL string, nameQueries ... return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) } - var mds []*provider.ResourceInfo - mds = make([]*provider.ResourceInfo, len(infos)) + mds := []*provider.ResourceInfo{} for _, fi := range infos { info := fi.(gowebdav.File) mds = append(mds, normalize(&info)) From 97381a2c675358a025f847da25e3412375063eff Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 18 Aug 2020 17:48:33 +0200 Subject: [PATCH 6/6] Fallback to Upload-Length header --- examples/storage-references/gateway.toml | 3 --- internal/grpc/services/gateway/storageprovider.go | 4 ++-- .../grpc/services/gateway/webdavstorageprovider.go | 11 ++++++----- internal/http/services/dataprovider/put.go | 10 +++++++--- internal/http/services/owncloud/ocdav/put.go | 12 ++++++++---- pkg/storage/utils/localfs/localfs.go | 2 +- pkg/token/manager/jwt/jwt.go | 2 +- 7 files changed, 25 insertions(+), 19 deletions(-) diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 5b53a7ce76..35d7b429cf 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -3,9 +3,6 @@ commit_share_to_storage_grant = true commit_share_to_storage_ref = true -[grpc.services.gateway.token_managers.jwt] -expires = 86400 - [grpc.services.storageregistry] [grpc.services.storageregistry.drivers.static] home_provider = "/home" diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 6cacf6ce10..b67e74a07f 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -193,7 +193,7 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi ep, opaque, err := s.webdavRefTransferEndpoint(ctx, statRes.Info.Target, shareChild) if err != nil { return &gateway.InitiateFileDownloadResponse{ - Status: status.NewInternal(ctx, err, "gateway: error creating container on webdav host: "+p), + Status: status.NewInternal(ctx, err, "gateway: error downloading from webdav host: "+p), }, nil } return &gateway.InitiateFileDownloadResponse{ @@ -343,7 +343,7 @@ func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFile ep, opaque, err := s.webdavRefTransferEndpoint(ctx, statRes.Info.Target, shareChild) if err != nil { return &gateway.InitiateFileUploadResponse{ - Status: status.NewInternal(ctx, err, "gateway: error resolving webdav host: "+p), + Status: status.NewInternal(ctx, err, "gateway: error uploading to webdav host: "+p), }, nil } return &gateway.InitiateFileUploadResponse{ diff --git a/internal/grpc/services/gateway/webdavstorageprovider.go b/internal/grpc/services/gateway/webdavstorageprovider.go index ba12c9fc80..5c27014571 100644 --- a/internal/grpc/services/gateway/webdavstorageprovider.go +++ b/internal/grpc/services/gateway/webdavstorageprovider.go @@ -20,6 +20,7 @@ package gateway import ( "context" + "fmt" "net/url" "path" "strings" @@ -55,7 +56,7 @@ func (s *svc) webdavRefStat(ctx context.Context, targetURL string, nameQueries . // ownloud-specific fields to get the resource ID and permissions. info, err := c.Stat(ep.filePath) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) + return nil, errors.Wrap(err, fmt.Sprintf("gateway: error statting %s at the webdav endpoint: %s", ep.filePath, ep.endpoint)) } return normalize(info.(*gowebdav.File)), nil } @@ -77,7 +78,7 @@ func (s *svc) webdavRefLs(ctx context.Context, targetURL string, nameQueries ... // ownloud-specific fields to get the resource ID and permissions. infos, err := c.ReadDir(ep.filePath) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling stat at the webdav endpoint: "+ep.endpoint) + return nil, errors.Wrap(err, fmt.Sprintf("gateway: error listing %s at the webdav endpoint: %s", ep.filePath, ep.endpoint)) } mds := []*provider.ResourceInfo{} @@ -103,7 +104,7 @@ func (s *svc) webdavRefMkdir(ctx context.Context, targetURL string, nameQueries err = c.Mkdir(ep.filePath, 0700) if err != nil { - return errors.Wrap(err, "gateway: error calling mkdir at the webdav endpoint: "+ep.endpoint) + return errors.Wrap(err, fmt.Sprintf("gateway: error creating dir %s at the webdav endpoint: %s", ep.filePath, ep.endpoint)) } return nil } @@ -132,7 +133,7 @@ func (s *svc) webdavRefMove(ctx context.Context, targetURL, src, destination str err = c.Rename(srcEP.filePath, destEP.filePath, true) if err != nil { - return errors.Wrap(err, "gateway: error calling rename at the webdav endpoint: "+srcEP.endpoint) + return errors.Wrap(err, fmt.Sprintf("gateway: error renaming %s to %s at the webdav endpoint: %s", srcEP.filePath, destEP.filePath, srcEP.endpoint)) } return nil } @@ -152,7 +153,7 @@ func (s *svc) webdavRefDelete(ctx context.Context, targetURL string, nameQueries err = c.Remove(ep.filePath) if err != nil { - return errors.Wrap(err, "gateway: error calling remove at the webdav endpoint: "+ep.endpoint) + return errors.Wrap(err, fmt.Sprintf("gateway: error removing %s at the webdav endpoint: %s", ep.filePath, ep.endpoint)) } return nil } diff --git a/internal/http/services/dataprovider/put.go b/internal/http/services/dataprovider/put.go index b38d153084..1bcbfbb65b 100644 --- a/internal/http/services/dataprovider/put.go +++ b/internal/http/services/dataprovider/put.go @@ -63,10 +63,14 @@ func (s *svc) doTusPut(w http.ResponseWriter, r *http.Request) { return } - length, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) if err != nil { - w.WriteHeader(http.StatusBadRequest) - return + // Fallback to Upload-Length + length, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } } dataServerURL := fmt.Sprintf("http://%s%s", r.Host, r.RequestURI) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index e1f7c2e7a7..718e896bad 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -202,16 +202,20 @@ func (s *svc) handlePut(w http.ResponseWriter, r *http.Request, ns string) { } } - length, err := strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + length, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) if err != nil { - w.WriteHeader(http.StatusBadRequest) - return + // Fallback to Upload-Length + length, err = strconv.ParseInt(r.Header.Get("Upload-Length"), 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } } opaqueMap := map[string]*typespb.OpaqueEntry{ "Upload-Length": { Decoder: "plain", - Value: []byte(r.Header.Get("Upload-Length")), + Value: []byte(strconv.FormatInt(length, 10)), }, } diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index 054cd79bed..5f643499d5 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -1163,7 +1163,7 @@ func (fs *localfs) propagate(ctx context.Context, leafPath string) error { } parts := strings.Split(strings.TrimPrefix(leafPath, root), "/") - // root never ents in / so the split returns an empty first element, which we can skip + // root never ends in / so the split returns an empty first element, which we can skip // we do not need to chmod the last element because it is the leaf path (< and not <= comparison) for i := 1; i < len(parts); i++ { if err := os.Chtimes(root, fi.ModTime(), fi.ModTime()); err != nil { diff --git a/pkg/token/manager/jwt/jwt.go b/pkg/token/manager/jwt/jwt.go index c92b8ff34b..2a48fef2b6 100644 --- a/pkg/token/manager/jwt/jwt.go +++ b/pkg/token/manager/jwt/jwt.go @@ -32,7 +32,7 @@ import ( "github.com/pkg/errors" ) -const defaultExpiration int64 = 3600 // 1 hour +const defaultExpiration int64 = 86400 // 1 day func init() { registry.Register("jwt", New)