From 12fb33af48ba871e910961b14eec740df558a10b Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Tue, 17 Aug 2021 11:04:56 +0200 Subject: [PATCH] Nextcloud storage driver (take 2) (#1998) --- .../unreleased/nextcloud-storage-driver.md | 6 + pkg/storage/fs/loader/loader.go | 1 + pkg/storage/fs/nextcloud/nextcloud.go | 548 ++++++++++++++++++ .../fs/nextcloud/nextcloud_server_mock.go | 168 ++++++ .../fs/nextcloud/nextcloud_suite_test.go | 31 + pkg/storage/fs/nextcloud/nextcloud_test.go | 90 +++ tests/helpers/helpers.go | 19 + .../fixtures/storageprovider-nextcloud.toml | 9 + .../integration/grpc/storageprovider_test.go | 58 ++ 9 files changed, 930 insertions(+) create mode 100644 changelog/unreleased/nextcloud-storage-driver.md create mode 100644 pkg/storage/fs/nextcloud/nextcloud.go create mode 100644 pkg/storage/fs/nextcloud/nextcloud_server_mock.go create mode 100644 pkg/storage/fs/nextcloud/nextcloud_suite_test.go create mode 100644 pkg/storage/fs/nextcloud/nextcloud_test.go create mode 100644 tests/integration/grpc/fixtures/storageprovider-nextcloud.toml diff --git a/changelog/unreleased/nextcloud-storage-driver.md b/changelog/unreleased/nextcloud-storage-driver.md new file mode 100644 index 0000000000..df57c4fd58 --- /dev/null +++ b/changelog/unreleased/nextcloud-storage-driver.md @@ -0,0 +1,6 @@ +Enhancement: Initial version of the Nextcloud storage driver + +This is not usable yet in isolation, but it's a first component of +https://github.com/pondersource/sciencemesh-nextcloud + +https://github.com/cs3org/reva/pull/1998 diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 2ccb0d7c9b..f9ff86af04 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -26,6 +26,7 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/eoshome" _ "github.com/cs3org/reva/pkg/storage/fs/local" _ "github.com/cs3org/reva/pkg/storage/fs/localhome" + _ "github.com/cs3org/reva/pkg/storage/fs/nextcloud" _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/owncloudsql" diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go new file mode 100644 index 0000000000..d7e6e56abd --- /dev/null +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -0,0 +1,548 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package nextcloud + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/tests/helpers" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("nextcloud", New) +} + +// StorageDriverConfig is the configuration struct for a NextcloudStorageDriver +type StorageDriverConfig struct { + EndPoint string `mapstructure:"end_point"` // e.g. "http://nc/apps/sciencemesh/~alice/" + MockHTTP bool `mapstructure:"mock_http"` +} + +// StorageDriver implements the storage.FS interface +// and connects with a StorageDriver server as its backend +type StorageDriver struct { + endPoint string + client *http.Client +} + +func parseConfig(m map[string]interface{}) (*StorageDriverConfig, error) { + c := &StorageDriverConfig{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return c, nil +} + +// New returns an implementation to of the storage.FS interface that talks to +// a Nextcloud instance over http. +func New(m map[string]interface{}) (storage.FS, error) { + conf, err := parseConfig(m) + if err != nil { + return nil, err + } + + return NewStorageDriver(conf) +} + +// NewStorageDriver returns a new NextcloudStorageDriver +func NewStorageDriver(c *StorageDriverConfig) (*StorageDriver, error) { + var client *http.Client + if c.MockHTTP { + nextcloudServerMock := GetNextcloudServerMock() + client, _ = helpers.TestingHTTPClient(nextcloudServerMock) + } else { + client = &http.Client{} + } + return &StorageDriver{ + endPoint: c.EndPoint, // e.g. "http://nc/apps/sciencemesh/~alice/" + client: client, + }, nil +} + +// Action describes a REST request to forward to the Nextcloud backend +type Action struct { + verb string + argS string +} + +// SetHTTPClient sets the HTTP client +func (nc *StorageDriver) SetHTTPClient(c *http.Client) { + nc.client = c +} + +func (nc *StorageDriver) doUpload(r io.ReadCloser) error { + filePath := "test.txt" + + // initialize http client + client := &http.Client{} + url := nc.endPoint + "Upload/" + filePath + req, err := http.NewRequest(http.MethodPut, url, r) + if err != nil { + panic(err) + } + + // set the request header Content-Type for the upload + // FIXME: get the actual content type from somewhere + req.Header.Set("Content-Type", "text/plain") + resp, err := client.Do(req) + if err != nil { + panic(err) + } + + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + return err +} + +func (nc *StorageDriver) do(a Action, endPoint string) (int, []byte, error) { + url := endPoint + a.verb + req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(a.argS)) + if err != nil { + panic(err) + } + + req.Header.Set("Content-Type", "application/json") + resp, err := nc.client.Do(req) + if err != nil { + panic(err) + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return resp.StatusCode, body, err +} + +// GetHome as defined in the storage.FS interface +func (nc *StorageDriver) GetHome(ctx context.Context) (string, error) { + _, respBody, err := nc.do(Action{"GetHome", ""}, nc.endPoint) + return string(respBody), err +} + +// CreateHome as defined in the storage.FS interface +func (nc *StorageDriver) CreateHome(ctx context.Context) error { + _, _, err := nc.do(Action{"CreateHome", ""}, nc.endPoint) + return err +} + +// CreateDir as defined in the storage.FS interface +func (nc *StorageDriver) CreateDir(ctx context.Context, ref *provider.Reference) error { + bodyStr, err := json.Marshal(ref) + if err != nil { + return err + } + _, _, err = nc.do(Action{"CreateDir", string(bodyStr)}, nc.endPoint) + return err +} + +// Delete as defined in the storage.FS interface +func (nc *StorageDriver) Delete(ctx context.Context, ref *provider.Reference) error { + bodyStr, err := json.Marshal(ref) + if err != nil { + return err + } + _, _, err = nc.do(Action{"Delete", string(bodyStr)}, nc.endPoint) + return err +} + +// Move as defined in the storage.FS interface +func (nc *StorageDriver) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { + data := make(map[string]string) + data["from"] = oldRef.Path + data["to"] = newRef.Path + bodyStr, _ := json.Marshal(data) + _, _, err := nc.do(Action{"Move", string(bodyStr)}, nc.endPoint) + return err +} + +// GetMD as defined in the storage.FS interface +func (nc *StorageDriver) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { + bodyStr, err := json.Marshal(ref) + if err != nil { + return nil, err + } + status, body, err := nc.do(Action{"GetMD", string(bodyStr)}, nc.endPoint) + if err != nil { + return nil, err + } + if status == 404 { + return nil, errtypes.NotFound("") + } + var respMap map[string]interface{} + err = json.Unmarshal(body, &respMap) + if err != nil { + return nil, err + } + size := int(respMap["size"].(float64)) + mdMap, ok := respMap["metadata"].(map[string]interface{}) + mdMapString := make(map[string]string) + if ok { + for key, value := range mdMap { + mdMapString[key] = value.(string) + } + } + md := &provider.ResourceInfo{ + Opaque: &types.Opaque{}, + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Id: &provider.ResourceId{OpaqueId: "fileid-" + url.QueryEscape(ref.Path)}, + Checksum: &provider.ResourceChecksum{}, + Etag: "some-etag", + MimeType: "application/octet-stream", + Mtime: &types.Timestamp{Seconds: 1234567890}, + Path: ref.Path, + PermissionSet: &provider.ResourcePermissions{}, + Size: uint64(size), + Owner: nil, + Target: "", + CanonicalMetadata: &provider.CanonicalMetadata{}, + ArbitraryMetadata: &provider.ArbitraryMetadata{ + Metadata: mdMapString, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + }, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + + return md, nil +} + +// ListFolder as defined in the storage.FS interface +func (nc *StorageDriver) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { + bodyStr, err := json.Marshal(ref) + if err != nil { + return nil, err + } + status, body, err := nc.do(Action{"ListFolder", string(bodyStr)}, nc.endPoint) + if err != nil { + return nil, err + } + if status == 404 { + return nil, errtypes.NotFound("") + } + var bodyArr []string + err = json.Unmarshal(body, &bodyArr) + var infos = make([]*provider.ResourceInfo, len(bodyArr)) + for i := 0; i < len(bodyArr); i++ { + infos[i] = &provider.ResourceInfo{ + Opaque: &types.Opaque{}, + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Id: &provider.ResourceId{OpaqueId: "fileid-" + url.QueryEscape(bodyArr[i])}, + Checksum: &provider.ResourceChecksum{}, + Etag: "some-etag", + MimeType: "application/octet-stream", + Mtime: &types.Timestamp{Seconds: 1234567890}, + Path: "/subdir", // FIXME: bodyArr[i], + PermissionSet: &provider.ResourcePermissions{}, + Size: 0, + Owner: &user.UserId{OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}, + Target: "", + CanonicalMetadata: &provider.CanonicalMetadata{}, + ArbitraryMetadata: nil, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + } + return infos, err +} + +// InitiateUpload as defined in the storage.FS interface +func (nc *StorageDriver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + bodyStr, _ := json.Marshal(ref) + _, respBody, err := nc.do(Action{"InitiateUpload", string(bodyStr)}, nc.endPoint) + if err != nil { + return nil, err + } + respMap := make(map[string]string) + err = json.Unmarshal(respBody, &respMap) + if err != nil { + return nil, err + } + return respMap, err +} + +// Upload as defined in the storage.FS interface +func (nc *StorageDriver) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + bodyStr, _ := json.Marshal(ref) + err := nc.doUpload(r) + if err != nil { + return err + } + _, _, err = nc.do(Action{"Upload", string(bodyStr)}, nc.endPoint) + return err +} + +// Download as defined in the storage.FS interface +func (nc *StorageDriver) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"Download", string(bodyStr)}, nc.endPoint) + return nil, err +} + +// ListRevisions as defined in the storage.FS interface +func (nc *StorageDriver) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { + bodyStr, _ := json.Marshal(ref) + _, respBody, err := nc.do(Action{"ListRevisions", string(bodyStr)}, nc.endPoint) + if err != nil { + return nil, err + } + var m []int + err = json.Unmarshal(respBody, &m) + if err != nil { + return nil, err + } + revs := make([]*provider.FileVersion, len(m)) + for i := 0; i < len(m); i++ { + revs[i] = &provider.FileVersion{ + Opaque: &types.Opaque{}, + Key: fmt.Sprint(i), + Size: uint64(m[i]), + Mtime: 0, + Etag: "", + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + } + return revs, err +} + +// DownloadRevision as defined in the storage.FS interface +func (nc *StorageDriver) DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"DownloadRevision", string(bodyStr)}, nc.endPoint) + return nil, err +} + +// RestoreRevision as defined in the storage.FS interface +func (nc *StorageDriver) RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"RestoreRevision", string(bodyStr)}, nc.endPoint) + return err +} + +// ListRecycle as defined in the storage.FS interface +func (nc *StorageDriver) ListRecycle(ctx context.Context, key string, path string) ([]*provider.RecycleItem, error) { + _, respBody, err := nc.do(Action{"ListRecycle", ""}, nc.endPoint) + if err != nil { + return nil, err + } + var m []string + err = json.Unmarshal(respBody, &m) + if err != nil { + return nil, err + } + items := make([]*provider.RecycleItem, len(m)) + for i := 0; i < len(m); i++ { + items[i] = &provider.RecycleItem{ + Opaque: &types.Opaque{}, + Type: 0, + Key: "", + Ref: &provider.Reference{ + ResourceId: &provider.ResourceId{}, + Path: m[i], + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + }, + Size: 0, + DeletionTime: &types.Timestamp{}, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + } + return items, err +} + +// RestoreRecycleItem as defined in the storage.FS interface +func (nc *StorageDriver) RestoreRecycleItem(ctx context.Context, key string, path string, restoreRef *provider.Reference) error { + bodyStr, _ := json.Marshal(restoreRef) + _, _, err := nc.do(Action{"RestoreRecycleItem", string(bodyStr)}, nc.endPoint) + return err +} + +// PurgeRecycleItem as defined in the storage.FS interface +func (nc *StorageDriver) PurgeRecycleItem(ctx context.Context, key string, path string) error { + bodyStr, _ := json.Marshal(key) + _, _, err := nc.do(Action{"PurgeRecycleItem", string(bodyStr)}, nc.endPoint) + return err +} + +// EmptyRecycle as defined in the storage.FS interface +func (nc *StorageDriver) EmptyRecycle(ctx context.Context) error { + _, _, err := nc.do(Action{"EmptyRecycle", ""}, nc.endPoint) + return err +} + +// GetPathByID as defined in the storage.FS interface +func (nc *StorageDriver) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { + bodyStr, _ := json.Marshal(id) + _, respBody, err := nc.do(Action{"GetPathByID", string(bodyStr)}, nc.endPoint) + return string(respBody), err +} + +// AddGrant as defined in the storage.FS interface +func (nc *StorageDriver) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"AddGrant", string(bodyStr)}, nc.endPoint) + return err +} + +// RemoveGrant as defined in the storage.FS interface +func (nc *StorageDriver) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"RemoveGrant", string(bodyStr)}, nc.endPoint) + return err +} + +// DenyGrant as defined in the storage.FS interface +func (nc *StorageDriver) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"DenyGrant", string(bodyStr)}, nc.endPoint) + return err +} + +// UpdateGrant as defined in the storage.FS interface +func (nc *StorageDriver) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"UpdateGrant", string(bodyStr)}, nc.endPoint) + return err +} + +// ListGrants as defined in the storage.FS interface +func (nc *StorageDriver) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { + bodyStr, _ := json.Marshal(ref) + _, respBody, err := nc.do(Action{"ListGrants", string(bodyStr)}, nc.endPoint) + if err != nil { + return nil, err + } + var m []map[string]bool + err = json.Unmarshal(respBody, &m) + if err != nil { + return nil, err + } + grants := make([]*provider.Grant, len(m)) + for i := 0; i < len(m); i++ { + var perms = &provider.ResourcePermissions{ + AddGrant: false, + CreateContainer: false, + Delete: false, + GetPath: false, + GetQuota: false, + InitiateFileDownload: false, + InitiateFileUpload: false, + ListGrants: false, + ListContainer: false, + ListFileVersions: false, + ListRecycle: false, + Move: false, + RemoveGrant: false, + PurgeRecycle: false, + RestoreFileVersion: false, + RestoreRecycleItem: false, + Stat: false, + UpdateGrant: false, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + for key, element := range m[i] { + if key == "stat" { + perms.Stat = element + } + if key == "move" { + perms.Move = element + } + if key == "delete" { + perms.Delete = element + } + } + grants[i] = &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: nil, + Opaque: &types.Opaque{}, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + }, + Permissions: perms, + XXX_NoUnkeyedLiteral: struct{}{}, + XXX_unrecognized: []byte{}, + XXX_sizecache: 0, + } + } + return grants, err +} + +// GetQuota as defined in the storage.FS interface +func (nc *StorageDriver) GetQuota(ctx context.Context) (uint64, uint64, error) { + _, _, err := nc.do(Action{"GetQuota", ""}, nc.endPoint) + return 0, 0, err +} + +// CreateReference as defined in the storage.FS interface +func (nc *StorageDriver) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { + _, _, err := nc.do(Action{"CreateReference", fmt.Sprintf(`{"path":"%s"}`, path)}, nc.endPoint) + return err +} + +// Shutdown as defined in the storage.FS interface +func (nc *StorageDriver) Shutdown(ctx context.Context) error { + _, _, err := nc.do(Action{"Shutdown", ""}, nc.endPoint) + return err +} + +// SetArbitraryMetadata as defined in the storage.FS interface +func (nc *StorageDriver) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { + bodyStr, _ := json.Marshal(md) + _, _, err := nc.do(Action{"SetArbitraryMetadata", string(bodyStr)}, nc.endPoint) + return err +} + +// UnsetArbitraryMetadata as defined in the storage.FS interface +func (nc *StorageDriver) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error { + bodyStr, _ := json.Marshal(ref) + _, _, err := nc.do(Action{"UnsetArbitraryMetadata", string(bodyStr)}, nc.endPoint) + return err +} + +// ListStorageSpaces :as defined in the storage.FS interface +func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { + _, _, err := nc.do(Action{"ListStorageSpaces", ""}, nc.endPoint) + return nil, err +} diff --git a/pkg/storage/fs/nextcloud/nextcloud_server_mock.go b/pkg/storage/fs/nextcloud/nextcloud_server_mock.go new file mode 100644 index 0000000000..1b438894e6 --- /dev/null +++ b/pkg/storage/fs/nextcloud/nextcloud_server_mock.go @@ -0,0 +1,168 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package nextcloud + +import ( + "fmt" + "io" + "net/http" + "strings" +) + +// Response contains data for the Nextcloud mock server to respond +// and to switch to a new server state +type Response struct { + code int + body string + newServerState string +} + +const serverStateError = "ERROR" +const serverStateEmpty = "EMPTY" +const serverStateHome = "HOME" +const serverStateSubdir = "SUBDIR" +const serverStateNewdir = "NEWDIR" +const serverStateSubdirNewdir = "SUBDIR-NEWDIR" +const serverStateFileRestored = "FILE-RESTORED" +const serverStateGrantAdded = "GRANT-ADDED" +const serverStateGrantUpdated = "GRANT-UPDATED" +const serverStateRecycle = "RECYCLE" +const serverStateReference = "REFERENCE" +const serverStateMetadata = "METADATA" + +var serverState = serverStateEmpty + +var responses = map[string]Response{ + `POST /apps/sciencemesh/~alice/AddGrant {"path":"/subdir"}`: {200, ``, serverStateGrantAdded}, + + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/subdir"} EMPTY`: {200, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/subdir"} HOME`: {200, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/subdir"} NEWDIR`: {200, ``, serverStateSubdirNewdir}, + + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/newdir"} EMPTY`: {200, ``, serverStateNewdir}, + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/newdir"} HOME`: {200, ``, serverStateNewdir}, + `POST /apps/sciencemesh/~alice/CreateDir {"path":"/newdir"} SUBDIR`: {200, ``, serverStateSubdirNewdir}, + + `POST /apps/sciencemesh/~alice/CreateHome `: {200, ``, serverStateHome}, + `POST /apps/sciencemesh/~alice/CreateHome {}`: {200, ``, serverStateHome}, + + `POST /apps/sciencemesh/~alice/CreateReference {"path":"/Shares/reference"}`: {200, `[]`, serverStateReference}, + + `POST /apps/sciencemesh/~alice/Delete {"path":"/subdir"}`: {200, ``, serverStateRecycle}, + + `POST /apps/sciencemesh/~alice/EmptyRecycle `: {200, ``, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/"} EMPTY`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/"} HOME`: {200, `{ "size": 1 }`, serverStateHome}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/newdir"} EMPTY`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/newdir"} HOME`: {404, ``, serverStateHome}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/newdir"} SUBDIR`: {404, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/newdir"} NEWDIR`: {200, `{ "size": 1 }`, serverStateNewdir}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/newdir"} SUBDIR-NEWDIR`: {200, `{ "size": 1 }`, serverStateSubdirNewdir}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/new_subdir"}`: {200, `{ "size": 1 }`, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} EMPTY`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} HOME`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} NEWDIR`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} RECYCLE`: {404, ``, serverStateRecycle}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} SUBDIR`: {200, `{ "size": 1 }`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} SUBDIR-NEWDIR`: {200, `{ "size": 1 }`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdir"} METADATA`: {200, `{ "size": 1, "metadata": { "foo": "bar" } }`, serverStateMetadata}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdirRestored"} EMPTY`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdirRestored"} RECYCLE`: {404, ``, serverStateRecycle}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdirRestored"} SUBDIR`: {404, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/subdirRestored"} FILE-RESTORED`: {200, `{ "size": 1 }`, serverStateFileRestored}, + + `POST /apps/sciencemesh/~alice/GetMD {"path":"/versionedFile"} EMPTY`: {200, `{ "size": 2 }`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/GetMD {"path":"/versionedFile"} FILE-RESTORED`: {200, `{ "size": 1 }`, serverStateFileRestored}, + + `POST /apps/sciencemesh/~alice/GetPathByID {"storage_id":"00000000-0000-0000-0000-000000000000","opaque_id":"fileid-%2Fsubdir"}`: {200, "/subdir", serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/InitiateUpload {"path":"/file"}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/ListFolder {"path":"/"}`: {200, `["/subdir"]`, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/ListFolder {"path":"/Shares"} EMPTY`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/ListFolder {"path":"/Shares"} SUBDIR`: {404, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/ListFolder {"path":"/Shares"} REFERENCE`: {200, `["reference"]`, serverStateReference}, + + `POST /apps/sciencemesh/~alice/ListGrants {"path":"/subdir"} SUBDIR`: {200, `[]`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/ListGrants {"path":"/subdir"} GRANT-ADDED`: {200, `[ { "stat": true, "move": true, "delete": false } ]`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/ListGrants {"path":"/subdir"} GRANT-UPDATED`: {200, `[ { "stat": true, "move": true, "delete": true } ]`, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/ListRecycle EMPTY`: {200, `[]`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/ListRecycle RECYCLE`: {200, `["/subdir"]`, serverStateRecycle}, + + `POST /apps/sciencemesh/~alice/ListRevisions {"path":"/versionedFile"} EMPTY`: {500, `[1]`, serverStateEmpty}, + `POST /apps/sciencemesh/~alice/ListRevisions {"path":"/versionedFile"} FILE-RESTORED`: {500, `[1, 2]`, serverStateFileRestored}, + + `POST /apps/sciencemesh/~alice/Move {"from":"/subdir","to":"/new_subdir"}`: {200, ``, serverStateEmpty}, + + `POST /apps/sciencemesh/~alice/RemoveGrant {"path":"/subdir"} GRANT-ADDED`: {200, ``, serverStateGrantUpdated}, + + `POST /apps/sciencemesh/~alice/RestoreRecycleItem null`: {200, ``, serverStateSubdir}, + `POST /apps/sciencemesh/~alice/RestoreRecycleItem {"path":"/subdirRestored"}`: {200, ``, serverStateFileRestored}, + + `POST /apps/sciencemesh/~alice/RestoreRevision {"path":"/versionedFile"}`: {200, ``, serverStateFileRestored}, + + `POST /apps/sciencemesh/~alice/SetArbitraryMetadata {"metadata":{"foo":"bar"}}`: {200, ``, serverStateMetadata}, + + `POST /apps/sciencemesh/~alice/UnsetArbitraryMetadata {"path":"/subdir"}`: {200, ``, serverStateSubdir}, + + `POST /apps/sciencemesh/~alice/UpdateGrant {"path":"/subdir"}`: {200, ``, serverStateGrantUpdated}, +} + +// GetNextcloudServerMock returns a handler that pretends to be a remote Nextcloud server +func GetNextcloudServerMock() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := new(strings.Builder) + _, err := io.Copy(buf, r.Body) + if err != nil { + panic("Error reading response into buffer") + } + var key = fmt.Sprintf("%s %s %s", r.Method, r.URL, buf.String()) + fmt.Printf("Nextcloud Server Mock key %s\n", key) + response := responses[key] + if (response == Response{}) { + key = fmt.Sprintf("%s %s %s %s", r.Method, r.URL, buf.String(), serverState) + fmt.Printf("Nextcloud Server Mock key with State %s\n", key) + response = responses[key] + } + if (response == Response{}) { + fmt.Println("ERROR!!") + fmt.Println("ERROR!!") + fmt.Printf("Nextcloud Server Mock key not found! %s\n", key) + fmt.Println("ERROR!!") + fmt.Println("ERROR!!") + response = Response{200, fmt.Sprintf("response not defined! %s", key), serverStateEmpty} + } + serverState = responses[key].newServerState + if serverState == `` { + serverState = serverStateError + } + w.WriteHeader(response.code) + _, err = w.Write([]byte(responses[key].body)) + if err != nil { + panic(err) + } + }) +} diff --git a/pkg/storage/fs/nextcloud/nextcloud_suite_test.go b/pkg/storage/fs/nextcloud/nextcloud_suite_test.go new file mode 100644 index 0000000000..7d75b64879 --- /dev/null +++ b/pkg/storage/fs/nextcloud/nextcloud_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package nextcloud_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestNextcloud(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Nextcloud Suite") +} diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go new file mode 100644 index 0000000000..a13fe20f05 --- /dev/null +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -0,0 +1,90 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package nextcloud_test + +import ( + "context" + "net/http" + "os" + + "github.com/cs3org/reva/pkg/storage/fs/nextcloud" + "github.com/cs3org/reva/tests/helpers" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Nextcloud", func() { + var ( + options map[string]interface{} + tmpRoot string + ) + + BeforeEach(func() { + tmpRoot, err := helpers.TempDir("reva-unit-tests-*-root") + Expect(err).ToNot(HaveOccurred()) + + options = map[string]interface{}{ + "root": tmpRoot, + "enable_home": true, + "share_folder": "/Shares", + } + }) + + AfterEach(func() { + if tmpRoot != "" { + os.RemoveAll(tmpRoot) + } + }) + + Describe("New", func() { + It("returns a new instance", func() { + _, err := nextcloud.New(options) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Describe("CreateHome", func() { + It("calls the CreateHome endpoint", func() { + nc, _ := nextcloud.NewStorageDriver(&nextcloud.StorageDriverConfig{ + EndPoint: "http://mock.com", + MockHTTP: true, + }) + + const ( + okResponse = `{ + "users": [ + {"id": 1, "name": "Roman"}, + {"id": 2, "name": "Dmitry"} + ] + }` + ) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(okResponse)) + if err != nil { + panic(err) + } + }) + mock, teardown := helpers.TestingHTTPClient(h) + defer teardown() + nc.SetHTTPClient(mock) + err2 := nc.CreateHome(context.TODO()) + Expect(err2).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/tests/helpers/helpers.go b/tests/helpers/helpers.go index 6b6ba37248..a7dc7c9bb0 100644 --- a/tests/helpers/helpers.go +++ b/tests/helpers/helpers.go @@ -19,7 +19,11 @@ package helpers import ( + "context" "io/ioutil" + "net" + "net/http" + "net/http/httptest" "os" "path/filepath" "runtime" @@ -44,3 +48,18 @@ func TempDir(name string) (string, error) { return tmpRoot, nil } + +// TestingHTTPClient thanks to https://itnext.io/how-to-stub-requests-to-remote-hosts-with-go-6c2c1db32bf2 +func TestingHTTPClient(handler http.Handler) (*http.Client, func()) { + s := httptest.NewServer(handler) + + cli := &http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, network, _ string) (net.Conn, error) { + return net.Dial(network, s.Listener.Addr().String()) + }, + }, + } + + return cli, s.Close +} diff --git a/tests/integration/grpc/fixtures/storageprovider-nextcloud.toml b/tests/integration/grpc/fixtures/storageprovider-nextcloud.toml new file mode 100644 index 0000000000..99ae4c85bd --- /dev/null +++ b/tests/integration/grpc/fixtures/storageprovider-nextcloud.toml @@ -0,0 +1,9 @@ +[grpc] +address = "{{grpc_address}}" + +[grpc.services.storageprovider] +driver = "nextcloud" + +[grpc.services.storageprovider.drivers.nextcloud] +end_point = "http://localhost:8080/apps/sciencemesh/~alice/" +mock_http = true diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index d869ee2c94..f2d5ca1a4e 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -465,6 +465,64 @@ var _ = Describe("storage providers", func() { }) } + Describe("nextcloud", func() { + BeforeEach(func() { + dependencies = map[string]string{ + "storage": "storageprovider-nextcloud.toml", + } + }) + + assertCreateHome() + + Context("with a home and a subdirectory", func() { + JustBeforeEach(func() { + res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + + assertCreateContainer() + assertListContainer() + assertGetPath() + assertDelete() + assertMove() + assertGrants() + assertUploads() + assertDownloads() + assertRecycle() + assertReferences() + assertMetadata() + }) + + Context("with an existing file /versioned_file", func() { + JustBeforeEach(func() { + fs, err := ocis.New(map[string]interface{}{ + "root": revads["storage"].TmpRoot, + "enable_home": true, + }) + Expect(err).ToNot(HaveOccurred()) + + content1 := ioutil.NopCloser(bytes.NewReader([]byte("1"))) + content2 := ioutil.NopCloser(bytes.NewReader([]byte("22"))) + + ctx := ctxpkg.ContextSetUser(context.Background(), user) + + err = fs.CreateHome(ctx) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content1) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content2) + Expect(err).ToNot(HaveOccurred()) + }) + + assertFileVersions() + }) + }) + Describe("ocis", func() { BeforeEach(func() { dependencies = map[string]string{