From 1727105ca30363e91d7a696b903968bc5a3ef6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= <61736220+Daniel-WWU-IT@users.noreply.github.com> Date: Tue, 27 Oct 2020 15:00:04 +0100 Subject: [PATCH] Add a Reva SDK (#1280) --- changelog/unreleased/reva-sdk.md | 5 + .../content/en/docs/tutorials/sdk-tutorial.md | 70 ++++++ examples/sdk/sdk.go | 139 ++++++++++++ go.sum | 2 - pkg/sdk/action/action.go | 38 ++++ pkg/sdk/action/action_test.go | 114 ++++++++++ pkg/sdk/action/download.go | 117 ++++++++++ pkg/sdk/action/enumfiles.go | 117 ++++++++++ pkg/sdk/action/fileops.go | 162 ++++++++++++++ pkg/sdk/action/upload.go | 199 ++++++++++++++++++ pkg/sdk/common/common_test.go | 182 ++++++++++++++++ pkg/sdk/common/crypto/crypto.go | 50 +++++ pkg/sdk/common/crypto/crypto_test.go | 76 +++++++ pkg/sdk/common/datadesc.go | 69 ++++++ pkg/sdk/common/net/httpreq.go | 110 ++++++++++ pkg/sdk/common/net/net.go | 35 +++ pkg/sdk/common/net/net_test.go | 159 ++++++++++++++ pkg/sdk/common/net/rpc.go | 49 +++++ pkg/sdk/common/net/tus.go | 134 ++++++++++++ pkg/sdk/common/net/webdav.go | 121 +++++++++++ pkg/sdk/common/opaque.go | 59 ++++++ pkg/sdk/common/testing/testing.go | 62 ++++++ pkg/sdk/common/util.go | 45 ++++ pkg/sdk/sdk_test.go | 74 +++++++ pkg/sdk/session.go | 170 +++++++++++++++ 25 files changed, 2356 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/reva-sdk.md create mode 100644 docs/content/en/docs/tutorials/sdk-tutorial.md create mode 100644 examples/sdk/sdk.go create mode 100644 pkg/sdk/action/action.go create mode 100644 pkg/sdk/action/action_test.go create mode 100644 pkg/sdk/action/download.go create mode 100644 pkg/sdk/action/enumfiles.go create mode 100644 pkg/sdk/action/fileops.go create mode 100644 pkg/sdk/action/upload.go create mode 100644 pkg/sdk/common/common_test.go create mode 100644 pkg/sdk/common/crypto/crypto.go create mode 100644 pkg/sdk/common/crypto/crypto_test.go create mode 100644 pkg/sdk/common/datadesc.go create mode 100644 pkg/sdk/common/net/httpreq.go create mode 100644 pkg/sdk/common/net/net.go create mode 100644 pkg/sdk/common/net/net_test.go create mode 100644 pkg/sdk/common/net/rpc.go create mode 100644 pkg/sdk/common/net/tus.go create mode 100644 pkg/sdk/common/net/webdav.go create mode 100644 pkg/sdk/common/opaque.go create mode 100644 pkg/sdk/common/testing/testing.go create mode 100644 pkg/sdk/common/util.go create mode 100644 pkg/sdk/sdk_test.go create mode 100644 pkg/sdk/session.go diff --git a/changelog/unreleased/reva-sdk.md b/changelog/unreleased/reva-sdk.md new file mode 100644 index 0000000000..491ac06a4f --- /dev/null +++ b/changelog/unreleased/reva-sdk.md @@ -0,0 +1,5 @@ +Enhancement: Add a Reva SDK + +A Reva SDK has been added to make working with a remote Reva instance much easier by offering a high-level API that hides all the underlying details of the CS3API. + +https://github.com/cs3org/reva/pull/1280 diff --git a/docs/content/en/docs/tutorials/sdk-tutorial.md b/docs/content/en/docs/tutorials/sdk-tutorial.md new file mode 100644 index 0000000000..359e7f1cbd --- /dev/null +++ b/docs/content/en/docs/tutorials/sdk-tutorial.md @@ -0,0 +1,70 @@ +--- +title: "Reva SDK" +linkTitle: "Reva SDK" +weight: 5 +description: > + Use the Reva SDK to easily access and work with a remote Reva instance. +--- +The Reva SDK (located under `/pkg/sdk/`) is a simple software development kit to work with Reva through the [CS3API](https://github.com/cs3org/go-cs3apis). It's goal is to make working with Reva as easy as possible by providing a high-level API which hides all the details of the underlying CS3API. + +## Design +The main design goal of the SDK is _simplicity_. This means that the code is extremely easy to use instead of being particularly fancy or feature-rich. + +There are two central kinds of objects you'll be using: a _session_ and various _actions_. The session represents the connection to the Reva server through its gRPC gateway client; the actions are then used to perform operations like up- and downloading or enumerating all files located at a specific remote path. + +## Using the SDK +### 1. Session creation +The first step when using the SDK is to create a session and establish a connection to Reva (which actually results in a token-creation and not a permanent connection, but this should not bother you in any way): + +``` +session := sdk.MustNewSession() // Panics if this fails (should usually not happen) +session.Initiate("reva.host.com:443", false) +session.BasicLogin("my-login", "my-pass") +``` + +Note that error checking is omitted here for brevity, but nearly all methods in the SDK return an error which should be checked upon. + +If the session has been created successfully - which can also be verified by calling `session.IsValid()` -, you can use one of the various actions to perform the actual operations. + +### 2. Performing operations +An overview of all currently supported operations can be found below; here is an example of how to upload a file using the `UploadAction`: + +``` +act := action.MustNewUploadAction(session) +info, err := act.UploadBytes([]byte("HELLO WORLD!\n"), "/home/mytest/hello.txt") +// Check error... +fmt.Printf("Uploaded file: %s [%db] -- %s", info.Path, info.Size, info.Type) +``` + +As you can see, you first need to create an instance of the desired action by either calling its corresponding `New...Action` or `MustNew...Action` function; these creators always require you to pass the previously created session object. The actual operations are then performed by using the appropriate methods offered by the action object. + +A more extensive example of how to use the SDK can also be found in `/examples/sdk/sdk.go`. + +## Supported operations +An action object often bundles various operations; the `FileOperationsAction`, for example, allows you to create directories, check if a file exists or remove an entire path. Below is an alphabetically sorted table of the available actions and their supported operations: + +| Action | Operation | Description | +| --- | --- | --- | +| `DownloadAction` | `Download` | Downloads a specific resource identified by a `ResourceInfo` object | +| | `DownloadFile` | Downloads a specific file | +| `EnumFilesAction`1 | `ListAll` | Lists all files and directories in a given path | +| | `ListAllWithFilter` | Lists all files and directories in a given path that fulfill a given predicate | +| | `ListDirs` | Lists all directories in a given path | +| | `ListFiles` | Lists all files in a given path | +| `FileOperationsAction` | `DirExists` | Checks whether the specified directory exists | +| | `FileExists` | Checks whether the specified file exists | +| | `MakePath` | Creates the entire directory tree specified by a path | +| | `Move` | Moves a specified resource to a new target | +| | `MoveTo` | Moves a specified resource to a new directory, creating it if necessary | +| | `Remove` | Deletes the specified resource | +| | `ResourceExists` | Checks whether the specified resource exists | +| | `Stat` | Queries information of a resource | +| `UploadAction`2 | `Upload` | Uploads data from a reader to a target file | +| | `UploadBytes` | Uploads byte data to a target file | +| | `UploadFile` | Uploads a file to a target file | +| | `UploadFileTo` | Uploads a file to a target directory | + +* 1 All enumeration operations support recursion. +* 2 The `UploadAction` creates the target directory automatically if necessary. + +_Note that not all features of the CS3API are currently implemented._ diff --git a/examples/sdk/sdk.go b/examples/sdk/sdk.go new file mode 100644 index 0000000000..362e432e97 --- /dev/null +++ b/examples/sdk/sdk.go @@ -0,0 +1,139 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package main + +import ( + "fmt" + "log" + + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/action" +) + +func runActions(session *sdk.Session) { + // Try creating a directory + { + act := action.MustNewFileOperationsAction(session) + if err := act.MakePath("/home/subdir/subsub"); err == nil { + log.Println("Created path /home/subdir/subsub") + } else { + log.Println("Could not create path /home/subdir/subsub") + } + fmt.Println() + } + + // Try deleting a directory + { + act := action.MustNewFileOperationsAction(session) + if err := act.Remove("/home/subdir/subsub"); err == nil { + log.Println("Removed path /home/subdir/subsub") + } else { + log.Println("Could not remove path /home/subdir/subsub") + } + fmt.Println() + } + + // Try uploading + { + act := action.MustNewUploadAction(session) + act.EnableTUS = true + if info, err := act.UploadBytes([]byte("HELLO WORLD!\n"), "/home/subdir/tests.txt"); err == nil { + log.Printf("Uploaded file: %s [%db] -- %s", info.Path, info.Size, info.Type) + } else { + log.Printf("Can't upload file: %v", err) + } + fmt.Println() + } + + // Try moving + { + act := action.MustNewFileOperationsAction(session) + if err := act.MoveTo("/home/subdir/tests.txt", "/home/sub2"); err == nil { + log.Println("Moved tests.txt around") + } else { + log.Println("Could not move tests.txt around") + } + fmt.Println() + } + + // Try listing and downloading + { + act := action.MustNewEnumFilesAction(session) + if files, err := act.ListFiles("/home", true); err == nil { + for _, info := range files { + log.Printf("%s [%db] -- %s", info.Path, info.Size, info.Type) + + // Download the file + actDl := action.MustNewDownloadAction(session) + if data, err := actDl.Download(info); err == nil { + log.Printf("Downloaded %d bytes for '%v'", len(data), info.Path) + } else { + log.Printf("Unable to download data for '%v': %v", info.Path, err) + } + + log.Println("---") + } + } else { + log.Printf("Can't list files: %v", err) + } + fmt.Println() + } + + // Try accessing some files and directories + { + act := action.MustNewFileOperationsAction(session) + if act.FileExists("/home/blargh.txt") { + log.Println("File '/home/blargh.txt' found") + } else { + log.Println("File '/home/blargh.txt' NOT found") + } + + if act.DirExists("/home") { + log.Println("Directory '/home' found") + } else { + log.Println("Directory '/home' NOT found") + } + fmt.Println() + } +} + +func main() { + session := sdk.MustNewSession() + if err := session.Initiate("sciencemesh-test.uni-muenster.de:9600", false); err != nil { + log.Fatalf("Can't initiate Reva session: %v", err) + } + + if methods, err := session.GetLoginMethods(); err == nil { + fmt.Println("Supported login methods:") + for _, m := range methods { + fmt.Printf("* %v\n", m) + } + fmt.Println() + } else { + log.Fatalf("Can't list login methods: %v", err) + } + + if err := session.BasicLogin("daniel", "danielpass"); err == nil { + log.Printf("Successfully logged into Reva (token=%v)", session.Token()) + fmt.Println() + runActions(session) + } else { + log.Fatalf("Can't log in to Reva: %v", err) + } +} diff --git a/go.sum b/go.sum index 4111ce49c8..7830ee4b46 100644 --- a/go.sum +++ b/go.sum @@ -88,8 +88,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20200929101248-821df597ec8d h1:YDnGz3eTIYQDXzJd/zefEsl0qbz/P63e8KWjSjYlb5Q= -github.com/cs3org/go-cs3apis v0.0.0-20200929101248-821df597ec8d/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20201007120910-416ed6cf8b00 h1:LVl25JaflluOchVvaHWtoCynm5OaM+VNai0IYkcCSe0= github.com/cs3org/go-cs3apis v0.0.0-20201007120910-416ed6cf8b00/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/sdk/action/action.go b/pkg/sdk/action/action.go new file mode 100644 index 0000000000..c84809738e --- /dev/null +++ b/pkg/sdk/action/action.go @@ -0,0 +1,38 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "fmt" + + "github.com/cs3org/reva/pkg/sdk" +) + +type action struct { + session *sdk.Session +} + +func (act *action) initAction(session *sdk.Session) error { + if !session.IsValid() { + return fmt.Errorf("no valid session provided") + } + act.session = session + + return nil +} diff --git a/pkg/sdk/action/action_test.go b/pkg/sdk/action/action_test.go new file mode 100644 index 0000000000..4052e99f89 --- /dev/null +++ b/pkg/sdk/action/action_test.go @@ -0,0 +1,114 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action_test + +import ( + "fmt" + "testing" + + "github.com/cs3org/reva/pkg/sdk/action" + testintl "github.com/cs3org/reva/pkg/sdk/common/testing" +) + +func TestActions(t *testing.T) { + tests := []struct { + host string + username string + password string + }{ + {"sciencemesh-test.uni-muenster.de:9600", "test", "testpass"}, + } + + for _, test := range tests { + t.Run(test.host, func(t *testing.T) { + // Prepare the session + if session, err := testintl.CreateTestSession(test.host, test.username, test.password); err == nil { + // Try creating a directory + if act, err := action.NewFileOperationsAction(session); err == nil { + if err := act.MakePath("/home/subdir/subsub"); err != nil { + t.Errorf(testintl.FormatTestError("FileOperationsAction.MakePath", err, "/home/subdir/subsub")) + } + } else { + t.Errorf(testintl.FormatTestError("NewFileOperationsAction", err, session)) + } + + // Try uploading + if act, err := action.NewUploadAction(session); err == nil { + act.EnableTUS = true + if _, err := act.UploadBytes([]byte("HELLO WORLD!\n"), "/home/subdir/tests.txt"); err != nil { + t.Errorf(testintl.FormatTestError("UploadAction.UploadBytes", err, []byte("HELLO WORLD!\n"), "/home/subdir/tests.txt")) + } + } else { + t.Errorf(testintl.FormatTestError("NewUploadAction", err, session)) + } + + // Try moving + if act, err := action.NewFileOperationsAction(session); err == nil { + if err := act.MoveTo("/home/subdir/tests.txt", "/home/subdir/subtest"); err != nil { + t.Errorf(testintl.FormatTestError("FileOperationsAction.MoveTo", err, "/home/subdir/tests.txt", "/home/subdir/subtest")) + } + } else { + t.Errorf(testintl.FormatTestError("NewFileOperationsAction", err, session)) + } + + // Try downloading + if act, err := action.NewDownloadAction(session); err == nil { + if _, err := act.DownloadFile("/home/subdir/subtest/tests.txt"); err != nil { + t.Errorf(testintl.FormatTestError("DownloadAction.DownloadFile", err, "/home/subdir/subtest/tests.txt")) + } + } else { + t.Errorf(testintl.FormatTestError("NewDownloadAction", err, session)) + } + + // Try listing + if act, err := action.NewEnumFilesAction(session); err == nil { + if _, err := act.ListFiles("/home", true); err != nil { + t.Errorf(testintl.FormatTestError("EnumFilesAction.ListFiles", err, "/home", true)) + } + } else { + t.Errorf(testintl.FormatTestError("NewEnumFilesAction", err, session)) + } + + // Try deleting a directory + if act, err := action.NewFileOperationsAction(session); err == nil { + if err := act.Remove("/home/subdir"); err != nil { + t.Errorf(testintl.FormatTestError("FileOperationsAction.Remove", err, "/home/subdir")) + } + } else { + t.Errorf(testintl.FormatTestError("NewFileOperationsAction", err, session)) + } + + // Try accessing some files and directories + if act, err := action.NewFileOperationsAction(session); err == nil { + if act.FileExists("/home/blargh.txt") { + t.Errorf(testintl.FormatTestError("FileOperationsAction.FileExists", fmt.Errorf("non-existing file reported as existing"), "/home/blargh.txt")) + } + + if !act.DirExists("/home") { + t.Errorf(testintl.FormatTestError("FileOperationsAction.DirExists", fmt.Errorf("/home dir reported as non-existing"), "/home")) + } + } else { + t.Errorf(testintl.FormatTestError("NewFileOperationsAction", err, session)) + } + } else { + t.Errorf(testintl.FormatTestError("CreateTestSession", err)) + } + }) + } +} diff --git a/pkg/sdk/action/download.go b/pkg/sdk/action/download.go new file mode 100644 index 0000000000..22293fbc6c --- /dev/null +++ b/pkg/sdk/action/download.go @@ -0,0 +1,117 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "fmt" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// DownloadAction is used to download files through Reva. +// WebDAV will be used automatically if the endpoint supports it. +type DownloadAction struct { + action +} + +// DownloadFile retrieves the data of the provided file path. +// The method first tries to retrieve information about the remote file by performing a "stat" on it. +func (action *DownloadAction) DownloadFile(path string) ([]byte, error) { + // Get the ResourceInfo object of the specified path + fileInfoAct := MustNewFileOperationsAction(action.session) + info, err := fileInfoAct.Stat(path) + if err != nil { + return nil, fmt.Errorf("the path '%v' was not found: %v", path, err) + } + + return action.Download(info) +} + +// Download retrieves the data of the provided resource. +func (action *DownloadAction) Download(fileInfo *storage.ResourceInfo) ([]byte, error) { + if fileInfo.Type != storage.ResourceType_RESOURCE_TYPE_FILE { + return nil, fmt.Errorf("resource is not a file") + } + + // Issue a file download request to Reva; this will provide the endpoint to read the file data from + download, err := action.initiateDownload(fileInfo) + if err != nil { + return nil, err + } + + // Try to get the file via WebDAV first + if client, values, err := net.NewWebDAVClientWithOpaque(download.DownloadEndpoint, download.Opaque); err == nil { + data, err := client.Read(values[net.WebDAVPathName]) + if err != nil { + return nil, fmt.Errorf("error while reading from '%v' via WebDAV: %v", download.DownloadEndpoint, err) + } + return data, nil + } + + // WebDAV is not supported, so directly read the HTTP endpoint + request, err := action.session.NewHTTPRequest(download.DownloadEndpoint, "GET", download.Token, nil) + if err != nil { + return nil, fmt.Errorf("unable to create an HTTP request for '%v': %v", download.DownloadEndpoint, err) + } + + data, err := request.Do(true) + if err != nil { + return nil, fmt.Errorf("error while reading from '%v' via HTTP: %v", download.DownloadEndpoint, err) + } + return data, nil +} + +func (action *DownloadAction) initiateDownload(fileInfo *storage.ResourceInfo) (*gateway.InitiateFileDownloadResponse, error) { + // Initiating a download request gets us the download endpoint for the specified resource + req := &provider.InitiateFileDownloadRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: fileInfo.Path, + }, + }, + } + res, err := action.session.Client().InitiateFileDownload(action.session.Context(), req) + if err := net.CheckRPCInvocation("initiating download", res, err); err != nil { + return nil, err + } + return res, nil +} + +// NewDownloadAction creates a new download action. +func NewDownloadAction(session *sdk.Session) (*DownloadAction, error) { + action := &DownloadAction{} + if err := action.initAction(session); err != nil { + return nil, fmt.Errorf("unable to create the DownloadAction: %v", err) + } + return action, nil +} + +// MustNewDownloadAction creates a new download action and panics on failure. +func MustNewDownloadAction(session *sdk.Session) *DownloadAction { + action, err := NewDownloadAction(session) + if err != nil { + panic(err) + } + return action +} diff --git a/pkg/sdk/action/enumfiles.go b/pkg/sdk/action/enumfiles.go new file mode 100644 index 0000000000..111cff4bcd --- /dev/null +++ b/pkg/sdk/action/enumfiles.go @@ -0,0 +1,117 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "fmt" + + storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// EnumFilesAction offers functions to enumerate files and directories. +type EnumFilesAction struct { + action +} + +// ListAll retrieves all files and directories contained in the provided path. +func (action *EnumFilesAction) ListAll(path string, includeSubdirectories bool) ([]*storage.ResourceInfo, error) { + ref := &storage.Reference{ + Spec: &storage.Reference_Path{Path: path}, + } + req := &storage.ListContainerRequest{Ref: ref} + res, err := action.session.Client().ListContainer(action.session.Context(), req) + if err := net.CheckRPCInvocation("listing container", res, err); err != nil { + return nil, err + } + + fileList := make([]*storage.ResourceInfo, 0, len(res.Infos)*64) + for _, fi := range res.Infos { + // Ignore resources that are neither files nor directories + if fi.Type <= storage.ResourceType_RESOURCE_TYPE_INVALID || fi.Type >= storage.ResourceType_RESOURCE_TYPE_INTERNAL { + continue + } + + fileList = append(fileList, fi) + + if fi.Type == storage.ResourceType_RESOURCE_TYPE_CONTAINER && includeSubdirectories { + subFileList, err := action.ListAll(fi.Path, includeSubdirectories) + if err != nil { + return nil, err + } + + fileList = append(fileList, subFileList...) + } + } + + return fileList, nil +} + +// ListAllWithFilter retrieves all files and directories that fulfill the provided predicate. +func (action *EnumFilesAction) ListAllWithFilter(path string, includeSubdirectories bool, filter func(*storage.ResourceInfo) bool) ([]*storage.ResourceInfo, error) { + all, err := action.ListAll(path, includeSubdirectories) + if err != nil { + return nil, err + } + + fileList := make([]*storage.ResourceInfo, 0, len(all)) + + for _, fi := range all { + // Add only those entries that fulfill the predicate + if filter(fi) { + fileList = append(fileList, fi) + } + } + + return fileList, nil +} + +// ListFiles retrieves all files contained in the provided path. +func (action *EnumFilesAction) ListFiles(path string, includeSubdirectories bool) ([]*storage.ResourceInfo, error) { + return action.ListAllWithFilter(path, includeSubdirectories, func(fi *storage.ResourceInfo) bool { + return fi.Type == storage.ResourceType_RESOURCE_TYPE_FILE || fi.Type == storage.ResourceType_RESOURCE_TYPE_SYMLINK + }) +} + +// ListDirs retrieves all directories contained in the provided path. +func (action *EnumFilesAction) ListDirs(path string, includeSubdirectories bool) ([]*storage.ResourceInfo, error) { + return action.ListAllWithFilter(path, includeSubdirectories, func(fi *storage.ResourceInfo) bool { + return fi.Type == storage.ResourceType_RESOURCE_TYPE_CONTAINER + }) +} + +// NewEnumFilesAction creates a new enum files action. +func NewEnumFilesAction(session *sdk.Session) (*EnumFilesAction, error) { + action := &EnumFilesAction{} + if err := action.initAction(session); err != nil { + return nil, fmt.Errorf("unable to create the EnumFilesAction: %v", err) + } + return action, nil +} + +// MustNewEnumFilesAction creates a new enum files action and panics on failure. +func MustNewEnumFilesAction(session *sdk.Session) *EnumFilesAction { + action, err := NewEnumFilesAction(session) + if err != nil { + panic(err) + } + return action +} diff --git a/pkg/sdk/action/fileops.go b/pkg/sdk/action/fileops.go new file mode 100644 index 0000000000..601227b1fa --- /dev/null +++ b/pkg/sdk/action/fileops.go @@ -0,0 +1,162 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "fmt" + p "path" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// FileOperationsAction offers basic file operations. +type FileOperationsAction struct { + action +} + +// Stat queries the file information of the specified remote resource. +func (action *FileOperationsAction) Stat(path string) (*storage.ResourceInfo, error) { + ref := &provider.Reference{ + Spec: &provider.Reference_Path{Path: path}, + } + req := &provider.StatRequest{Ref: ref} + res, err := action.session.Client().Stat(action.session.Context(), req) + if err := net.CheckRPCInvocation("querying resource information", res, err); err != nil { + return nil, err + } + return res.Info, nil +} + +// FileExists checks whether the specified file exists. +func (action *FileOperationsAction) FileExists(path string) bool { + // Stat the file and see if that succeeds; if so, check if the resource is indeed a file + info, err := action.Stat(path) + if err != nil { + return false + } + return info.Type == provider.ResourceType_RESOURCE_TYPE_FILE +} + +// DirExists checks whether the specified directory exists. +func (action *FileOperationsAction) DirExists(path string) bool { + // Stat the file and see if that succeeds; if so, check if the resource is indeed a directory + info, err := action.Stat(path) + if err != nil { + return false + } + return info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER +} + +// ResourceExists checks whether the specified resource exists (w/o checking for its actual type). +func (action *FileOperationsAction) ResourceExists(path string) bool { + // Stat the file and see if that succeeds + _, err := action.Stat(path) + return err == nil +} + +// MakePath creates the entire directory tree specified by the given path. +func (action *FileOperationsAction) MakePath(path string) error { + path = strings.TrimPrefix(path, "/") + + var curPath string + for _, token := range strings.Split(path, "/") { + curPath = p.Join(curPath, "/"+token) + + fileInfo, err := action.Stat(curPath) + if err != nil { // Stating failed, so the path probably doesn't exist yet + ref := &provider.Reference{ + Spec: &provider.Reference_Path{Path: curPath}, + } + req := &provider.CreateContainerRequest{Ref: ref} + res, err := action.session.Client().CreateContainer(action.session.Context(), req) + if err := net.CheckRPCInvocation("creating container", res, err); err != nil { + return err + } + } else if fileInfo.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER { + // The path exists, so make sure that is actually a directory + return fmt.Errorf("'%v' is not a directory", curPath) + } + } + + return nil +} + +// Move moves the specified source to a new location. The caller must ensure that the target directory exists. +func (action *FileOperationsAction) Move(source string, target string) error { + sourceRef := &provider.Reference{ + Spec: &provider.Reference_Path{Path: source}, + } + targetRef := &provider.Reference{ + Spec: &provider.Reference_Path{Path: target}, + } + req := &provider.MoveRequest{Source: sourceRef, Destination: targetRef} + res, err := action.session.Client().Move(action.session.Context(), req) + if err := net.CheckRPCInvocation("moving resource", res, err); err != nil { + return err + } + + return nil +} + +// MoveTo moves the specified source to the target directory, creating it if necessary. +func (action *FileOperationsAction) MoveTo(source string, path string) error { + if err := action.MakePath(path); err != nil { + return fmt.Errorf("unable to create the target directory '%v': %v", path, err) + } + + path = p.Join(path, p.Base(source)) // Keep the original resource base name + return action.Move(source, path) +} + +// Remove deletes the specified resource. +func (action *FileOperationsAction) Remove(path string) error { + ref := &provider.Reference{ + Spec: &provider.Reference_Path{Path: path}, + } + req := &provider.DeleteRequest{Ref: ref} + res, err := action.session.Client().Delete(action.session.Context(), req) + if err := net.CheckRPCInvocation("deleting resource", res, err); err != nil { + return err + } + + return nil +} + +// NewFileOperationsAction creates a new file operations action. +func NewFileOperationsAction(session *sdk.Session) (*FileOperationsAction, error) { + action := &FileOperationsAction{} + if err := action.initAction(session); err != nil { + return nil, fmt.Errorf("unable to create the FileOperationsAction: %v", err) + } + return action, nil +} + +// MustNewFileOperationsAction creates a new file operations action and panics on failure. +func MustNewFileOperationsAction(session *sdk.Session) *FileOperationsAction { + action, err := NewFileOperationsAction(session) + if err != nil { + panic(err) + } + return action +} diff --git a/pkg/sdk/action/upload.go b/pkg/sdk/action/upload.go new file mode 100644 index 0000000000..7e517cc93c --- /dev/null +++ b/pkg/sdk/action/upload.go @@ -0,0 +1,199 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package action + +import ( + "bytes" + "fmt" + "io" + "math" + "os" + p "path" + "strconv" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + + "github.com/cs3org/reva/pkg/sdk" + "github.com/cs3org/reva/pkg/sdk/common" + "github.com/cs3org/reva/pkg/sdk/common/crypto" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// UploadAction is used to upload files through Reva. +// WebDAV will be used automatically if the endpoint supports it. The EnableTUS flag specifies whether to use TUS if WebDAV is not supported. +type UploadAction struct { + action + + EnableTUS bool +} + +// UploadFile uploads the provided file to the target. +func (action *UploadAction) UploadFile(file *os.File, target string) (*storage.ResourceInfo, error) { + fileInfo, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("unable to stat the specified file: %v", err) + } + + return action.upload(file, fileInfo, target) +} + +// UploadFileTo uploads the provided file to the target directory, keeping the original file name. +func (action *UploadAction) UploadFileTo(file *os.File, path string) (*storage.ResourceInfo, error) { + return action.UploadFile(file, p.Join(path, p.Base(file.Name()))) +} + +// UploadBytes uploads the provided byte data to the target. +func (action *UploadAction) UploadBytes(data []byte, target string) (*storage.ResourceInfo, error) { + return action.Upload(bytes.NewReader(data), int64(len(data)), target) +} + +// Upload uploads data from the provided reader to the target. +func (action *UploadAction) Upload(data io.Reader, size int64, target string) (*storage.ResourceInfo, error) { + dataDesc := common.CreateDataDescriptor(p.Base(target), size) + return action.upload(data, &dataDesc, target) +} + +func (action *UploadAction) upload(data io.Reader, dataInfo os.FileInfo, target string) (*storage.ResourceInfo, error) { + fileOpsAct := MustNewFileOperationsAction(action.session) + + dir := p.Dir(target) + if err := fileOpsAct.MakePath(dir); err != nil { + return nil, fmt.Errorf("unable to create target directory '%v': %v", dir, err) + } + + // Issue a file upload request to Reva; this will provide the endpoint to write the file data to + upload, err := action.initiateUpload(target, dataInfo.Size()) + if err != nil { + return nil, err + } + + // Try to upload the file via WebDAV first + if client, values, err := net.NewWebDAVClientWithOpaque(upload.UploadEndpoint, upload.Opaque); err == nil { + if err := client.Write(values[net.WebDAVPathName], data, dataInfo.Size()); err != nil { + return nil, fmt.Errorf("error while writing to '%v' via WebDAV: %v", upload.UploadEndpoint, err) + } + } else { + // WebDAV is not supported, so directly write to the HTTP endpoint + checksumType := action.selectChecksumType(upload.AvailableChecksums) + checksumTypeName := crypto.GetChecksumTypeName(checksumType) + checksum, err := crypto.ComputeChecksum(checksumType, data) + if err != nil { + return nil, fmt.Errorf("unable to compute data checksum: %v", err) + } + + // Check if the data object can be seeked; if so, reset it to its beginning + if seeker, ok := data.(io.Seeker); ok { + _, _ = seeker.Seek(0, 0) + } + + if action.EnableTUS { + if err := action.uploadFileTUS(upload, target, data, dataInfo, checksum, checksumTypeName); err != nil { + return nil, fmt.Errorf("error while writing to '%v' via TUS: %v", upload.UploadEndpoint, err) + } + } else { + if err := action.uploadFilePUT(upload, data, checksum, checksumTypeName); err != nil { + return nil, fmt.Errorf("error while writing to '%v' via HTTP: %v", upload.UploadEndpoint, err) + } + } + } + + // Return information about the just-uploaded file + return fileOpsAct.Stat(target) +} + +func (action *UploadAction) initiateUpload(target string, size int64) (*gateway.InitiateFileUploadResponse, error) { + // Initiating an upload request gets us the upload endpoint for the specified target + req := &provider.InitiateFileUploadRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: target, + }, + }, + Opaque: &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "Upload-Length": { + Decoder: "plain", + Value: []byte(strconv.FormatInt(size, 10)), + }, + }, + }, + } + res, err := action.session.Client().InitiateFileUpload(action.session.Context(), req) + if err := net.CheckRPCInvocation("initiating upload", res, err); err != nil { + return nil, err + } + + return res, nil +} + +func (action *UploadAction) selectChecksumType(checksumTypes []*provider.ResourceChecksumPriority) provider.ResourceChecksumType { + var selChecksumType provider.ResourceChecksumType + var maxPrio uint32 = math.MaxUint32 + for _, xs := range checksumTypes { + if xs.Priority < maxPrio { + maxPrio = xs.Priority + selChecksumType = xs.Type + } + } + return selChecksumType +} + +func (action *UploadAction) uploadFilePUT(upload *gateway.InitiateFileUploadResponse, data io.Reader, checksum string, checksumType string) error { + request, err := action.session.NewHTTPRequest(upload.UploadEndpoint, "PUT", upload.Token, data) + if err != nil { + return fmt.Errorf("unable to create HTTP request for '%v': %v", upload.UploadEndpoint, err) + } + + request.AddParameters(map[string]string{ + "xs": checksum, + "xs_type": checksumType, + }) + + _, err = request.Do(true) + return err +} + +func (action *UploadAction) uploadFileTUS(upload *gateway.InitiateFileUploadResponse, target string, data io.Reader, fileInfo os.FileInfo, checksum string, checksumType string) error { + tusClient, err := net.NewTUSClient(upload.UploadEndpoint, action.session.Token(), upload.Token) + if err != nil { + return fmt.Errorf("unable to create TUS client: %v", err) + } + return tusClient.Write(data, target, fileInfo, checksumType, checksum) +} + +// NewUploadAction creates a new upload action. +func NewUploadAction(session *sdk.Session) (*UploadAction, error) { + action := &UploadAction{} + if err := action.initAction(session); err != nil { + return nil, fmt.Errorf("unable to create the UploadAction: %v", err) + } + return action, nil +} + +// MustNewUploadAction creates a new upload action and panics on failure. +func MustNewUploadAction(session *sdk.Session) *UploadAction { + action, err := NewUploadAction(session) + if err != nil { + panic(err) + } + return action +} diff --git a/pkg/sdk/common/common_test.go b/pkg/sdk/common/common_test.go new file mode 100644 index 0000000000..7736be5332 --- /dev/null +++ b/pkg/sdk/common/common_test.go @@ -0,0 +1,182 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package common_test + +import ( + "fmt" + "testing" + "time" + + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + + "github.com/cs3org/reva/pkg/sdk/common" + testintl "github.com/cs3org/reva/pkg/sdk/common/testing" +) + +func TestDataDescriptor(t *testing.T) { + const name = "DATA_DESC" + const size = 42 + + dataDesc := common.CreateDataDescriptor(name, size) + now := time.Now().Round(time.Millisecond) + if v := dataDesc.Name(); v != name { + t.Errorf(testintl.FormatTestResult("DataDescriptor.Name", name, v)) + } + if v := dataDesc.Size(); v != size { + t.Errorf(testintl.FormatTestResult("DataDescriptor.Size", size, v)) + } + if v := dataDesc.Mode(); v != 0700 { + t.Errorf(testintl.FormatTestResult("DataDescriptor.Mode", 0700, v)) + } + if v := dataDesc.IsDir(); v != false { + t.Errorf(testintl.FormatTestResult("DataDescriptor.IsDir", false, v)) + } + if v := dataDesc.ModTime(); !v.Round(time.Millisecond).Equal(now) { + // Since there's always a slight chance that the rounded times won't match, just log this mismatch + t.Logf(testintl.FormatTestResult("DataDescriptor.ModTime", now, v)) + } + if v := dataDesc.Sys(); v != nil { + t.Errorf(testintl.FormatTestResult("DataDescriptor.Sys", nil, v)) + } +} + +func TestFindString(t *testing.T) { + tests := []struct { + input []string + needle string + wants int + }{ + {[]string{}, "so empty", -1}, + {[]string{"12345", "hello", "goodbye"}, "hello", 1}, + {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND"}, "Wüste", 2}, + {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND", "Sandiger Grund"}, "Sandiger Grund", 4}, + {[]string{"Nachahmer", "Roger", "k thx bye"}, "thx", -1}, + {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet Under", 0}, + {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet UNDER", -1}, + } + + for _, test := range tests { + found := common.FindString(test.input, test.needle) + if found != test.wants { + t.Errorf(testintl.FormatTestResult("FindString", test.wants, found, test.input, test.needle)) + } + } +} + +func TestFindStringNoCase(t *testing.T) { + tests := []struct { + input []string + needle string + wants int + }{ + {[]string{}, "so empty", -1}, + {[]string{"12345", "hello", "goodbye"}, "hello", 1}, + {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND"}, "Wüste", 2}, + {[]string{"Rudimentär", "Ich bin du", "Wüste", "SANDIGER GRUND", "Sandiger Grund"}, "Sandiger Grund", 3}, + {[]string{"Nachahmer", "Roger", "k thx bye"}, "thx", -1}, + {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet Under", 0}, + {[]string{"Six Feet Under", "Rock&Roll", "k thx bye"}, "Six Feet UNDER", 0}, + } + + for _, test := range tests { + found := common.FindStringNoCase(test.input, test.needle) + if found != test.wants { + t.Errorf(testintl.FormatTestResult("FindString", test.wants, found, test.input, test.needle)) + } + } +} + +func TestDecodeOpaqueMap(t *testing.T) { + opaque := types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "magic": { + Decoder: "plain", + Value: []byte("42"), + }, + "json": { + Decoder: "json", + Value: []byte("[]"), + }, + }, + } + + tests := []struct { + key string + wants string + shouldSucceed bool + }{ + {"magic", "42", true}, + {"json", "[]", false}, + {"somekey", "", false}, + } + + decodedMap := common.DecodeOpaqueMap(&opaque) + for _, test := range tests { + value, ok := decodedMap[test.key] + if ok == test.shouldSucceed { + if ok { + if value != test.wants { + t.Errorf(testintl.FormatTestResult("DecodeOpaqueMap", test.wants, value, opaque)) + } + } + } else { + t.Errorf(testintl.FormatTestResult("DecodeOpaqueMap", test.shouldSucceed, ok, opaque)) + } + } +} + +func TestGetValuesFromOpaque(t *testing.T) { + opaque := types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "magic": { + Decoder: "plain", + Value: []byte("42"), + }, + "stuff": { + Decoder: "plain", + Value: []byte("Some stuff"), + }, + "json": { + Decoder: "json", + Value: []byte("[]"), + }, + }, + } + + tests := []struct { + keys []string + mandatory bool + shouldSucceed bool + }{ + {[]string{"magic", "stuff"}, true, true}, + {[]string{"magic", "stuff", "json"}, false, true}, + {[]string{"magic", "stuff", "json"}, true, false}, + {[]string{"notfound"}, false, true}, + {[]string{"notfound"}, true, false}, + } + + for _, test := range tests { + _, err := common.GetValuesFromOpaque(&opaque, test.keys, test.mandatory) + if err != nil && test.shouldSucceed { + t.Errorf(testintl.FormatTestError("GetValuesFromOpaque", err, opaque, test.keys, test.mandatory)) + } else if err == nil && !test.shouldSucceed { + t.Errorf(testintl.FormatTestError("GetValuesFromOpaque", fmt.Errorf("getting values from an invalid opaque succeeded"), opaque, test.keys, test.mandatory)) + } + } +} diff --git a/pkg/sdk/common/crypto/crypto.go b/pkg/sdk/common/crypto/crypto.go new file mode 100644 index 0000000000..31d7d54e60 --- /dev/null +++ b/pkg/sdk/common/crypto/crypto.go @@ -0,0 +1,50 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package crypto + +import ( + "fmt" + "io" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/internal/grpc/services/storageprovider" + "github.com/cs3org/reva/pkg/crypto" +) + +// ComputeChecksum calculates the checksum of the given data using the specified checksum type. +func ComputeChecksum(checksumType provider.ResourceChecksumType, data io.Reader) (string, error) { + switch checksumType { + case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32: + return crypto.ComputeAdler32XS(data) + case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5: + return crypto.ComputeMD5XS(data) + case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_SHA1: + return crypto.ComputeSHA1XS(data) + case provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET: + return "", nil + default: + return "", fmt.Errorf("invalid checksum type: %s", checksumType) + } +} + +// GetChecksumTypeName returns a stringified name of the given checksum type. +func GetChecksumTypeName(checksumType provider.ResourceChecksumType) string { + return string(storageprovider.GRPC2PKGXS(checksumType)) +} diff --git a/pkg/sdk/common/crypto/crypto_test.go b/pkg/sdk/common/crypto/crypto_test.go new file mode 100644 index 0000000000..e5f3c5d91b --- /dev/null +++ b/pkg/sdk/common/crypto/crypto_test.go @@ -0,0 +1,76 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package crypto_test + +import ( + "fmt" + "strings" + "testing" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/pkg/sdk/common/crypto" + testintl "github.com/cs3org/reva/pkg/sdk/common/testing" +) + +func TestComputeChecksum(t *testing.T) { + tests := map[string]struct { + checksumType provider.ResourceChecksumType + input string + wants string + }{ + "Unset": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET, "Hello World!", ""}, + "Adler32": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32, "Hello World!", "1c49043e"}, + "SHA1": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_SHA1, "Hello World!", "2ef7bde608ce5404e97d5f042f95f89f1c232871"}, + "MD5": {provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5, "Hello World!", "ed076287532e86365e841e92bfc50d8c"}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if checksum, err := crypto.ComputeChecksum(test.checksumType, strings.NewReader(test.input)); err == nil { + if checksum != test.wants { + t.Errorf(testintl.FormatTestResult("ComputeChecksum", test.wants, checksum, test.checksumType, test.input)) + } + } else { + t.Errorf(testintl.FormatTestError("ComputeChecksum", err)) + } + }) + } + + // Check how ComputeChecksum reacts to an invalid checksum type + if _, err := crypto.ComputeChecksum(provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, nil); err == nil { + t.Errorf(testintl.FormatTestError("ComputeChecksum", fmt.Errorf("accepted an invalid checksum type w/o erring"), provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID, nil)) + } +} + +func TestGetChecksumTypeName(t *testing.T) { + tests := map[provider.ResourceChecksumType]string{ + provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET: "unset", + provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_SHA1: "sha1", + provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_ADLER32: "adler32", + provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5: "md5", + provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID: "invalid", + } + + for input, wants := range tests { + if got := crypto.GetChecksumTypeName(input); got != wants { + t.Errorf(testintl.FormatTestResult("GetChecksumTypeName", wants, got, input)) + } + } +} diff --git a/pkg/sdk/common/datadesc.go b/pkg/sdk/common/datadesc.go new file mode 100644 index 0000000000..85262a33c1 --- /dev/null +++ b/pkg/sdk/common/datadesc.go @@ -0,0 +1,69 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package common + +import ( + "os" + "time" +) + +// DataDescriptor implements os.FileInfo to provide file information for non-file data objects. +// This is used, for example, when uploading data that doesn't come from a local file. +type DataDescriptor struct { + name string + size int64 +} + +// Name returns the quasi-filename of this object. +func (ddesc *DataDescriptor) Name() string { + return ddesc.name +} + +// Size returns the specified data size. +func (ddesc *DataDescriptor) Size() int64 { + return ddesc.size +} + +// Mode always returns a 0700 file mode. +func (ddesc *DataDescriptor) Mode() os.FileMode { + return 0700 +} + +// ModTime always returns the current time as the modification time. +func (ddesc *DataDescriptor) ModTime() time.Time { + return time.Now() +} + +// IsDir always returns false. +func (ddesc *DataDescriptor) IsDir() bool { + return false +} + +// Sys returns nil, as this object doesn't represent a system object. +func (ddesc *DataDescriptor) Sys() interface{} { + return nil +} + +// CreateDataDescriptor creates a new descriptor for non-file data objects. +func CreateDataDescriptor(name string, size int64) DataDescriptor { + return DataDescriptor{ + name: name, + size: size, + } +} diff --git a/pkg/sdk/common/net/httpreq.go b/pkg/sdk/common/net/httpreq.go new file mode 100644 index 0000000000..0edcda82c7 --- /dev/null +++ b/pkg/sdk/common/net/httpreq.go @@ -0,0 +1,110 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "time" +) + +// HTTPRequest performs Reva-specific requests through an HTTP endpoint. +type HTTPRequest struct { + endpoint string + data io.Reader + + client *http.Client + request *http.Request +} + +func (request *HTTPRequest) initRequest(ctx context.Context, endpoint string, method string, accessToken string, transportToken string, data io.Reader) error { + request.endpoint = endpoint + request.data = data + + // Initialize the HTTP client + request.client = &http.Client{ + Timeout: time.Duration(24 * int64(time.Hour)), + } + + // Initialize the HTTP request + httpReq, err := http.NewRequestWithContext(ctx, method, endpoint, data) + if err != nil { + return fmt.Errorf("unable to create the HTTP request: %v", err) + } + request.request = httpReq + + // Set mandatory header values + request.request.Header.Set(AccessTokenName, accessToken) + request.request.Header.Set(TransportTokenName, transportToken) + + return nil +} + +func (request *HTTPRequest) do() (*http.Response, error) { + httpRes, err := request.client.Do(request.request) + if err != nil { + return nil, fmt.Errorf("unable to do the HTTP request: %v", err) + } + if httpRes.StatusCode != http.StatusOK { + return nil, fmt.Errorf("performing the HTTP request failed: %v", httpRes.Status) + } + return httpRes, nil +} + +// AddParameters adds the specified parameters to the request. +// The parameters are passed in the query URL. +func (request *HTTPRequest) AddParameters(params map[string]string) { + query := request.request.URL.Query() + for k, v := range params { + query.Add(k, v) + } + request.request.URL.RawQuery = query.Encode() +} + +// Do performs the request on the HTTP endpoint and returns the body data. +// If checkStatus is set to true, the call will only succeed if the server returns a status code of 200. +func (request *HTTPRequest) Do(checkStatus bool) ([]byte, error) { + httpRes, err := request.do() + if err != nil { + return nil, fmt.Errorf("unable to perform the HTTP request for '%v': %v", request.endpoint, err) + } + defer httpRes.Body.Close() + + if checkStatus && httpRes.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received invalid response from '%v': %s", request.endpoint, httpRes.Status) + } + + data, err := ioutil.ReadAll(httpRes.Body) + if err != nil { + return nil, fmt.Errorf("reading response data from '%v' failed: %v", request.endpoint, err) + } + return data, nil +} + +// NewHTTPRequest creates a new HTTP request. +func NewHTTPRequest(ctx context.Context, endpoint string, method string, accessToken string, transportToken string, data io.Reader) (*HTTPRequest, error) { + request := &HTTPRequest{} + if err := request.initRequest(ctx, endpoint, method, accessToken, transportToken, data); err != nil { + return nil, fmt.Errorf("unable to initialize the HTTP request: %v", err) + } + return request, nil +} diff --git a/pkg/sdk/common/net/net.go b/pkg/sdk/common/net/net.go new file mode 100644 index 0000000000..2de9ac83b9 --- /dev/null +++ b/pkg/sdk/common/net/net.go @@ -0,0 +1,35 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import ( + "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cs3org/reva/pkg/token" +) + +type ctxKey int + +const ( + // AccessTokenIndex specifies the index of the Reva access token in a context. + AccessTokenIndex ctxKey = iota + // AccessTokenName specifies the name of the Reva access token used during requests. + AccessTokenName = token.TokenHeader + // TransportTokenName specifies the name of the Reva transport token used during data transfers. + TransportTokenName = datagateway.TokenTransportHeader +) diff --git a/pkg/sdk/common/net/net_test.go b/pkg/sdk/common/net/net_test.go new file mode 100644 index 0000000000..989a1f093e --- /dev/null +++ b/pkg/sdk/common/net/net_test.go @@ -0,0 +1,159 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net_test + +import ( + "fmt" + "strings" + "testing" + + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/cs3org/reva/pkg/sdk/common" + "github.com/cs3org/reva/pkg/sdk/common/crypto" + "github.com/cs3org/reva/pkg/sdk/common/net" + testintl "github.com/cs3org/reva/pkg/sdk/common/testing" +) + +type rpcStatusTest struct { + status rpc.Code +} + +func (r *rpcStatusTest) GetStatus() *rpc.Status { + return &rpc.Status{ + Code: r.status, + } +} + +func TestCheckRPCInvocation(t *testing.T) { + tests := []struct { + operation string + status rpcStatusTest + shouldSucceed bool + callError error + }{ + {"ok-check", rpcStatusTest{rpc.Code_CODE_OK}, true, nil}, + {"fail-status", rpcStatusTest{rpc.Code_CODE_NOT_FOUND}, false, nil}, + {"fail-err", rpcStatusTest{rpc.Code_CODE_OK}, false, fmt.Errorf("failed")}, + } + + for _, test := range tests { + err := net.CheckRPCInvocation(test.operation, &test.status, test.callError) + if err != nil && test.shouldSucceed { + t.Errorf(testintl.FormatTestError("CheckRPCInvocation", err, test.operation, test.status, test.callError)) + } else if err == nil && !test.shouldSucceed { + t.Errorf(testintl.FormatTestError("CheckRPCInvocation", fmt.Errorf("accepted an invalid RPC invocation"), test.operation, test.status, test.callError)) + } + } +} + +func TestTUSClient(t *testing.T) { + tests := []struct { + endpoint string + shouldSucceed bool + }{ + {"https://tusd.tusdemo.net/files/", true}, + {"https://google.de", false}, + } + + for _, test := range tests { + t.Run(test.endpoint, func(t *testing.T) { + if client, err := net.NewTUSClient(test.endpoint, "", ""); err == nil { + data := strings.NewReader("This is a simple TUS test") + dataDesc := common.CreateDataDescriptor("tus-test.txt", data.Size()) + checksumTypeName := crypto.GetChecksumTypeName(provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5) + + if err := client.Write(data, dataDesc.Name(), &dataDesc, checksumTypeName, ""); err != nil && test.shouldSucceed { + t.Errorf(testintl.FormatTestError("TUSClient.Write", err, data, dataDesc.Name(), &dataDesc, checksumTypeName, "")) + } else if err == nil && !test.shouldSucceed { + t.Errorf(testintl.FormatTestError("TUSClient.Write", fmt.Errorf("writing to a non-TUS host succeeded"), data, dataDesc.Name(), &dataDesc, checksumTypeName, "")) + } + } else { + t.Errorf(testintl.FormatTestError("NewTUSClient", err, test.endpoint, "", "")) + } + }) + } +} + +func TestWebDAVClient(t *testing.T) { + tests := []struct { + endpoint string + shouldSucceed bool + }{ + {"https://zivowncloud2.uni-muenster.de/owncloud/remote.php/dav/files/testUser/", true}, + {"https://google.de", false}, + } + + for _, test := range tests { + t.Run(test.endpoint, func(t *testing.T) { + if client, err := net.NewWebDAVClient(test.endpoint, "testUser", "test12345"); err == nil { + const fileName = "webdav-test.txt" + + data := strings.NewReader("This is a simple WebDAV test") + if err := client.Write(fileName, data, data.Size()); err == nil { + if test.shouldSucceed { + if _, err := client.Read(fileName); err != nil { + t.Errorf(testintl.FormatTestError("WebDAVClient.Read", err)) + } + + if err := client.Remove(fileName); err != nil { + t.Errorf(testintl.FormatTestError("WebDAVClient.Remove", err)) + } + } else { + t.Errorf(testintl.FormatTestError("WebDAVClient.Write", fmt.Errorf("writing to a non-WebDAV host succeeded"), fileName, data, data.Size())) + } + } else if test.shouldSucceed { + t.Errorf(testintl.FormatTestError("WebDAVClient.Write", err, fileName, data, data.Size())) + } + } else { + t.Errorf(testintl.FormatTestError("NewWebDavClient", err, test.endpoint, "testUser", "test12345")) + } + }) + } +} + +func TestHTTPRequest(t *testing.T) { + tests := []struct { + url string + shouldSucceed bool + }{ + {"https://google.de", true}, + {"https://ujhwrgobniwoeo.de", false}, + } + + // Prepare the session + if session, err := testintl.CreateTestSession("sciencemesh-test.uni-muenster.de:9600", "test", "testpass"); err == nil { + for _, test := range tests { + t.Run(test.url, func(t *testing.T) { + if request, err := session.NewHTTPRequest(test.url, "GET", "", nil); err == nil { + if _, err := request.Do(true); err != nil && test.shouldSucceed { + t.Errorf(testintl.FormatTestError("HTTPRequest.Do", err)) + } else if err == nil && !test.shouldSucceed { + t.Errorf(testintl.FormatTestError("HTTPRequest.Do", fmt.Errorf("send request to an invalid host succeeded"))) + } + } else { + t.Errorf(testintl.FormatTestError("Session.NewHTTPRequest", err, test.url, "GET", "", nil)) + } + }) + } + } else { + t.Errorf(testintl.FormatTestError("CreateTestSession", err)) + } +} diff --git a/pkg/sdk/common/net/rpc.go b/pkg/sdk/common/net/rpc.go new file mode 100644 index 0000000000..c8dcc35bee --- /dev/null +++ b/pkg/sdk/common/net/rpc.go @@ -0,0 +1,49 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import ( + "fmt" + + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" +) + +type rpcStatusGetter interface { + GetStatus() *rpc.Status +} + +// CheckRPCInvocation checks if an RPC invocation has succeeded. +// For this, the error from the original call is first checked; after that, the actual RPC response status is checked. +func CheckRPCInvocation(operation string, res rpcStatusGetter, callErr error) error { + if callErr != nil { + return fmt.Errorf("%s: %v", operation, callErr) + } + + return CheckRPCStatus(operation, res) +} + +// CheckRPCStatus checks the returned status of an RPC call. +func CheckRPCStatus(operation string, res rpcStatusGetter) error { + status := res.GetStatus() + if status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s: %q (code=%+v, trace=%q)", operation, status.Message, status.Code, status.Trace) + } + + return nil +} diff --git a/pkg/sdk/common/net/tus.go b/pkg/sdk/common/net/tus.go new file mode 100644 index 0000000000..09d96e4892 --- /dev/null +++ b/pkg/sdk/common/net/tus.go @@ -0,0 +1,134 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this filePath except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/eventials/go-tus" + "github.com/eventials/go-tus/memorystore" + + "github.com/cs3org/reva/pkg/sdk/common" +) + +// TUSClient is a simple client wrapper for uploading files via TUS. +type TUSClient struct { + config *tus.Config + client *tus.Client + + supportsResourceCreation bool +} + +func (client *TUSClient) initClient(endpoint string, accessToken string, transportToken string) error { + // Create the TUS configuration + client.config = tus.DefaultConfig() + client.config.Resume = true + + memStore, err := memorystore.NewMemoryStore() + if err != nil { + return fmt.Errorf("unable to create a TUS memory store: %v", err) + } + client.config.Store = memStore + + if accessToken != "" { + client.config.Header.Add(AccessTokenName, accessToken) + } + + if transportToken != "" { + client.config.Header.Add(TransportTokenName, transportToken) + } + + // Create the TUS client + tusClient, err := tus.NewClient(endpoint, client.config) + if err != nil { + return fmt.Errorf("error creating the TUS client: %v", err) + } + client.client = tusClient + + // Check if the TUS server supports resource creation + client.supportsResourceCreation = client.checkEndpointCreationOption(endpoint) + + return nil +} + +func (client *TUSClient) checkEndpointCreationOption(endpoint string) bool { + // Perform an OPTIONS request to the endpoint; if this succeeds, check if the header "Tus-Extension" contains the "creation" flag + httpClient := &http.Client{ + Timeout: time.Duration(1.5 * float64(time.Second)), + } + + if httpReq, err := http.NewRequest("OPTIONS", endpoint, nil); err == nil { + if res, err := httpClient.Do(httpReq); err == nil { + defer res.Body.Close() + + if res.StatusCode == http.StatusOK { + ext := strings.Split(res.Header.Get("Tus-Extension"), ",") + return common.FindStringNoCase(ext, "creation") != -1 + } + } + } + + return false +} + +// Write writes the provided data to the endpoint. +// The target is used as the filename on the remote site. The file information and checksum are used to create a fingerprint. +func (client *TUSClient) Write(data io.Reader, target string, fileInfo os.FileInfo, checksumType string, checksum string) error { + metadata := map[string]string{ + "filename": path.Base(target), + "dir": path.Dir(target), + "checksum": fmt.Sprintf("%s %s", checksumType, checksum), + } + fingerprint := fmt.Sprintf("%s-%d-%s-%s", path.Base(target), fileInfo.Size(), fileInfo.ModTime(), checksum) + + upload := tus.NewUpload(data, fileInfo.Size(), metadata, fingerprint) + client.config.Store.Set(upload.Fingerprint, client.client.Url) + + var uploader *tus.Uploader + if client.supportsResourceCreation { + upldr, err := client.client.CreateUpload(upload) + if err != nil { + return fmt.Errorf("unable to perform the TUS resource creation for '%v': %v", client.client.Url, err) + } + uploader = upldr + } else { + uploader = tus.NewUploader(client.client, client.client.Url, upload, 0) + } + + if err := uploader.Upload(); err != nil { + return fmt.Errorf("unable to perform the TUS upload for '%v': %v", client.client.Url, err) + } + + return nil +} + +// NewTUSClient creates a new TUS client. +func NewTUSClient(endpoint string, accessToken string, transportToken string) (*TUSClient, error) { + client := &TUSClient{} + if err := client.initClient(endpoint, accessToken, transportToken); err != nil { + return nil, fmt.Errorf("unable to create the TUS client: %v", err) + } + return client, nil +} diff --git a/pkg/sdk/common/net/webdav.go b/pkg/sdk/common/net/webdav.go new file mode 100644 index 0000000000..63cd2289ec --- /dev/null +++ b/pkg/sdk/common/net/webdav.go @@ -0,0 +1,121 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this filePath except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import ( + "fmt" + "io" + "io/ioutil" + "strconv" + + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/studio-b12/gowebdav" + + "github.com/cs3org/reva/pkg/sdk/common" +) + +const ( + // WebDAVTokenName is the header name of the WebDAV token. + WebDAVTokenName = "webdav-token" + // WebDAVPathName is the header name of the WebDAV file path. + WebDAVPathName = "webdav-file-path" +) + +// WebDAVClient is a simple client wrapper for down- and uploading files via WebDAV. +type WebDAVClient struct { + client *gowebdav.Client +} + +func (webdav *WebDAVClient) initClient(endpoint string, userName string, password string, accessToken string) error { + // Create the WebDAV client + webdav.client = gowebdav.NewClient(endpoint, userName, password) + + if accessToken != "" { + webdav.client.SetHeader(AccessTokenName, accessToken) + } + + return nil +} + +// Read reads all data of the specified remote file. +func (webdav *WebDAVClient) Read(file string) ([]byte, error) { + reader, err := webdav.client.ReadStream(file) + if err != nil { + return nil, fmt.Errorf("unable to create reader: %v", err) + } + defer reader.Close() + + data, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("unable to read the data: %v", err) + } + return data, nil +} + +// Write writes data to the specified remote file. +func (webdav *WebDAVClient) Write(file string, data io.Reader, size int64) error { + webdav.client.SetHeader("Upload-Length", strconv.FormatInt(size, 10)) + + if err := webdav.client.WriteStream(file, data, 0700); err != nil { + return fmt.Errorf("unable to write the data: %v", err) + } + + return nil +} + +// Remove deletes the entire file/path. +func (webdav *WebDAVClient) Remove(path string) error { + if err := webdav.client.Remove(path); err != nil { + return fmt.Errorf("error removing '%v' :%v", path, err) + } + + return nil +} + +func newWebDAVClient(endpoint string, userName string, password string, accessToken string) (*WebDAVClient, error) { + client := &WebDAVClient{} + if err := client.initClient(endpoint, userName, password, accessToken); err != nil { + return nil, fmt.Errorf("unable to create the WebDAV client: %v", err) + } + return client, nil +} + +// NewWebDAVClientWithAccessToken creates a new WebDAV client using an access token. +func NewWebDAVClientWithAccessToken(endpoint string, accessToken string) (*WebDAVClient, error) { + return newWebDAVClient(endpoint, "", "", accessToken) +} + +// NewWebDAVClientWithOpaque creates a new WebDAV client using the information stored in the opaque. +func NewWebDAVClientWithOpaque(endpoint string, opaque *types.Opaque) (*WebDAVClient, map[string]string, error) { + values, err := common.GetValuesFromOpaque(opaque, []string{WebDAVTokenName, WebDAVPathName}, true) + if err != nil { + return nil, nil, fmt.Errorf("invalid opaque object: %v", err) + } + + client, err := NewWebDAVClientWithAccessToken(endpoint, values[WebDAVTokenName]) + if err != nil { + return nil, nil, err + } + return client, values, nil +} + +// NewWebDAVClient creates a new WebDAV client with user credentials. +func NewWebDAVClient(endpoint string, userName string, password string) (*WebDAVClient, error) { + return newWebDAVClient(endpoint, userName, password, "") +} diff --git a/pkg/sdk/common/opaque.go b/pkg/sdk/common/opaque.go new file mode 100644 index 0000000000..7ce3ce7685 --- /dev/null +++ b/pkg/sdk/common/opaque.go @@ -0,0 +1,59 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package common + +import ( + "fmt" + + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" +) + +// DecodeOpaqueMap decodes a Reva opaque object into a map of strings. +// Only plain decoders are currently supported. +func DecodeOpaqueMap(opaque *types.Opaque) map[string]string { + entries := make(map[string]string) + + if opaque != nil { + for k, v := range opaque.GetMap() { + // Only plain values are currently supported + if v.Decoder == "plain" { + entries[k] = string(v.Value) + } + } + } + + return entries +} + +// GetValuesFromOpaque extracts the given keys from the opaque object. +// If mandatory is set to true, all specified keys must be available in the opaque object. +func GetValuesFromOpaque(opaque *types.Opaque, keys []string, mandatory bool) (map[string]string, error) { + values := make(map[string]string) + entries := DecodeOpaqueMap(opaque) + + for _, key := range keys { + if value, ok := entries[key]; ok { + values[key] = value + } else if mandatory { + return map[string]string{}, fmt.Errorf("missing opaque entry '%v'", key) + } + } + + return values, nil +} diff --git a/pkg/sdk/common/testing/testing.go b/pkg/sdk/common/testing/testing.go new file mode 100644 index 0000000000..5a83c45c31 --- /dev/null +++ b/pkg/sdk/common/testing/testing.go @@ -0,0 +1,62 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package testing + +import ( + "fmt" + "strings" + + "github.com/cs3org/reva/pkg/sdk" +) + +func formatTestMessage(funcName string, msg string, params ...interface{}) string { + // Format parameter list + paramList := make([]string, 0, len(params)) + for _, param := range params { + paramList = append(paramList, fmt.Sprintf("%#v", param)) + } + + return fmt.Sprintf("%s(%s) -> %s", funcName, strings.Join(paramList, ", "), msg) +} + +// FormatTestResult pretty-formats a function call along with its parameters, result and expected result. +func FormatTestResult(funcName string, wants interface{}, got interface{}, params ...interface{}) string { + msg := fmt.Sprintf("Got: %#v; Wants: %#v", got, wants) + return formatTestMessage(funcName, msg, params...) +} + +// FormatTestError pretty-formats a function error. +func FormatTestError(funcName string, err error, params ...interface{}) string { + msg := fmt.Sprintf("Error: %v", err) + return formatTestMessage(funcName, msg, params...) +} + +// CreateTestSession creates a Reva session for testing. +// For this, it performs a basic login using the specified credentials. +func CreateTestSession(host string, username string, password string) (*sdk.Session, error) { + if session, err := sdk.NewSession(); err == nil { + if err := session.Initiate(host, false); err == nil { + if err := session.BasicLogin(username, password); err == nil { + return session, nil + } + } + } + + return nil, fmt.Errorf("unable to create the test session") +} diff --git a/pkg/sdk/common/util.go b/pkg/sdk/common/util.go new file mode 100644 index 0000000000..478279dc93 --- /dev/null +++ b/pkg/sdk/common/util.go @@ -0,0 +1,45 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package common + +import ( + "strings" +) + +// FindString performs a case-sensitive string search in a string vector and returns its index or -1 if it couldn't be found. +func FindString(a []string, x string) int { + for i, n := range a { + if x == n { + return i + } + } + + return -1 +} + +// FindStringNoCase performs a case-insensitive string search in a string vector and returns its index or -1 if it couldn't be found. +func FindStringNoCase(a []string, x string) int { + for i, n := range a { + if strings.EqualFold(x, n) { + return i + } + } + + return -1 +} diff --git a/pkg/sdk/sdk_test.go b/pkg/sdk/sdk_test.go new file mode 100644 index 0000000000..b2c5a1c3cc --- /dev/null +++ b/pkg/sdk/sdk_test.go @@ -0,0 +1,74 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sdk_test + +import ( + "fmt" + "testing" + + "github.com/cs3org/reva/pkg/sdk" + testintl "github.com/cs3org/reva/pkg/sdk/common/testing" +) + +func TestSession(t *testing.T) { + tests := []struct { + host string + username string + password string + shouldList bool + shouldLogin bool + }{ + {"sciencemesh-test.uni-muenster.de:9600", "test", "testpass", true, true}, + {"sciencemesh.cernbox.cern.ch:443", "invalid", "invalid", true, false}, + {"google.de:443", "invalid", "invalid", false, false}, + } + + for _, test := range tests { + t.Run(test.host, func(t *testing.T) { + if session, err := sdk.NewSession(); err == nil { + if err := session.Initiate(test.host, false); err == nil { + if _, err := session.GetLoginMethods(); err != nil && test.shouldList { + t.Errorf(testintl.FormatTestError("Session.GetLoginMethods", err)) + } else if err == nil && !test.shouldList { + t.Errorf(testintl.FormatTestError("Session.GetLoginMethods", fmt.Errorf("listing of login methods with an invalid host succeeded"))) + } + + if err := session.BasicLogin(test.username, test.password); err == nil { + if test.shouldLogin { + if !session.IsValid() { + t.Errorf(testintl.FormatTestError("Session.BasicLogin", fmt.Errorf("logged in, but session is invalid"), test.username, test.password)) + } + if session.Token() == "" { + t.Errorf(testintl.FormatTestError("Session.BasicLogin", fmt.Errorf("logged in, but received no token"), test.username, test.password)) + } + } else { + t.Errorf(testintl.FormatTestError("Session.BasicLogin", fmt.Errorf("logging in with invalid credentials succeeded"), test.username, test.password)) + } + } else if test.shouldLogin { + t.Errorf(testintl.FormatTestError("Session.BasicLogin", err, test.username, test.password)) + } + } else { + t.Errorf(testintl.FormatTestError("Session.Initiate", err, test.host, false)) + } + } else { + t.Errorf(testintl.FormatTestError("NewSession", err)) + } + }) + } +} diff --git a/pkg/sdk/session.go b/pkg/sdk/session.go new file mode 100644 index 0000000000..5c6f1f3d56 --- /dev/null +++ b/pkg/sdk/session.go @@ -0,0 +1,170 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sdk + +import ( + "context" + "crypto/tls" + "fmt" + "io" + + registry "github.com/cs3org/go-cs3apis/cs3/auth/registry/v1beta1" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + + "github.com/cs3org/reva/pkg/sdk/common" + "github.com/cs3org/reva/pkg/sdk/common/net" +) + +// Session stores information about a Reva session. +// It is also responsible for managing the Reva gateway client. +type Session struct { + ctx context.Context + client gateway.GatewayAPIClient + + token string +} + +func (session *Session) initSession(ctx context.Context) error { + session.ctx = ctx + + return nil +} + +// Initiate initiates the session by creating a connection to the host and preparing the gateway client. +func (session *Session) Initiate(host string, insecure bool) error { + conn, err := session.getConnection(host, insecure) + if err != nil { + return fmt.Errorf("unable to establish a gRPC connection to '%v': %v", host, err) + } + session.client = gateway.NewGatewayAPIClient(conn) + + return nil +} + +func (session *Session) getConnection(host string, insecure bool) (*grpc.ClientConn, error) { + if insecure { + return grpc.Dial(host, grpc.WithInsecure()) + } + + tlsconf := &tls.Config{InsecureSkipVerify: false} + creds := credentials.NewTLS(tlsconf) + return grpc.Dial(host, grpc.WithTransportCredentials(creds)) +} + +// GetLoginMethods returns a list of all available login methods supported by the Reva instance. +func (session *Session) GetLoginMethods() ([]string, error) { + req := ®istry.ListAuthProvidersRequest{} + res, err := session.client.ListAuthProviders(session.ctx, req) + if err := net.CheckRPCInvocation("listing authorization providers", res, err); err != nil { + return []string{}, err + } + + return res.Types, nil +} + +// Login logs into Reva using the specified method and user credentials. +func (session *Session) Login(method string, username string, password string) error { + req := &gateway.AuthenticateRequest{ + Type: method, + ClientId: username, + ClientSecret: password, + } + res, err := session.client.Authenticate(session.ctx, req) + if err := net.CheckRPCInvocation("authenticating", res, err); err != nil { + return err + } + + if res.Token == "" { + return fmt.Errorf("invalid token received: %q", res.Token) + } + session.token = res.Token + + // Now that we have a valid token, we can append this to our context + session.ctx = context.WithValue(session.ctx, net.AccessTokenIndex, session.token) + session.ctx = metadata.AppendToOutgoingContext(session.ctx, net.AccessTokenName, session.token) + + return nil +} + +// BasicLogin tries to log into Reva using basic authentication. +// Before the actual login attempt, the method verifies that the Reva instance does support the "basic" login method. +func (session *Session) BasicLogin(username string, password string) error { + // Check if the 'basic' method is actually supported by the Reva instance; only continue if this is the case + supportedMethods, err := session.GetLoginMethods() + if err != nil { + return fmt.Errorf("unable to get a list of all supported login methods: %v", err) + } + + if common.FindStringNoCase(supportedMethods, "basic") == -1 { + return fmt.Errorf("'basic' login method is not supported") + } + + return session.Login("basic", username, password) +} + +// NewHTTPRequest returns an HTTP request instance. +func (session *Session) NewHTTPRequest(endpoint string, method string, transportToken string, data io.Reader) (*net.HTTPRequest, error) { + return net.NewHTTPRequest(session.ctx, endpoint, method, session.token, transportToken, data) +} + +// Client gets the gateway client instance. +func (session *Session) Client() gateway.GatewayAPIClient { + return session.client +} + +// Context returns the session context. +func (session *Session) Context() context.Context { + return session.ctx +} + +// Token returns the session token. +func (session *Session) Token() string { + return session.token +} + +// IsValid checks whether the session has been initialized and fully established. +func (session *Session) IsValid() bool { + return session.client != nil && session.ctx != nil && session.token != "" +} + +// NewSessionWithContext creates a new Reva session using the provided context. +func NewSessionWithContext(ctx context.Context) (*Session, error) { + session := &Session{} + if err := session.initSession(ctx); err != nil { + return nil, fmt.Errorf("unable to initialize the session: %v", err) + } + return session, nil +} + +// NewSession creates a new Reva session using a default background context. +func NewSession() (*Session, error) { + return NewSessionWithContext(context.Background()) +} + +// MustNewSession creates a new session and panics on failure. +func MustNewSession() *Session { + session, err := NewSession() + if err != nil { + panic(err) + } + return session +}