From d29febf750a2c50df34199741ba4976affe1a74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 18 Feb 2021 14:08:05 +0100 Subject: [PATCH 01/24] Port grpc tests to ginkgo, increase test coverage --- grpc-tests/userprovider_test.go | 220 --------------- .../grpc/fixtures/userprovider-demo.toml | 0 .../grpc/fixtures/userprovider-json.toml | 2 +- .../grpc/fixtures}/users.demo.json | 10 +- tests/integration/grpc/grpc_suite_test.go | 124 +++++++++ tests/integration/grpc/userprovider_test.go | 260 ++++++++++++++++++ 6 files changed, 394 insertions(+), 222 deletions(-) delete mode 100644 grpc-tests/userprovider_test.go rename grpc-tests/userproviders/demo.toml => tests/integration/grpc/fixtures/userprovider-demo.toml (100%) rename grpc-tests/userproviders/json.toml => tests/integration/grpc/fixtures/userprovider-json.toml (75%) rename {grpc-tests/userproviders => tests/integration/grpc/fixtures}/users.demo.json (89%) create mode 100644 tests/integration/grpc/grpc_suite_test.go create mode 100644 tests/integration/grpc/userprovider_test.go diff --git a/grpc-tests/userprovider_test.go b/grpc-tests/userprovider_test.go deleted file mode 100644 index 979f29c2c9..0000000000 --- a/grpc-tests/userprovider_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package grpctests - -import ( - "context" - "errors" - "net" - "os" - "os/exec" - "testing" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/stretchr/testify/assert" -) - -const grpcAddress = "localhost:19000" -const timeoutMs = 30000 - -func Test_service_GetUser(t *testing.T) { - providers := []struct { - name string - existingIdp string - }{ - { - name: "json", - existingIdp: "localhost:20080", - }, - { - name: "demo", - existingIdp: "http://localhost:9998", - }, - } - - for _, tt := range providers { - t.Run(tt.name, func(t *testing.T) { - // start revad with the specific provider - cmd := exec.Command("../cmd/revad/revad", "-c", "userproviders/"+tt.name+".toml") - err := cmd.Start() - - if err != nil { - t.Fatalf("Could not start revad! ERROR: %v", err) - } - - // wait till port is open - _ = waitForPort("open") - - // even the port is open the service might not be available yet - time.Sleep(1 * time.Second) - - GetUser(t, tt.existingIdp) - - // kill revad - err = cmd.Process.Signal(os.Kill) - if err != nil { - t.Fatalf("Could not kill revad! ERROR: %v", err) - } - _ = waitForPort("close") - }) - } -} - -func GetUser(t *testing.T, existingIdp string) { - tests := []struct { - name string - userID *userpb.UserId - want *userpb.GetUserResponse - }{ - { - name: "simple", - userID: &userpb.UserId{ - Idp: existingIdp, - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - }, - want: &userpb.GetUserResponse{ - Status: &v1beta11.Status{ - Code: 1, - }, - User: &userpb.User{ - Username: "marie", - Mail: "marie@example.org", - DisplayName: "Marie Curie", - Groups: []string{ - "radium-lovers", - "polonium-lovers", - "physics-lovers", - }, - }, - }, - }, - { - name: "not-existing opaqueId", - userID: &userpb.UserId{ - Idp: existingIdp, - OpaqueId: "doesnote-xist-4376-b307-cf0a8c2d0d9c", - }, - want: &userpb.GetUserResponse{ - Status: &v1beta11.Status{ - Code: 15, - }, - }, - }, - { - name: "no opaqueId", - userID: &userpb.UserId{ - Idp: existingIdp, - OpaqueId: "", - }, - want: &userpb.GetUserResponse{ - Status: &v1beta11.Status{ - Code: 15, - }, - }, - }, - { - name: "not-existing idp", - userID: &userpb.UserId{ - Idp: "http://does-not-exist:12345", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - }, - want: &userpb.GetUserResponse{ - Status: &v1beta11.Status{ - Code: 15, - }, - }, - }, - { - name: "no idp", - userID: &userpb.UserId{ - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - }, - want: &userpb.GetUserResponse{ - Status: &v1beta11.Status{ - Code: 1, - }, - User: &userpb.User{ - Username: "marie", - Mail: "marie@example.org", - DisplayName: "Marie Curie", - Groups: []string{ - "radium-lovers", - "polonium-lovers", - "physics-lovers", - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - - serviceClient, err := pool.GetUserProviderServiceClient(grpcAddress) - if err != nil { - t.Fatalf("cannot get UserProviderServiceClient! ERROR: %v", err) - } - - userResp, err := serviceClient.GetUser(ctx, &userpb.GetUserRequest{ - UserId: tt.userID, - }) - if err != nil { - t.Fatalf("cannot get user! ERROR: %v", err) - } - assert.Equal(t, tt.want.Status.Code, userResp.Status.Code) - if tt.want.User == nil { - assert.Nil(t, userResp.User) - } else { - // make sure not to run into a nil pointer error - if userResp.User == nil { - t.Fatalf("no user in response %v", userResp) - } - assert.Equal(t, tt.want.User.Username, userResp.User.Username) - assert.Equal(t, tt.want.User.Mail, userResp.User.Mail) - assert.Equal(t, tt.want.User.DisplayName, userResp.User.DisplayName) - assert.Equal(t, tt.want.User.Groups, userResp.User.Groups) - } - }) - } -} - -func waitForPort(expectedStatus string) error { - if expectedStatus != "open" && expectedStatus != "close" { - return errors.New("status can only be 'open' or 'close'") - } - timoutCounter := 0 - for timoutCounter <= timeoutMs { - conn, err := net.Dial("tcp", grpcAddress) - if err == nil { - _ = conn.Close() - if expectedStatus == "open" { - break - } - } else if expectedStatus == "close" { - break - } - - time.Sleep(1 * time.Millisecond) - timoutCounter++ - } - return nil -} diff --git a/grpc-tests/userproviders/demo.toml b/tests/integration/grpc/fixtures/userprovider-demo.toml similarity index 100% rename from grpc-tests/userproviders/demo.toml rename to tests/integration/grpc/fixtures/userprovider-demo.toml diff --git a/grpc-tests/userproviders/json.toml b/tests/integration/grpc/fixtures/userprovider-json.toml similarity index 75% rename from grpc-tests/userproviders/json.toml rename to tests/integration/grpc/fixtures/userprovider-json.toml index 13e2ea7599..93e57bead5 100644 --- a/grpc-tests/userproviders/json.toml +++ b/tests/integration/grpc/fixtures/userprovider-json.toml @@ -5,4 +5,4 @@ address = "0.0.0.0:19000" driver = "json" [grpc.services.userprovider.drivers.json] -users = "userproviders/users.demo.json" +users = "fixtures/users.demo.json" diff --git a/grpc-tests/userproviders/users.demo.json b/tests/integration/grpc/fixtures/users.demo.json similarity index 89% rename from grpc-tests/userproviders/users.demo.json rename to tests/integration/grpc/fixtures/users.demo.json index d13a252b9b..cf3ef4b2c8 100644 --- a/grpc-tests/userproviders/users.demo.json +++ b/tests/integration/grpc/fixtures/users.demo.json @@ -8,7 +8,15 @@ "secret": "relativity", "mail": "einstein@example.org", "display_name": "Albert Einstein", - "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] + "groups": ["sailing-lovers", "violin-haters", "physics-lovers"], + "opaque": { + "map": { + "uid": { + "decoder": "plain", + "value": "MTIz" + } + } + } }, { "id": { diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go new file mode 100644 index 0000000000..5bb75b4fe7 --- /dev/null +++ b/tests/integration/grpc/grpc_suite_test.go @@ -0,0 +1,124 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package grpc_test + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "sync" + "sync/atomic" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const grpcAddress = "localhost:19000" +const timeoutMs = 30000 + +var revads = map[string]*Revad{} +var mutex = sync.Mutex{} + +func TestGprc(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Gprc Suite") +} + +type shutdownFunc func() error + +type Revad struct { + shutdown shutdownFunc + references int32 +} + +func (r *Revad) Use() { + atomic.AddInt32(&r.references, 1) +} + +func (r *Revad) Cleanup() { + references := atomic.AddInt32(&r.references, -1) + if references == 0 { + r.shutdown() + } +} + +func startRevad(config string) (*Revad, error) { + mutex.Lock() + defer mutex.Unlock() + + if revads[config] != nil { + revad := revads[config] + revad.Use() + return revad, nil + } + + cmd := exec.Command("../../../cmd/revad/revad", "-c", config) + err := cmd.Start() + if err != nil { + return nil, fmt.Errorf("Could not start revad! ERROR: %v", err) + } + + err = waitForPort("open") + if err != nil { + return nil, err + } + + //even the port is open the service might not be available yet + time.Sleep(1 * time.Second) + + revad := &Revad{ + references: 1, + } + revad.shutdown = func() error { + err := cmd.Process.Signal(os.Kill) + if err != nil { + return fmt.Errorf("Could not kill revad! ERROR: %v", err) + } + waitForPort("close") + return nil + } + + return revad, nil +} + +func waitForPort(expectedStatus string) error { + if expectedStatus != "open" && expectedStatus != "close" { + return errors.New("status can only be 'open' or 'close'") + } + timoutCounter := 0 + for timoutCounter <= timeoutMs { + conn, err := net.Dial("tcp", grpcAddress) + if err == nil { + _ = conn.Close() + if expectedStatus == "open" { + break + } + } else if expectedStatus == "close" { + break + } + + time.Sleep(1 * time.Millisecond) + timoutCounter++ + } + return nil +} diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go new file mode 100644 index 0000000000..9954001351 --- /dev/null +++ b/tests/integration/grpc/userprovider_test.go @@ -0,0 +1,260 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package grpc_test + +import ( + "context" + "path" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "google.golang.org/grpc/metadata" + + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" + jwt "github.com/cs3org/reva/pkg/token/manager/jwt" + ruser "github.com/cs3org/reva/pkg/user" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("user providers", func() { + var ( + provider string + existingIdp string + revad *Revad + + ctx context.Context + serviceClient userpb.UserAPIClient + ) + + BeforeEach(func() { + var err error + serviceClient, err = pool.GetUserProviderServiceClient(grpcAddress) + Expect(err).ToNot(HaveOccurred()) + }) + + JustBeforeEach(func() { + var err error + ctx = context.Background() + + // Add auth token + user := &userpb.User{ + Id: &userpb.UserId{ + Idp: existingIdp, + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + } + tokenManager, err := jwt.New(map[string]interface{}{"secret": "changemeplease"}) + Expect(err).ToNot(HaveOccurred()) + t, err := tokenManager.MintToken(ctx, user) + Expect(err).ToNot(HaveOccurred()) + ctx = token.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + ctx = ruser.ContextSetUser(ctx, user) + + revad, err = startRevad(path.Join("fixtures", "userprovider-"+provider+".toml")) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + revad.Cleanup() + }) + + var assertGetUserByClaimResponses = func() { + It("gets users as expected", func() { + tests := map[string]string{ + "mail": "einstein@example.org", + "username": "einstein", + "uid": "123", + } + + for claim, value := range tests { + user, err := serviceClient.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{Claim: claim, Value: value}) + Expect(err).ToNot(HaveOccurred()) + Expect(user.User.Mail).To(Equal("einstein@example.org")) + } + }) + } + + var assertGetUserResponses = func() { + It("gets users as expected", func() { + tests := []struct { + name string + userID *userpb.UserId + want *userpb.GetUserResponse + }{ + { + name: "simple", + userID: &userpb.UserId{ + Idp: existingIdp, + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + want: &userpb.GetUserResponse{ + Status: &rpc.Status{ + Code: 1, + }, + User: &userpb.User{ + Username: "marie", + Mail: "marie@example.org", + DisplayName: "Marie Curie", + Groups: []string{ + "radium-lovers", + "polonium-lovers", + "physics-lovers", + }, + }, + }, + }, + { + name: "not-existing opaqueId", + userID: &userpb.UserId{ + Idp: existingIdp, + OpaqueId: "doesnote-xist-4376-b307-cf0a8c2d0d9c", + }, + want: &userpb.GetUserResponse{ + Status: &rpc.Status{ + Code: 15, + }, + }, + }, + { + name: "no opaqueId", + userID: &userpb.UserId{ + Idp: existingIdp, + OpaqueId: "", + }, + want: &userpb.GetUserResponse{ + Status: &rpc.Status{ + Code: 15, + }, + }, + }, + { + name: "not-existing idp", + userID: &userpb.UserId{ + Idp: "http://does-not-exist:12345", + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + want: &userpb.GetUserResponse{ + Status: &rpc.Status{ + Code: 15, + }, + }, + }, + { + name: "no idp", + userID: &userpb.UserId{ + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + want: &userpb.GetUserResponse{ + Status: &rpc.Status{ + Code: 1, + }, + User: &userpb.User{ + Username: "marie", + Mail: "marie@example.org", + DisplayName: "Marie Curie", + Groups: []string{ + "radium-lovers", + "polonium-lovers", + "physics-lovers", + }, + }, + }, + }, + } + + for _, t := range tests { + userResp, err := serviceClient.GetUser(ctx, &userpb.GetUserRequest{ + UserId: t.userID, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(t.want.Status.Code).To(Equal(userResp.Status.Code)) + if t.want.User == nil { + Expect(userResp.User).To(BeNil()) + } else { + //make sure not to run into a nil pointer error + Expect(userResp.User).ToNot(BeNil()) + Expect(t.want.User.Username).To(Equal(userResp.User.Username)) + Expect(t.want.User.Mail).To(Equal(userResp.User.Mail)) + Expect(t.want.User.DisplayName).To(Equal(userResp.User.DisplayName)) + Expect(t.want.User.Groups).To(Equal(userResp.User.Groups)) + } + } + }) + } + + var assertFindUsersResponses = func() { + It("finds users by email", func() { + res, err := serviceClient.FindUsers(ctx, &userpb.FindUsersRequest{Filter: "marie@example.org"}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Users)).To(Equal(1)) + user := res.Users[0] + Expect(user.DisplayName).To(Equal("Marie Curie")) + }) + + It("finds users by displayname", func() { + res, err := serviceClient.FindUsers(ctx, &userpb.FindUsersRequest{Filter: "Marie Curie"}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Users)).To(Equal(1)) + user := res.Users[0] + Expect(user.Mail).To(Equal("marie@example.org")) + }) + + It("finds users by username", func() { + res, err := serviceClient.FindUsers(ctx, &userpb.FindUsersRequest{Filter: "marie"}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Users)).To(Equal(1)) + user := res.Users[0] + Expect(user.Mail).To(Equal("marie@example.org")) + }) + + It("finds users by id", func() { + res, err := serviceClient.FindUsers(ctx, &userpb.FindUsersRequest{Filter: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Users)).To(Equal(1)) + user := res.Users[0] + Expect(user.Mail).To(Equal("marie@example.org")) + }) + } + + Describe("the json userprovider", func() { + BeforeEach(func() { + provider = "json" + existingIdp = "localhost:20080" + }) + + assertFindUsersResponses() + assertGetUserResponses() + assertGetUserByClaimResponses() + }) + + Describe("the demo userprovider", func() { + BeforeEach(func() { + provider = "demo" + existingIdp = "http://localhost:9998" + }) + + assertGetUserResponses() + assertFindUsersResponses() + assertGetUserByClaimResponses() + }) +}) From a321809eb9f2c54deaaf5dcbe9f24af1425e4a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 18 Feb 2021 14:08:43 +0100 Subject: [PATCH 02/24] Also consider the opaque id when searching for users --- pkg/user/manager/demo/demo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/user/manager/demo/demo.go b/pkg/user/manager/demo/demo.go index 77447f71cc..3eadeab718 100644 --- a/pkg/user/manager/demo/demo.go +++ b/pkg/user/manager/demo/demo.go @@ -80,9 +80,9 @@ func extractClaim(u *userpb.User, claim string) (string, error) { return "", errors.New("demo: invalid field") } -// TODO(jfd) search Opaque? compare sub? +// TODO(jfd) compare sub? func userContains(u *userpb.User, query string) bool { - return strings.Contains(u.Username, query) || strings.Contains(u.DisplayName, query) || strings.Contains(u.Mail, query) + return strings.Contains(u.Username, query) || strings.Contains(u.DisplayName, query) || strings.Contains(u.Mail, query) || strings.Contains(u.Id.OpaqueId, query) } func (m *manager) FindUsers(ctx context.Context, query string) ([]*userpb.User, error) { From c90864bc5936fe68b582d6f1c9d3454fa36826ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 19 Feb 2021 14:16:03 +0100 Subject: [PATCH 03/24] Add grpc integration tests for the storage provider --- .../grpc/fixtures/storageprovider-ocis.toml | 12 ++ tests/integration/grpc/grpc_suite_test.go | 30 ++++- .../integration/grpc/storageprovider_test.go | 106 ++++++++++++++++++ 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 tests/integration/grpc/fixtures/storageprovider-ocis.toml create mode 100644 tests/integration/grpc/storageprovider_test.go diff --git a/tests/integration/grpc/fixtures/storageprovider-ocis.toml b/tests/integration/grpc/fixtures/storageprovider-ocis.toml new file mode 100644 index 0000000000..9e39de98a5 --- /dev/null +++ b/tests/integration/grpc/fixtures/storageprovider-ocis.toml @@ -0,0 +1,12 @@ +[grpc] +address = "0.0.0.0:19000" + +[grpc.services.storageprovider] +driver = "ocis" + +[grpc.services.storageprovider.drivers.ocis] +root = "{{root}}" +treetime_accounting = true +treesize_accounting = true +enable_home = true +userprovidersvc = "localhost:18000" \ No newline at end of file diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 5bb75b4fe7..58f5b3a578 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -19,16 +19,20 @@ package grpc_test import ( - "errors" "fmt" + "io/ioutil" "net" "os" "os/exec" + "path" + "strings" "sync" "sync/atomic" "testing" "time" + "github.com/pkg/errors" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -72,10 +76,27 @@ func startRevad(config string) (*Revad, error) { return revad, nil } - cmd := exec.Command("../../../cmd/revad/revad", "-c", config) - err := cmd.Start() + // Create a temporary root for this revad + tmpRoot, err := ioutil.TempDir("", "reva-grpc-integration-tests-*-root") + if err != nil { + return nil, errors.Wrapf(err, "Could not create tmpdir") + } + newCfgPath := path.Join(tmpRoot, "config.toml") + rawCfg, err := ioutil.ReadFile(config) + if err != nil { + return nil, errors.Wrapf(err, "Could not read config file") + } + cfg := string(rawCfg) + strings.ReplaceAll(cfg, "{{root}}", tmpRoot) + err = ioutil.WriteFile(newCfgPath, []byte(cfg), 0600) + if err != nil { + return nil, errors.Wrapf(err, "Could not write config file") + } + + cmd := exec.Command("../../../cmd/revad/revad", "-c", newCfgPath) + err = cmd.Start() if err != nil { - return nil, fmt.Errorf("Could not start revad! ERROR: %v", err) + return nil, errors.Wrapf(err, "Could not start revad") } err = waitForPort("open") @@ -95,6 +116,7 @@ func startRevad(config string) (*Revad, error) { return fmt.Errorf("Could not kill revad! ERROR: %v", err) } waitForPort("close") + os.RemoveAll(tmpRoot) return nil } diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go new file mode 100644 index 0000000000..232c8ba036 --- /dev/null +++ b/tests/integration/grpc/storageprovider_test.go @@ -0,0 +1,106 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package grpc_test + +import ( + "context" + "path" + + "google.golang.org/grpc/metadata" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + storagep "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" + jwt "github.com/cs3org/reva/pkg/token/manager/jwt" + ruser "github.com/cs3org/reva/pkg/user" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("storage providers", func() { + var ( + provider string + revad *Revad + + ctx context.Context + serviceClient storagep.ProviderAPIClient + ) + + BeforeEach(func() { + var err error + serviceClient, err = pool.GetStorageProviderServiceClient(grpcAddress) + Expect(err).ToNot(HaveOccurred()) + }) + + JustBeforeEach(func() { + var err error + ctx = context.Background() + + // Add auth token + user := &userpb.User{ + Id: &userpb.UserId{ + Idp: "0.0.0.0:19000", + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + } + tokenManager, err := jwt.New(map[string]interface{}{"secret": "changemeplease"}) + Expect(err).ToNot(HaveOccurred()) + t, err := tokenManager.MintToken(ctx, user) + Expect(err).ToNot(HaveOccurred()) + ctx = token.ContextSetToken(ctx, t) + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + ctx = ruser.ContextSetUser(ctx, user) + + revad, err = startRevad(path.Join("fixtures", "storageprovider-"+provider+".toml")) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + revad.Cleanup() + }) + + assertCreateHomeResponses := func() { + It("creates a home directory", func() { + homeRef := &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/"}, + } + res, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + + _, err = serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + Expect(err).ToNot(HaveOccurred()) + + res, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + } + + Describe("ocis", func() { + BeforeEach(func() { + provider = "ocis" + }) + + assertCreateHomeResponses() + }) +}) From 9454cb837a56ac1ce8adf09ec6ec0d32f932a057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 09:46:59 +0100 Subject: [PATCH 04/24] Add grpc tests for GetPath and CreateContainer --- .../grpc/fixtures/storageprovider-ocis.toml | 2 +- .../grpc/fixtures/userprovider-demo.toml | 2 +- .../grpc/fixtures/userprovider-json.toml | 2 +- tests/integration/grpc/grpc_suite_test.go | 29 ++++--- .../integration/grpc/storageprovider_test.go | 76 ++++++++++++++++--- tests/integration/grpc/userprovider_test.go | 8 +- 6 files changed, 89 insertions(+), 30 deletions(-) diff --git a/tests/integration/grpc/fixtures/storageprovider-ocis.toml b/tests/integration/grpc/fixtures/storageprovider-ocis.toml index 9e39de98a5..126a9d9aed 100644 --- a/tests/integration/grpc/fixtures/storageprovider-ocis.toml +++ b/tests/integration/grpc/fixtures/storageprovider-ocis.toml @@ -1,5 +1,5 @@ [grpc] -address = "0.0.0.0:19000" +address = "{{grpc_address}}" [grpc.services.storageprovider] driver = "ocis" diff --git a/tests/integration/grpc/fixtures/userprovider-demo.toml b/tests/integration/grpc/fixtures/userprovider-demo.toml index 1c3276e6da..840829b703 100644 --- a/tests/integration/grpc/fixtures/userprovider-demo.toml +++ b/tests/integration/grpc/fixtures/userprovider-demo.toml @@ -1,5 +1,5 @@ [grpc] -address = "0.0.0.0:19000" +address = "{{grpc_address}}" [grpc.services.userprovider] driver = "demo" diff --git a/tests/integration/grpc/fixtures/userprovider-json.toml b/tests/integration/grpc/fixtures/userprovider-json.toml index 93e57bead5..fc542ee9c8 100644 --- a/tests/integration/grpc/fixtures/userprovider-json.toml +++ b/tests/integration/grpc/fixtures/userprovider-json.toml @@ -1,5 +1,5 @@ [grpc] -address = "0.0.0.0:19000" +address = "{{grpc_address}}" [grpc.services.userprovider] driver = "json" diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 58f5b3a578..84d32eab17 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -37,11 +37,11 @@ import ( . "github.com/onsi/gomega" ) -const grpcAddress = "localhost:19000" const timeoutMs = 30000 var revads = map[string]*Revad{} var mutex = sync.Mutex{} +var port = 19000 func TestGprc(t *testing.T) { RegisterFailHandler(Fail) @@ -51,8 +51,9 @@ func TestGprc(t *testing.T) { type shutdownFunc func() error type Revad struct { - shutdown shutdownFunc - references int32 + GrpcAddress string + shutdown shutdownFunc + references int32 } func (r *Revad) Use() { @@ -71,11 +72,16 @@ func startRevad(config string) (*Revad, error) { defer mutex.Unlock() if revads[config] != nil { + fmt.Println("Reusing revad for config", config) revad := revads[config] revad.Use() return revad, nil } + // Define a grpc address + grpcAddress := fmt.Sprintf("localhost:%d", port) + port += 1 + // Create a temporary root for this revad tmpRoot, err := ioutil.TempDir("", "reva-grpc-integration-tests-*-root") if err != nil { @@ -87,19 +93,21 @@ func startRevad(config string) (*Revad, error) { return nil, errors.Wrapf(err, "Could not read config file") } cfg := string(rawCfg) - strings.ReplaceAll(cfg, "{{root}}", tmpRoot) + cfg = strings.ReplaceAll(cfg, "{{root}}", tmpRoot) + cfg = strings.ReplaceAll(cfg, "{{grpc_address}}", grpcAddress) err = ioutil.WriteFile(newCfgPath, []byte(cfg), 0600) if err != nil { return nil, errors.Wrapf(err, "Could not write config file") } + // Run revad cmd := exec.Command("../../../cmd/revad/revad", "-c", newCfgPath) err = cmd.Start() if err != nil { return nil, errors.Wrapf(err, "Could not start revad") } - err = waitForPort("open") + err = waitForPort(grpcAddress, "open") if err != nil { return nil, err } @@ -108,22 +116,25 @@ func startRevad(config string) (*Revad, error) { time.Sleep(1 * time.Second) revad := &Revad{ - references: 1, + GrpcAddress: grpcAddress, + references: 1, } revad.shutdown = func() error { err := cmd.Process.Signal(os.Kill) if err != nil { - return fmt.Errorf("Could not kill revad! ERROR: %v", err) + return errors.Wrap(err, "Could not kill revad") } - waitForPort("close") + waitForPort(grpcAddress, "close") os.RemoveAll(tmpRoot) + revads[config] = nil return nil } + revads[config] = revad return revad, nil } -func waitForPort(expectedStatus string) error { +func waitForPort(grpcAddress, expectedStatus string) error { if expectedStatus != "open" && expectedStatus != "close" { return errors.New("status can only be 'open' or 'close'") } diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 232c8ba036..656a25a573 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -43,12 +43,18 @@ var _ = Describe("storage providers", func() { ctx context.Context serviceClient storagep.ProviderAPIClient + + homeRef *storagep.Reference + subdirRef *storagep.Reference ) BeforeEach(func() { - var err error - serviceClient, err = pool.GetStorageProviderServiceClient(grpcAddress) - Expect(err).ToNot(HaveOccurred()) + homeRef = &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/"}, + } + subdirRef = &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/subdir"}, + } }) JustBeforeEach(func() { @@ -72,27 +78,58 @@ var _ = Describe("storage providers", func() { revad, err = startRevad(path.Join("fixtures", "storageprovider-"+provider+".toml")) Expect(err).ToNot(HaveOccurred()) + serviceClient, err = pool.GetStorageProviderServiceClient(revad.GrpcAddress) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { revad.Cleanup() }) - assertCreateHomeResponses := func() { + assertCreateHome := func() { It("creates a home directory", func() { - homeRef := &storagep.Reference{ - Spec: &storagep.Reference_Path{Path: "/"}, - } - res, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) Expect(err).ToNot(HaveOccurred()) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) - _, err = serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + } + + assertCreateContainer := func() { + It("creates a new directory", func() { + newRef := &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/newdir"}, + } - res, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: newRef}) Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + + res, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: newRef}) Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: newRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + } + + assertGetPath := func() { + It("gets the path to an ID", func() { + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + + res, err := serviceClient.GetPath(ctx, &storagep.GetPathRequest{ResourceId: statRes.Info.Id}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Path).To(Equal(subdirRef.Spec.(*storagep.Reference_Path).Path)) }) } @@ -101,6 +138,21 @@ var _ = Describe("storage providers", func() { provider = "ocis" }) - assertCreateHomeResponses() + assertCreateHome() + + Context("with a home and a subdirectory", func() { + JustBeforeEach(func() { + res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + + assertCreateContainer() + assertGetPath() + }) }) }) diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 9954001351..6e121a5481 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -45,12 +45,6 @@ var _ = Describe("user providers", func() { serviceClient userpb.UserAPIClient ) - BeforeEach(func() { - var err error - serviceClient, err = pool.GetUserProviderServiceClient(grpcAddress) - Expect(err).ToNot(HaveOccurred()) - }) - JustBeforeEach(func() { var err error ctx = context.Background() @@ -72,6 +66,8 @@ var _ = Describe("user providers", func() { revad, err = startRevad(path.Join("fixtures", "userprovider-"+provider+".toml")) Expect(err).ToNot(HaveOccurred()) + serviceClient, err = pool.GetUserProviderServiceClient(revad.GrpcAddress) + Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { From 26c175b6508f019d96c92625bbc8a1f214d7a777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 16:06:32 +0100 Subject: [PATCH 05/24] Add testsuite for the ace package --- pkg/storage/utils/ace/ace_suite_test.go | 13 ++ pkg/storage/utils/ace/ace_test.go | 230 ++++++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 pkg/storage/utils/ace/ace_suite_test.go create mode 100644 pkg/storage/utils/ace/ace_test.go diff --git a/pkg/storage/utils/ace/ace_suite_test.go b/pkg/storage/utils/ace/ace_suite_test.go new file mode 100644 index 0000000000..348b65fab2 --- /dev/null +++ b/pkg/storage/utils/ace/ace_suite_test.go @@ -0,0 +1,13 @@ +package ace_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestAce(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Ace Suite") +} diff --git a/pkg/storage/utils/ace/ace_test.go b/pkg/storage/utils/ace/ace_test.go new file mode 100644 index 0000000000..724cbe1c87 --- /dev/null +++ b/pkg/storage/utils/ace/ace_test.go @@ -0,0 +1,230 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ace_test + +import ( + "fmt" + + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/storage/utils/ace" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("ACE", func() { + + var ( + userGrant = &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{ + UserId: &userpb.UserId{ + OpaqueId: "foo", + }, + }, + }, + Permissions: &provider.ResourcePermissions{}, + } + + groupGrant = &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_GROUP, + Id: &provider.Grantee_GroupId{ + GroupId: &grouppb.GroupId{ + OpaqueId: "foo", + }, + }, + }, + Permissions: &provider.ResourcePermissions{}, + } + ) + + Describe("FromGrant", func() { + It("creates an ACE from a user grant", func() { + ace := ace.FromGrant(userGrant) + Expect(ace.Principal()).To(Equal("u:foo")) + }) + + It("creates an ACE from a group grant", func() { + ace := ace.FromGrant(groupGrant) + Expect(ace.Principal()).To(Equal("g:foo")) + }) + }) + + Describe("Grant", func() { + It("returns a proper Grant", func() { + ace := ace.FromGrant(userGrant) + grant := ace.Grant() + Expect(grant).To(Equal(userGrant)) + }) + }) + + Describe("marshalling", func() { + It("works", func() { + a := ace.FromGrant(userGrant) + + marshalled, principal := a.Marshal() + unmarshalled, err := ace.Unmarshal(marshalled, principal) + Expect(err).ToNot(HaveOccurred()) + + Expect(unmarshalled).To(Equal(a)) + }) + }) + + Describe("converting permissions", func() { + It("converts r", func() { + userGrant.Permissions.Stat = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.Stat = false + Expect(newGrant.Permissions.Stat).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.ListContainer = true + newGrant = ace.FromGrant(userGrant).Grant() + userGrant.Permissions.ListContainer = false + Expect(newGrant.Permissions.ListContainer).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.InitiateFileDownload = true + newGrant = ace.FromGrant(userGrant).Grant() + userGrant.Permissions.InitiateFileDownload = false + Expect(newGrant.Permissions.InitiateFileDownload).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.GetPath = true + newGrant = ace.FromGrant(userGrant).Grant() + fmt.Println(newGrant.Permissions) + userGrant.Permissions.GetPath = false + Expect(newGrant.Permissions.GetPath).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts w", func() { + userGrant.Permissions.InitiateFileUpload = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.InitiateFileUpload = false + Expect(newGrant.Permissions.InitiateFileUpload).To(BeTrue()) + Expect(newGrant.Permissions.Move).To(BeFalse()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.InitiateFileUpload = true + userGrant.Permissions.InitiateFileDownload = true + newGrant = ace.FromGrant(userGrant).Grant() + userGrant.Permissions.InitiateFileUpload = false + Expect(newGrant.Permissions.InitiateFileUpload).To(BeTrue()) + Expect(newGrant.Permissions.Move).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts a", func() { + userGrant.Permissions.CreateContainer = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.CreateContainer = false + Expect(newGrant.Permissions.CreateContainer).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts d", func() { + userGrant.Permissions.Delete = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.Delete = false + Expect(newGrant.Permissions.Delete).To(BeTrue()) + Expect(newGrant.Permissions.Move).To(BeFalse()) + }) + + It("converts C", func() { + userGrant.Permissions.AddGrant = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.AddGrant = false + Expect(newGrant.Permissions.AddGrant).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.RemoveGrant = true + newGrant = ace.FromGrant(userGrant).Grant() + userGrant.Permissions.RemoveGrant = false + Expect(newGrant.Permissions.RemoveGrant).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + + userGrant.Permissions.UpdateGrant = true + newGrant = ace.FromGrant(userGrant).Grant() + userGrant.Permissions.UpdateGrant = false + Expect(newGrant.Permissions.UpdateGrant).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts c", func() { + userGrant.Permissions.ListGrants = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.ListGrants = false + Expect(newGrant.Permissions.ListGrants).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts u", func() { + userGrant.Permissions.ListRecycle = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.ListRecycle = false + Expect(newGrant.Permissions.ListRecycle).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts U", func() { + userGrant.Permissions.RestoreRecycleItem = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.RestoreRecycleItem = false + Expect(newGrant.Permissions.RestoreRecycleItem).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts P", func() { + userGrant.Permissions.PurgeRecycle = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.PurgeRecycle = false + Expect(newGrant.Permissions.PurgeRecycle).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts v", func() { + userGrant.Permissions.ListFileVersions = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.ListFileVersions = false + Expect(newGrant.Permissions.ListFileVersions).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts V", func() { + userGrant.Permissions.RestoreFileVersion = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.RestoreFileVersion = false + Expect(newGrant.Permissions.RestoreFileVersion).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + + It("converts q", func() { + userGrant.Permissions.GetQuota = true + newGrant := ace.FromGrant(userGrant).Grant() + userGrant.Permissions.GetQuota = false + Expect(newGrant.Permissions.GetQuota).To(BeTrue()) + Expect(newGrant.Permissions.Delete).To(BeFalse()) + }) + }) +}) From 44a8483311f2c29271f8a8b4bcc78c9547f0f736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 16:07:28 +0100 Subject: [PATCH 06/24] Make the GetPath permission translate to a "r" ace permission It always works that way in the other direction ("r" -> GetPath) --- pkg/storage/utils/ace/ace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storage/utils/ace/ace.go b/pkg/storage/utils/ace/ace.go index 7408e5e497..29817e8896 100644 --- a/pkg/storage/utils/ace/ace.go +++ b/pkg/storage/utils/ace/ace.go @@ -324,7 +324,7 @@ func unmarshalKV(s string) (*ACE, error) { func getACEPerm(set *provider.ResourcePermissions) string { var b strings.Builder - if set.Stat || set.InitiateFileDownload || set.ListContainer { + if set.Stat || set.InitiateFileDownload || set.ListContainer || set.GetPath { b.WriteString("r") } if set.InitiateFileUpload || set.Move { From fadf79d73ba4ff51fce2dbd6cc7ab29f831e3dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 16:10:17 +0100 Subject: [PATCH 07/24] Fix setting the user and group ids in the grants created from an ace --- pkg/storage/utils/ace/ace.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/storage/utils/ace/ace.go b/pkg/storage/utils/ace/ace.go index 29817e8896..0123bce377 100644 --- a/pkg/storage/utils/ace/ace.go +++ b/pkg/storage/utils/ace/ace.go @@ -187,10 +187,11 @@ func (e *ACE) Grant() *provider.Grant { }, Permissions: e.grantPermissionSet(), } + id := e.principal[2:] if e.granteeType() == provider.GranteeType_GRANTEE_TYPE_GROUP { - g.Grantee.Id = &provider.Grantee_GroupId{GroupId: &grouppb.GroupId{OpaqueId: e.principal}} + g.Grantee.Id = &provider.Grantee_GroupId{GroupId: &grouppb.GroupId{OpaqueId: id}} } else if e.granteeType() == provider.GranteeType_GRANTEE_TYPE_USER { - g.Grantee.Id = &provider.Grantee_UserId{UserId: &userpb.UserId{OpaqueId: e.principal}} + g.Grantee.Id = &provider.Grantee_UserId{UserId: &userpb.UserId{OpaqueId: id}} } return g } From 9cf6016bfc8f1259bed7de6dfbdc0ce16bfeee04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 16:32:14 +0100 Subject: [PATCH 08/24] Assert grpc integration tests for Delete and *Grant(s) --- .../integration/grpc/storageprovider_test.go | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 656a25a573..3824fe019c 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -122,6 +122,22 @@ var _ = Describe("storage providers", func() { }) } + assertDelete := func() { + It("deletes a directory", func() { + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + }) + } + assertGetPath := func() { It("gets the path to an ID", func() { statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) @@ -133,6 +149,54 @@ var _ = Describe("storage providers", func() { }) } + assertGrants := func() { + It("lists, adds and removes grants", func() { + By("there are no grants initially") + listRes, err := serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(listRes.Grants)).To(Equal(0)) + + By("adding a grant") + grant := &storagep.Grant{ + Grantee: &storagep.Grantee{ + Type: storagep.GranteeType_GRANTEE_TYPE_USER, + Id: &storagep.Grantee_UserId{ + UserId: &userpb.UserId{ + OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51", + }, + }, + }, + Permissions: &storagep.ResourcePermissions{ + Stat: true, + Move: true, + Delete: false, + }, + } + addRes, err := serviceClient.AddGrant(ctx, &storagep.AddGrantRequest{Ref: subdirRef, Grant: grant}) + Expect(err).ToNot(HaveOccurred()) + Expect(addRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + By("listing the new grant") + listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(listRes.Grants)).To(Equal(1)) + readGrant := listRes.Grants[0] + Expect(readGrant.Permissions.Stat).To(BeTrue()) + Expect(readGrant.Permissions.Move).To(BeTrue()) + Expect(readGrant.Permissions.Delete).To(BeFalse()) + + By("deleting a grant") + delRes, err := serviceClient.RemoveGrant(ctx, &storagep.RemoveGrantRequest{Ref: subdirRef, Grant: readGrant}) + Expect(err).ToNot(HaveOccurred()) + Expect(delRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + By("the grant is gone") + listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(listRes.Grants)).To(Equal(0)) + }) + } + Describe("ocis", func() { BeforeEach(func() { provider = "ocis" @@ -153,6 +217,8 @@ var _ = Describe("storage providers", func() { assertCreateContainer() assertGetPath() + assertDelete() + assertGrants() }) }) }) From af32650ed841649c03c4b55777b942c26e5d9f0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 16:36:58 +0100 Subject: [PATCH 09/24] Add unit test for grants --- pkg/storage/utils/decomposedfs/grants_test.go | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 pkg/storage/utils/decomposedfs/grants_test.go diff --git a/pkg/storage/utils/decomposedfs/grants_test.go b/pkg/storage/utils/decomposedfs/grants_test.go new file mode 100644 index 0000000000..80fca9a806 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/grants_test.go @@ -0,0 +1,147 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package decomposedfs_test + +import ( + "path" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" + "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" + "github.com/pkg/xattr" + "github.com/stretchr/testify/mock" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Grants", func() { + var ( + env *helpers.TestEnv + + ref *provider.Reference + grant *provider.Grant + ) + + BeforeEach(func() { + ref = &provider.Reference{ + Spec: &provider.Reference_Path{ + Path: "dir1", + }, + } + + grant = &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{ + UserId: &userpb.UserId{ + OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51", + }, + }, + }, + Permissions: &provider.ResourcePermissions{ + Stat: true, + Move: true, + Delete: false, + }, + } + }) + + JustBeforeEach(func() { + var err error + env, err = helpers.NewTestEnv() + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + if env != nil { + env.Cleanup() + } + }) + + Context("with insufficient permissions", func() { + JustBeforeEach(func() { + env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) + }) + + Describe("AddGrant", func() { + It("adds grants", func() { + err := env.Fs.AddGrant(env.Ctx, ref, grant) + Expect(err).To(MatchError(ContainSubstring("permission denied"))) + }) + }) + }) + + Context("with sufficient permissions", func() { + JustBeforeEach(func() { + env.Permissions.On("HasPermission", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + }) + + Describe("AddGrant", func() { + It("adds grants", func() { + n, err := env.Lookup.NodeFromPath(env.Ctx, "dir1") + Expect(err).ToNot(HaveOccurred()) + + err = env.Fs.AddGrant(env.Ctx, ref, grant) + Expect(err).ToNot(HaveOccurred()) + + localPath := path.Join(env.Root, "nodes", n.ID) + attr, err := xattr.Get(localPath, xattrs.GrantPrefix+xattrs.UserAcePrefix+grant.Grantee.GetUserId().OpaqueId) + Expect(err).ToNot(HaveOccurred()) + Expect(string(attr)).To(Equal("\x00t=A:f=:p=rw")) + }) + }) + + Describe("ListGrants", func() { + It("lists existing grants", func() { + err := env.Fs.AddGrant(env.Ctx, ref, grant) + Expect(err).ToNot(HaveOccurred()) + + grants, err := env.Fs.ListGrants(env.Ctx, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(len(grants)).To(Equal(1)) + + g := grants[0] + Expect(g.Grantee.GetUserId().OpaqueId).To(Equal(grant.Grantee.GetUserId().OpaqueId)) + Expect(g.Permissions.Stat).To(BeTrue()) + Expect(g.Permissions.Move).To(BeTrue()) + Expect(g.Permissions.Delete).To(BeFalse()) + }) + }) + + Describe("RemoveGrants", func() { + It("removes the grant", func() { + err := env.Fs.AddGrant(env.Ctx, ref, grant) + Expect(err).ToNot(HaveOccurred()) + + grants, err := env.Fs.ListGrants(env.Ctx, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(len(grants)).To(Equal(1)) + + err = env.Fs.RemoveGrant(env.Ctx, ref, grant) + Expect(err).ToNot(HaveOccurred()) + + grants, err = env.Fs.ListGrants(env.Ctx, ref) + Expect(err).ToNot(HaveOccurred()) + Expect(len(grants)).To(Equal(0)) + }) + }) + }) +}) From 807bfc4d2c62c06152fad4f4ac6244d3679d7d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Feb 2021 17:02:42 +0100 Subject: [PATCH 10/24] Add more grpc integration tests --- .../integration/grpc/storageprovider_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 3824fe019c..5365430118 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -45,6 +45,7 @@ var _ = Describe("storage providers", func() { serviceClient storagep.ProviderAPIClient homeRef *storagep.Reference + fileRef *storagep.Reference subdirRef *storagep.Reference ) @@ -52,6 +53,9 @@ var _ = Describe("storage providers", func() { homeRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: "/"}, } + fileRef = &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/file"}, + } subdirRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: "/subdir"}, } @@ -122,6 +126,20 @@ var _ = Describe("storage providers", func() { }) } + assertListContainer := func() { + It("lists a directory", func() { + listRes, err := serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: homeRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(listRes.Infos)).To(Equal(1)) + + info := listRes.Infos[0] + Expect(info.Type).To(Equal(storagep.ResourceType_RESOURCE_TYPE_CONTAINER)) + Expect(info.Path).To(Equal(subdirRef.Spec.(*storagep.Reference_Path).Path)) + Expect(info.Owner.OpaqueId).To(Equal("f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c")) + }) + } + assertDelete := func() { It("deletes a directory", func() { statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) @@ -197,6 +215,24 @@ var _ = Describe("storage providers", func() { }) } + assertUploads := func() { + It("returns upload URLs for simple and tus", func() { + res, err := serviceClient.InitiateFileUpload(ctx, &storagep.InitiateFileUploadRequest{Ref: fileRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(res.Protocols)).To(Equal(2)) + }) + } + + assertDownloads := func() { + It("returns 'simple' download URLs", func() { + res, err := serviceClient.InitiateFileDownload(ctx, &storagep.InitiateFileDownloadRequest{Ref: fileRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(res.Protocols)).To(Equal(1)) + }) + } + Describe("ocis", func() { BeforeEach(func() { provider = "ocis" @@ -216,9 +252,12 @@ var _ = Describe("storage providers", func() { }) assertCreateContainer() + assertListContainer() assertGetPath() assertDelete() assertGrants() + assertUploads() + assertDownloads() }) }) }) From fd406fc18f94659265b98d62c35a0ee005969304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 23 Feb 2021 15:42:51 +0100 Subject: [PATCH 11/24] Paths always start with a /, also in RecycleItems --- .../utils/decomposedfs/decomposedfs_test.go | 2 +- pkg/storage/utils/decomposedfs/grants_test.go | 4 +- pkg/storage/utils/decomposedfs/lookup.go | 1 + pkg/storage/utils/decomposedfs/lookup_test.go | 37 ++++++++++++++++ .../utils/decomposedfs/node/node_test.go | 10 ++--- .../utils/decomposedfs/testhelpers/helpers.go | 6 +-- .../utils/decomposedfs/tree/tree_test.go | 4 +- .../integration/grpc/storageprovider_test.go | 42 ++++++++++++++----- 8 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 pkg/storage/utils/decomposedfs/lookup_test.go diff --git a/pkg/storage/utils/decomposedfs/decomposedfs_test.go b/pkg/storage/utils/decomposedfs/decomposedfs_test.go index 2f9043dccd..118bb1f87a 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs_test.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs_test.go @@ -40,7 +40,7 @@ var _ = Describe("Decomposed", func() { BeforeEach(func() { ref = &provider.Reference{ Spec: &provider.Reference_Path{ - Path: "dir1", + Path: "/dir1", }, } }) diff --git a/pkg/storage/utils/decomposedfs/grants_test.go b/pkg/storage/utils/decomposedfs/grants_test.go index 80fca9a806..e8f93cbe89 100644 --- a/pkg/storage/utils/decomposedfs/grants_test.go +++ b/pkg/storage/utils/decomposedfs/grants_test.go @@ -43,7 +43,7 @@ var _ = Describe("Grants", func() { BeforeEach(func() { ref = &provider.Reference{ Spec: &provider.Reference_Path{ - Path: "dir1", + Path: "/dir1", }, } @@ -96,7 +96,7 @@ var _ = Describe("Grants", func() { Describe("AddGrant", func() { It("adds grants", func() { - n, err := env.Lookup.NodeFromPath(env.Ctx, "dir1") + n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1") Expect(err).ToNot(HaveOccurred()) err = env.Fs.AddGrant(env.Ctx, ref, grant) diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 7b1024d426..0de4106b73 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -101,6 +101,7 @@ func (lu *Lookup) Path(ctx context.Context, n *node.Node) (p string, err error) return } } + p = filepath.Join("/", p) return } diff --git a/pkg/storage/utils/decomposedfs/lookup_test.go b/pkg/storage/utils/decomposedfs/lookup_test.go new file mode 100644 index 0000000000..0e2a3b5966 --- /dev/null +++ b/pkg/storage/utils/decomposedfs/lookup_test.go @@ -0,0 +1,37 @@ +package decomposedfs_test + +import ( + helpers "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/testhelpers" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Lookup", func() { + var ( + env *helpers.TestEnv + ) + + JustBeforeEach(func() { + var err error + env, err = helpers.NewTestEnv() + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + if env != nil { + env.Cleanup() + } + }) + + Describe("Path", func() { + It("returns the path including a leading slash", func() { + n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1") + Expect(err).ToNot(HaveOccurred()) + + path, err := env.Lookup.Path(env.Ctx, n) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal("/dir1/file1")) + }) + }) +}) diff --git a/pkg/storage/utils/decomposedfs/node/node_test.go b/pkg/storage/utils/decomposedfs/node/node_test.go index f75cee3ef7..13565a2543 100644 --- a/pkg/storage/utils/decomposedfs/node/node_test.go +++ b/pkg/storage/utils/decomposedfs/node/node_test.go @@ -62,7 +62,7 @@ var _ = Describe("Node", func() { Describe("ReadNode", func() { It("reads the blobID from the xattrs", func() { - lookupNode, err := env.Lookup.NodeFromPath(env.Ctx, "dir1/file1") + lookupNode, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1") Expect(err).ToNot(HaveOccurred()) n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.ID) @@ -73,7 +73,7 @@ var _ = Describe("Node", func() { Describe("WriteMetadata", func() { It("writes all xattrs", func() { - n, err := env.Lookup.NodeFromPath(env.Ctx, "dir1/file1") + n, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1") Expect(err).ToNot(HaveOccurred()) blobsize := 239485734 @@ -87,7 +87,7 @@ var _ = Describe("Node", func() { err = n.WriteMetadata(owner) Expect(err).ToNot(HaveOccurred()) - n2, err := env.Lookup.NodeFromPath(env.Ctx, "dir1/file1") + n2, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1") Expect(err).ToNot(HaveOccurred()) Expect(n2.Name).To(Equal("TestName")) Expect(n2.BlobID).To(Equal("TestBlobID")) @@ -97,7 +97,7 @@ var _ = Describe("Node", func() { Describe("Parent", func() { It("returns the parent node", func() { - child, err := env.Lookup.NodeFromPath(env.Ctx, "dir1/subdir1") + child, err := env.Lookup.NodeFromPath(env.Ctx, "/dir1/subdir1") Expect(err).ToNot(HaveOccurred()) Expect(child).ToNot(BeNil()) @@ -115,7 +115,7 @@ var _ = Describe("Node", func() { BeforeEach(func() { var err error - parent, err = env.Lookup.NodeFromPath(env.Ctx, "dir1") + parent, err = env.Lookup.NodeFromPath(env.Ctx, "/dir1") Expect(err).ToNot(HaveOccurred()) Expect(parent).ToNot(BeNil()) }) diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index f7d950ca71..808ef0215d 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -53,9 +53,9 @@ type TestEnv struct { // NewTestEnv prepares a test environment on disk // The storage contains some directories and a file: // -// dir1/ -// dir1/file1 -// dir1/subdir1/ +// /dir1/ +// /dir1/file1 +// /dir1/subdir1/ func NewTestEnv() (*TestEnv, error) { tmpRoot, err := ioutil.TempDir("", "reva-unit-tests-*-root") if err != nil { diff --git a/pkg/storage/utils/decomposedfs/tree/tree_test.go b/pkg/storage/utils/decomposedfs/tree/tree_test.go index 1fc1ac835b..eae66faea6 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree_test.go +++ b/pkg/storage/utils/decomposedfs/tree/tree_test.go @@ -60,7 +60,7 @@ var _ = Describe("Tree", func() { JustBeforeEach(func() { var err error - n, err = env.Lookup.NodeFromPath(env.Ctx, "dir1/file1") + n, err = env.Lookup.NodeFromPath(env.Ctx, "/dir1/file1") Expect(err).ToNot(HaveOccurred()) }) @@ -90,7 +90,7 @@ var _ = Describe("Tree", func() { trashPath := path.Join(env.Root, "trash", env.Owner.Id.OpaqueId, n.ID) attr, err := xattr.Get(trashPath, xattrs.TrashOriginAttr) Expect(err).ToNot(HaveOccurred()) - Expect(string(attr)).To(Equal("dir1/file1")) + Expect(string(attr)).To(Equal("/dir1/file1")) }) It("does not delete the blob from the blobstore", func() { diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 5365430118..178be3db19 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -20,7 +20,6 @@ package grpc_test import ( "context" - "path" "google.golang.org/grpc/metadata" @@ -38,27 +37,33 @@ import ( var _ = Describe("storage providers", func() { var ( - provider string - revad *Revad + dependencies map[string]string + revads map[string]*Revad ctx context.Context serviceClient storagep.ProviderAPIClient - homeRef *storagep.Reference - fileRef *storagep.Reference - subdirRef *storagep.Reference + homeRef *storagep.Reference + filePath string + fileRef *storagep.Reference + subdirPath string + subdirRef *storagep.Reference ) BeforeEach(func() { homeRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: "/"}, } + filePath = "/file" fileRef = &storagep.Reference{ - Spec: &storagep.Reference_Path{Path: "/file"}, + Spec: &storagep.Reference_Path{Path: filePath}, } + subdirPath = "/subdir" subdirRef = &storagep.Reference{ - Spec: &storagep.Reference_Path{Path: "/subdir"}, + Spec: &storagep.Reference_Path{Path: subdirPath}, } + revads = map[string]*Revad{} + dependencies = map[string]string{} }) JustBeforeEach(func() { @@ -135,7 +140,7 @@ var _ = Describe("storage providers", func() { info := listRes.Infos[0] Expect(info.Type).To(Equal(storagep.ResourceType_RESOURCE_TYPE_CONTAINER)) - Expect(info.Path).To(Equal(subdirRef.Spec.(*storagep.Reference_Path).Path)) + Expect(info.Path).To(Equal(subdirPath)) Expect(info.Owner.OpaqueId).To(Equal("f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c")) }) } @@ -163,7 +168,7 @@ var _ = Describe("storage providers", func() { res, err := serviceClient.GetPath(ctx, &storagep.GetPathRequest{ResourceId: statRes.Info.Id}) Expect(err).ToNot(HaveOccurred()) - Expect(res.Path).To(Equal(subdirRef.Spec.(*storagep.Reference_Path).Path)) + Expect(res.Path).To(Equal(subdirPath)) }) } @@ -233,6 +238,22 @@ var _ = Describe("storage providers", func() { }) } + assertRecycle := func() { + It("lists and restores resources", func() { + res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{}) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(listRes.RecycleItems)).To(Equal(1)) + item := listRes.RecycleItems[0] + Expect(item.Path).To(Equal(subdirPath)) + }) + } + Describe("ocis", func() { BeforeEach(func() { provider = "ocis" @@ -258,6 +279,7 @@ var _ = Describe("storage providers", func() { assertGrants() assertUploads() assertDownloads() + assertRecycle() }) }) }) From c2edcee4347ab32f2b9a51dcdc1af30f16a4ff9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 23 Feb 2021 15:51:01 +0100 Subject: [PATCH 12/24] Allow for starting multiple revads for grpc integration tests --- tests/integration/grpc/grpc_suite_test.go | 129 +++++++++--------- .../integration/grpc/storageprovider_test.go | 12 +- tests/integration/grpc/userprovider_test.go | 22 +-- 3 files changed, 84 insertions(+), 79 deletions(-) diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 84d32eab17..60766f9ccf 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -27,7 +27,6 @@ import ( "path" "strings" "sync" - "sync/atomic" "testing" "time" @@ -48,90 +47,86 @@ func TestGprc(t *testing.T) { RunSpecs(t, "Gprc Suite") } -type shutdownFunc func() error +type cleanupFunc func() error type Revad struct { GrpcAddress string - shutdown shutdownFunc - references int32 + Cleanup cleanupFunc } -func (r *Revad) Use() { - atomic.AddInt32(&r.references, 1) -} - -func (r *Revad) Cleanup() { - references := atomic.AddInt32(&r.references, -1) - if references == 0 { - r.shutdown() - } -} - -func startRevad(config string) (*Revad, error) { +func startRevads(configs map[string]string) (map[string]*Revad, error) { mutex.Lock() defer mutex.Unlock() - if revads[config] != nil { - fmt.Println("Reusing revad for config", config) - revad := revads[config] - revad.Use() - return revad, nil + revads := map[string]*Revad{} + addresses := map[string]string{} + for name, _ := range configs { + addresses[name] = fmt.Sprintf("localhost:%d", port) + port += 1 } - // Define a grpc address - grpcAddress := fmt.Sprintf("localhost:%d", port) - port += 1 + for name, config := range configs { + ownAddress := addresses[name] - // Create a temporary root for this revad - tmpRoot, err := ioutil.TempDir("", "reva-grpc-integration-tests-*-root") - if err != nil { - return nil, errors.Wrapf(err, "Could not create tmpdir") - } - newCfgPath := path.Join(tmpRoot, "config.toml") - rawCfg, err := ioutil.ReadFile(config) - if err != nil { - return nil, errors.Wrapf(err, "Could not read config file") - } - cfg := string(rawCfg) - cfg = strings.ReplaceAll(cfg, "{{root}}", tmpRoot) - cfg = strings.ReplaceAll(cfg, "{{grpc_address}}", grpcAddress) - err = ioutil.WriteFile(newCfgPath, []byte(cfg), 0600) - if err != nil { - return nil, errors.Wrapf(err, "Could not write config file") - } + // Create a temporary root for this revad + tmpRoot, err := ioutil.TempDir("", "reva-grpc-integration-tests-*-root") + if err != nil { + return nil, errors.Wrapf(err, "Could not create tmpdir") + } + newCfgPath := path.Join(tmpRoot, "config.toml") + rawCfg, err := ioutil.ReadFile(path.Join("fixtures", config)) + if err != nil { + return nil, errors.Wrapf(err, "Could not read config file") + } + cfg := string(rawCfg) + cfg = strings.ReplaceAll(cfg, "{{root}}", tmpRoot) + cfg = strings.ReplaceAll(cfg, "{{grpc_address}}", ownAddress) + for name, address := range addresses { + cfg = strings.ReplaceAll(cfg, "{{"+name+"_address}}", address) + } + err = ioutil.WriteFile(newCfgPath, []byte(cfg), 0600) + if err != nil { + return nil, errors.Wrapf(err, "Could not write config file") + } - // Run revad - cmd := exec.Command("../../../cmd/revad/revad", "-c", newCfgPath) - err = cmd.Start() - if err != nil { - return nil, errors.Wrapf(err, "Could not start revad") - } + // Run revad + cmd := exec.Command("../../../cmd/revad/revad", "-c", newCfgPath) - err = waitForPort(grpcAddress, "open") - if err != nil { - return nil, err - } + outfile, err := os.Create(path.Join(tmpRoot, name+"-out.log")) + if err != nil { + panic(err) + } + defer outfile.Close() + cmd.Stdout = outfile - //even the port is open the service might not be available yet - time.Sleep(1 * time.Second) + err = cmd.Start() + if err != nil { + return nil, errors.Wrapf(err, "Could not start revad") + } - revad := &Revad{ - GrpcAddress: grpcAddress, - references: 1, - } - revad.shutdown = func() error { - err := cmd.Process.Signal(os.Kill) + err = waitForPort(ownAddress, "open") if err != nil { - return errors.Wrap(err, "Could not kill revad") + return nil, err } - waitForPort(grpcAddress, "close") - os.RemoveAll(tmpRoot) - revads[config] = nil - return nil - } - revads[config] = revad - return revad, nil + //even the port is open the service might not be available yet + time.Sleep(1 * time.Second) + + revad := &Revad{ + GrpcAddress: ownAddress, + Cleanup: func() error { + err := cmd.Process.Signal(os.Kill) + if err != nil { + return errors.Wrap(err, "Could not kill revad") + } + waitForPort(ownAddress, "close") + os.RemoveAll(tmpRoot) + return nil + }, + } + revads[name] = revad + } + return revads, nil } func waitForPort(grpcAddress, expectedStatus string) error { diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 178be3db19..3751d975af 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -85,14 +85,16 @@ var _ = Describe("storage providers", func() { ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) ctx = ruser.ContextSetUser(ctx, user) - revad, err = startRevad(path.Join("fixtures", "storageprovider-"+provider+".toml")) + revads, err := startRevads(dependencies) Expect(err).ToNot(HaveOccurred()) - serviceClient, err = pool.GetStorageProviderServiceClient(revad.GrpcAddress) + serviceClient, err = pool.GetStorageProviderServiceClient(revads["storage"].GrpcAddress) Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - revad.Cleanup() + for _, r := range revads { + Expect(r.Cleanup()).To(Succeed()) + } }) assertCreateHome := func() { @@ -256,7 +258,9 @@ var _ = Describe("storage providers", func() { Describe("ocis", func() { BeforeEach(func() { - provider = "ocis" + dependencies = map[string]string{ + "storage": "storageprovider-ocis.toml", + } }) assertCreateHome() diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 6e121a5481..21a7d5b916 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -20,7 +20,6 @@ package grpc_test import ( "context" - "path" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -37,9 +36,10 @@ import ( var _ = Describe("user providers", func() { var ( - provider string + dependencies map[string]string + revads map[string]*Revad + existingIdp string - revad *Revad ctx context.Context serviceClient userpb.UserAPIClient @@ -64,14 +64,16 @@ var _ = Describe("user providers", func() { ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) ctx = ruser.ContextSetUser(ctx, user) - revad, err = startRevad(path.Join("fixtures", "userprovider-"+provider+".toml")) + revads, err = startRevads(dependencies) Expect(err).ToNot(HaveOccurred()) - serviceClient, err = pool.GetUserProviderServiceClient(revad.GrpcAddress) + serviceClient, err = pool.GetUserProviderServiceClient(revads["users"].GrpcAddress) Expect(err).ToNot(HaveOccurred()) }) AfterEach(func() { - revad.Cleanup() + for _, r := range revads { + r.Cleanup() + } }) var assertGetUserByClaimResponses = func() { @@ -234,7 +236,9 @@ var _ = Describe("user providers", func() { Describe("the json userprovider", func() { BeforeEach(func() { - provider = "json" + dependencies = map[string]string{ + "users": "userprovider-json.toml", + } existingIdp = "localhost:20080" }) @@ -245,7 +249,9 @@ var _ = Describe("user providers", func() { Describe("the demo userprovider", func() { BeforeEach(func() { - provider = "demo" + dependencies = map[string]string{ + "users": "userprovider-demo.toml", + } existingIdp = "http://localhost:9998" }) From 3b3aac8f806ed8f479cfad2593f4ce95450c242a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 24 Feb 2021 10:06:07 +0100 Subject: [PATCH 13/24] Add grpc integration tests for purging a recycle item --- .../integration/grpc/storageprovider_test.go | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 3751d975af..f9c0108352 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -85,7 +85,7 @@ var _ = Describe("storage providers", func() { ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) ctx = ruser.ContextSetUser(ctx, user) - revads, err := startRevads(dependencies) + revads, err = startRevads(dependencies) Expect(err).ToNot(HaveOccurred()) serviceClient, err = pool.GetStorageProviderServiceClient(revads["storage"].GrpcAddress) Expect(err).ToNot(HaveOccurred()) @@ -242,17 +242,60 @@ var _ = Describe("storage providers", func() { assertRecycle := func() { It("lists and restores resources", func() { + By("deleting an item") res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) - Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + By("listing the recycle items") listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{}) - Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) Expect(len(listRes.RecycleItems)).To(Equal(1)) item := listRes.RecycleItems[0] Expect(item.Path).To(Equal(subdirPath)) + + By("restoring a recycle item") + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + + restoreRes, err := serviceClient.RestoreRecycleItem(ctx, + &storagep.RestoreRecycleItemRequest{ + Ref: subdirRef, + Key: item.Key, + }, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(restoreRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + + It("purges recycle items resources", func() { + By("deleting an item") + res, err := serviceClient.Delete(ctx, &storagep.DeleteRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + By("listing recycle items") + listRes, err := serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(listRes.RecycleItems)).To(Equal(1)) + + By("purging a recycle item") + purgeRes, err := serviceClient.PurgeRecycle(ctx, &storagep.PurgeRecycleRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(purgeRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + listRes, err = serviceClient.ListRecycle(ctx, &storagep.ListRecycleRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(listRes.RecycleItems)).To(Equal(0)) }) } From 89c061861bce6c3d83ffc7b7352a7fb18b472f42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 24 Feb 2021 10:09:20 +0100 Subject: [PATCH 14/24] Add grpc integration test for UpdateGrant --- tests/integration/grpc/storageprovider_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index f9c0108352..041bba05f3 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -210,6 +210,21 @@ var _ = Describe("storage providers", func() { Expect(readGrant.Permissions.Move).To(BeTrue()) Expect(readGrant.Permissions.Delete).To(BeFalse()) + By("updating the grant") + grant.Permissions.Delete = true + updateRes, err := serviceClient.UpdateGrant(ctx, &storagep.UpdateGrantRequest{Ref: subdirRef, Grant: grant}) + Expect(err).ToNot(HaveOccurred()) + Expect(updateRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + By("listing the update grant") + listRes, err = serviceClient.ListGrants(ctx, &storagep.ListGrantsRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(listRes.Grants)).To(Equal(1)) + readGrant = listRes.Grants[0] + Expect(readGrant.Permissions.Stat).To(BeTrue()) + Expect(readGrant.Permissions.Move).To(BeTrue()) + Expect(readGrant.Permissions.Delete).To(BeTrue()) + By("deleting a grant") delRes, err := serviceClient.RemoveGrant(ctx, &storagep.RemoveGrantRequest{Ref: subdirRef, Grant: readGrant}) Expect(err).ToNot(HaveOccurred()) From c6956b1fd56fc6e7cb567be3620cc528180c1871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 24 Feb 2021 10:17:27 +0100 Subject: [PATCH 15/24] Add grpc integration test for Move --- .../integration/grpc/storageprovider_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 041bba05f3..577e92489c 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -163,6 +163,29 @@ var _ = Describe("storage providers", func() { }) } + assertMove := func() { + It("moves a directory", func() { + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + targetRef := &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: "/new_subdir"}, + } + res, err := serviceClient.Move(ctx, &storagep.MoveRequest{Source: subdirRef, Destination: targetRef}) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(err).ToNot(HaveOccurred()) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: targetRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + } + assertGetPath := func() { It("gets the path to an ID", func() { statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) @@ -338,6 +361,7 @@ var _ = Describe("storage providers", func() { assertListContainer() assertGetPath() assertDelete() + assertMove() assertGrants() assertUploads() assertDownloads() From 6656a01d23764fa162d500853548f35dfa669335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 24 Feb 2021 12:53:49 +0100 Subject: [PATCH 16/24] Add grpc integration test for file versions --- tests/integration/grpc/grpc_suite_test.go | 2 + .../integration/grpc/storageprovider_test.go | 78 +++++++++++++++++-- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 60766f9ccf..800f93839a 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -50,6 +50,7 @@ func TestGprc(t *testing.T) { type cleanupFunc func() error type Revad struct { + TmpRoot string GrpcAddress string Cleanup cleanupFunc } @@ -113,6 +114,7 @@ func startRevads(configs map[string]string) (map[string]*Revad, error) { time.Sleep(1 * time.Second) revad := &Revad{ + TmpRoot: tmpRoot, GrpcAddress: ownAddress, Cleanup: func() error { err := cmd.Process.Signal(os.Kill) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index 577e92489c..fe71931206 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -19,7 +19,9 @@ package grpc_test import ( + "bytes" "context" + "io/ioutil" "google.golang.org/grpc/metadata" @@ -27,6 +29,7 @@ import ( rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" storagep "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/storage/fs/ocis" "github.com/cs3org/reva/pkg/token" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" ruser "github.com/cs3org/reva/pkg/user" @@ -40,14 +43,17 @@ var _ = Describe("storage providers", func() { dependencies map[string]string revads map[string]*Revad + user *userpb.User ctx context.Context serviceClient storagep.ProviderAPIClient - homeRef *storagep.Reference - filePath string - fileRef *storagep.Reference - subdirPath string - subdirRef *storagep.Reference + homeRef *storagep.Reference + versionedFilePath string + versionedFileRef *storagep.Reference + filePath string + fileRef *storagep.Reference + subdirPath string + subdirRef *storagep.Reference ) BeforeEach(func() { @@ -58,6 +64,10 @@ var _ = Describe("storage providers", func() { fileRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: filePath}, } + versionedFilePath = "/versionedFile" + versionedFileRef = &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: versionedFilePath}, + } subdirPath = "/subdir" subdirRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: subdirPath}, @@ -71,7 +81,7 @@ var _ = Describe("storage providers", func() { ctx = context.Background() // Add auth token - user := &userpb.User{ + user = &userpb.User{ Id: &userpb.UserId{ Idp: "0.0.0.0:19000", OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", @@ -147,6 +157,38 @@ var _ = Describe("storage providers", func() { }) } + assertFileVersions := func() { + It("lists file versions", func() { + listRes, err := serviceClient.ListFileVersions(ctx, &storagep.ListFileVersionsRequest{Ref: versionedFileRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(listRes.Versions)).To(Equal(1)) + Expect(listRes.Versions[0].Size).To(Equal(uint64(1))) + }) + + It("restores a file version", func() { + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: versionedFileRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(statRes.Info.Size).To(Equal(uint64(2))) // second version contains 2 bytes + + listRes, err := serviceClient.ListFileVersions(ctx, &storagep.ListFileVersionsRequest{Ref: versionedFileRef}) + Expect(err).ToNot(HaveOccurred()) + restoreRes, err := serviceClient.RestoreFileVersion(ctx, + &storagep.RestoreFileVersionRequest{ + Ref: versionedFileRef, + Key: listRes.Versions[0].Key, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(restoreRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: versionedFileRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(statRes.Info.Size).To(Equal(uint64(1))) // initial version contains 1 byte + }) + } + assertDelete := func() { It("deletes a directory", func() { statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) @@ -367,5 +409,29 @@ var _ = Describe("storage providers", func() { assertDownloads() assertRecycle() }) + + Context("with an existing file /versioned_file", func() { + JustBeforeEach(func() { + fs, err := ocis.New(map[string]interface{}{ + "root": revads["storage"].TmpRoot, + "enable_home": true, + }) + Expect(err).ToNot(HaveOccurred()) + + content1 := ioutil.NopCloser(bytes.NewReader([]byte("1"))) + content2 := ioutil.NopCloser(bytes.NewReader([]byte("22"))) + + ctx := ruser.ContextSetUser(context.Background(), user) + + err = fs.CreateHome(ctx) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content1) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content2) + Expect(err).ToNot(HaveOccurred()) + }) + + assertFileVersions() + }) }) }) From d5cca97d846c6a9a0a8982acb9f191e5a17ae019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 24 Feb 2021 14:40:23 +0100 Subject: [PATCH 17/24] Add grpc integration tests for arbitrary metadata --- .../integration/grpc/storageprovider_test.go | 104 +++++++++++++----- 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index fe71931206..b2ba157072 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -40,53 +40,44 @@ import ( var _ = Describe("storage providers", func() { var ( - dependencies map[string]string - revads map[string]*Revad + dependencies = map[string]string{} + revads = map[string]*Revad{} - user *userpb.User ctx context.Context serviceClient storagep.ProviderAPIClient + user = &userpb.User{ + Id: &userpb.UserId{ + Idp: "0.0.0.0:19000", + OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + }, + } - homeRef *storagep.Reference - versionedFilePath string - versionedFileRef *storagep.Reference - filePath string - fileRef *storagep.Reference - subdirPath string - subdirRef *storagep.Reference - ) - - BeforeEach(func() { homeRef = &storagep.Reference{ - Spec: &storagep.Reference_Path{Path: "/"}, + Spec: &storagep.Reference_Path{Path: "/home"}, } - filePath = "/file" - fileRef = &storagep.Reference{ + filePath = "/home/file" + fileRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: filePath}, } - versionedFilePath = "/versionedFile" - versionedFileRef = &storagep.Reference{ + versionedFilePath = "/home/versionedFile" + versionedFileRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: versionedFilePath}, } - subdirPath = "/subdir" - subdirRef = &storagep.Reference{ + subdirPath = "/home/subdir" + subdirRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: subdirPath}, } - revads = map[string]*Revad{} - dependencies = map[string]string{} - }) + sharesPath = "/Shares" + sharesRef = &storagep.Reference{ + Spec: &storagep.Reference_Path{Path: sharesPath}, + } + ) JustBeforeEach(func() { var err error ctx = context.Background() // Add auth token - user = &userpb.User{ - Id: &userpb.UserId{ - Idp: "0.0.0.0:19000", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - }, - } tokenManager, err := jwt.New(map[string]interface{}{"secret": "changemeplease"}) Expect(err).ToNot(HaveOccurred()) t, err := tokenManager.MintToken(ctx, user) @@ -379,6 +370,59 @@ var _ = Describe("storage providers", func() { }) } + assertReferences := func() { + It("creates references", func() { + listRes, err := serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: sharesRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_NOT_FOUND)) + Expect(len(listRes.Infos)).To(Equal(0)) + + res, err := serviceClient.CreateReference(ctx, &storagep.CreateReferenceRequest{Path: "/Shares/reference", TargetUri: "scheme://target"}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + listRes, err = serviceClient.ListContainer(ctx, &storagep.ListContainerRequest{Ref: sharesRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(listRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(len(listRes.Infos)).To(Equal(1)) + }) + } + + assertMetadata := func() { + It("sets and unsets metadata", func() { + statRes, err := serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(BeEmpty()) + + By("setting arbitrary metadata") + samRes, err := serviceClient.SetArbitraryMetadata(ctx, &storagep.SetArbitraryMetadataRequest{ + Ref: subdirRef, + ArbitraryMetadata: &storagep.ArbitraryMetadata{Metadata: map[string]string{"foo": "bar"}}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(samRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(Equal("bar")) + + By("unsetting arbitrary metadata") + uamRes, err := serviceClient.UnsetArbitraryMetadata(ctx, &storagep.UnsetArbitraryMetadataRequest{ + Ref: subdirRef, + ArbitraryMetadataKeys: []string{"foo"}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(uamRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + Expect(statRes.Info.ArbitraryMetadata.Metadata["foo"]).To(BeEmpty()) + }) + } + Describe("ocis", func() { BeforeEach(func() { dependencies = map[string]string{ @@ -408,6 +452,8 @@ var _ = Describe("storage providers", func() { assertUploads() assertDownloads() assertRecycle() + assertReferences() + assertMetadata() }) Context("with an existing file /versioned_file", func() { From bb1467bde838b98d0422faeed48e0e0127eebc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 25 Feb 2021 14:49:41 +0100 Subject: [PATCH 18/24] Add grpc tests for the owncloud driver --- .../fixtures/storageprovider-owncloud.toml | 11 +++ tests/integration/grpc/grpc_suite_test.go | 45 +++++++-- .../integration/grpc/storageprovider_test.go | 94 +++++++++++++++++-- tests/integration/grpc/userprovider_test.go | 4 +- 4 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 tests/integration/grpc/fixtures/storageprovider-owncloud.toml diff --git a/tests/integration/grpc/fixtures/storageprovider-owncloud.toml b/tests/integration/grpc/fixtures/storageprovider-owncloud.toml new file mode 100644 index 0000000000..a65851c005 --- /dev/null +++ b/tests/integration/grpc/fixtures/storageprovider-owncloud.toml @@ -0,0 +1,11 @@ +[grpc] +address = "{{grpc_address}}" + +[grpc.services.storageprovider] +driver = "owncloud" + +[grpc.services.storageprovider.drivers.owncloud] +enable_home = true +datadirectory = "{{root}}" +userprovidersvc = "{{users_address}}" +redis = "{{redis_address}}" diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 800f93839a..1dbbed4a9f 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -38,7 +38,6 @@ import ( const timeoutMs = 30000 -var revads = map[string]*Revad{} var mutex = sync.Mutex{} var port = 19000 @@ -47,15 +46,37 @@ func TestGprc(t *testing.T) { RunSpecs(t, "Gprc Suite") } -type cleanupFunc func() error +type cleanupFunc func(bool) error +// Revad represents a running revad process type Revad struct { - TmpRoot string - GrpcAddress string - Cleanup cleanupFunc + TmpRoot string // Temporary directory on disk. Will be cleaned up by the Cleanup func. + GrpcAddress string // Address of the grpc service + Cleanup cleanupFunc // Function to kill the process and cleanup the temp. root. If the given parameter is true the files will be kept to make debugging failures easier. } -func startRevads(configs map[string]string) (map[string]*Revad, error) { +// stardRevads takes a list of revad configuration files plus a map of +// variables that need to be substituted in them and starts them. +// +// A unique port is assigned to each spawned instance. +// Placeholders in the config files can be replaced the variables from the +// `variables` map, e.g. the config +// +// redis = "{{redis_address}}" +// +// and the variables map +// +// variables = map[string]string{"redis_address": "localhost:6379"} +// +// will result in the config +// +// redis = "localhost:6379" +// +// Special variables are created for the revad addresses, e.g. having a +// `storage` and a `users` revad will make `storage_address` and +// `users_address` available wit the dynamically assigned ports so that +// the services can be made available to each other. +func startRevads(configs map[string]string, variables map[string]string) (map[string]*Revad, error) { mutex.Lock() defer mutex.Unlock() @@ -82,6 +103,9 @@ func startRevads(configs map[string]string) (map[string]*Revad, error) { cfg := string(rawCfg) cfg = strings.ReplaceAll(cfg, "{{root}}", tmpRoot) cfg = strings.ReplaceAll(cfg, "{{grpc_address}}", ownAddress) + for v, value := range variables { + cfg = strings.ReplaceAll(cfg, "{{"+v+"}}", value) + } for name, address := range addresses { cfg = strings.ReplaceAll(cfg, "{{"+name+"_address}}", address) } @@ -99,6 +123,7 @@ func startRevads(configs map[string]string) (map[string]*Revad, error) { } defer outfile.Close() cmd.Stdout = outfile + cmd.Stderr = outfile err = cmd.Start() if err != nil { @@ -116,13 +141,17 @@ func startRevads(configs map[string]string) (map[string]*Revad, error) { revad := &Revad{ TmpRoot: tmpRoot, GrpcAddress: ownAddress, - Cleanup: func() error { + Cleanup: func(keepLogs bool) error { err := cmd.Process.Signal(os.Kill) if err != nil { return errors.Wrap(err, "Could not kill revad") } waitForPort(ownAddress, "close") - os.RemoveAll(tmpRoot) + if keepLogs { + fmt.Println("Test failed, keeping root", tmpRoot, "around for debugging") + } else { + os.RemoveAll(tmpRoot) + } return nil }, } diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index b2ba157072..cce656b3af 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -22,6 +22,7 @@ import ( "bytes" "context" "io/ioutil" + "os" "google.golang.org/grpc/metadata" @@ -30,6 +31,7 @@ import ( storagep "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/storage/fs/ocis" + "github.com/cs3org/reva/pkg/storage/fs/owncloud" "github.com/cs3org/reva/pkg/token" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" ruser "github.com/cs3org/reva/pkg/user" @@ -38,9 +40,17 @@ import ( . "github.com/onsi/gomega" ) +// This test suite tests the gprc storageprovider interface using different +// storage backends +// +// It uses the `startRevads` helper to spawn the according reva daemon and +// other dependencies like a userprovider if needed. +// It also sets up an authenticated context and a service client to the storage +// provider to be used in the assertion functions. var _ = Describe("storage providers", func() { var ( dependencies = map[string]string{} + variables = map[string]string{} revads = map[string]*Revad{} ctx context.Context @@ -53,17 +63,17 @@ var _ = Describe("storage providers", func() { } homeRef = &storagep.Reference{ - Spec: &storagep.Reference_Path{Path: "/home"}, + Spec: &storagep.Reference_Path{Path: "/"}, } - filePath = "/home/file" + filePath = "/file" fileRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: filePath}, } - versionedFilePath = "/home/versionedFile" + versionedFilePath = "/versionedFile" versionedFileRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: versionedFilePath}, } - subdirPath = "/home/subdir" + subdirPath = "/subdir" subdirRef = &storagep.Reference{ Spec: &storagep.Reference_Path{Path: subdirPath}, } @@ -86,7 +96,7 @@ var _ = Describe("storage providers", func() { ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) ctx = ruser.ContextSetUser(ctx, user) - revads, err = startRevads(dependencies) + revads, err = startRevads(dependencies, variables) Expect(err).ToNot(HaveOccurred()) serviceClient, err = pool.GetStorageProviderServiceClient(revads["storage"].GrpcAddress) Expect(err).ToNot(HaveOccurred()) @@ -94,7 +104,7 @@ var _ = Describe("storage providers", func() { AfterEach(func() { for _, r := range revads { - Expect(r.Cleanup()).To(Succeed()) + Expect(r.Cleanup(CurrentGinkgoTestDescription().Failed)).To(Succeed()) } }) @@ -111,6 +121,10 @@ var _ = Describe("storage providers", func() { statRes, err = serviceClient.Stat(ctx, &storagep.StatRequest{Ref: homeRef}) Expect(err).ToNot(HaveOccurred()) Expect(statRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + ghRes, err := serviceClient.GetHome(ctx, &storagep.GetHomeRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(ghRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) }) } @@ -480,4 +494,72 @@ var _ = Describe("storage providers", func() { assertFileVersions() }) }) + + Describe("owncloud", func() { + BeforeEach(func() { + dependencies = map[string]string{ + "users": "userprovider-json.toml", + "storage": "storageprovider-owncloud.toml", + } + + redisAddress := os.Getenv("REDIS_ADDRESS") + if redisAddress == "" { + Fail("REDIS_ADDRESS not set") + } + variables = map[string]string{ + "redis_address": redisAddress, + } + }) + + assertCreateHome() + + Context("with a home and a subdirectory", func() { + JustBeforeEach(func() { + res, err := serviceClient.CreateHome(ctx, &storagep.CreateHomeRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + + subdirRes, err := serviceClient.CreateContainer(ctx, &storagep.CreateContainerRequest{Ref: subdirRef}) + Expect(err).ToNot(HaveOccurred()) + Expect(subdirRes.Status.Code).To(Equal(rpcv1beta1.Code_CODE_OK)) + }) + + assertCreateContainer() + assertListContainer() + assertGetPath() + assertDelete() + assertMove() + assertGrants() + assertUploads() + assertDownloads() + assertRecycle() + assertReferences() + assertMetadata() + }) + + Context("with an existing file /versioned_file", func() { + JustBeforeEach(func() { + fs, err := owncloud.New(map[string]interface{}{ + "datadirectory": revads["storage"].TmpRoot, + "userprovidersvc": revads["users"].GrpcAddress, + "enable_home": true, + }) + Expect(err).ToNot(HaveOccurred()) + + content1 := ioutil.NopCloser(bytes.NewReader([]byte("1"))) + content2 := ioutil.NopCloser(bytes.NewReader([]byte("22"))) + + ctx := ruser.ContextSetUser(context.Background(), user) + + err = fs.CreateHome(ctx) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content1) + Expect(err).ToNot(HaveOccurred()) + err = fs.Upload(ctx, versionedFileRef, content2) + Expect(err).ToNot(HaveOccurred()) + }) + + assertFileVersions() + }) + }) }) diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 21a7d5b916..24bb7a77a9 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -64,7 +64,7 @@ var _ = Describe("user providers", func() { ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) ctx = ruser.ContextSetUser(ctx, user) - revads, err = startRevads(dependencies) + revads, err = startRevads(dependencies, map[string]string{}) Expect(err).ToNot(HaveOccurred()) serviceClient, err = pool.GetUserProviderServiceClient(revads["users"].GrpcAddress) Expect(err).ToNot(HaveOccurred()) @@ -72,7 +72,7 @@ var _ = Describe("user providers", func() { AfterEach(func() { for _, r := range revads { - r.Cleanup() + r.Cleanup(CurrentGinkgoTestDescription().Failed) } }) From a300361895d0f3b13fab260b1948e00555cc8c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 25 Feb 2021 14:49:58 +0100 Subject: [PATCH 19/24] Return CODE_NOT_FOUND when listing the non-existent shared folder As defined in the protocol. --- pkg/storage/fs/owncloud/owncloud.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index a7a0f47152..78e378a45f 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -1814,6 +1814,9 @@ func (fs *ocfs) listShareFolderRoot(ctx context.Context, sp string, mdKeys []str mds, err := ioutil.ReadDir(ip) if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(fs.toStoragePath(ctx, filepath.Dir(ip))) + } return nil, errors.Wrap(err, "ocfs: error listing shadow_files") } From 2c129ee67e885c94d4c7d1d42503acf3cc1a1967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 4 Mar 2021 15:38:04 +0100 Subject: [PATCH 20/24] Fix hound issues --- tests/integration/grpc/grpc_suite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index 1dbbed4a9f..fdc41b23ab 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -82,9 +82,9 @@ func startRevads(configs map[string]string, variables map[string]string) (map[st revads := map[string]*Revad{} addresses := map[string]string{} - for name, _ := range configs { + for name := range configs { addresses[name] = fmt.Sprintf("localhost:%d", port) - port += 1 + port++ } for name, config := range configs { From f09b63f48ef7d840fc9476c91b6012378aec9bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Mar 2021 11:03:19 +0100 Subject: [PATCH 21/24] Fix license headers --- pkg/storage/utils/ace/ace_suite_test.go | 18 ++++++++++++++++++ pkg/storage/utils/decomposedfs/lookup_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/pkg/storage/utils/ace/ace_suite_test.go b/pkg/storage/utils/ace/ace_suite_test.go index 348b65fab2..62778e0ba3 100644 --- a/pkg/storage/utils/ace/ace_suite_test.go +++ b/pkg/storage/utils/ace/ace_suite_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package ace_test import ( diff --git a/pkg/storage/utils/decomposedfs/lookup_test.go b/pkg/storage/utils/decomposedfs/lookup_test.go index 0e2a3b5966..e6283abf56 100644 --- a/pkg/storage/utils/decomposedfs/lookup_test.go +++ b/pkg/storage/utils/decomposedfs/lookup_test.go @@ -1,3 +1,21 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package decomposedfs_test import ( From 72e19b9ef19db2707f0e98da60763a3dee1594d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Mar 2021 11:03:41 +0100 Subject: [PATCH 22/24] Move the grpc integration tests to a separate pipeline --- .drone.star | 34 ++++++++++++++++++++++++++++++++++ Makefile | 5 ++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index fff80f1d63..42e158f24b 100644 --- a/.drone.star +++ b/.drone.star @@ -75,6 +75,7 @@ def main(ctx): return [ buildAndPublishDocker(), buildOnly(), + testIntegration(), release(), litmusOcisOldWebdav(), litmusOcisNewWebdav(), @@ -256,6 +257,39 @@ def buildOnly(): ] } +def testIntegration(): + return { + "kind": "pipeline", + "type": "docker", + "name": "test-integration", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + ], + }, + }, + "steps": [ + { + "name": "test", + "image": "registry.cern.ch/docker.io/library/golang:1.13", + "commands": [ + "make test-integration", + ], + "environment": { + "REDIS_ADDRESS": "redis:6379", + }, + } + ], + "services": [ + redisService(), + ], + } + def release(): return { "kind": "pipeline", diff --git a/Makefile b/Makefile index 374b0e756d..90d93adbcd 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,10 @@ build-reva: imports go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva test: off - go test -race ./... + go test -race $$(go list ./... | grep -v /tests/integration) + +test-integration: build-ci + cd tests/integration && go test -race ./... lint: go run tools/check-license/check-license.go From a1ef1cf7ff1ddd5ddb05bfeaf277aaed43d57bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 5 Mar 2021 11:40:58 +0100 Subject: [PATCH 23/24] Fix linter issues --- tests/integration/grpc/grpc_suite_test.go | 2 +- tests/integration/grpc/userprovider_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/grpc/grpc_suite_test.go b/tests/integration/grpc/grpc_suite_test.go index fdc41b23ab..b387e390a6 100644 --- a/tests/integration/grpc/grpc_suite_test.go +++ b/tests/integration/grpc/grpc_suite_test.go @@ -146,7 +146,7 @@ func startRevads(configs map[string]string, variables map[string]string) (map[st if err != nil { return errors.Wrap(err, "Could not kill revad") } - waitForPort(ownAddress, "close") + _ = waitForPort(ownAddress, "close") if keepLogs { fmt.Println("Test failed, keeping root", tmpRoot, "around for debugging") } else { diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 24bb7a77a9..3af9633786 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -72,7 +72,7 @@ var _ = Describe("user providers", func() { AfterEach(func() { for _, r := range revads { - r.Cleanup(CurrentGinkgoTestDescription().Failed) + Expect(r.Cleanup(CurrentGinkgoTestDescription().Failed)) } }) From 21c536e0f822c1ba39fe0580fb6e999ff71c8a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 11 Mar 2021 15:17:47 +0100 Subject: [PATCH 24/24] Add changelog --- changelog/unreleased/add-grpc-testsuite.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/add-grpc-testsuite.md diff --git a/changelog/unreleased/add-grpc-testsuite.md b/changelog/unreleased/add-grpc-testsuite.md new file mode 100644 index 0000000000..c3671432af --- /dev/null +++ b/changelog/unreleased/add-grpc-testsuite.md @@ -0,0 +1,6 @@ +Enhancement: Add grpc test suite for the storage provider + +A new test suite has been added which tests the grpc interface to the storage +provider. It currently runs against the ocis and the owncloud storage drivers. + +https://github.com/cs3org/reva/pull/1514 \ No newline at end of file