Skip to content

Commit

Permalink
Add s3ng integration tests (#1466)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aduffeck
aduffeck authored Mar 15, 2021
1 parent 3b7e1f0 commit 7d251bf
Show file tree
Hide file tree
Showing 9 changed files with 2,388 additions and 34 deletions.
91 changes: 90 additions & 1 deletion .drone.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,25 @@ def redisService():
},
}

def cephService():
return {
"name": "ceph",
"image": "ceph/daemon",
"pull": "always",
"environment": {
"CEPH_DAEMON": "demo",
"NETWORK_AUTO_DETECT": "4",
"MON_IP": "0.0.0.0",
"CEPH_PUBLIC_NETWORK": "0.0.0.0/0",
"RGW_CIVETWEB_PORT": "4000 ",
"RGW_NAME": "ceph",
"CEPH_DEMO_UID": "test-user",
"CEPH_DEMO_ACCESS_KEY": "test",
"CEPH_DEMO_SECRET_KEY": "test",
"CEPH_DEMO_BUCKET": "test",
}
}

# Pipeline definitions
def main(ctx):
return [
Expand All @@ -80,7 +99,7 @@ def main(ctx):
litmusOcisNewWebdav(),
localIntegrationTestsOwncloud(),
localIntegrationTestsOcis(),
] + ocisIntegrationTests(6) + owncloudIntegrationTests(6)
] + ocisIntegrationTests(6) + owncloudIntegrationTests(6) + s3ngIntegrationTests(12)


def buildAndPublishDocker():
Expand Down Expand Up @@ -718,4 +737,74 @@ def owncloudIntegrationTests(parallelRuns):
}
)

return pipelines

def s3ngIntegrationTests(parallelRuns):
pipelines = []
for runPart in range(1, parallelRuns + 1):
pipelines.append(
{
"kind": "pipeline",
"type": "docker",
"name": "s3ng-integration-tests-%s" % runPart,
"platform": {
"os": "linux",
"arch": "amd64",
},
"trigger": {
"event": {
"include": [
"pull_request",
"tag",
],
},
},
"steps": [
makeStep("build-ci"),
{
"name": "revad-services",
"image": "registry.cern.ch/docker.io/library/golang:1.13",
"detach": True,
"commands": [
"cd /drone/src/tests/oc-integration-tests/drone/",
"/drone/src/cmd/revad/revad -c frontend.toml &",
"/drone/src/cmd/revad/revad -c gateway.toml &",
"/drone/src/cmd/revad/revad -c shares.toml &",
"/drone/src/cmd/revad/revad -c storage-home-s3ng.toml &",
"/drone/src/cmd/revad/revad -c storage-oc-s3ng.toml &",
"/drone/src/cmd/revad/revad -c storage-publiclink-s3ng.toml &",
"/drone/src/cmd/revad/revad -c ldap-users.toml",
],
},
cloneOc10TestReposStep(),
{
"name": "oC10APIAcceptanceTestsS3ngStorage",
"image": "registry.cern.ch/docker.io/owncloudci/php:7.4",
"commands": [
"cd /drone/src/tmp/testrunner",
"make test-acceptance-api",
],
"environment": {
"TEST_SERVER_URL": "http://revad-services:20080",
"OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
"DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-*",
"STORAGE_DRIVER": "S3NG",
"SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
"TEST_WITH_LDAP": "true",
"REVA_LDAP_HOSTNAME": "ldap",
"TEST_REVA": "true",
"BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage",
"DIVIDE_INTO_NUM_PARTS": parallelRuns,
"RUN_PART": runPart,
"EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md",
},
},
],
"services": [
ldapService(),
cephService(),
],
}
)

return pipelines
5 changes: 5 additions & 0 deletions changelog/unreleased/add-s3ng-integration-tests.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Enhancement: Add integration tests for the s3ng driver

We extended the integration test suite to also run all tests against the s3ng driver.

https://github.com/cs3org/reva/pull/1466
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ require (
github.com/imdario/mergo v0.3.8 // indirect
github.com/jedib0t/go-pretty v4.3.0+incompatible
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/minio/minio-go/v7 v7.0.9
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.4.1
github.com/onsi/ginkgo v1.15.1
Expand Down
16 changes: 16 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
Expand All @@ -656,6 +657,9 @@ github.com/karrick/godirwalk v1.15.5/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -742,10 +746,17 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.9 h1:v+RS2/dpRq+XaarlZItHd3MVjjQcN2noRn4HxmVdmg4=
github.com/minio/minio-go/v7 v7.0.9/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
Expand All @@ -761,8 +772,10 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
Expand Down Expand Up @@ -938,6 +951,7 @@ github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
Expand Down Expand Up @@ -1150,6 +1164,7 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c h1:9HhBz5L/UjnK9XLtiZhYAdue5BVKep3PMmS2LuPDt8k=
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -1232,6 +1247,7 @@ golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
Expand Down
56 changes: 23 additions & 33 deletions pkg/storage/fs/s3ng/blobstore/blobstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,51 +19,49 @@
package blobstore

import (
"context"
"io"
"net/url"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/pkg/errors"
)

// Blobstore provides an interface to an s3 compatible blobstore
type Blobstore struct {
s3 *s3.S3
uploader *s3manager.Uploader
client *minio.Client

bucket string
}

// New returns a new Blobstore
func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, error) {
sess, err := session.NewSession(&aws.Config{
Endpoint: aws.String(endpoint),
Region: aws.String(region),
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
S3ForcePathStyle: aws.Bool(true),
u, err := url.Parse(endpoint)
if err != nil {
return nil, errors.Wrap(err, "failed to parse s3 endpoint")
}

useSSL := u.Scheme != "http"
client, err := minio.New(u.Host, &minio.Options{
Region: region,
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: useSSL,
})
if err != nil {
return nil, errors.Wrap(err, "failed to setup s3 session")
return nil, errors.Wrap(err, "failed to setup s3 client")
}
uploader := s3manager.NewUploader(sess)

return &Blobstore{
uploader: uploader,
s3: s3.New(sess),
bucket: bucket,
client: client,
bucket: bucket,
}, nil
}

// Upload stores some data in the blobstore under the given key
func (bs *Blobstore) Upload(key string, reader io.Reader) error {
_, err := bs.uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bs.bucket),
Key: aws.String(key),
Body: reader,
})
_, err := bs.client.PutObject(context.Background(), bs.bucket, key, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"})

if err != nil {
return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", key, bs.bucket)
}
Expand All @@ -72,24 +70,16 @@ func (bs *Blobstore) Upload(key string, reader io.Reader) error {

// Download retrieves a blob from the blobstore for reading
func (bs *Blobstore) Download(key string) (io.ReadCloser, error) {
input := &s3.GetObjectInput{
Bucket: aws.String(bs.bucket),
Key: aws.String(key),
}
result, err := bs.s3.GetObject(input)
reader, err := bs.client.GetObject(context.Background(), bs.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", key, bs.bucket)
}
return result.Body, nil
return reader, nil
}

// Delete deletes a blob from the blobstore
func (bs *Blobstore) Delete(key string) error {
input := &s3.DeleteObjectInput{
Bucket: aws.String(bs.bucket),
Key: aws.String(key),
}
_, err := bs.s3.DeleteObject(input)
err := bs.client.RemoveObject(context.Background(), bs.bucket, key, minio.RemoveObjectOptions{})
if err != nil {
return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", key, bs.bucket)
}
Expand Down
Loading

0 comments on commit 7d251bf

Please sign in to comment.