Skip to content

Commit

Permalink
Merge pull request cs3org#4562 from aduffeck/decomposed-based-posixfs
Browse files Browse the repository at this point in the history 
Decomposed based posix filesystem driver
  • Loading branch information
@aduffeck
aduffeck authored May 29, 2024
2 parents 0a513de + 5ff2e82 commit 5d7302c
Show file tree
Hide file tree
Showing 39 changed files with 4,569 additions and 420 deletions.
83 changes: 80 additions & 3 deletions .drone.star
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# images
OC_CI_GOLANG = "owncloudci/golang:1.21"
OC_CI_GOLANG = "owncloudci/golang:1.22@sha256:bc1ff4ac994a432146b0424207ca89985491496fcc534156cad5ad5d9e7e216d"
OC_CI_ALPINE = "owncloudci/alpine:latest"
OSIXIA_OPEN_LDAP = "osixia/openldap:1.3.0"
REDIS = "redis:6-alpine"
Expand Down Expand Up @@ -103,7 +103,7 @@ def cephService():
def main(ctx):
# In order to run specific parts only, specify the parts as
# ocisIntegrationTests(6, [1, 4]) - this will only run 1st and 4th parts
# implemented for: ocisIntegrationTests and s3ngIntegrationTests
# implemented for: ocisIntegrationTests, posixfsIntegrationTests and s3ngIntegrationTests
return [
checkStarlark(),
checkGoGenerate(),
Expand All @@ -117,7 +117,7 @@ def main(ctx):
cs3ApiValidatorS3NG(),
# virtual views don't work on edge at the moment
#virtualViews(),
] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)
] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) + posixfsIntegrationTests(6)

def coverage():
return {
Expand Down Expand Up @@ -705,6 +705,83 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):

return pipelines

def posixfsIntegrationTests(parallelRuns, skipExceptParts = []):
pipelines = []
debugPartsEnabled = (len(skipExceptParts) != 0)
for runPart in range(1, parallelRuns + 1):
if debugPartsEnabled and runPart not in skipExceptParts:
continue

pipelines.append(
{
"kind": "pipeline",
"type": "docker",
"name": "posixfs-integration-tests-%s" % runPart,
"platform": {
"os": "linux",
"arch": "amd64",
},
"trigger": {
"ref": [
"refs/heads/master",
"refs/heads/edge",
"refs/pull/**",
],
},
"steps": [
makeStep("build-ci"),
{
"name": "revad-services",
"image": OC_CI_GOLANG,
"detach": True,
"commands": [
"cd /drone/src/tests/oc-integration-tests/drone/",
"/drone/src/cmd/revad/revad -c frontend.toml &",
"/drone/src/cmd/revad/revad -c gateway.toml &",
"/drone/src/cmd/revad/revad -c shares.toml &",
"/drone/src/cmd/revad/revad -c storage-shares.toml &",
"/drone/src/cmd/revad/revad -c machine-auth.toml &",
"/drone/src/cmd/revad/revad -c storage-users-posixfs.toml &",
"/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
"/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
"/drone/src/cmd/revad/revad -c ldap-users.toml",
],
},
cloneApiTestReposStep(),
{
"name": "APIAcceptanceTestsPosixStorage",
"image": OC_CI_PHP,
"commands": [
"cd /drone/src/tmp/testrunner",
"make test-acceptance-from-core-api",
],
"environment": {
"TEST_SERVER_URL": "http://revad-services:20080",
"OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
"DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/*",
"STORAGE_DRIVER": "ocis",
"SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
"TEST_WITH_LDAP": "true",
"REVA_LDAP_HOSTNAME": "ldap",
"TEST_REVA": "true",
"SEND_SCENARIO_LINE_REFERENCES": "true",
"BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@env-config",
"DIVIDE_INTO_NUM_PARTS": parallelRuns,
"RUN_PART": runPart,
"EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-POSIX-storage.md",
},
},
],
"services": [
redisService(),
ldapService(),
],
"depends_on": ["unit-test-coverage"],
},
)

return pipelines

def checkStarlark():
return {
"kind": "pipeline",
Expand Down
3 changes: 3 additions & 0 deletions .mockery.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ packages:
github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree:
interfaces:
Blobstore:
github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/usermapper:
interfaces:
Mapper:
github.com/cs3org/reva/v2/pkg/storage/utils/indexer:
interfaces:
Indexer:
Expand Down
5 changes: 5 additions & 0 deletions changelog/unreleased/improve-posixfs.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Enhancement: Improve posixfs storage driver

Improve the posixfs storage driver by fixing several issues and adding missing features.

https://github.com/cs3org/reva/pull/4562
12 changes: 10 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ require (
github.com/go-micro/plugins/v4/events/natsjs v1.2.2-0.20231215124540-f7f8d3274bf9
github.com/go-micro/plugins/v4/server/http v1.2.2
github.com/go-micro/plugins/v4/store/nats-js v1.2.0
github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e
github.com/go-micro/plugins/v4/store/redis v1.2.1
github.com/go-playground/locales v0.14.1
github.com/go-playground/universal-translator v0.18.1
Expand Down Expand Up @@ -61,13 +62,15 @@ require (
github.com/onsi/ginkgo/v2 v2.15.0
github.com/onsi/gomega v1.31.1
github.com/owncloud/ocis/v2 v2.0.0
github.com/pablodz/inotifywaitgo v0.0.6
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.9
github.com/prometheus/alertmanager v0.26.0
github.com/prometheus/client_golang v1.18.0
github.com/rogpeppe/go-internal v1.12.0
github.com/rs/cors v1.10.1
github.com/rs/zerolog v1.32.0
github.com/segmentio/kafka-go v0.4.47
github.com/sethvargo/go-password v0.2.0
github.com/shamaton/msgpack/v2 v2.1.1
github.com/stretchr/testify v1.8.4
Expand All @@ -88,7 +91,7 @@ require (
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
golang.org/x/oauth2 v0.17.0
golang.org/x/sync v0.6.0
golang.org/x/sys v0.17.0
golang.org/x/sys v0.19.0
golang.org/x/term v0.17.0
golang.org/x/text v0.14.0
google.golang.org/genproto v0.0.0-20240205150955-31a09d347014
Expand Down Expand Up @@ -136,7 +139,7 @@ require (
github.com/go-micro/plugins/v4/registry/mdns v1.2.0 // indirect
github.com/go-micro/plugins/v4/registry/memory v1.2.0 // indirect
github.com/go-micro/plugins/v4/registry/nats v1.2.1 // indirect
github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/errors v0.20.4 // indirect
github.com/go-openapi/strfmt v0.21.7 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
Expand Down Expand Up @@ -185,6 +188,7 @@ require (
github.com/oklog/ulid v1.3.1 // indirect
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pkg/term v1.2.0-beta.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
Expand All @@ -195,14 +199,18 @@ require (
github.com/rs/xid v1.5.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/urfave/cli/v2 v2.25.7 // indirect
github.com/xanzy/ssh-agent v0.3.2 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.etcd.io/etcd/api/v3 v3.5.12 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
Expand Down
Loading

0 comments on commit 5d7302c

Please sign in to comment.