Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[tests-only][full-ci] Test pipeline fix #5406

Closed
wants to merge 6 commits into from
Closed

[tests-only][full-ci] Test pipeline fix #5406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1e0c5a1
pass estream to storageprovider
kobergj Jan 13, 2023
1139036
use feature reva
kobergj Jan 13, 2023
80c962c
event config
kobergj Jan 18, 2023
c722434
reva bump
kobergj Jan 26, 2023
dc18940
don't configure events for dataprovider dcfs
kobergj Jan 31, 2023
041bd5b
no events for storage provider
kobergj Jan 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,3 +289,5 @@ require (
)

replace github.com/cs3org/go-cs3apis => github.com/c0rby/go-cs3apis v0.0.0-20230110100311-5b424f1baa35

replace github.com/cs3org/reva/v2 => github.com/kobergj/reva/v2 v2.0.0-20230126140608-584cca852cee
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,10 +344,6 @@ github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3p
github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A=
github.com/crewjam/saml v0.4.9 h1:X2jDv4dv3IvfT9t+RhADavzNFAcq3fVxzTCIH3G605U=
github.com/crewjam/saml v0.4.9/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4=
github.com/cs3org/reva/v2 v2.12.1-0.20230125142028-2d125a5354d0 h1:FCKt9i38oNgs6oQYpdWaIj7KfJykCNer22HwTUi+h3Y=
github.com/cs3org/reva/v2 v2.12.1-0.20230125142028-2d125a5354d0/go.mod h1:u73Df9JAZsDj43GIjQIb3DO1PLJuPutZXkRqQH0oGXA=
github.com/cs3org/reva/v2 v2.12.1-0.20230126045302-72336e6ececa h1:adJ1cFJ2QA+ZaloXEsEAd5jO++qFmEUavv8J2TyvSYY=
github.com/cs3org/reva/v2 v2.12.1-0.20230126045302-72336e6ececa/go.mod h1:u73Df9JAZsDj43GIjQIb3DO1PLJuPutZXkRqQH0oGXA=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
Expand Down Expand Up @@ -875,6 +871,8 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/kobergj/reva/v2 v2.0.0-20230126140608-584cca852cee h1:KkOrwQg0+prZCEjgJCc3PyivH3tkobt3vIWholtD7/E=
github.com/kobergj/reva/v2 v2.0.0-20230126140608-584cca852cee/go.mod h1:u73Df9JAZsDj43GIjQIb3DO1PLJuPutZXkRqQH0oGXA=
github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down
4 changes: 2 additions & 2 deletions services/storage-users/pkg/command/uploads.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func ListUploads(cfg *config.Config) *cli.Command {
os.Exit(1)
}
drivers := revaconfig.UserDrivers(cfg)
fs, err := f(drivers[cfg.Driver].(map[string]interface{}))
fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver)
return err
Expand Down Expand Up @@ -86,7 +86,7 @@ func PurgeExpiredUploads(cfg *config.Config) *cli.Command {
os.Exit(1)
}
drivers := revaconfig.UserDrivers(cfg)
fs, err := f(drivers[cfg.Driver].(map[string]interface{}))
fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver)
return err
Expand Down
9 changes: 8 additions & 1 deletion services/storage-users/pkg/revaconfig/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,13 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} {
"expose_data_server": cfg.ExposeDataServer,
"data_server_url": cfg.DataServerURL,
"upload_expiration": cfg.UploadExpiration,
"events": map[string]interface{}{
"nats_address": cfg.Events.Addr,
"nats_clusterid": cfg.Events.ClusterID,
"nats_enable_tls": cfg.Events.EnableTLS,
"tls_insecure": cfg.Events.TLSInsecure,
"tls_root_ca_cert": cfg.Events.TLSRootCaCertPath,
},
},
},
"interceptors": map[string]interface{}{
Expand Down Expand Up @@ -65,7 +72,7 @@ func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} {
"dataprovider": map[string]interface{}{
"prefix": cfg.HTTP.Prefix,
"driver": cfg.Driver,
"drivers": UserDrivers(cfg),
"drivers": DataProviderDrivers(cfg),
"nats_address": cfg.Events.Addr,
"nats_clusterID": cfg.Events.ClusterID,
"nats_tls_insecure": cfg.Events.TLSInsecure,
Expand Down
169 changes: 169 additions & 0 deletions services/storage-users/pkg/revaconfig/dataprovider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
package revaconfig

import "github.com/owncloud/ocis/v2/services/storage-users/pkg/config"

// DataProviderDrivers are the storage drivers for the data provider
func DataProviderDrivers(cfg *config.Config) map[string]interface{} {
return map[string]interface{}{
"eos": map[string]interface{}{
"namespace": cfg.Drivers.EOS.Root,
"shadow_namespace": cfg.Drivers.EOS.ShadowNamespace,
"uploads_namespace": cfg.Drivers.EOS.UploadsNamespace,
"share_folder": cfg.Drivers.EOS.ShareFolder,
"eos_binary": cfg.Drivers.EOS.EosBinary,
"xrdcopy_binary": cfg.Drivers.EOS.XrdcopyBinary,
"master_url": cfg.Drivers.EOS.MasterURL,
"slave_url": cfg.Drivers.EOS.SlaveURL,
"cache_directory": cfg.Drivers.EOS.CacheDirectory,
"sec_protocol": cfg.Drivers.EOS.SecProtocol,
"keytab": cfg.Drivers.EOS.Keytab,
"single_username": cfg.Drivers.EOS.SingleUsername,
"enable_logging": cfg.Drivers.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Drivers.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Drivers.EOS.ForceSingleUserMode,
"use_keytab": cfg.Drivers.EOS.UseKeytab,
"gatewaysvc": cfg.Drivers.EOS.GatewaySVC,
},
"eoshome": map[string]interface{}{
"namespace": cfg.Drivers.EOS.Root,
"shadow_namespace": cfg.Drivers.EOS.ShadowNamespace,
"uploads_namespace": cfg.Drivers.EOS.UploadsNamespace,
"share_folder": cfg.Drivers.EOS.ShareFolder,
"eos_binary": cfg.Drivers.EOS.EosBinary,
"xrdcopy_binary": cfg.Drivers.EOS.XrdcopyBinary,
"master_url": cfg.Drivers.EOS.MasterURL,
"slave_url": cfg.Drivers.EOS.SlaveURL,
"cache_directory": cfg.Drivers.EOS.CacheDirectory,
"sec_protocol": cfg.Drivers.EOS.SecProtocol,
"keytab": cfg.Drivers.EOS.Keytab,
"single_username": cfg.Drivers.EOS.SingleUsername,
"user_layout": cfg.Drivers.EOS.UserLayout,
"enable_logging": cfg.Drivers.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Drivers.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Drivers.EOS.ForceSingleUserMode,
"use_keytab": cfg.Drivers.EOS.UseKeytab,
"gatewaysvc": cfg.Drivers.EOS.GatewaySVC,
},
"eosgrpc": map[string]interface{}{
"namespace": cfg.Drivers.EOS.Root,
"shadow_namespace": cfg.Drivers.EOS.ShadowNamespace,
"share_folder": cfg.Drivers.EOS.ShareFolder,
"eos_binary": cfg.Drivers.EOS.EosBinary,
"xrdcopy_binary": cfg.Drivers.EOS.XrdcopyBinary,
"master_url": cfg.Drivers.EOS.MasterURL,
"master_grpc_uri": cfg.Drivers.EOS.GRPCURI,
"slave_url": cfg.Drivers.EOS.SlaveURL,
"cache_directory": cfg.Drivers.EOS.CacheDirectory,
"sec_protocol": cfg.Drivers.EOS.SecProtocol,
"keytab": cfg.Drivers.EOS.Keytab,
"single_username": cfg.Drivers.EOS.SingleUsername,
"user_layout": cfg.Drivers.EOS.UserLayout,
"enable_logging": cfg.Drivers.EOS.EnableLogging,
"show_hidden_sys_files": cfg.Drivers.EOS.ShowHiddenSysFiles,
"force_single_user_mode": cfg.Drivers.EOS.ForceSingleUserMode,
"use_keytab": cfg.Drivers.EOS.UseKeytab,
"enable_home": false,
"gatewaysvc": cfg.Drivers.EOS.GatewaySVC,
},
"local": map[string]interface{}{
"root": cfg.Drivers.Local.Root,
"share_folder": cfg.Drivers.Local.ShareFolder,
},
"localhome": map[string]interface{}{
"root": cfg.Drivers.Local.Root,
"share_folder": cfg.Drivers.Local.ShareFolder,
"user_layout": cfg.Drivers.Local.UserLayout,
},
"owncloudsql": map[string]interface{}{
"datadirectory": cfg.Drivers.OwnCloudSQL.Root,
"upload_info_dir": cfg.Drivers.OwnCloudSQL.UploadInfoDir,
"share_folder": cfg.Drivers.OwnCloudSQL.ShareFolder,
"user_layout": cfg.Drivers.OwnCloudSQL.UserLayout,
"enable_home": false,
"dbusername": cfg.Drivers.OwnCloudSQL.DBUsername,
"dbpassword": cfg.Drivers.OwnCloudSQL.DBPassword,
"dbhost": cfg.Drivers.OwnCloudSQL.DBHost,
"dbport": cfg.Drivers.OwnCloudSQL.DBPort,
"dbname": cfg.Drivers.OwnCloudSQL.DBName,
"userprovidersvc": cfg.Drivers.OwnCloudSQL.UsersProviderEndpoint,
},
"ocis": map[string]interface{}{
"root": cfg.Drivers.OCIS.Root,
"user_layout": cfg.Drivers.OCIS.UserLayout,
"share_folder": cfg.Drivers.OCIS.ShareFolder,
"personalspacealias_template": cfg.Drivers.OCIS.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.OCIS.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.Cache.Store,
"cache_nodes": cfg.Cache.Nodes,
"cache_database": cfg.Cache.Database,
},
// "events": map[string]interface{}{
// "natsaddress": cfg.Events.Addr,
// "natsclusterid": cfg.Events.ClusterID,
// "tlsinsecure": cfg.Events.TLSInsecure,
// "tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
// "numconsumers": cfg.Events.NumConsumers,
// },
"tokens": map[string]interface{}{
"transfer_shared_secret": cfg.Commons.TransferSecret,
"transfer_expires": cfg.TransferExpires,
"download_endpoint": cfg.DataServerURL,
"datagateway_endpoint": cfg.DataGatewayURL,
},
},
"s3": map[string]interface{}{
"enable_home": false,
"region": cfg.Drivers.S3.Region,
"access_key": cfg.Drivers.S3.AccessKey,
"secret_key": cfg.Drivers.S3.SecretKey,
"endpoint": cfg.Drivers.S3.Endpoint,
"bucket": cfg.Drivers.S3.Bucket,
"prefix": cfg.Drivers.S3.Root,
},
"s3ng": map[string]interface{}{
"root": cfg.Drivers.S3NG.Root,
"user_layout": cfg.Drivers.S3NG.UserLayout,
"share_folder": cfg.Drivers.S3NG.ShareFolder,
"personalspacealias_template": cfg.Drivers.S3NG.PersonalSpaceAliasTemplate,
"generalspacealias_template": cfg.Drivers.S3NG.GeneralSpaceAliasTemplate,
"treetime_accounting": true,
"treesize_accounting": true,
"permissionssvc": cfg.Drivers.S3NG.PermissionsEndpoint,
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"s3.region": cfg.Drivers.S3NG.Region,
"s3.access_key": cfg.Drivers.S3NG.AccessKey,
"s3.secret_key": cfg.Drivers.S3NG.SecretKey,
"s3.endpoint": cfg.Drivers.S3NG.Endpoint,
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
// "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.Cache.Store,
"cache_nodes": cfg.Cache.Nodes,
"cache_database": cfg.Cache.Database,
},
// "events": map[string]interface{}{
// "natsaddress": cfg.Events.Addr,
// "natsclusterid": cfg.Events.ClusterID,
// "tlsinsecure": cfg.Events.TLSInsecure,
// "tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
// "numconsumers": cfg.Events.NumConsumers,
// },
"tokens": map[string]interface{}{
"transfer_shared_secret": cfg.Commons.TransferSecret,
"transfer_expires": cfg.TransferExpires,
"download_endpoint": cfg.DataServerURL,
"datagateway_endpoint": cfg.DataGatewayURL,
},
},
}
}
56 changes: 28 additions & 28 deletions services/storage-users/pkg/revaconfig/user.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,25 +98,25 @@ func UserDrivers(cfg *config.Config) map[string]interface{} {
"permissionssvc_tls_mode": cfg.Commons.GRPCClientTLS.Mode,
"max_acquire_lock_cycles": cfg.Drivers.OCIS.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.OCIS.LockCycleDurationFactor,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
// "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.Cache.Store,
"cache_nodes": cfg.Cache.Nodes,
"cache_database": cfg.Cache.Database,
},
"events": map[string]interface{}{
"natsaddress": cfg.Events.Addr,
"natsclusterid": cfg.Events.ClusterID,
"tlsinsecure": cfg.Events.TLSInsecure,
"tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
"numconsumers": cfg.Events.NumConsumers,
},
"tokens": map[string]interface{}{
"transfer_shared_secret": cfg.Commons.TransferSecret,
"transfer_expires": cfg.TransferExpires,
"download_endpoint": cfg.DataServerURL,
"datagateway_endpoint": cfg.DataGatewayURL,
},
// "events": map[string]interface{}{
// "natsaddress": cfg.Events.Addr,
// "natsclusterid": cfg.Events.ClusterID,
// "tlsinsecure": cfg.Events.TLSInsecure,
// "tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
// "numconsumers": cfg.Events.NumConsumers,
// },
// "tokens": map[string]interface{}{
// "transfer_shared_secret": cfg.Commons.TransferSecret,
// "transfer_expires": cfg.TransferExpires,
// "download_endpoint": cfg.DataServerURL,
// "datagateway_endpoint": cfg.DataGatewayURL,
// },
},
"s3": map[string]interface{}{
"enable_home": false,
Expand Down Expand Up @@ -144,25 +144,25 @@ func UserDrivers(cfg *config.Config) map[string]interface{} {
"s3.bucket": cfg.Drivers.S3NG.Bucket,
"max_acquire_lock_cycles": cfg.Drivers.S3NG.MaxAcquireLockCycles,
"lock_cycle_duration_factor": cfg.Drivers.S3NG.LockCycleDurationFactor,
"asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
// "asyncfileuploads": cfg.Drivers.OCIS.AsyncUploads,
"statcache": map[string]interface{}{
"cache_store": cfg.Cache.Store,
"cache_nodes": cfg.Cache.Nodes,
"cache_database": cfg.Cache.Database,
},
"events": map[string]interface{}{
"natsaddress": cfg.Events.Addr,
"natsclusterid": cfg.Events.ClusterID,
"tlsinsecure": cfg.Events.TLSInsecure,
"tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
"numconsumers": cfg.Events.NumConsumers,
},
"tokens": map[string]interface{}{
"transfer_shared_secret": cfg.Commons.TransferSecret,
"transfer_expires": cfg.TransferExpires,
"download_endpoint": cfg.DataServerURL,
"datagateway_endpoint": cfg.DataGatewayURL,
},
// "events": map[string]interface{}{
// "natsaddress": cfg.Events.Addr,
// "natsclusterid": cfg.Events.ClusterID,
// "tlsinsecure": cfg.Events.TLSInsecure,
// "tlsrootcacertificate": cfg.Events.TLSRootCaCertPath,
// "numconsumers": cfg.Events.NumConsumers,
// },
// "tokens": map[string]interface{}{
// "transfer_shared_secret": cfg.Commons.TransferSecret,
// "transfer_expires": cfg.TransferExpires,
// "download_endpoint": cfg.DataServerURL,
// "datagateway_endpoint": cfg.DataGatewayURL,
// },
},
}
}