Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[full-ci] Refactor storage configuration to make it similar to other oCIS configuration #3569

Merged
merged 29 commits into from
Apr 26, 2022
Merged
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
57e4e70
remove auth basic command to improve config code
Apr 19, 2022
b31b708
make auth-basic config similar to other services
Apr 19, 2022
ed4e9ee
make user provider config similar to other services
Apr 19, 2022
9e20992
make group provider config similar to other services
Apr 19, 2022
c70a60d
make app provider config similar to other services
Apr 20, 2022
7c87934
make auth bearer config similar to other services
Apr 20, 2022
5013d71
make sharing config similar to other services
Apr 20, 2022
d303bb6
make storage metadata config similar to other services
Apr 21, 2022
4c6848e
make auth machine config similar to other services
Apr 21, 2022
12c14e3
make storage publiclink config similar to other services
Apr 21, 2022
f40ec3c
make storage users config similar to other services
Apr 21, 2022
f39e354
make storage shares config similar to other services
Apr 21, 2022
430e7f6
make ocdav config similar to other services
Apr 22, 2022
76a6588
make frontend config similar to other services
Apr 22, 2022
60bf000
make gateway config similar to other services
Apr 22, 2022
01e307f
clean up the new commands
Apr 22, 2022
f51bdd8
fix config docs generation by adding the Sanitize function everywhere
wkloucek Apr 25, 2022
b26b68b
fix sharing yaml tag
wkloucek Apr 25, 2022
c29f69d
add missing configs
Apr 25, 2022
89b925f
fix environment variables
Apr 25, 2022
82bbb9f
fix archiver url
Apr 25, 2022
5311c31
add environment variables for ldap config
Apr 25, 2022
4f2ea7e
add missing archiver defaults
Apr 25, 2022
40320fb
start ocdav delayed
Apr 25, 2022
5919d25
add missing insecure environment variables
Apr 26, 2022
3c1f43e
add missing environment variables
Apr 26, 2022
c7844e2
remove unused env variables and increase log level
Apr 26, 2022
4ae93e6
skip parallel deployment tests
Apr 26, 2022
e2ea74c
remove commented out code
Apr 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -6,14 +6,15 @@ import (
"os"
"path"

"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/owncloud/ocis/ocis-pkg/tracing"

"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/storage-shares/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
@@ -24,14 +25,18 @@ func StorageShares(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "storage-shares",
Usage: "start storage-shares service",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-shares")
},
// Before: func(c *cli.Context) error {
// return ParseConfig(c, cfg, "storage-shares")
// },
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)

tracing.Configure(cfg, logger)

logCfg := cfg.Logging
logger := log.NewLogger(
log.Level(logCfg.Level),
log.File(logCfg.File),
log.Pretty(logCfg.Pretty),
log.Color(logCfg.Color),
)
tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -58,10 +63,12 @@ func StorageShares(cfg *config.Config) *cli.Command {

debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.StorageShares.DebugAddr),
debug.Addr(cfg.Debug.Addr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
debug.Pprof(cfg.Debug.Pprof),
debug.Zpages(cfg.Debug.Zpages),
debug.Token(cfg.Debug.Token),
)

if err != nil {
@@ -73,7 +80,7 @@ func StorageShares(cfg *config.Config) *cli.Command {
cancel()
})

if !cfg.Reva.StorageShares.Supervised {
if !cfg.Supervised {
sync.Trap(&gr, cancel)
}

@@ -86,29 +93,27 @@ func StorageShares(cfg *config.Config) *cli.Command {
func storageSharesConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.StorageShares.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
"jwt_secret": cfg.JWTSecret,
"gatewaysvc": cfg.GatewayEndpoint,
"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.StorageShares.GRPCNetwork,
"address": cfg.Reva.StorageShares.GRPCAddr,
"network": cfg.GRPC.Protocol,
"address": cfg.GRPC.Addr,
"services": map[string]interface{}{
"sharesstorageprovider": map[string]interface{}{
"usershareprovidersvc": cfg.Reva.Sharing.Endpoint,
"gateway_addr": cfg.Reva.Gateway.Endpoint,
"usershareprovidersvc": cfg.SharesProviderEndpoint,
},
},
},
}
if cfg.Reva.StorageShares.ReadOnly {
if cfg.ReadOnly {
gcfg := rcfg["grpc"].(map[string]interface{})
gcfg["interceptors"] = map[string]interface{}{
"readonly": map[string]interface{}{},
@@ -124,28 +129,29 @@ type StorageSharesSutureService struct {

// NewStorageShares creates a new storage.StorageSharesSutureService
func NewStorageShares(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
cfg.StorageShares.Commons = cfg.Commons
return StorageSharesSutureService{
cfg: cfg.Storage,
cfg: cfg.StorageShares,
}
}

func (s StorageSharesSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.StorageShares.Context = ctx
// s.cfg.Reva.StorageShares.Context = ctx
cmd := StorageShares(s.cfg)
f := &flag.FlagSet{}
cmdFlags := StorageShares(s.cfg).Flags
cmdFlags := cmd.Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if StorageShares(s.cfg).Before != nil {
if err := StorageShares(s.cfg).Before(cliCtx); err != nil {
if cmd.Before != nil {
if err := cmd.Before(cliCtx); err != nil {
return err
}
}
if err := StorageShares(s.cfg).Action(cliCtx); err != nil {
if err := cmd.Action(cliCtx); err != nil {
return err
}

60 changes: 60 additions & 0 deletions extensions/storage-shares/pkg/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package config

import (
"context"

"github.com/owncloud/ocis/ocis-pkg/shared"
)

type Config struct {
*shared.Commons `yaml:"-"`
Service Service `yaml:"-"`
Tracing *Tracing `yaml:"tracing"`
Logging *Logging `yaml:"log"`
Debug Debug `yaml:"debug"`
Supervised bool

GRPC GRPCConfig `yaml:"grpc"`
HTTP HTTPConfig `yaml:"http"`

Context context.Context
JWTSecret string
GatewayEndpoint string
SkipUserGroupsInToken bool
ReadOnly bool
SharesProviderEndpoint string
}
type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
}

type Logging struct {
Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
}

type Service struct {
Name string `yaml:"-"`
}

type Debug struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
Token string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
Pprof bool `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
Zpages bool `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
}

type GRPCConfig struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}

type HTTPConfig struct {
Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}
64 changes: 64 additions & 0 deletions extensions/storage-shares/pkg/config/defaults/defaultconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package defaults

import (
"github.com/owncloud/ocis/extensions/storage-shares/pkg/config"
)

func FullDefaultConfig() *config.Config {
cfg := DefaultConfig()

EnsureDefaults(cfg)

return cfg
}

func DefaultConfig() *config.Config {
return &config.Config{
Debug: config.Debug{
Addr: "127.0.0.1:9156",
Token: "",
Pprof: false,
Zpages: false,
},
GRPC: config.GRPCConfig{
Addr: "127.0.0.1:9154",
Protocol: "tcp",
},
HTTP: config.HTTPConfig{
Addr: "127.0.0.1:9155",
Protocol: "tcp",
},
Service: config.Service{
Name: "storage-metadata",
},
GatewayEndpoint: "127.0.0.1:9142",
JWTSecret: "Pive-Fumkiu4",
ReadOnly: false,
SharesProviderEndpoint: "localhost:9150",
}
}

func EnsureDefaults(cfg *config.Config) {
// provide with defaults for shared logging, since we need a valid destination address for BindEnv.
if cfg.Logging == nil && cfg.Commons != nil && cfg.Commons.Log != nil {
cfg.Logging = &config.Logging{
Level: cfg.Commons.Log.Level,
Pretty: cfg.Commons.Log.Pretty,
Color: cfg.Commons.Log.Color,
File: cfg.Commons.Log.File,
}
} else if cfg.Logging == nil {
cfg.Logging = &config.Logging{}
}
// provide with defaults for shared tracing, since we need a valid destination address for BindEnv.
if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil {
cfg.Tracing = &config.Tracing{
Enabled: cfg.Commons.Tracing.Enabled,
Type: cfg.Commons.Tracing.Type,
Endpoint: cfg.Commons.Tracing.Endpoint,
Collector: cfg.Commons.Tracing.Collector,
}
} else if cfg.Tracing == nil {
cfg.Tracing = &config.Tracing{}
}
}
2 changes: 1 addition & 1 deletion extensions/storage/pkg/command/root.go
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ func GetCommands(cfg *config.Config) cli.Commands {
// AuthMachine(cfg),
// Sharing(cfg),
// StoragePublicLink(cfg),
StorageShares(cfg),
// StorageShares(cfg),
// StorageUsers(cfg),
// StorageMetadata(cfg),
Health(cfg),
79 changes: 0 additions & 79 deletions extensions/storage/pkg/command/storagedrivers/metadata.go

This file was deleted.

Loading