[full-ci] Refactor storage configuration to make it similar to other oCIS configuration

.drone.star

@@ -1624,8 +1624,8 @@ def ocisServerWithIdp():
         "GRAPH_LDAP_SERVER_WRITE_ENABLED": "true",
         "LDAP_URI": "ldaps://0.0.0.0:9235",
         "LDAP_INSECURE": "true",
-        "LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
-        "LDAP_BIND_PASSWORD": "idm",
+        "GRAPH_LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
+        "GRAPH_LDAP_BIND_PASSWORD": "idm",
         "LDAP_USER_BASE_DN": "ou=users,o=libregraph-idm",
         "LDAP_USER_SCHEMA_ID": "ownclouduuid",
         "LDAP_USER_SCHEMA_MAIL": "mail",
@@ -1644,8 +1644,8 @@ def ocisServerWithIdp():
         "IDP_LDAP_LOGIN_ATTRIBUTE": "uid",
         "PROXY_ACCOUNT_BACKEND_TYPE": "cs3",
         "PROXY_ENABLE_BASIC_AUTH": "true",
-        "STORAGE_LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
-        "STORAGE_LDAP_BIND_PASSWORD": "reva",
+        "LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
+        "LDAP_BIND_PASSWORD": "reva",
         "OCS_ACCOUNT_BACKEND_TYPE": "cs3",
         "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,idm,ocdav",
         "OCIS_LOG_LEVEL": "error",
@@ -1679,13 +1679,13 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
         user = "0:0"
         environment = {
             "OCIS_URL": "https://ocis-server:9200",
-            "STORAGE_GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
+            "GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
             "STORAGE_HOME_DRIVER": "%s" % (storage),
             "STORAGE_USERS_DRIVER": "%s" % (storage),
             "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
             "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
             "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
-            "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
+            "SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
             "PROXY_ENABLE_BASIC_AUTH": True,
             "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
             "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
@@ -1711,39 +1711,38 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud",
             "WEB_OIDC_CLIENT_ID": "ocis-web",
             "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration",
-            "STORAGE_OIDC_ISSUER": "https://keycloak",
-            "STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud",
+            "AUTH_BEARER_OIDC_ISSUER": "https://keycloak",
+            "LDAP_IDP": "https://keycloak/auth/realms/owncloud",
             "WEB_OIDC_SCOPE": "openid profile email owncloud",
             # LDAP bind
-            "STORAGE_LDAP_URI": "ldaps://openldap",
-            "STORAGE_LDAP_INSECURE": "true",
-            "STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
-            "STORAGE_LDAP_BIND_PASSWORD": "admin",
+            "LDAP_URI": "ldaps://openldap",
+            "LDAP_INSECURE": "true",
+            "LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
+            "LDAP_BIND_PASSWORD": "admin",
             # LDAP user settings
             "PROXY_AUTOPROVISION_ACCOUNTS": "true",  # automatically create users when they login
             "PROXY_ACCOUNT_BACKEND_TYPE": "cs3",  # proxy should get users from CS3APIS (which gets it from LDAP)
             "PROXY_USER_OIDC_CLAIM": "ocis.user.uuid",  # claim was added in Keycloak
             "PROXY_USER_CS3_CLAIM": "userid",  # equals STORAGE_LDAP_USER_SCHEMA_UID
-            "STORAGE_LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
-            "STORAGE_LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
-            "STORAGE_LDAP_GROUPFILTER": "(objectclass=owncloud)",
-            "STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
-            "STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber",
-            "STORAGE_LDAP_GROUP_SCHEMA_ID": "cn",
-            "STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail",
-            "STORAGE_LDAP_GROUP_SCHEMA_MEMBER": "cn",
-            "STORAGE_LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
-            "STORAGE_LDAP_USER_OBJECTCLASS": "posixAccount",
-            "STORAGE_LDAP_USERFILTER": "(objectclass=owncloud)",
-            "STORAGE_LDAP_USER_SCHEMA_USERNAME": "cn",
-            "STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
-            "STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber",
-            "STORAGE_LDAP_USER_SCHEMA_MAIL": "mail",
-            "STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber",
-            "STORAGE_LDAP_USER_SCHEMA_ID": "ownclouduuid",
-            "STORAGE_LDAP_LOGIN_ATTRIBUTES": "uid,mail",
+            "LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
+            "LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
+            "LDAP_GROUPFILTER": "(objectclass=owncloud)",
+            "LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
+            "LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber",
+            "LDAP_GROUP_SCHEMA_ID": "cn",
+            "LDAP_GROUP_SCHEMA_MAIL": "mail",
+            "LDAP_GROUP_SCHEMA_MEMBER": "cn",
+            "LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
+            "LDAP_USER_OBJECTCLASS": "posixAccount",
+            "LDAP_USERFILTER": "(objectclass=owncloud)",
+            "LDAP_USER_SCHEMA_USERNAME": "cn",
+            "LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
+            "LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber",
+            "LDAP_USER_SCHEMA_MAIL": "mail",
+            "LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber",
+            "LDAP_USER_SCHEMA_ID": "ownclouduuid",
+            "LDAP_LOGIN_ATTRIBUTES": "uid,mail",
             # ownCloudSQL storage driver
-            "STORAGE_HOME_DRIVER": "owncloudsql",
             "STORAGE_USERS_DRIVER": "owncloudsql",
             "STORAGE_METADATA_DRIVER": "ocis",
             "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files",
@@ -1758,12 +1757,12 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             # TODO: redis is not yet supported
             "STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379",
             # ownCloudSQL sharing driver
-            "STORAGE_SHARING_USER_DRIVER": "owncloudsql",
-            "STORAGE_SHARING_USER_SQL_USERNAME": "owncloud",
-            "STORAGE_SHARING_USER_SQL_PASSWORD": "owncloud",
-            "STORAGE_SHARING_USER_SQL_HOST": "oc10-db",
-            "STORAGE_SHARING_USER_SQL_PORT": 3306,
-            "STORAGE_SHARING_USER_SQL_NAME": "owncloud",
+            "SHARING_USER_DRIVER": "owncloudsql",
+            "SHARING_USER_SQL_USERNAME": "owncloud",
+            "SHARING_USER_SQL_PASSWORD": "owncloud",
+            "SHARING_USER_SQL_HOST": "oc10-db",
+            "SHARING_USER_SQL_PORT": 3306,
+            "SHARING_USER_SQL_NAME": "owncloud",
             # ownCloud storage readonly
             # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
             "OCIS_STORAGE_READ_ONLY": "false",

extensions/storage/pkg/command/appprovider.go → extensions/appprovider/pkg/command/command.go

@@ -9,11 +9,12 @@ import (
 	"github.com/cs3org/reva/v2/cmd/revad/runtime"
 	"github.com/gofrs/uuid"
 	"github.com/oklog/run"
-	"github.com/owncloud/ocis/extensions/storage/pkg/config"
+	"github.com/owncloud/ocis/extensions/appprovider/pkg/config"
 	"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
-	"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
 	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/log"
 	"github.com/owncloud/ocis/ocis-pkg/sync"
+	"github.com/owncloud/ocis/ocis-pkg/tracing"
 	"github.com/thejerf/suture/v4"
 	"github.com/urfave/cli/v2"
 )
@@ -23,12 +24,18 @@ func AppProvider(cfg *config.Config) *cli.Command {
 	return &cli.Command{
 		Name:  "app-provider",
 		Usage: "start appprovider for providing apps",
-		Before: func(c *cli.Context) error {
-			return ParseConfig(c, cfg, "storage-app-provider")
-		},
+		// Before: func(c *cli.Context) error {
+		// 	return ParseConfig(c, cfg, "storage-app-provider")
+		// },
 		Action: func(c *cli.Context) error {
-			logger := NewLogger(cfg)
-			tracing.Configure(cfg, logger)
+			logCfg := cfg.Logging
+			logger := log.NewLogger(
+				log.Level(logCfg.Level),
+				log.File(logCfg.File),
+				log.Pretty(logCfg.Pretty),
+				log.Color(logCfg.Color),
+			)
+			tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger)
 			gr := run.Group{}
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
@@ -51,10 +58,12 @@ func AppProvider(cfg *config.Config) *cli.Command {
 
 			debugServer, err := debug.Server(
 				debug.Name(c.Command.Name+"-debug"),
-				debug.Addr(cfg.Reva.AppProvider.DebugAddr),
+				debug.Addr(cfg.Debug.Addr),
 				debug.Logger(logger),
 				debug.Context(ctx),
-				debug.Config(cfg),
+				debug.Pprof(cfg.Debug.Pprof),
+				debug.Zpages(cfg.Debug.Zpages),
+				debug.Token(cfg.Debug.Token),
 			)
 
 			if err != nil {
@@ -66,7 +75,7 @@ func AppProvider(cfg *config.Config) *cli.Command {
 				cancel()
 			})
 
-			if !cfg.Reva.AppProvider.Supervised {
+			if !cfg.Supervised {
 				sync.Trap(&gr, cancel)
 			}
 
@@ -80,38 +89,37 @@ func appProviderConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]
 
 	rcfg := map[string]interface{}{
 		"core": map[string]interface{}{
-			"max_cpus":             cfg.Reva.AppProvider.MaxCPUs,
 			"tracing_enabled":      cfg.Tracing.Enabled,
 			"tracing_endpoint":     cfg.Tracing.Endpoint,
 			"tracing_collector":    cfg.Tracing.Collector,
 			"tracing_service_name": c.Command.Name,
 		},
 		"shared": map[string]interface{}{
-			"jwt_secret":                cfg.Reva.JWTSecret,
-			"gatewaysvc":                cfg.Reva.Gateway.Endpoint,
-			"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
+			"jwt_secret":                cfg.JWTSecret,
+			"gatewaysvc":                cfg.GatewayEndpoint,
+			"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
 		},
 		"grpc": map[string]interface{}{
-			"network": cfg.Reva.AppProvider.GRPCNetwork,
-			"address": cfg.Reva.AppProvider.GRPCAddr,
+			"network": cfg.GRPC.Protocol,
+			"address": cfg.GRPC.Addr,
 			// TODO build services dynamically
 			"services": map[string]interface{}{
 				"appprovider": map[string]interface{}{
-					"gatewaysvc":       cfg.Reva.Gateway.Endpoint,
-					"app_provider_url": cfg.Reva.AppProvider.ExternalAddr,
-					"driver":           cfg.Reva.AppProvider.Driver,
+					// "gatewaysvc":       cfg.Reva.Gateway.Endpoint,
+					"app_provider_url": cfg.ExternalAddr,
+					"driver":           cfg.Driver,
 					"drivers": map[string]interface{}{
 						"wopi": map[string]interface{}{
-							"app_api_key":          cfg.Reva.AppProvider.WopiDriver.AppAPIKey,
-							"app_desktop_only":     cfg.Reva.AppProvider.WopiDriver.AppDesktopOnly,
-							"app_icon_uri":         cfg.Reva.AppProvider.WopiDriver.AppIconURI,
-							"app_int_url":          cfg.Reva.AppProvider.WopiDriver.AppInternalURL,
-							"app_name":             cfg.Reva.AppProvider.WopiDriver.AppName,
-							"app_url":              cfg.Reva.AppProvider.WopiDriver.AppURL,
-							"insecure_connections": cfg.Reva.AppProvider.WopiDriver.Insecure,
-							"iop_secret":           cfg.Reva.AppProvider.WopiDriver.IopSecret,
-							"jwt_secret":           cfg.Reva.AppProvider.WopiDriver.JWTSecret,
-							"wopi_url":             cfg.Reva.AppProvider.WopiDriver.WopiURL,
+							"app_api_key":          cfg.Drivers.WOPI.AppAPIKey,
+							"app_desktop_only":     cfg.Drivers.WOPI.AppDesktopOnly,
+							"app_icon_uri":         cfg.Drivers.WOPI.AppIconURI,
+							"app_int_url":          cfg.Drivers.WOPI.AppInternalURL,
+							"app_name":             cfg.Drivers.WOPI.AppName,
+							"app_url":              cfg.Drivers.WOPI.AppURL,
+							"insecure_connections": cfg.Drivers.WOPI.Insecure,
+							"iop_secret":           cfg.Drivers.WOPI.IopSecret,
+							"jwt_secret":           cfg.JWTSecret,
+							"wopi_url":             cfg.Drivers.WOPI.WopiURL,
 						},
 					},
 				},
@@ -128,28 +136,29 @@ type AppProviderSutureService struct {
 
 // NewAppProvider creates a new store.AppProviderSutureService
 func NewAppProvider(cfg *ociscfg.Config) suture.Service {
-	cfg.Storage.Commons = cfg.Commons
+	cfg.AppProvider.Commons = cfg.Commons
 	return AppProviderSutureService{
-		cfg: cfg.Storage,
+		cfg: cfg.AppProvider,
 	}
 }
 
 func (s AppProviderSutureService) Serve(ctx context.Context) error {
-	s.cfg.Reva.AppProvider.Context = ctx
+	// s.cfg.Reva.AppProvider.Context = ctx
+	cmd := AppProvider(s.cfg)
 	f := &flag.FlagSet{}
-	cmdFlags := AppProvider(s.cfg).Flags
+	cmdFlags := cmd.Flags
 	for k := range cmdFlags {
 		if err := cmdFlags[k].Apply(f); err != nil {
 			return err
 		}
 	}
 	cliCtx := cli.NewContext(nil, f, nil)
-	if AppProvider(s.cfg).Before != nil {
-		if err := AppProvider(s.cfg).Before(cliCtx); err != nil {
+	if cmd.Before != nil {
+		if err := cmd.Before(cliCtx); err != nil {
 			return err
 		}
 	}
-	if err := AppProvider(s.cfg).Action(cliCtx); err != nil {
+	if err := cmd.Action(cliCtx); err != nil {
 		return err
 	}
 

extensions/appprovider/pkg/config/config.go

@@ -0,0 +1,67 @@
+package config
+
+import "github.com/owncloud/ocis/ocis-pkg/shared"
+
+type Config struct {
+	*shared.Commons `yaml:"-"`
+	Service         Service  `yaml:"-"`
+	Tracing         *Tracing `yaml:"tracing"`
+	Logging         *Logging `yaml:"log"`
+	Debug           Debug    `yaml:"debug"`
+	Supervised      bool
+
+	GRPC GRPCConfig `yaml:"grpc"`
+
+	JWTSecret             string
+	GatewayEndpoint       string
+	SkipUserGroupsInToken bool
+	ExternalAddr          string
+	Driver                string
+	Drivers               Drivers
+}
+
+type Tracing struct {
+	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;APP_PROVIDER_TRACING_ENABLED" desc:"Activates tracing."`
+	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;APP_PROVIDER_TRACING_TYPE"`
+	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;APP_PROVIDER_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
+	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;APP_PROVIDER_TRACING_COLLECTOR"`
+}
+
+type Logging struct {
+	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;APP_PROVIDER_LOG_LEVEL" desc:"The log level."`
+	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;APP_PROVIDER_LOG_PRETTY" desc:"Activates pretty log output."`
+	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;APP_PROVIDER_LOG_COLOR" desc:"Activates colorized log output."`
+	File   string `yaml:"file" env:"OCIS_LOG_FILE;APP_PROVIDER_LOG_FILE" desc:"The target log file."`
+}
+
+type Service struct {
+	Name string `yaml:"-"`
+}
+
+type Debug struct {
+	Addr   string `yaml:"addr" env:"APP_PROVIDER_DEBUG_ADDR"`
+	Token  string `yaml:"token" env:"APP_PROVIDER_DEBUG_TOKEN"`
+	Pprof  bool   `yaml:"pprof" env:"APP_PROVIDER_DEBUG_PPROF"`
+	Zpages bool   `yaml:"zpages" env:"APP_PROVIDER_DEBUG_ZPAGES"`
+}
+
+type GRPCConfig struct {
+	Addr     string `yaml:"addr" env:"APP_PROVIDER_GRPC_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"APP_PROVIDER_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+}
+
+type Drivers struct {
+	WOPI WOPIDriver
+}
+
+type WOPIDriver struct {
+	AppAPIKey      string `yaml:"app_api_key"`
+	AppDesktopOnly bool   `yaml:"app_desktop_only"`
+	AppIconURI     string `yaml:"app_icon_uri"`
+	AppInternalURL string `yaml:"app_internal_url"`
+	AppName        string `yaml:"app_name"`
+	AppURL         string `yaml:"app_url"`
+	Insecure       bool   `yaml:"insecure"`
+	IopSecret      string `yaml:"ipo_secret"`
+	WopiURL        string `yaml:"wopi_url"`
+}