Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[full-ci] Refactor storage configuration to make it similar to other oCIS configuration #3569

Merged
merged 29 commits into from
Apr 26, 2022
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
57e4e70
remove auth basic command to improve config code
Apr 19, 2022
b31b708
make auth-basic config similar to other services
Apr 19, 2022
ed4e9ee
make user provider config similar to other services
Apr 19, 2022
9e20992
make group provider config similar to other services
Apr 19, 2022
c70a60d
make app provider config similar to other services
Apr 20, 2022
7c87934
make auth bearer config similar to other services
Apr 20, 2022
5013d71
make sharing config similar to other services
Apr 20, 2022
d303bb6
make storage metadata config similar to other services
Apr 21, 2022
4c6848e
make auth machine config similar to other services
Apr 21, 2022
12c14e3
make storage publiclink config similar to other services
Apr 21, 2022
f40ec3c
make storage users config similar to other services
Apr 21, 2022
f39e354
make storage shares config similar to other services
Apr 21, 2022
430e7f6
make ocdav config similar to other services
Apr 22, 2022
76a6588
make frontend config similar to other services
Apr 22, 2022
60bf000
make gateway config similar to other services
Apr 22, 2022
01e307f
clean up the new commands
Apr 22, 2022
f51bdd8
fix config docs generation by adding the Sanitize function everywhere
wkloucek Apr 25, 2022
b26b68b
fix sharing yaml tag
wkloucek Apr 25, 2022
c29f69d
add missing configs
Apr 25, 2022
89b925f
fix environment variables
Apr 25, 2022
82bbb9f
fix archiver url
Apr 25, 2022
5311c31
add environment variables for ldap config
Apr 25, 2022
4f2ea7e
add missing archiver defaults
Apr 25, 2022
40320fb
start ocdav delayed
Apr 25, 2022
5919d25
add missing insecure environment variables
Apr 26, 2022
3c1f43e
add missing environment variables
Apr 26, 2022
c7844e2
remove unused env variables and increase log level
Apr 26, 2022
4ae93e6
skip parallel deployment tests
Apr 26, 2022
e2ea74c
remove commented out code
Apr 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 34 additions & 40 deletions .drone.star
Original file line number Diff line number Diff line change
Expand Up @@ -1624,8 +1624,8 @@ def ocisServerWithIdp():
"GRAPH_LDAP_SERVER_WRITE_ENABLED": "true",
"LDAP_URI": "ldaps://0.0.0.0:9235",
"LDAP_INSECURE": "true",
"LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
"LDAP_BIND_PASSWORD": "idm",
"GRAPH_LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
"GRAPH_LDAP_BIND_PASSWORD": "idm",
"LDAP_USER_BASE_DN": "ou=users,o=libregraph-idm",
"LDAP_USER_SCHEMA_ID": "ownclouduuid",
"LDAP_USER_SCHEMA_MAIL": "mail",
Expand All @@ -1644,8 +1644,8 @@ def ocisServerWithIdp():
"IDP_LDAP_LOGIN_ATTRIBUTE": "uid",
"PROXY_ACCOUNT_BACKEND_TYPE": "cs3",
"PROXY_ENABLE_BASIC_AUTH": "true",
"STORAGE_LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
"STORAGE_LDAP_BIND_PASSWORD": "reva",
"LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
"LDAP_BIND_PASSWORD": "reva",
"OCS_ACCOUNT_BACKEND_TYPE": "cs3",
"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,idm,ocdav",
"OCIS_LOG_LEVEL": "error",
Expand Down Expand Up @@ -1679,13 +1679,13 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
user = "0:0"
environment = {
"OCIS_URL": "https://ocis-server:9200",
"STORAGE_GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
"STORAGE_HOME_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER": "%s" % (storage),
"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
"STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
"PROXY_ENABLE_BASIC_AUTH": True,
"WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
"IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
Expand All @@ -1708,42 +1708,38 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
environment = {
# Keycloak IDP specific configuration
"PROXY_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud",
"LDAP_IDP": "https://keycloak/auth/realms/owncloud",
"WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud",
"WEB_OIDC_CLIENT_ID": "ocis-web",
"WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration",
"STORAGE_OIDC_ISSUER": "https://keycloak",
"STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud",
"AUTH_BEARER_OIDC_ISSUER": "https://keycloak",
"WEB_OIDC_SCOPE": "openid profile email owncloud",
# LDAP bind
"STORAGE_LDAP_URI": "ldaps://openldap",
"STORAGE_LDAP_INSECURE": "true",
"STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
"STORAGE_LDAP_BIND_PASSWORD": "admin",
"LDAP_URI": "ldaps://openldap",
"LDAP_INSECURE": "true",
"LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
"LDAP_BIND_PASSWORD": "admin",
# LDAP user settings
"PROXY_AUTOPROVISION_ACCOUNTS": "true", # automatically create users when they login
"PROXY_ACCOUNT_BACKEND_TYPE": "cs3", # proxy should get users from CS3APIS (which gets it from LDAP)
"PROXY_USER_OIDC_CLAIM": "ocis.user.uuid", # claim was added in Keycloak
"PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID
"STORAGE_LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
"STORAGE_LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
"STORAGE_LDAP_GROUPFILTER": "(objectclass=owncloud)",
"STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
"STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber",
"STORAGE_LDAP_GROUP_SCHEMA_ID": "cn",
"STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail",
"STORAGE_LDAP_GROUP_SCHEMA_MEMBER": "cn",
"STORAGE_LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
"STORAGE_LDAP_USER_OBJECTCLASS": "posixAccount",
"STORAGE_LDAP_USERFILTER": "(objectclass=owncloud)",
"STORAGE_LDAP_USER_SCHEMA_USERNAME": "cn",
"STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
"STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber",
"STORAGE_LDAP_USER_SCHEMA_MAIL": "mail",
"STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber",
"STORAGE_LDAP_USER_SCHEMA_ID": "ownclouduuid",
"STORAGE_LDAP_LOGIN_ATTRIBUTES": "uid,mail",
"LDAP_GROUP_BASE_DN": "ou=testgroups,dc=owncloud,dc=com",
"LDAP_GROUP_OBJECTCLASS": "groupOfUniqueNames",
"LDAP_GROUPFILTER": "(objectclass=owncloud)",
"LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
"LDAP_GROUP_SCHEMA_ID": "cn",
"LDAP_GROUP_SCHEMA_MAIL": "mail",
"LDAP_GROUP_SCHEMA_MEMBER": "cn",
"LDAP_USER_BASE_DN": "ou=testusers,dc=owncloud,dc=com",
"LDAP_USER_OBJECTCLASS": "posixAccount",
"LDAP_USERFILTER": "(objectclass=owncloud)",
"LDAP_USER_SCHEMA_USERNAME": "cn",
"LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
"LDAP_USER_SCHEMA_MAIL": "mail",
"LDAP_USER_SCHEMA_ID": "ownclouduuid",
"LDAP_LOGIN_ATTRIBUTES": "uid,mail",
# ownCloudSQL storage driver
"STORAGE_HOME_DRIVER": "owncloudsql",
"STORAGE_USERS_DRIVER": "owncloudsql",
"STORAGE_METADATA_DRIVER": "ocis",
"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files",
Expand All @@ -1758,19 +1754,19 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
# TODO: redis is not yet supported
"STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379",
# ownCloudSQL sharing driver
"STORAGE_SHARING_USER_DRIVER": "owncloudsql",
"STORAGE_SHARING_USER_SQL_USERNAME": "owncloud",
"STORAGE_SHARING_USER_SQL_PASSWORD": "owncloud",
"STORAGE_SHARING_USER_SQL_HOST": "oc10-db",
"STORAGE_SHARING_USER_SQL_PORT": 3306,
"STORAGE_SHARING_USER_SQL_NAME": "owncloud",
"SHARING_USER_DRIVER": "owncloudsql",
"SHARING_USER_SQL_USERNAME": "owncloud",
"SHARING_USER_SQL_PASSWORD": "owncloud",
"SHARING_USER_SQL_HOST": "oc10-db",
"SHARING_USER_SQL_PORT": 3306,
"SHARING_USER_SQL_NAME": "owncloud",
# ownCloud storage readonly
# TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
"OCIS_STORAGE_READ_ONLY": "false",
# General oCIS config
# OCIS_RUN_EXTENSIONS specifies to start all extensions except glauth, idp and accounts. These are replaced by external services
"OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,nats,ocdav",
"OCIS_LOG_LEVEL": "error",
"OCIS_LOG_LEVEL": "info",
"OCIS_URL": OCIS_URL,
"PROXY_TLS": "true",
"OCIS_BASE_DATA_PATH": "/mnt/data/ocis",
Expand All @@ -1780,8 +1776,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
"OCIS_MACHINE_AUTH_API_KEY": "change-me-please",
"OCIS_INSECURE": "true",
"PROXY_ENABLE_BASIC_AUTH": "true",
"ACCOUNTS_DEMO_USERS_AND_GROUPS": True, # deprecated, remove after switching to LibreIDM
"IDM_CREATE_DEMO_USERS": True,
}
wait_for_ocis = {
"name": "wait-for-ocis-server",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ import (
"github.com/cs3org/reva/v2/cmd/revad/runtime"
"github.com/gofrs/uuid"
"github.com/oklog/run"
"github.com/owncloud/ocis/extensions/storage/pkg/config"
"github.com/owncloud/ocis/extensions/appprovider/pkg/config"
"github.com/owncloud/ocis/extensions/storage/pkg/server/debug"
"github.com/owncloud/ocis/extensions/storage/pkg/tracing"
ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
"github.com/owncloud/ocis/ocis-pkg/log"
"github.com/owncloud/ocis/ocis-pkg/sync"
"github.com/owncloud/ocis/ocis-pkg/tracing"
"github.com/thejerf/suture/v4"
"github.com/urfave/cli/v2"
)
Expand All @@ -23,12 +24,18 @@ func AppProvider(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "app-provider",
Usage: "start appprovider for providing apps",
Before: func(c *cli.Context) error {
return ParseConfig(c, cfg, "storage-app-provider")
},
// Before: func(c *cli.Context) error {
// return ParseConfig(c, cfg, "storage-app-provider")
// },
phil-davis marked this conversation as resolved.
Show resolved Hide resolved
Action: func(c *cli.Context) error {
logger := NewLogger(cfg)
tracing.Configure(cfg, logger)
logCfg := cfg.Logging
logger := log.NewLogger(
log.Level(logCfg.Level),
log.File(logCfg.File),
log.Pretty(logCfg.Pretty),
log.Color(logCfg.Color),
)
tracing.Configure(cfg.Tracing.Enabled, cfg.Tracing.Type, logger)
gr := run.Group{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -51,10 +58,12 @@ func AppProvider(cfg *config.Config) *cli.Command {

debugServer, err := debug.Server(
debug.Name(c.Command.Name+"-debug"),
debug.Addr(cfg.Reva.AppProvider.DebugAddr),
debug.Addr(cfg.Debug.Addr),
debug.Logger(logger),
debug.Context(ctx),
debug.Config(cfg),
debug.Pprof(cfg.Debug.Pprof),
debug.Zpages(cfg.Debug.Zpages),
debug.Token(cfg.Debug.Token),
)

if err != nil {
Expand All @@ -66,7 +75,7 @@ func AppProvider(cfg *config.Config) *cli.Command {
cancel()
})

if !cfg.Reva.AppProvider.Supervised {
if !cfg.Supervised {
sync.Trap(&gr, cancel)
}

Expand All @@ -80,38 +89,37 @@ func appProviderConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]

rcfg := map[string]interface{}{
"core": map[string]interface{}{
"max_cpus": cfg.Reva.AppProvider.MaxCPUs,
"tracing_enabled": cfg.Tracing.Enabled,
"tracing_endpoint": cfg.Tracing.Endpoint,
"tracing_collector": cfg.Tracing.Collector,
"tracing_service_name": c.Command.Name,
},
"shared": map[string]interface{}{
"jwt_secret": cfg.Reva.JWTSecret,
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"skip_user_groups_in_token": cfg.Reva.SkipUserGroupsInToken,
"jwt_secret": cfg.JWTSecret,
"gatewaysvc": cfg.GatewayEndpoint,
"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
},
"grpc": map[string]interface{}{
"network": cfg.Reva.AppProvider.GRPCNetwork,
"address": cfg.Reva.AppProvider.GRPCAddr,
"network": cfg.GRPC.Protocol,
"address": cfg.GRPC.Addr,
// TODO build services dynamically
"services": map[string]interface{}{
"appprovider": map[string]interface{}{
"gatewaysvc": cfg.Reva.Gateway.Endpoint,
"app_provider_url": cfg.Reva.AppProvider.ExternalAddr,
"driver": cfg.Reva.AppProvider.Driver,
// "gatewaysvc": cfg.Reva.Gateway.Endpoint,
phil-davis marked this conversation as resolved.
Show resolved Hide resolved
"app_provider_url": cfg.ExternalAddr,
"driver": cfg.Driver,
"drivers": map[string]interface{}{
"wopi": map[string]interface{}{
"app_api_key": cfg.Reva.AppProvider.WopiDriver.AppAPIKey,
"app_desktop_only": cfg.Reva.AppProvider.WopiDriver.AppDesktopOnly,
"app_icon_uri": cfg.Reva.AppProvider.WopiDriver.AppIconURI,
"app_int_url": cfg.Reva.AppProvider.WopiDriver.AppInternalURL,
"app_name": cfg.Reva.AppProvider.WopiDriver.AppName,
"app_url": cfg.Reva.AppProvider.WopiDriver.AppURL,
"insecure_connections": cfg.Reva.AppProvider.WopiDriver.Insecure,
"iop_secret": cfg.Reva.AppProvider.WopiDriver.IopSecret,
"jwt_secret": cfg.Reva.AppProvider.WopiDriver.JWTSecret,
"wopi_url": cfg.Reva.AppProvider.WopiDriver.WopiURL,
"app_api_key": cfg.Drivers.WOPI.AppAPIKey,
"app_desktop_only": cfg.Drivers.WOPI.AppDesktopOnly,
"app_icon_uri": cfg.Drivers.WOPI.AppIconURI,
"app_int_url": cfg.Drivers.WOPI.AppInternalURL,
"app_name": cfg.Drivers.WOPI.AppName,
"app_url": cfg.Drivers.WOPI.AppURL,
"insecure_connections": cfg.Drivers.WOPI.Insecure,
"iop_secret": cfg.Drivers.WOPI.IopSecret,
"jwt_secret": cfg.JWTSecret,
"wopi_url": cfg.Drivers.WOPI.WopiURL,
},
},
},
Expand All @@ -128,28 +136,29 @@ type AppProviderSutureService struct {

// NewAppProvider creates a new store.AppProviderSutureService
func NewAppProvider(cfg *ociscfg.Config) suture.Service {
cfg.Storage.Commons = cfg.Commons
cfg.AppProvider.Commons = cfg.Commons
return AppProviderSutureService{
cfg: cfg.Storage,
cfg: cfg.AppProvider,
}
}

func (s AppProviderSutureService) Serve(ctx context.Context) error {
s.cfg.Reva.AppProvider.Context = ctx
// s.cfg.Reva.AppProvider.Context = ctx
phil-davis marked this conversation as resolved.
Show resolved Hide resolved
cmd := AppProvider(s.cfg)
f := &flag.FlagSet{}
cmdFlags := AppProvider(s.cfg).Flags
cmdFlags := cmd.Flags
for k := range cmdFlags {
if err := cmdFlags[k].Apply(f); err != nil {
return err
}
}
cliCtx := cli.NewContext(nil, f, nil)
if AppProvider(s.cfg).Before != nil {
if err := AppProvider(s.cfg).Before(cliCtx); err != nil {
if cmd.Before != nil {
if err := cmd.Before(cliCtx); err != nil {
return err
}
}
if err := AppProvider(s.cfg).Action(cliCtx); err != nil {
if err := cmd.Action(cliCtx); err != nil {
return err
}

Expand Down
67 changes: 67 additions & 0 deletions extensions/appprovider/pkg/config/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package config

import "github.com/owncloud/ocis/ocis-pkg/shared"

type Config struct {
*shared.Commons `yaml:"-"`
Service Service `yaml:"-"`
Tracing *Tracing `yaml:"tracing"`
Logging *Logging `yaml:"log"`
Debug Debug `yaml:"debug"`
Supervised bool

GRPC GRPCConfig `yaml:"grpc"`

JWTSecret string
GatewayEndpoint string
SkipUserGroupsInToken bool
ExternalAddr string
Driver string
Drivers Drivers
}

type Tracing struct {
Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;APP_PROVIDER_TRACING_ENABLED" desc:"Activates tracing."`
Type string `yaml:"type" env:"OCIS_TRACING_TYPE;APP_PROVIDER_TRACING_TYPE"`
Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;APP_PROVIDER_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;APP_PROVIDER_TRACING_COLLECTOR"`
}

type Logging struct {
Level string `yaml:"level" env:"OCIS_LOG_LEVEL;APP_PROVIDER_LOG_LEVEL" desc:"The log level."`
Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;APP_PROVIDER_LOG_PRETTY" desc:"Activates pretty log output."`
Color bool `yaml:"color" env:"OCIS_LOG_COLOR;APP_PROVIDER_LOG_COLOR" desc:"Activates colorized log output."`
File string `yaml:"file" env:"OCIS_LOG_FILE;APP_PROVIDER_LOG_FILE" desc:"The target log file."`
}

type Service struct {
Name string `yaml:"-"`
}

type Debug struct {
Addr string `yaml:"addr" env:"APP_PROVIDER_DEBUG_ADDR"`
Token string `yaml:"token" env:"APP_PROVIDER_DEBUG_TOKEN"`
Pprof bool `yaml:"pprof" env:"APP_PROVIDER_DEBUG_PPROF"`
Zpages bool `yaml:"zpages" env:"APP_PROVIDER_DEBUG_ZPAGES"`
}

type GRPCConfig struct {
Addr string `yaml:"addr" env:"APP_PROVIDER_GRPC_ADDR" desc:"The address of the grpc service."`
Protocol string `yaml:"protocol" env:"APP_PROVIDER_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
}

type Drivers struct {
WOPI WOPIDriver
}

type WOPIDriver struct {
AppAPIKey string `yaml:"app_api_key"`
AppDesktopOnly bool `yaml:"app_desktop_only"`
AppIconURI string `yaml:"app_icon_uri"`
AppInternalURL string `yaml:"app_internal_url"`
AppName string `yaml:"app_name"`
AppURL string `yaml:"app_url"`
Insecure bool `yaml:"insecure"`
IopSecret string `yaml:"ipo_secret"`
WopiURL string `yaml:"wopi_url"`
}
Loading