From e1fad5c6e932cfad1eee8e56cc64facc82b97f73 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 19 Dec 2023 10:43:36 +0100 Subject: [PATCH] make nats-js-kv the default registry Signed-off-by: jkoberg --- changelog/unreleased/default-registry.md | 5 + docs/services/general-info/registry.md | 42 ++++++ ocis-pkg/registry/registry.go | 120 ++++++++++++------ ocis/pkg/command/server.go | 2 - .../graph/pkg/service/v0/graph_suite_test.go | 3 +- .../pkg/service/notification_suite_test.go | 3 +- .../search/pkg/search/search_suite_test.go | 3 +- .../storage-users/pkg/task/task_suite_test.go | 3 +- .../userlog/pkg/service/service_suit_test.go | 3 +- 9 files changed, 131 insertions(+), 53 deletions(-) create mode 100644 changelog/unreleased/default-registry.md create mode 100644 docs/services/general-info/registry.md diff --git a/changelog/unreleased/default-registry.md b/changelog/unreleased/default-registry.md new file mode 100644 index 00000000000..49df8520e9b --- /dev/null +++ b/changelog/unreleased/default-registry.md @@ -0,0 +1,5 @@ +Enhancement: Make nats-js-kv the default registry + +The previously used default `mdns` is faulty. Deprecated it and `consul`, `nats` and `etcd` implementations + +https://github.com/owncloud/ocis/pull/8011 diff --git a/docs/services/general-info/registry.md b/docs/services/general-info/registry.md new file mode 100644 index 00000000000..f3466b643bc --- /dev/null +++ b/docs/services/general-info/registry.md @@ -0,0 +1,42 @@ +--- +title: Registry +date: 2023-12-19T00:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/general-info +geekdocFilePath: registry.md +geekdocCollapseSection: true +--- + +To be able to communicate with each other, services need to register in a common registry. + +## Configuration + +The type of registry to use can be configured with the `MICRO_REGISTRY` environment variable. Supported values are: + +### `memory` + +Setting the environment variable to `memory` starts an inmemory registry. This only works with the single binary deployment. + +### `nats-js-kv` + +Set the environment variable to `nats-js-kv` (or leave it empty) to use a nats-js key value store as registry. +- Note: If not running build-in nats, `MICRO_REGISTRY_ADDRESS` needs to be set to the address of the nats-js cluster. (Same as `OCIS_EVENTS_ENDPOINT`) +- Optional: Use `MICRO_REGISTRY_AUTH_USERNAME` and `MICRO_REGISTRY_AUTH_PASSWORD` to authenticate with the nats cluster. + +This is the default. + +### `kubernetes` + +When deploying in a kubernetes cluster, the kubernetes registry can be used. Additionally the `MICRO_REGISTRY_ADDRESS` environment +variable needs to be set to the url of the kubernetes registry. + +### Deprecated registries + +These registries are currently working but will be removed in a later version. It is recommended to switch to a supported one. + +- `nats`. Uses a registry based on nats streams. Requires `MICRO_REGISTRY_ADDRESS` to bet set. +- `etcd`. Uses an etcd cluster as registry. Requires `MICRO_REGISTRY_ADDRESS` to bet set. +- `consul`. Uses `HashiCorp Consul` as registry. Requires `MICRO_REGISTRY_ADDRESS` to bet set. +- `mdns`. Uses multicast dns for registration. This type can have unwanted side effects when other devices in the local network use mdns too. + diff --git a/ocis-pkg/registry/registry.go b/ocis-pkg/registry/registry.go index cee918bf81c..3d1b2b80e3f 100644 --- a/ocis-pkg/registry/registry.go +++ b/ocis-pkg/registry/registry.go @@ -1,6 +1,7 @@ package registry import ( + "fmt" "os" "strings" "sync" @@ -21,73 +22,110 @@ import ( const ( registryEnv = "MICRO_REGISTRY" registryAddressEnv = "MICRO_REGISTRY_ADDRESS" - regisryUsernameEnv = "MICRO_REGISTRY_AUTH_USERNAME" + registryUsernameEnv = "MICRO_REGISTRY_AUTH_USERNAME" registryPasswordEnv = "MICRO_REGISTRY_AUTH_PASSWORD" ) var ( - once sync.Once - regPlugin string - reg mRegistry.Registry + _once sync.Once + _reg mRegistry.Registry ) -// Configure configures the registry -func Configure(plugin string) { - if reg == nil { - regPlugin = plugin +// Config is the config for a registry +type Config struct { + Type string `mapstructure:"type"` + Addresses []string `mapstructure:"addresses"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + DisableCache bool `mapstructure:"disable_cache"` +} + +// Option allows configuring the registry +type Option func(*Config) + +// Inmemory overrides env values to use an in-memory registry +func Inmemory() Option { + return func(c *Config) { + c.Type = "memory" } } // GetRegistry returns a configured micro registry based on Micro env vars. // It defaults to mDNS, so mind that systems with mDNS disabled by default (i.e SUSE) will have a hard time // and it needs to explicitly use etcd. Os awareness for providing a working registry out of the box should be done. -func GetRegistry() mRegistry.Registry { - once.Do(func() { - addresses := strings.Split(os.Getenv(registryAddressEnv), ",") - // prefer env of setting from Configure() - plugin := os.Getenv(registryEnv) - if plugin == "" { - plugin = regPlugin - } +func GetRegistry(opts ...Option) mRegistry.Registry { + _once.Do(func() { + cfg := getEnvs(opts...) - switch plugin { - case "nats": - reg = natsr.NewRegistry( - mRegistry.Addrs(addresses...), - natsr.RegisterAction("put"), + switch cfg.Type { + default: + fmt.Println("Attention: unknown registry type, using default nats-js-kv") + fallthrough + case "natsjs", "nats-js", "nats-js-kv": // for backwards compatibility - we will stick with one of those + _reg = natsjsregistry.NewRegistry( + mRegistry.Addrs(cfg.Addresses...), + natsjsregistry.Authenticate(cfg.Username, cfg.Password), ) case "kubernetes": - reg = kubernetesr.NewRegistry( - mRegistry.Addrs(addresses...), + _reg = kubernetesr.NewRegistry( + mRegistry.Addrs(cfg.Addresses...), + ) + case "memory": + _reg = memr.NewRegistry() + cfg.DisableCache = true // no cache needed for in-memory registry + case "nats": + fmt.Println("Attention: nats registry is deprecated, use nats-js-kv instead") + _reg = natsr.NewRegistry( + mRegistry.Addrs(cfg.Addresses...), + natsr.RegisterAction("put"), ) case "etcd": - reg = etcdr.NewRegistry( - mRegistry.Addrs(addresses...), + fmt.Println("Attention: etcd registry is deprecated, use nats-js-kv instead") + _reg = etcdr.NewRegistry( + mRegistry.Addrs(cfg.Addresses...), ) case "consul": - reg = consulr.NewRegistry( - mRegistry.Addrs(addresses...), - ) - case "memory": - reg = memr.NewRegistry() - case "natsjs", "nats-js", "nats-js-kv": // for backwards compatibility - we will stick with one of those - reg = natsjsregistry.NewRegistry( - mRegistry.Addrs(addresses...), - natsjsregistry.Authenticate(os.Getenv(regisryUsernameEnv), os.Getenv(registryPasswordEnv)), + fmt.Println("Attention: consul registry is deprecated, use nats-js-kv instead") + _reg = consulr.NewRegistry( + mRegistry.Addrs(cfg.Addresses...), ) - default: - reg = mdnsr.NewRegistry() + case "mdns": + fmt.Println("Attention: mdns registry is deprecated, use nats-js-kv instead") + _reg = mdnsr.NewRegistry() } - // No cache needed for in-memory registry - if plugin != "memory" { - reg = cache.New(reg, cache.WithTTL(30*time.Second)) + // Disable cache if wanted + if !cfg.DisableCache { + _reg = cache.New(_reg, cache.WithTTL(30*time.Second)) } // fixme: lazy initialization of reva registry, needs refactor to a explicit call per service - _ = rRegistry.Init(reg) + _ = rRegistry.Init(_reg) }) // always use cached registry to prevent registry // lookup for every request - return reg + return _reg +} + +func getEnvs(opts ...Option) *Config { + cfg := &Config{ + Type: "nats-js-kv", + Addresses: []string{"127.0.0.1:9233"}, + Username: os.Getenv(registryUsernameEnv), + Password: os.Getenv(registryPasswordEnv), + } + + if s := os.Getenv(registryEnv); s != "" { + cfg.Type = s + } + + if s := strings.Split(os.Getenv(registryAddressEnv), ","); len(s) > 0 { + cfg.Addresses = s + } + + for _, o := range opts { + o(cfg) + } + + return cfg } diff --git a/ocis/pkg/command/server.go b/ocis/pkg/command/server.go index 60d450cf9e2..108038ce72d 100644 --- a/ocis/pkg/command/server.go +++ b/ocis/pkg/command/server.go @@ -4,7 +4,6 @@ import ( "github.com/owncloud/ocis/v2/ocis-pkg/config" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" - "github.com/owncloud/ocis/v2/ocis-pkg/registry" "github.com/owncloud/ocis/v2/ocis/pkg/register" "github.com/owncloud/ocis/v2/ocis/pkg/runtime" "github.com/urfave/cli/v2" @@ -21,7 +20,6 @@ func Server(cfg *config.Config) *cli.Command { }, Action: func(c *cli.Context) error { // Prefer the in-memory registry as the default when running in single-binary mode - registry.Configure("memory") r := runtime.New(cfg) return r.Start() }, diff --git a/services/graph/pkg/service/v0/graph_suite_test.go b/services/graph/pkg/service/v0/graph_suite_test.go index 7b6ed5b234f..70dfcce1450 100644 --- a/services/graph/pkg/service/v0/graph_suite_test.go +++ b/services/graph/pkg/service/v0/graph_suite_test.go @@ -11,8 +11,7 @@ import ( ) func init() { - registry.Configure("memory") - r := registry.GetRegistry() + r := registry.GetRegistry(registry.Inmemory()) service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", diff --git a/services/notifications/pkg/service/notification_suite_test.go b/services/notifications/pkg/service/notification_suite_test.go index dc3c1f42511..7f282054fb6 100644 --- a/services/notifications/pkg/service/notification_suite_test.go +++ b/services/notifications/pkg/service/notification_suite_test.go @@ -11,8 +11,7 @@ import ( ) func init() { - registry.Configure("memory") - r := registry.GetRegistry() + r := registry.GetRegistry(registry.Inmemory()) service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", diff --git a/services/search/pkg/search/search_suite_test.go b/services/search/pkg/search/search_suite_test.go index 75873d6daaa..9422179fc53 100644 --- a/services/search/pkg/search/search_suite_test.go +++ b/services/search/pkg/search/search_suite_test.go @@ -11,8 +11,7 @@ import ( ) func init() { - registry.Configure("memory") - r := registry.GetRegistry() + r := registry.GetRegistry(registry.Inmemory()) service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", diff --git a/services/storage-users/pkg/task/task_suite_test.go b/services/storage-users/pkg/task/task_suite_test.go index 5b904e7e302..ac0c8a44b9d 100644 --- a/services/storage-users/pkg/task/task_suite_test.go +++ b/services/storage-users/pkg/task/task_suite_test.go @@ -11,8 +11,7 @@ import ( ) func init() { - registry.Configure("memory") - r := registry.GetRegistry() + r := registry.GetRegistry(registry.Inmemory()) service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any", diff --git a/services/userlog/pkg/service/service_suit_test.go b/services/userlog/pkg/service/service_suit_test.go index ab7fb7b1989..9da9654d4e9 100644 --- a/services/userlog/pkg/service/service_suit_test.go +++ b/services/userlog/pkg/service/service_suit_test.go @@ -11,8 +11,7 @@ import ( ) func init() { - registry.Configure("memory") - r := registry.GetRegistry() + r := registry.GetRegistry(registry.Inmemory()) service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") service.Nodes = []*mRegistry.Node{{ Address: "any",