From d79dafee0106d2dbad971d5a68a2f7adebe25dd9 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 10:46:40 +0000 Subject: [PATCH 01/24] wip Signed-off-by: James Milligan --- cmd/start.go | 17 ++++- pkg/runtime/from_config.go | 96 ++++++++++++++++++-------- pkg/runtime/runtime.go | 8 +-- pkg/sync/file/filepath_sync.go | 6 +- pkg/sync/grpc/grpc_sync.go | 1 + pkg/sync/http/http_sync.go | 19 +++-- pkg/sync/http/http_sync_test.go | 9 +-- pkg/sync/isync.go | 9 ++- pkg/sync/kubernetes/kubernetes_sync.go | 10 +-- 9 files changed, 113 insertions(+), 62 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index ee6f1b597..0fb864955 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -7,6 +7,7 @@ import ( "github.com/open-feature/flagd/pkg/logger" "github.com/open-feature/flagd/pkg/runtime" + "github.com/open-feature/flagd/pkg/sync" "github.com/spf13/cobra" "github.com/spf13/viper" "go.uber.org/zap" @@ -105,17 +106,27 @@ var startCmd = &cobra.Command{ rtLogger.Warn("DEPRECATED: The --evaluator flag has been deprecated. " + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") } + + syncProvidersFromURI, err := runtime.SyncProvidersFromArgs(viper.GetStringSlice(uriFlagName)) + if err != nil { + log.Fatal(err) + } + + syncProviders := []sync.SyncProviderConfig{} + if err := viper.UnmarshalKey(syncProviderFlagName, &syncProviders); err != nil { + log.Fatal(err) + } + syncProviders = append(syncProviders, syncProvidersFromURI...) + // Build Runtime ----------------------------------------------------------- rt, err := runtime.FromConfig(logger, runtime.Config{ CORS: viper.GetStringSlice(corsFlagName), MetricsPort: viper.GetInt32(metricsPortFlagName), - ProviderArgs: viper.GetStringMapString(providerArgsFlagName), ServiceCertPath: viper.GetString(serverCertPathFlagName), ServiceKeyPath: viper.GetString(serverKeyPathFlagName), ServicePort: viper.GetInt32(portFlagName), ServiceSocketPath: viper.GetString(socketPathFlagName), - SyncBearerToken: viper.GetString(bearerTokenFlagName), - SyncURI: viper.GetStringSlice(uriFlagName), + SyncProviders: syncProviders, }) if err != nil { rtLogger.Fatal(err.Error()) diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 7e313de9c..b530e3210 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -19,6 +19,13 @@ import ( "go.uber.org/zap" ) +const ( + syncProviderFile = "file" + syncProviderGrpc = "grpc" + syncProviderKubernetes = "kubernetes" + syncProviderHttp = "http" +) + var ( regCrd *regexp.Regexp regURL *regexp.Regexp @@ -64,43 +71,43 @@ func (r *Runtime) setService(logger *logger.Logger) { func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { rtLogger := logger.WithFields(zap.String("component", "runtime")) - r.SyncImpl = make([]sync.ISync, 0, len(r.config.SyncURI)) - for _, uri := range r.config.SyncURI { - switch uriB := []byte(uri); { - case regFile.Match(uriB): + r.SyncImpl = make([]sync.ISync, 0, len(r.config.SyncProviders)) + for _, syncProvider := range r.config.SyncProviders { + switch syncProvider.Provider { + case syncProviderFile: r.SyncImpl = append( r.SyncImpl, - r.newFile(uri, logger), + r.newFile(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", uri)) - case regCrd.Match(uriB): + rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", syncProvider.URI)) + case syncProviderKubernetes: r.SyncImpl = append( r.SyncImpl, - r.newK8s(uri, logger), + r.newK8s(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using kubernetes sync-provider for: %s", uri)) - case regURL.Match(uriB): + rtLogger.Debug(fmt.Sprintf("using kubernetes sync-provider for: %s", syncProvider.URI)) + case syncProviderHttp: r.SyncImpl = append( r.SyncImpl, - r.newHTTP(uri, logger), + r.newHTTP(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %q", uri)) - case regGRPC.Match(uriB): + rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %q", syncProvider.URI)) + case syncProviderGrpc: r.SyncImpl = append( r.SyncImpl, - r.newGRPC(uri, logger), + r.newGRPC(syncProvider, logger), ) default: return fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', 'http(s)://', 'grpc://',"+ - " or 'core.openfeature.dev'", uri) + " or 'core.openfeature.dev'", syncProvider.URI) } } return nil } -func (r *Runtime) newGRPC(uri string, logger *logger.Logger) *grpc.Sync { +func (r *Runtime) newGRPC(config sync.SyncProviderConfig, logger *logger.Logger) *grpc.Sync { return &grpc.Sync{ - Target: grpc.URLToGRPCTarget(uri), + Target: grpc.URLToGRPCTarget(config.URI), Logger: logger.WithFields( zap.String("component", "sync"), zap.String("sync", "grpc"), @@ -109,10 +116,9 @@ func (r *Runtime) newGRPC(uri string, logger *logger.Logger) *grpc.Sync { } } -func (r *Runtime) newHTTP(uri string, logger *logger.Logger) *httpSync.Sync { +func (r *Runtime) newHTTP(config sync.SyncProviderConfig, logger *logger.Logger) *httpSync.Sync { return &httpSync.Sync{ - URI: uri, - BearerToken: r.config.SyncBearerToken, + URI: config.URI, Client: &http.Client{ Timeout: time.Second * 10, }, @@ -120,30 +126,62 @@ func (r *Runtime) newHTTP(uri string, logger *logger.Logger) *httpSync.Sync { zap.String("component", "sync"), zap.String("sync", "remote"), ), - ProviderArgs: r.config.ProviderArgs, - Cron: cron.New(), + Config: config, + Cron: cron.New(), } } -func (r *Runtime) newK8s(uri string, logger *logger.Logger) *kubernetes.Sync { +func (r *Runtime) newK8s(config sync.SyncProviderConfig, logger *logger.Logger) *kubernetes.Sync { return &kubernetes.Sync{ Logger: logger.WithFields( zap.String("component", "sync"), zap.String("sync", "kubernetes"), ), - URI: regCrd.ReplaceAllString(uri, ""), - ProviderArgs: r.config.ProviderArgs, + URI: config.URI, + Config: config, } } -func (r *Runtime) newFile(uri string, logger *logger.Logger) *file.Sync { +func (r *Runtime) newFile(config sync.SyncProviderConfig, logger *logger.Logger) *file.Sync { return &file.Sync{ - URI: regFile.ReplaceAllString(uri, ""), + URI: config.URI, Logger: logger.WithFields( zap.String("component", "sync"), zap.String("sync", "filepath"), ), - ProviderArgs: r.config.ProviderArgs, - Mux: &msync.RWMutex{}, + Config: config, + Mux: &msync.RWMutex{}, + } +} + +func SyncProvidersFromArgs(uris []string) ([]sync.SyncProviderConfig, error) { + syncProvidersParsed := []sync.SyncProviderConfig{} + for _, uri := range uris { + switch uriB := []byte(uri); { + case regFile.Match(uriB): + syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + URI: regFile.ReplaceAllString(uri, ""), + Provider: syncProviderFile, + }) + case regCrd.Match(uriB): + syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + URI: regCrd.ReplaceAllString(uri, ""), + Provider: syncProviderKubernetes, + }) + case regURL.Match(uriB): + syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + URI: uri, + Provider: syncProviderHttp, + }) + case regGRPC.Match(uriB): + syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + URI: uri, + Provider: syncProviderGrpc, + }) + default: + return syncProvidersParsed, fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', 'http(s)://', 'grpc://',"+ + " or 'core.openfeature.dev'", uri) + } } + return syncProvidersParsed, nil } diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 4772eaef3..41b150bfa 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -33,12 +33,8 @@ type Config struct { ServiceCertPath string ServiceKeyPath string - ProviderArgs sync.ProviderArgs - SyncURI []string - RemoteSyncType string - SyncBearerToken string - - CORS []string + SyncProviders []sync.SyncProviderConfig + CORS []string } func (r *Runtime) Start() error { diff --git a/pkg/sync/file/filepath_sync.go b/pkg/sync/file/filepath_sync.go index 5fc31ebfd..9763a2d1b 100644 --- a/pkg/sync/file/filepath_sync.go +++ b/pkg/sync/file/filepath_sync.go @@ -17,9 +17,9 @@ import ( ) type Sync struct { - URI string - Logger *logger.Logger - ProviderArgs sync.ProviderArgs + URI string + Logger *logger.Logger + Config sync.SyncProviderConfig // FileType indicates the file type e.g., json, yaml/yml etc., fileType string watcher *fsnotify.Watcher diff --git a/pkg/sync/grpc/grpc_sync.go b/pkg/sync/grpc/grpc_sync.go index a36c7ac35..c274a8c6a 100644 --- a/pkg/sync/grpc/grpc_sync.go +++ b/pkg/sync/grpc/grpc_sync.go @@ -39,6 +39,7 @@ type Sync struct { options []grpc.DialOption ready bool Mux *msync.RWMutex + Config sync.SyncProviderConfig } func (g *Sync) Init(ctx context.Context) error { diff --git a/pkg/sync/http/http_sync.go b/pkg/sync/http/http_sync.go index c2a56460f..8f2a786dd 100644 --- a/pkg/sync/http/http_sync.go +++ b/pkg/sync/http/http_sync.go @@ -16,14 +16,13 @@ import ( ) type Sync struct { - URI string - Client Client - Cron Cron - BearerToken string - LastBodySHA string - Logger *logger.Logger - ProviderArgs sync.ProviderArgs - ready bool + URI string + Client Client + Cron Cron + LastBodySHA string + Logger *logger.Logger + ready bool + Config sync.SyncProviderConfig } // Client defines the behaviour required of a http client @@ -108,8 +107,8 @@ func (hs *Sync) fetchBodyFromURL(ctx context.Context, url string) ([]byte, error req.Header.Add("Accept", "application/json") - if hs.BearerToken != "" { - bearer := "Bearer " + hs.BearerToken + if hs.Config.BearerToken != "" { + bearer := fmt.Sprintf("Bearer %s", hs.Config.BearerToken) req.Header.Set("Authorization", bearer) } diff --git a/pkg/sync/http/http_sync_test.go b/pkg/sync/http/http_sync_test.go index e7e1596c4..952f04fe9 100644 --- a/pkg/sync/http/http_sync_test.go +++ b/pkg/sync/http/http_sync_test.go @@ -32,7 +32,6 @@ func TestSimpleSync(t *testing.T) { URI: "http://localhost", Client: mockClient, Cron: mockCron, - BearerToken: "", LastBodySHA: "", Logger: logger.NewLogger(nil, false), } @@ -146,9 +145,11 @@ func TestHTTPSync_Fetch(t *testing.T) { tt.setup(t, mockClient) httpSync := Sync{ - URI: tt.uri, - Client: mockClient, - BearerToken: tt.bearerToken, + URI: tt.uri, + Client: mockClient, + Config: sync.SyncProviderConfig{ + BearerToken: tt.bearerToken, + }, LastBodySHA: tt.lastBodySHA, Logger: logger.NewLogger(nil, false), } diff --git a/pkg/sync/isync.go b/pkg/sync/isync.go index 7af60ff9d..c370c4f72 100644 --- a/pkg/sync/isync.go +++ b/pkg/sync/isync.go @@ -2,8 +2,6 @@ package sync import "context" -type ProviderArgs map[string]string - type Type int // Type of the sync operation @@ -55,3 +53,10 @@ type DataSync struct { Source string Type } + +type SyncProviderConfig struct { + URI string `json:"uri"` + Provider string `json:"provider"` + + BearerToken string `json:"bearerToken,omitempty"` +} diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index 632599d67..729fe95f7 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -26,11 +26,11 @@ import ( var resyncPeriod = 1 * time.Minute type Sync struct { - Logger *logger.Logger - ProviderArgs sync.ProviderArgs - client client.Client - URI string - ready bool + Logger *logger.Logger + Config sync.SyncProviderConfig + client client.Client + URI string + ready bool } func (k *Sync) Init(ctx context.Context) error { From ee8d8232c629ade7237c17a7c9e1b9ff416e8ffe Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 13:55:06 +0000 Subject: [PATCH 02/24] docs Signed-off-by: James Milligan --- pkg/runtime/from_config.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index b530e3210..dc9f2d317 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -1,6 +1,8 @@ package runtime import ( + "encoding/json" + "errors" "fmt" "net/http" "regexp" @@ -154,7 +156,23 @@ func (r *Runtime) newFile(config sync.SyncProviderConfig, logger *logger.Logger) } } -func SyncProvidersFromArgs(uris []string) ([]sync.SyncProviderConfig, error) { +func SyncProviderArgPass(syncProviders string) ([]sync.SyncProviderConfig, error) { + syncProvidersParsed := []sync.SyncProviderConfig{} + if err := json.Unmarshal([]byte(syncProviders), &syncProvidersParsed); err != nil { + return syncProvidersParsed, fmt.Errorf("unable to parse sync providers: %w", err) + } + for _, sp := range syncProvidersParsed { + if sp.URI == "" { + return syncProvidersParsed, errors.New("sync provider argument parse: uri is a required field") + } + if sp.Provider == "" { + return syncProvidersParsed, errors.New("sync provider argument parse: provider is a required field") + } + } + return syncProvidersParsed, nil +} + +func SyncProvidersFromURIs(uris []string) ([]sync.SyncProviderConfig, error) { syncProvidersParsed := []sync.SyncProviderConfig{} for _, uri := range uris { switch uriB := []byte(uri); { From 35a84b53804a903eba4ea49196237f47b5cd50a0 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 13:55:11 +0000 Subject: [PATCH 03/24] docs Signed-off-by: James Milligan --- cmd/start.go | 30 ++++++++++++++++++----------- docs/configuration/configuration.md | 28 +++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 0fb864955..aa921a630 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -25,6 +25,7 @@ const ( serverCertPathFlagName = "server-cert-path" serverKeyPathFlagName = "server-key-path" socketPathFlagName = "socket-path" + syncProvidersFlagName = "sync-providers" syncProviderFlagName = "sync-provider" uriFlagName = "uri" ) @@ -58,6 +59,10 @@ func init() { flags.StringP( syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", ) + flags.StringP( + syncProvidersFlagName, "y", "", "JSON representation of an array of SyncProviderConfig objects. This object contains "+ + "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: ", + ) flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ") _ = viper.BindPFlag(bearerTokenFlagName, flags.Lookup(bearerTokenFlagName)) @@ -71,6 +76,7 @@ func init() { _ = viper.BindPFlag(serverKeyPathFlagName, flags.Lookup(serverKeyPathFlagName)) _ = viper.BindPFlag(socketPathFlagName, flags.Lookup(socketPathFlagName)) _ = viper.BindPFlag(syncProviderFlagName, flags.Lookup(syncProviderFlagName)) + _ = viper.BindPFlag(syncProvidersFlagName, flags.Lookup(syncProviderFlagName)) _ = viper.BindPFlag(uriFlagName, flags.Lookup(uriFlagName)) } @@ -97,26 +103,28 @@ var startCmd = &cobra.Command{ rtLogger.Info(fmt.Sprintf("flagd version: %s (%s), built at: %s", Version, Commit, Date)) - if viper.GetString(syncProviderFlagName) != "" { - rtLogger.Warn("DEPRECATED: The --sync-provider flag has been deprecated. " + - "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") - } - if viper.GetString(evaluatorFlagName) != "json" { rtLogger.Warn("DEPRECATED: The --evaluator flag has been deprecated. " + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") } - syncProvidersFromURI, err := runtime.SyncProvidersFromArgs(viper.GetStringSlice(uriFlagName)) + syncProviders, err := runtime.SyncProvidersFromURIs(viper.GetStringSlice(uriFlagName)) if err != nil { log.Fatal(err) } - - syncProviders := []sync.SyncProviderConfig{} - if err := viper.UnmarshalKey(syncProviderFlagName, &syncProviders); err != nil { - log.Fatal(err) + syncProviders2 := []sync.SyncProviderConfig{} + if cfgFile == "" { + syncProviders2, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) + if err != nil { + log.Fatal(err) + } + } else { + err = viper.UnmarshalKey(syncProvidersFlagName, &syncProviders2) + if err != nil { + log.Fatal(err) + } } - syncProviders = append(syncProviders, syncProvidersFromURI...) + syncProviders = append(syncProviders, syncProviders2...) // Build Runtime ----------------------------------------------------------- rt, err := runtime.FromConfig(logger, runtime.Config{ diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 2a209bdef..6be2ecf27 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -10,7 +10,7 @@ Environment variable keys are uppercased, prefixed with `FLAGD_` and all `-` are Config file expects the keys to have the exact naming as the flags. -### URI patterns +### URI patterns Any URI passed to flagd via the `--uri` flag must follow one of the 4 following patterns to ensure that it is passed to the correct implementation: @@ -22,7 +22,6 @@ Any URI passed to flagd via the `--uri` flag must follow one of the 4 following | Grpc | `grpc://flag-source-url` | `grpc://my-flags-server` | - ### Customising sync providers Custom sync providers can be used to provide flag evaluation logic. @@ -35,4 +34,29 @@ To use an existing FeatureFlagConfiguration custom resource, start flagD with th ```shell flagd start --uri core.openfeature.dev/default/my_example +``` + +### Sync Provider Configuration + +While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sync-providers` flag should be used. +The flag takes a string argument, which should be a JSON representation of an array of SyncProviderConfig objects. Alternatively, these configurations should be passed to +flagd via config file, specified using the `--config` flag. + +| Field | Type | +|------------|------------------------------------| +| uri | required `string` | | +| provider | required `string` (`file`, `kubernetes`, `http` or `grpc`) | +| bearerToken | optional `string` | + +The `uri` field values do not need to follow the [URI patterns](#uri-patterns), the provider type is instead derived from the provider field. + +Example start command using a filepath sync provider and the equivalent config file definition: +```sh +./flagd start --sync-providers=\[{\"uri\":\"config/samples/example_flags.json\"\,\"provider\":\"file\"}\] +``` + +```yaml +sync-providers: +- uri: config/samples/example_flags.json + provider: file ``` \ No newline at end of file From e88e47b00b13d5e9c2375f6fa768c2428df77044 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 15:53:11 +0000 Subject: [PATCH 04/24] linting Signed-off-by: James Milligan --- cmd/start.go | 4 +-- docs/configuration/configuration.md | 2 +- pkg/runtime/from_config.go | 34 +++++++++++++------------- pkg/runtime/runtime.go | 2 +- pkg/sync/file/filepath_sync.go | 2 +- pkg/sync/grpc/grpc_sync.go | 2 +- pkg/sync/http/http_sync.go | 2 +- pkg/sync/http/http_sync_test.go | 2 +- pkg/sync/isync.go | 2 +- pkg/sync/kubernetes/kubernetes_sync.go | 2 +- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index aa921a630..ebdd854b0 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -60,7 +60,7 @@ func init() { syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", ) flags.StringP( - syncProvidersFlagName, "y", "", "JSON representation of an array of SyncProviderConfig objects. This object contains "+ + syncProvidersFlagName, "y", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: ", ) flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ") @@ -112,7 +112,7 @@ var startCmd = &cobra.Command{ if err != nil { log.Fatal(err) } - syncProviders2 := []sync.SyncProviderConfig{} + syncProviders2 := []sync.ProviderConfig{} if cfgFile == "" { syncProviders2, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) if err != nil { diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 6be2ecf27..087f81819 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -39,7 +39,7 @@ flagd start --uri core.openfeature.dev/default/my_example ### Sync Provider Configuration While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sync-providers` flag should be used. -The flag takes a string argument, which should be a JSON representation of an array of SyncProviderConfig objects. Alternatively, these configurations should be passed to +The flag takes a string argument, which should be a JSON representation of an array of `ProviderConfig` objects. Alternatively, these configurations should be passed to flagd via config file, specified using the `--config` flag. | Field | Type | diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index dc9f2d317..8b93c3773 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -25,7 +25,7 @@ const ( syncProviderFile = "file" syncProviderGrpc = "grpc" syncProviderKubernetes = "kubernetes" - syncProviderHttp = "http" + syncProviderHTTP = "http" ) var ( @@ -88,7 +88,7 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { r.newK8s(syncProvider, logger), ) rtLogger.Debug(fmt.Sprintf("using kubernetes sync-provider for: %s", syncProvider.URI)) - case syncProviderHttp: + case syncProviderHTTP: r.SyncImpl = append( r.SyncImpl, r.newHTTP(syncProvider, logger), @@ -107,7 +107,7 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { return nil } -func (r *Runtime) newGRPC(config sync.SyncProviderConfig, logger *logger.Logger) *grpc.Sync { +func (r *Runtime) newGRPC(config sync.ProviderConfig, logger *logger.Logger) *grpc.Sync { return &grpc.Sync{ Target: grpc.URLToGRPCTarget(config.URI), Logger: logger.WithFields( @@ -118,7 +118,7 @@ func (r *Runtime) newGRPC(config sync.SyncProviderConfig, logger *logger.Logger) } } -func (r *Runtime) newHTTP(config sync.SyncProviderConfig, logger *logger.Logger) *httpSync.Sync { +func (r *Runtime) newHTTP(config sync.ProviderConfig, logger *logger.Logger) *httpSync.Sync { return &httpSync.Sync{ URI: config.URI, Client: &http.Client{ @@ -133,7 +133,7 @@ func (r *Runtime) newHTTP(config sync.SyncProviderConfig, logger *logger.Logger) } } -func (r *Runtime) newK8s(config sync.SyncProviderConfig, logger *logger.Logger) *kubernetes.Sync { +func (r *Runtime) newK8s(config sync.ProviderConfig, logger *logger.Logger) *kubernetes.Sync { return &kubernetes.Sync{ Logger: logger.WithFields( zap.String("component", "sync"), @@ -144,7 +144,7 @@ func (r *Runtime) newK8s(config sync.SyncProviderConfig, logger *logger.Logger) } } -func (r *Runtime) newFile(config sync.SyncProviderConfig, logger *logger.Logger) *file.Sync { +func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *file.Sync { return &file.Sync{ URI: config.URI, Logger: logger.WithFields( @@ -156,8 +156,8 @@ func (r *Runtime) newFile(config sync.SyncProviderConfig, logger *logger.Logger) } } -func SyncProviderArgPass(syncProviders string) ([]sync.SyncProviderConfig, error) { - syncProvidersParsed := []sync.SyncProviderConfig{} +func SyncProviderArgPass(syncProviders string) ([]sync.ProviderConfig, error) { + syncProvidersParsed := []sync.ProviderConfig{} if err := json.Unmarshal([]byte(syncProviders), &syncProvidersParsed); err != nil { return syncProvidersParsed, fmt.Errorf("unable to parse sync providers: %w", err) } @@ -172,33 +172,33 @@ func SyncProviderArgPass(syncProviders string) ([]sync.SyncProviderConfig, error return syncProvidersParsed, nil } -func SyncProvidersFromURIs(uris []string) ([]sync.SyncProviderConfig, error) { - syncProvidersParsed := []sync.SyncProviderConfig{} +func SyncProvidersFromURIs(uris []string) ([]sync.ProviderConfig, error) { + syncProvidersParsed := []sync.ProviderConfig{} for _, uri := range uris { switch uriB := []byte(uri); { case regFile.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ URI: regFile.ReplaceAllString(uri, ""), Provider: syncProviderFile, }) case regCrd.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ URI: regCrd.ReplaceAllString(uri, ""), Provider: syncProviderKubernetes, }) case regURL.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ URI: uri, - Provider: syncProviderHttp, + Provider: syncProviderHTTP, }) case regGRPC.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.SyncProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ URI: uri, Provider: syncProviderGrpc, }) default: - return syncProvidersParsed, fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', 'http(s)://', 'grpc://',"+ - " or 'core.openfeature.dev'", uri) + return syncProvidersParsed, fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', "+ + "'http(s)://', 'grpc://', or 'core.openfeature.dev'", uri) } } return syncProvidersParsed, nil diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 41b150bfa..6526745a9 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -33,7 +33,7 @@ type Config struct { ServiceCertPath string ServiceKeyPath string - SyncProviders []sync.SyncProviderConfig + SyncProviders []sync.ProviderConfig CORS []string } diff --git a/pkg/sync/file/filepath_sync.go b/pkg/sync/file/filepath_sync.go index 9763a2d1b..551a3dc3f 100644 --- a/pkg/sync/file/filepath_sync.go +++ b/pkg/sync/file/filepath_sync.go @@ -19,7 +19,7 @@ import ( type Sync struct { URI string Logger *logger.Logger - Config sync.SyncProviderConfig + Config sync.ProviderConfig // FileType indicates the file type e.g., json, yaml/yml etc., fileType string watcher *fsnotify.Watcher diff --git a/pkg/sync/grpc/grpc_sync.go b/pkg/sync/grpc/grpc_sync.go index c274a8c6a..1a81d72d0 100644 --- a/pkg/sync/grpc/grpc_sync.go +++ b/pkg/sync/grpc/grpc_sync.go @@ -39,7 +39,7 @@ type Sync struct { options []grpc.DialOption ready bool Mux *msync.RWMutex - Config sync.SyncProviderConfig + Config sync.ProviderConfig } func (g *Sync) Init(ctx context.Context) error { diff --git a/pkg/sync/http/http_sync.go b/pkg/sync/http/http_sync.go index 8f2a786dd..3bee580a8 100644 --- a/pkg/sync/http/http_sync.go +++ b/pkg/sync/http/http_sync.go @@ -22,7 +22,7 @@ type Sync struct { LastBodySHA string Logger *logger.Logger ready bool - Config sync.SyncProviderConfig + Config sync.ProviderConfig } // Client defines the behaviour required of a http client diff --git a/pkg/sync/http/http_sync_test.go b/pkg/sync/http/http_sync_test.go index 952f04fe9..b43c7fe1c 100644 --- a/pkg/sync/http/http_sync_test.go +++ b/pkg/sync/http/http_sync_test.go @@ -147,7 +147,7 @@ func TestHTTPSync_Fetch(t *testing.T) { httpSync := Sync{ URI: tt.uri, Client: mockClient, - Config: sync.SyncProviderConfig{ + Config: sync.ProviderConfig{ BearerToken: tt.bearerToken, }, LastBodySHA: tt.lastBodySHA, diff --git a/pkg/sync/isync.go b/pkg/sync/isync.go index c370c4f72..6b011a6ce 100644 --- a/pkg/sync/isync.go +++ b/pkg/sync/isync.go @@ -54,7 +54,7 @@ type DataSync struct { Type } -type SyncProviderConfig struct { +type ProviderConfig struct { URI string `json:"uri"` Provider string `json:"provider"` diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index 729fe95f7..8118fa567 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -27,7 +27,7 @@ var resyncPeriod = 1 * time.Minute type Sync struct { Logger *logger.Logger - Config sync.SyncProviderConfig + Config sync.ProviderConfig client client.Client URI string ready bool From 484814fb8c3d9a7c3671199f4f969685984f5184 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 15:55:30 +0000 Subject: [PATCH 05/24] docs Signed-off-by: James Milligan --- cmd/start.go | 2 +- docs/configuration/flagd_start.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/start.go b/cmd/start.go index ebdd854b0..4c53ff5be 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -60,7 +60,7 @@ func init() { syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", ) flags.StringP( - syncProvidersFlagName, "y", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ + syncProvidersFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: ", ) flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ") diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 0cc8945f3..bf54b4b0e 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -21,6 +21,7 @@ flagd start [flags] -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString Sync provider arguments as key values separated by = (default []) + -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. Using multiple providers is supported however if flag keys are duplicated across multiple sources it may lead to unexpected behavior. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. ``` From f8aef135a9590588714c805b02280afa7abc6bf9 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 28 Feb 2023 16:04:16 +0000 Subject: [PATCH 06/24] bug fix Signed-off-by: James Milligan --- cmd/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/start.go b/cmd/start.go index 4c53ff5be..c6ff4101a 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -113,7 +113,7 @@ var startCmd = &cobra.Command{ log.Fatal(err) } syncProviders2 := []sync.ProviderConfig{} - if cfgFile == "" { + if cfgFile == "" && viper.GetString(syncProvidersFlagName) != "" { syncProviders2, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) if err != nil { log.Fatal(err) From 71f09e70c43686a39daf7e760e4a798c71e084a7 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 1 Mar 2023 12:06:23 +0000 Subject: [PATCH 07/24] test coverage Signed-off-by: James Milligan --- pkg/runtime/runtime_test.go | 156 ++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 pkg/runtime/runtime_test.go diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go new file mode 100644 index 000000000..f1321a985 --- /dev/null +++ b/pkg/runtime/runtime_test.go @@ -0,0 +1,156 @@ +package runtime_test + +import ( + "reflect" + "testing" + + "github.com/open-feature/flagd/pkg/runtime" + "github.com/open-feature/flagd/pkg/sync" +) + +func TestSyncProviderArgPass(t *testing.T) { + test := map[string]struct { + in string + expectErr bool + out []sync.ProviderConfig + }{ + "simple": { + in: "[{\"uri\":\"config/samples/example_flags.json\",\"provider\":\"file\"}]", + expectErr: false, + out: []sync.ProviderConfig{ + { + URI: "config/samples/example_flags.json", + Provider: "file", + }, + }, + }, + "multiple-syncs": { + in: `[ + {"uri":"config/samples/example_flags.json","provider":"file"}, + {"uri":"http://test.com","provider":"http","bearerToken":":)"}, + {"uri":"host:port","provider":"grpc"}, + {"uri":"default/my-crd","provider":"kubernetes"} + ]`, + expectErr: false, + out: []sync.ProviderConfig{ + { + URI: "config/samples/example_flags.json", + Provider: "file", + }, + { + URI: "http://test.com", + Provider: "http", + BearerToken: ":)", + }, + { + URI: "host:port", + Provider: "grpc", + }, + { + URI: "default/my-crd", + Provider: "kubernetes", + }, + }, + }, + "empty": { + in: `[]`, + expectErr: false, + out: []sync.ProviderConfig{}, + }, + "parse-failure": { + in: ``, + expectErr: true, + out: []sync.ProviderConfig{}, + }, + } + + for name, tt := range test { + t.Run(name, func(t *testing.T) { + out, err := runtime.SyncProviderArgPass(tt.in) + if tt.expectErr { + if err == nil { + t.Error("expected error, got none") + } + } else if err != nil { + t.Errorf("did not expect error: %s", err.Error()) + } + if !reflect.DeepEqual(out, tt.out) { + t.Errorf("unexpected output, expected %v, got %v", tt.out, out) + } + }) + } +} + +func TestSyncProvidersFromURIs(t *testing.T) { + test := map[string]struct { + in []string + expectErr bool + out []sync.ProviderConfig + }{ + "simple": { + in: []string{ + "file:my-file.json", + }, + expectErr: false, + out: []sync.ProviderConfig{ + { + URI: "my-file.json", + Provider: "file", + }, + }, + }, + "multiple-uris": { + in: []string{ + "file:my-file.json", + "https://test.com", + "grpc://host:port", + "core.openfeature.dev/default/my-crd", + }, + expectErr: false, + out: []sync.ProviderConfig{ + { + URI: "my-file.json", + Provider: "file", + }, + { + URI: "https://test.com", + Provider: "http", + }, + { + URI: "grpc://host:port", + Provider: "grpc", + }, + { + URI: "default/my-crd", + Provider: "kubernetes", + }, + }, + }, + "empty": { + in: []string{}, + expectErr: false, + out: []sync.ProviderConfig{}, + }, + "parse-failure": { + in: []string{"care.openfeature.dev/will/fail"}, + expectErr: true, + out: []sync.ProviderConfig{}, + }, + } + + for name, tt := range test { + t.Run(name, func(t *testing.T) { + out, err := runtime.SyncProvidersFromURIs(tt.in) + if tt.expectErr { + if err == nil { + t.Error("expected error, got none") + } + } else if err != nil { + t.Errorf("did not expect error: %s", err.Error()) + } + if !reflect.DeepEqual(out, tt.out) { + t.Errorf("unexpected output, expected %v, got %v", tt.out, out) + } + }) + } +} From a0c844e8f5fb8d7c86e66dc4d13eaa791c80fec1 Mon Sep 17 00:00:00 2001 From: James Milligan <75740990+james-milligan@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:51:21 +0000 Subject: [PATCH 08/24] Apply suggestions from code review Co-authored-by: Skye Gill Signed-off-by: James Milligan <75740990+james-milligan@users.noreply.github.com> --- pkg/runtime/from_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 8b93c3773..200fb1780 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -81,7 +81,7 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { r.SyncImpl, r.newFile(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", syncProvider.URI)) + rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %s", syncProvider.URI)) case syncProviderKubernetes: r.SyncImpl = append( r.SyncImpl, @@ -93,7 +93,7 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { r.SyncImpl, r.newHTTP(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %q", syncProvider.URI)) + rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %s", syncProvider.URI)) case syncProviderGrpc: r.SyncImpl = append( r.SyncImpl, From c4dee0ed3251ea6f9d882ee480a1440771ae05a5 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 1 Mar 2023 13:54:07 +0000 Subject: [PATCH 09/24] name update Signed-off-by: James Milligan --- cmd/start.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index c6ff4101a..633b63e3c 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -112,19 +112,19 @@ var startCmd = &cobra.Command{ if err != nil { log.Fatal(err) } - syncProviders2 := []sync.ProviderConfig{} + syncProvidersFromConfig := []sync.ProviderConfig{} if cfgFile == "" && viper.GetString(syncProvidersFlagName) != "" { - syncProviders2, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) + syncProvidersFromConfig, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) if err != nil { log.Fatal(err) } } else { - err = viper.UnmarshalKey(syncProvidersFlagName, &syncProviders2) + err = viper.UnmarshalKey(syncProvidersFlagName, &syncProvidersFromConfig) if err != nil { log.Fatal(err) } } - syncProviders = append(syncProviders, syncProviders2...) + syncProviders = append(syncProviders, syncProvidersFromConfig...) // Build Runtime ----------------------------------------------------------- rt, err := runtime.FromConfig(logger, runtime.Config{ From 73ce052ce9739adca8a39cb50d033a8a82bbd748 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 2 Mar 2023 09:37:59 +0000 Subject: [PATCH 10/24] conflict fix Signed-off-by: James Milligan --- pkg/runtime/from_config.go | 6 +++++- pkg/sync/http/http_sync_test.go | 8 +++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index fa269181b..0d38cdfd7 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -45,7 +45,11 @@ func init() { func FromConfig(logger *logger.Logger, config Config) (*Runtime, error) { s := store.NewFlags() - s.FlagSources = config.SyncURI + sources := []string{} + for _, sync := range config.SyncProviders { + sources = append(sources, sync.URI) + } + s.FlagSources = sources rt := Runtime{ config: config, Logger: logger.WithFields(zap.String("component", "runtime")), diff --git a/pkg/sync/http/http_sync_test.go b/pkg/sync/http/http_sync_test.go index 0daec0478..187fbdbf2 100644 --- a/pkg/sync/http/http_sync_test.go +++ b/pkg/sync/http/http_sync_test.go @@ -223,9 +223,11 @@ func TestHTTPSync_Resync(t *testing.T) { tt.setup(t, mockClient) httpSync := Sync{ - URI: tt.uri, - Client: mockClient, - BearerToken: tt.bearerToken, + URI: tt.uri, + Client: mockClient, + Config: sync.ProviderConfig{ + BearerToken: tt.bearerToken, + }, LastBodySHA: tt.lastBodySHA, Logger: logger.NewLogger(nil, false), } From 5c3942e340e4f8bf523651918892219dc412c840 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 2 Mar 2023 09:38:43 +0000 Subject: [PATCH 11/24] conflict fix Signed-off-by: James Milligan --- docs/configuration/flagd_start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 95770ff30..3944d3f2f 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -21,7 +21,7 @@ flagd start [flags] -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString Sync provider arguments as key values separated by = (default []) - -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: + -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: TODO -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. ``` From de6b6ba24d2ed3139249df1f577398b4cdc11ae4 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 2 Mar 2023 09:55:52 +0000 Subject: [PATCH 12/24] bug fix Signed-off-by: James Milligan --- cmd/start.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/start.go b/cmd/start.go index d631eec97..19c282c01 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -77,7 +77,7 @@ func init() { _ = viper.BindPFlag(serverKeyPathFlagName, flags.Lookup(serverKeyPathFlagName)) _ = viper.BindPFlag(socketPathFlagName, flags.Lookup(socketPathFlagName)) _ = viper.BindPFlag(syncProviderFlagName, flags.Lookup(syncProviderFlagName)) - _ = viper.BindPFlag(syncProvidersFlagName, flags.Lookup(syncProviderFlagName)) + _ = viper.BindPFlag(syncProvidersFlagName, flags.Lookup(syncProvidersFlagName)) _ = viper.BindPFlag(uriFlagName, flags.Lookup(uriFlagName)) } @@ -113,6 +113,7 @@ var startCmd = &cobra.Command{ if err != nil { log.Fatal(err) } + syncProvidersFromConfig := []sync.ProviderConfig{} if cfgFile == "" && viper.GetString(syncProvidersFlagName) != "" { syncProvidersFromConfig, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) From 86d392d2ca843b5ae5247775a48dcf8438fe9b86 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 2 Mar 2023 10:09:09 +0000 Subject: [PATCH 13/24] doc fix Signed-off-by: James Milligan --- docs/configuration/flagd_start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 3944d3f2f..95770ff30 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -21,7 +21,7 @@ flagd start [flags] -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString Sync provider arguments as key values separated by = (default []) - -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: TODO + -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. ``` From 3e7c41a41e13722d72ff2990a70b48744c89766f Mon Sep 17 00:00:00 2001 From: James Milligan Date: Thu, 2 Mar 2023 10:10:50 +0000 Subject: [PATCH 14/24] deprecation warning Signed-off-by: James Milligan --- cmd/start.go | 10 ++++++++-- docs/configuration/flagd_start.md | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 19c282c01..702d3cfa6 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -46,7 +46,7 @@ func init() { flags.StringP(serverCertPathFlagName, "c", "", "Server side tls certificate path") flags.StringP(serverKeyPathFlagName, "k", "", "Server side tls key path") flags.StringToStringP(providerArgsFlagName, - "a", nil, "Sync provider arguments as key values separated by =") + "a", nil, "DEPRECATED: Sync provider arguments as key values separated by =") flags.StringSliceP( uriFlagName, "f", []string{}, "Set a sync provider uri to read data from, this can be a filepath,"+ "url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the "+ @@ -62,7 +62,8 @@ func init() { ) flags.StringP( syncProvidersFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ - "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: ", + "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: "+ + "https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation", ) flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ") @@ -109,6 +110,11 @@ var startCmd = &cobra.Command{ "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") } + if viper.GetStringMapString(providerArgsFlagName) != nil { + rtLogger.Warn("DEPRECATED: The --sync-provider-args flag has been deprecated. " + + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") + } + syncProviders, err := runtime.SyncProvidersFromURIs(viper.GetStringSlice(uriFlagName)) if err != nil { log.Fatal(err) diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 95770ff30..3fe9e21ae 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -20,8 +20,8 @@ flagd start [flags] -k, --server-key-path string Server side tls key path -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote - -a, --sync-provider-args stringToString Sync provider arguments as key values separated by = (default []) - -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: + -a, --sync-provider-args stringToString DEPRECATED: Sync provider arguments as key values separated by = (default []) + -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. ``` From 1f7f7aec31c358230576f76502befd0783386e50 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 7 Mar 2023 16:41:26 +0000 Subject: [PATCH 15/24] removed config object from syncs Signed-off-by: James Milligan --- pkg/runtime/from_config.go | 10 ++++------ pkg/sync/file/filepath_sync.go | 1 - pkg/sync/grpc/grpc_sync.go | 1 - pkg/sync/http/http_sync.go | 6 +++--- pkg/sync/http/http_sync_test.go | 16 ++++++---------- pkg/sync/kubernetes/kubernetes_sync.go | 1 - 6 files changed, 13 insertions(+), 22 deletions(-) diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 0d38cdfd7..6b75016f1 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -135,8 +135,8 @@ func (r *Runtime) newHTTP(config sync.ProviderConfig, logger *logger.Logger) *ht zap.String("component", "sync"), zap.String("sync", "remote"), ), - Config: config, - Cron: cron.New(), + BearerToken: config.BearerToken, + Cron: cron.New(), } } @@ -146,8 +146,7 @@ func (r *Runtime) newK8s(config sync.ProviderConfig, logger *logger.Logger) *kub zap.String("component", "sync"), zap.String("sync", "kubernetes"), ), - URI: config.URI, - Config: config, + URI: config.URI, } } @@ -158,8 +157,7 @@ func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *fi zap.String("component", "sync"), zap.String("sync", "filepath"), ), - Config: config, - Mux: &msync.RWMutex{}, + Mux: &msync.RWMutex{}, } } diff --git a/pkg/sync/file/filepath_sync.go b/pkg/sync/file/filepath_sync.go index 6cb02c52a..7ec703c33 100644 --- a/pkg/sync/file/filepath_sync.go +++ b/pkg/sync/file/filepath_sync.go @@ -19,7 +19,6 @@ import ( type Sync struct { URI string Logger *logger.Logger - Config sync.ProviderConfig Source string // FileType indicates the file type e.g., json, yaml/yml etc., fileType string diff --git a/pkg/sync/grpc/grpc_sync.go b/pkg/sync/grpc/grpc_sync.go index 0241b014b..d1fb80641 100644 --- a/pkg/sync/grpc/grpc_sync.go +++ b/pkg/sync/grpc/grpc_sync.go @@ -39,7 +39,6 @@ type Sync struct { options []grpc.DialOption ready bool Mux *msync.RWMutex - Config sync.ProviderConfig } func (g *Sync) connectClient(ctx context.Context) error { diff --git a/pkg/sync/http/http_sync.go b/pkg/sync/http/http_sync.go index c3c59cc93..9f7ea3174 100644 --- a/pkg/sync/http/http_sync.go +++ b/pkg/sync/http/http_sync.go @@ -22,7 +22,7 @@ type Sync struct { LastBodySHA string Logger *logger.Logger ready bool - Config sync.ProviderConfig + BearerToken string } // Client defines the behaviour required of a http client @@ -116,8 +116,8 @@ func (hs *Sync) fetchBodyFromURL(ctx context.Context, url string) ([]byte, error req.Header.Add("Accept", "application/json") - if hs.Config.BearerToken != "" { - bearer := fmt.Sprintf("Bearer %s", hs.Config.BearerToken) + if hs.BearerToken != "" { + bearer := fmt.Sprintf("Bearer %s", hs.BearerToken) req.Header.Set("Authorization", bearer) } diff --git a/pkg/sync/http/http_sync_test.go b/pkg/sync/http/http_sync_test.go index 187fbdbf2..dc1c991cd 100644 --- a/pkg/sync/http/http_sync_test.go +++ b/pkg/sync/http/http_sync_test.go @@ -147,11 +147,9 @@ func TestHTTPSync_Fetch(t *testing.T) { tt.setup(t, mockClient) httpSync := Sync{ - URI: tt.uri, - Client: mockClient, - Config: sync.ProviderConfig{ - BearerToken: tt.bearerToken, - }, + URI: tt.uri, + Client: mockClient, + BearerToken: tt.bearerToken, LastBodySHA: tt.lastBodySHA, Logger: logger.NewLogger(nil, false), } @@ -223,11 +221,9 @@ func TestHTTPSync_Resync(t *testing.T) { tt.setup(t, mockClient) httpSync := Sync{ - URI: tt.uri, - Client: mockClient, - Config: sync.ProviderConfig{ - BearerToken: tt.bearerToken, - }, + URI: tt.uri, + Client: mockClient, + BearerToken: tt.bearerToken, LastBodySHA: tt.lastBodySHA, Logger: logger.NewLogger(nil, false), } diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index a406f0458..179889c02 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -27,7 +27,6 @@ var resyncPeriod = 1 * time.Minute type Sync struct { Logger *logger.Logger - Config sync.ProviderConfig client client.Client URI string ready bool From a522f566c8cc66bf78873f78c29d8a374596021b Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 7 Mar 2023 16:50:59 +0000 Subject: [PATCH 16/24] rename sync providers to sources Signed-off-by: James Milligan --- cmd/start.go | 12 ++++++------ docs/configuration/configuration.md | 8 ++++---- docs/configuration/flagd_start.md | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 702d3cfa6..a03623ada 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -25,7 +25,7 @@ const ( serverCertPathFlagName = "server-cert-path" serverKeyPathFlagName = "server-key-path" socketPathFlagName = "socket-path" - syncProvidersFlagName = "sync-providers" + sourcesFlagName = "sources" syncProviderFlagName = "sync-provider" uriFlagName = "uri" ) @@ -61,7 +61,7 @@ func init() { syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", ) flags.StringP( - syncProvidersFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ + sourcesFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: "+ "https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation", ) @@ -78,7 +78,7 @@ func init() { _ = viper.BindPFlag(serverKeyPathFlagName, flags.Lookup(serverKeyPathFlagName)) _ = viper.BindPFlag(socketPathFlagName, flags.Lookup(socketPathFlagName)) _ = viper.BindPFlag(syncProviderFlagName, flags.Lookup(syncProviderFlagName)) - _ = viper.BindPFlag(syncProvidersFlagName, flags.Lookup(syncProvidersFlagName)) + _ = viper.BindPFlag(sourcesFlagName, flags.Lookup(sourcesFlagName)) _ = viper.BindPFlag(uriFlagName, flags.Lookup(uriFlagName)) } @@ -121,13 +121,13 @@ var startCmd = &cobra.Command{ } syncProvidersFromConfig := []sync.ProviderConfig{} - if cfgFile == "" && viper.GetString(syncProvidersFlagName) != "" { - syncProvidersFromConfig, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName)) + if cfgFile == "" && viper.GetString(sourcesFlagName) != "" { + syncProvidersFromConfig, err = runtime.SyncProviderArgPass(viper.GetString(sourcesFlagName)) if err != nil { log.Fatal(err) } } else { - err = viper.UnmarshalKey(syncProvidersFlagName, &syncProvidersFromConfig) + err = viper.UnmarshalKey(sourcesFlagName, &syncProvidersFromConfig) if err != nil { log.Fatal(err) } diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 087f81819..a12cb7516 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -36,9 +36,9 @@ To use an existing FeatureFlagConfiguration custom resource, start flagD with th flagd start --uri core.openfeature.dev/default/my_example ``` -### Sync Provider Configuration +### Source Configuration -While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sync-providers` flag should be used. +While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sources` flag should be used. The flag takes a string argument, which should be a JSON representation of an array of `ProviderConfig` objects. Alternatively, these configurations should be passed to flagd via config file, specified using the `--config` flag. @@ -52,11 +52,11 @@ The `uri` field values do not need to follow the [URI patterns](#uri-patterns), Example start command using a filepath sync provider and the equivalent config file definition: ```sh -./flagd start --sync-providers=\[{\"uri\":\"config/samples/example_flags.json\"\,\"provider\":\"file\"}\] +./flagd start --sources=\[{\"uri\":\"config/samples/example_flags.json\"\,\"provider\":\"file\"}\] ``` ```yaml -sync-providers: +sources: - uri: config/samples/example_flags.json provider: file ``` \ No newline at end of file diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 3fe9e21ae..369ab46f3 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -19,9 +19,9 @@ flagd start [flags] -c, --server-cert-path string Server side tls certificate path -k, --server-key-path string Server side tls key path -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. + -s, --sources string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString DEPRECATED: Sync provider arguments as key values separated by = (default []) - -s, --sync-providers string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. ``` From 5019af86027fa5166b3e9583f18f56ce1a72ffe0 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 7 Mar 2023 16:53:29 +0000 Subject: [PATCH 17/24] reintroduce deprecation of --sync-provider Signed-off-by: James Milligan --- cmd/start.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/start.go b/cmd/start.go index a03623ada..086352a32 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -105,6 +105,11 @@ var startCmd = &cobra.Command{ rtLogger.Info(fmt.Sprintf("flagd version: %s (%s), built at: %s", Version, Commit, Date)) + if viper.GetString(syncProviderFlagName) != "" { + rtLogger.Warn("DEPRECATED: The --sync-provider flag has been deprecated. " + + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") + } + if viper.GetString(evaluatorFlagName) != "json" { rtLogger.Warn("DEPRECATED: The --evaluator flag has been deprecated. " + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") From 33644c33f42b7080c1e357020df7448b76f54c53 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Tue, 7 Mar 2023 16:54:34 +0000 Subject: [PATCH 18/24] rename func Signed-off-by: James Milligan --- cmd/start.go | 2 +- pkg/runtime/from_config.go | 2 +- pkg/runtime/runtime_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 086352a32..8d1188bf8 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -127,7 +127,7 @@ var startCmd = &cobra.Command{ syncProvidersFromConfig := []sync.ProviderConfig{} if cfgFile == "" && viper.GetString(sourcesFlagName) != "" { - syncProvidersFromConfig, err = runtime.SyncProviderArgPass(viper.GetString(sourcesFlagName)) + syncProvidersFromConfig, err = runtime.SyncProviderArgParse(viper.GetString(sourcesFlagName)) if err != nil { log.Fatal(err) } diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 6b75016f1..849533439 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -161,7 +161,7 @@ func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *fi } } -func SyncProviderArgPass(syncProviders string) ([]sync.ProviderConfig, error) { +func SyncProviderArgParse(syncProviders string) ([]sync.ProviderConfig, error) { syncProvidersParsed := []sync.ProviderConfig{} if err := json.Unmarshal([]byte(syncProviders), &syncProvidersParsed); err != nil { return syncProvidersParsed, fmt.Errorf("unable to parse sync providers: %w", err) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index f1321a985..8334d62a0 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -8,7 +8,7 @@ import ( "github.com/open-feature/flagd/pkg/sync" ) -func TestSyncProviderArgPass(t *testing.T) { +func TestSyncProviderArgParse(t *testing.T) { test := map[string]struct { in string expectErr bool @@ -66,7 +66,7 @@ func TestSyncProviderArgPass(t *testing.T) { for name, tt := range test { t.Run(name, func(t *testing.T) { - out, err := runtime.SyncProviderArgPass(tt.in) + out, err := runtime.SyncProviderArgParse(tt.in) if tt.expectErr { if err == nil { t.Error("expected error, got none") From 000509dcb74ba750f36874d25ed457355149da69 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 09:20:47 +0000 Subject: [PATCH 19/24] cleanup Signed-off-by: James Milligan --- cmd/start.go | 6 ++--- docs/configuration/configuration.md | 17 ++++++++++---- docs/configuration/flagd_start.md | 2 +- pkg/runtime/from_config.go | 32 +++++++++++++++----------- pkg/runtime/runtime.go | 2 +- pkg/runtime/runtime_test.go | 20 ++++++++-------- pkg/sync/grpc/grpc_sync.go | 2 +- pkg/sync/http/http_sync.go | 3 ++- pkg/sync/isync.go | 2 +- pkg/sync/kubernetes/kubernetes_sync.go | 7 +++--- 10 files changed, 55 insertions(+), 38 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 8d1188bf8..b9eff1110 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -61,7 +61,7 @@ func init() { syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", ) flags.StringP( - sourcesFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+ + sourcesFlagName, "s", "", "JSON representation of an array of SourceConfig objects. This object contains "+ "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: "+ "https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation", ) @@ -115,7 +115,7 @@ var startCmd = &cobra.Command{ "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") } - if viper.GetStringMapString(providerArgsFlagName) != nil { + if viper.GetString(providerArgsFlagName) != "" { rtLogger.Warn("DEPRECATED: The --sync-provider-args flag has been deprecated. " + "Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md") } @@ -125,7 +125,7 @@ var startCmd = &cobra.Command{ log.Fatal(err) } - syncProvidersFromConfig := []sync.ProviderConfig{} + syncProvidersFromConfig := []sync.SourceConfig{} if cfgFile == "" && viper.GetString(sourcesFlagName) != "" { syncProvidersFromConfig, err = runtime.SyncProviderArgParse(viper.GetString(sourcesFlagName)) if err != nil { diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index a12cb7516..3a107b808 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -39,7 +39,7 @@ flagd start --uri core.openfeature.dev/default/my_example ### Source Configuration While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sources` flag should be used. -The flag takes a string argument, which should be a JSON representation of an array of `ProviderConfig` objects. Alternatively, these configurations should be passed to +The flag takes a string argument, which should be a JSON representation of an array of `SourceConfig` objects. Alternatively, these configurations should be passed to flagd via config file, specified using the `--config` flag. | Field | Type | @@ -48,15 +48,24 @@ flagd via config file, specified using the `--config` flag. | provider | required `string` (`file`, `kubernetes`, `http` or `grpc`) | | bearerToken | optional `string` | -The `uri` field values do not need to follow the [URI patterns](#uri-patterns), the provider type is instead derived from the provider field. +The `uri` field values do not need to follow the [URI patterns](#uri-patterns), the provider type is instead derived from the provider field. If the prefix is supplied, it will be removed on startup without error. Example start command using a filepath sync provider and the equivalent config file definition: ```sh -./flagd start --sources=\[{\"uri\":\"config/samples/example_flags.json\"\,\"provider\":\"file\"}\] +./flagd start --sources='[{"uri":"config/samples/example_flags.json","provider":"file"},{"uri":"http://my-flag-source.json","provider":"http","bearerToken":"bearer-dji34ld2l"}]{"uri":"default/my-flag-config","provider":"kubernetes"},{"uri":"grpc://my-flag-source:8080","provider":"grpc"}' ``` ```yaml sources: - uri: config/samples/example_flags.json provider: file -``` \ No newline at end of file +- uri: http://my-flag-source.json + provider: http + bearerToken: bearer-dji34ld2l +- uri: default/my-flag-config + provider: kubernetes +- uri: http://my-flag-source.json + provider: kubernetes +- uri: grpc://my-flag-source:8080 + provider: grpc +``` diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 369ab46f3..286f3005e 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -19,7 +19,7 @@ flagd start [flags] -c, --server-cert-path string Server side tls certificate path -k, --server-key-path string Server side tls key path -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. - -s, --sources string JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation + -s, --sources string JSON representation of an array of SourceConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString DEPRECATED: Sync provider arguments as key values separated by = (default []) -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 849533439..60ffd2c2b 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -114,7 +114,7 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { return nil } -func (r *Runtime) newGRPC(config sync.ProviderConfig, logger *logger.Logger) *grpc.Sync { +func (r *Runtime) newGRPC(config sync.SourceConfig, logger *logger.Logger) *grpc.Sync { return &grpc.Sync{ Target: grpc.URLToGRPCTarget(config.URI), Logger: logger.WithFields( @@ -125,7 +125,7 @@ func (r *Runtime) newGRPC(config sync.ProviderConfig, logger *logger.Logger) *gr } } -func (r *Runtime) newHTTP(config sync.ProviderConfig, logger *logger.Logger) *httpSync.Sync { +func (r *Runtime) newHTTP(config sync.SourceConfig, logger *logger.Logger) *httpSync.Sync { return &httpSync.Sync{ URI: config.URI, Client: &http.Client{ @@ -140,7 +140,7 @@ func (r *Runtime) newHTTP(config sync.ProviderConfig, logger *logger.Logger) *ht } } -func (r *Runtime) newK8s(config sync.ProviderConfig, logger *logger.Logger) *kubernetes.Sync { +func (r *Runtime) newK8s(config sync.SourceConfig, logger *logger.Logger) *kubernetes.Sync { return &kubernetes.Sync{ Logger: logger.WithFields( zap.String("component", "sync"), @@ -150,7 +150,7 @@ func (r *Runtime) newK8s(config sync.ProviderConfig, logger *logger.Logger) *kub } } -func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *file.Sync { +func (r *Runtime) newFile(config sync.SourceConfig, logger *logger.Logger) *file.Sync { return &file.Sync{ URI: config.URI, Logger: logger.WithFields( @@ -161,43 +161,49 @@ func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *fi } } -func SyncProviderArgParse(syncProviders string) ([]sync.ProviderConfig, error) { - syncProvidersParsed := []sync.ProviderConfig{} +func SyncProviderArgParse(syncProviders string) ([]sync.SourceConfig, error) { + syncProvidersParsed := []sync.SourceConfig{} if err := json.Unmarshal([]byte(syncProviders), &syncProvidersParsed); err != nil { return syncProvidersParsed, fmt.Errorf("unable to parse sync providers: %w", err) } - for _, sp := range syncProvidersParsed { + for i, sp := range syncProvidersParsed { if sp.URI == "" { return syncProvidersParsed, errors.New("sync provider argument parse: uri is a required field") } if sp.Provider == "" { return syncProvidersParsed, errors.New("sync provider argument parse: provider is a required field") } + switch uriB := []byte(sp.URI); { + case regFile.Match(uriB): + syncProvidersParsed[i].URI = regFile.ReplaceAllString(syncProvidersParsed[i].URI, "") + case regCrd.Match(uriB): + syncProvidersParsed[i].URI = regCrd.ReplaceAllString(syncProvidersParsed[i].URI, "") + } } return syncProvidersParsed, nil } -func SyncProvidersFromURIs(uris []string) ([]sync.ProviderConfig, error) { - syncProvidersParsed := []sync.ProviderConfig{} +func SyncProvidersFromURIs(uris []string) ([]sync.SourceConfig, error) { + syncProvidersParsed := []sync.SourceConfig{} for _, uri := range uris { switch uriB := []byte(uri); { case regFile.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.SourceConfig{ URI: regFile.ReplaceAllString(uri, ""), Provider: syncProviderFile, }) case regCrd.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.SourceConfig{ URI: regCrd.ReplaceAllString(uri, ""), Provider: syncProviderKubernetes, }) case regURL.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.SourceConfig{ URI: uri, Provider: syncProviderHTTP, }) case regGRPC.Match(uriB): - syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{ + syncProvidersParsed = append(syncProvidersParsed, sync.SourceConfig{ URI: uri, Provider: syncProviderGrpc, }) diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 0a18f4e7e..d6d242dfc 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -33,7 +33,7 @@ type Config struct { ServiceCertPath string ServiceKeyPath string - SyncProviders []sync.ProviderConfig + SyncProviders []sync.SourceConfig CORS []string } diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 8334d62a0..84da5ebd0 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -12,12 +12,12 @@ func TestSyncProviderArgParse(t *testing.T) { test := map[string]struct { in string expectErr bool - out []sync.ProviderConfig + out []sync.SourceConfig }{ "simple": { in: "[{\"uri\":\"config/samples/example_flags.json\",\"provider\":\"file\"}]", expectErr: false, - out: []sync.ProviderConfig{ + out: []sync.SourceConfig{ { URI: "config/samples/example_flags.json", Provider: "file", @@ -32,7 +32,7 @@ func TestSyncProviderArgParse(t *testing.T) { {"uri":"default/my-crd","provider":"kubernetes"} ]`, expectErr: false, - out: []sync.ProviderConfig{ + out: []sync.SourceConfig{ { URI: "config/samples/example_flags.json", Provider: "file", @@ -55,12 +55,12 @@ func TestSyncProviderArgParse(t *testing.T) { "empty": { in: `[]`, expectErr: false, - out: []sync.ProviderConfig{}, + out: []sync.SourceConfig{}, }, "parse-failure": { in: ``, expectErr: true, - out: []sync.ProviderConfig{}, + out: []sync.SourceConfig{}, }, } @@ -85,14 +85,14 @@ func TestSyncProvidersFromURIs(t *testing.T) { test := map[string]struct { in []string expectErr bool - out []sync.ProviderConfig + out []sync.SourceConfig }{ "simple": { in: []string{ "file:my-file.json", }, expectErr: false, - out: []sync.ProviderConfig{ + out: []sync.SourceConfig{ { URI: "my-file.json", Provider: "file", @@ -107,7 +107,7 @@ func TestSyncProvidersFromURIs(t *testing.T) { "core.openfeature.dev/default/my-crd", }, expectErr: false, - out: []sync.ProviderConfig{ + out: []sync.SourceConfig{ { URI: "my-file.json", Provider: "file", @@ -129,12 +129,12 @@ func TestSyncProvidersFromURIs(t *testing.T) { "empty": { in: []string{}, expectErr: false, - out: []sync.ProviderConfig{}, + out: []sync.SourceConfig{}, }, "parse-failure": { in: []string{"care.openfeature.dev/will/fail"}, expectErr: true, - out: []sync.ProviderConfig{}, + out: []sync.SourceConfig{}, }, } diff --git a/pkg/sync/grpc/grpc_sync.go b/pkg/sync/grpc/grpc_sync.go index 892780c29..7d8f63daa 100644 --- a/pkg/sync/grpc/grpc_sync.go +++ b/pkg/sync/grpc/grpc_sync.go @@ -34,12 +34,12 @@ type Sync struct { Target string ProviderID string Logger *logger.Logger + Mux *msync.RWMutex syncClient syncv1grpc.FlagSyncService_SyncFlagsClient client syncv1grpc.FlagSyncServiceClient options []grpc.DialOption ready bool - Mux *msync.RWMutex } func (g *Sync) connectClient(ctx context.Context) error { diff --git a/pkg/sync/http/http_sync.go b/pkg/sync/http/http_sync.go index e2818e27f..8ec2a9b30 100644 --- a/pkg/sync/http/http_sync.go +++ b/pkg/sync/http/http_sync.go @@ -20,8 +20,9 @@ type Sync struct { Cron Cron LastBodySHA string Logger *logger.Logger - ready bool BearerToken string + + ready bool } // Client defines the behaviour required of a http client diff --git a/pkg/sync/isync.go b/pkg/sync/isync.go index 14154c223..82ca6ee27 100644 --- a/pkg/sync/isync.go +++ b/pkg/sync/isync.go @@ -58,7 +58,7 @@ type DataSync struct { Type } -type ProviderConfig struct { +type SourceConfig struct { URI string `json:"uri"` Provider string `json:"provider"` diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index dde57b766..bf7477071 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -28,10 +28,11 @@ var ( ) type Sync struct { - Logger *logger.Logger + Logger *logger.Logger + Source string + URI string + client client.Client - Source string - URI string ready bool namespace string crdName string From aa5468517360eeaabc55fd3a21e15f7805547f07 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 09:24:18 +0000 Subject: [PATCH 20/24] linting Signed-off-by: James Milligan --- pkg/sync/kubernetes/kubernetes_sync.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index bf7477071..e8a3fe92a 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -32,7 +32,6 @@ type Sync struct { Source string URI string - client client.Client ready bool namespace string crdName string From cfbef90168d2f065903180291efc4da16f6d23b6 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 11:50:21 +0000 Subject: [PATCH 21/24] remove source Signed-off-by: James Milligan --- pkg/sync/file/filepath_sync.go | 1 - pkg/sync/kubernetes/kubernetes_sync.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/sync/file/filepath_sync.go b/pkg/sync/file/filepath_sync.go index feb54afcf..947d84f97 100644 --- a/pkg/sync/file/filepath_sync.go +++ b/pkg/sync/file/filepath_sync.go @@ -19,7 +19,6 @@ import ( type Sync struct { URI string Logger *logger.Logger - Source string // FileType indicates the file type e.g., json, yaml/yml etc., fileType string watcher *fsnotify.Watcher diff --git a/pkg/sync/kubernetes/kubernetes_sync.go b/pkg/sync/kubernetes/kubernetes_sync.go index e8a3fe92a..bd2fdae4f 100644 --- a/pkg/sync/kubernetes/kubernetes_sync.go +++ b/pkg/sync/kubernetes/kubernetes_sync.go @@ -29,7 +29,6 @@ var ( type Sync struct { Logger *logger.Logger - Source string URI string ready bool From 12b8f877f3cd3d6942a73ecc61940804d46e3a49 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 11:55:09 +0000 Subject: [PATCH 22/24] deprecate bearer token Signed-off-by: James Milligan --- cmd/start.go | 2 +- docs/configuration/flagd_start.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index b9eff1110..65b3f450e 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -55,7 +55,7 @@ func init() { "Please note that if you are using filepath, flagd only supports files with `.yaml/.yml/.json` extension.", ) flags.StringP( - bearerTokenFlagName, "b", "", "Set a bearer token to use for remote sync") + bearerTokenFlagName, "b", "", "DEPRECATED: Superseded by --sources.") flags.StringSliceP(corsFlagName, "C", []string{}, "CORS allowed origins, * will allow all origins") flags.StringP( syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote", diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index 286f3005e..c68c17745 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -9,7 +9,7 @@ flagd start [flags] ### Options ``` - -b, --bearer-token string Set a bearer token to use for remote sync + -b, --bearer-token string DEPRECATED: Superseded by --sources. -C, --cors-origin strings CORS allowed origins, * will allow all origins -e, --evaluator string DEPRECATED: Set an evaluator e.g. json, yaml/yml.Please note that yaml/yml and json evaluations work the same (yaml/yml files are converted to json internally) (default "json") -h, --help help for start From 31e1126249a8db482c5692e6edb0577d3bfc7bac Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 11:56:47 +0000 Subject: [PATCH 23/24] flag doc fix Signed-off-by: James Milligan --- cmd/start.go | 2 +- docs/configuration/flagd_start.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/start.go b/cmd/start.go index 65b3f450e..2b1bf19ce 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -62,7 +62,7 @@ func init() { ) flags.StringP( sourcesFlagName, "s", "", "JSON representation of an array of SourceConfig objects. This object contains "+ - "2 required fields, uri (string) and provider (string). Documentation for this object can be found here: "+ + "2 required fields, uri (string) and provider (string). Documentation for this object: "+ "https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation", ) flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ") diff --git a/docs/configuration/flagd_start.md b/docs/configuration/flagd_start.md index c68c17745..aef1d746b 100644 --- a/docs/configuration/flagd_start.md +++ b/docs/configuration/flagd_start.md @@ -19,7 +19,7 @@ flagd start [flags] -c, --server-cert-path string Server side tls certificate path -k, --server-key-path string Server side tls key path -d, --socket-path string Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally. - -s, --sources string JSON representation of an array of SourceConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation + -s, --sources string JSON representation of an array of SourceConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md#sync-provider-customisation -y, --sync-provider string DEPRECATED: Set a sync provider e.g. filepath or remote -a, --sync-provider-args stringToString DEPRECATED: Sync provider arguments as key values separated by = (default []) -f, --uri .yaml/.yml/.json Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. When flag keys are duplicated across multiple providers the merge priority follows the index of the flag arguments, as such flags from the uri at index 0 take the lowest precedence, with duplicated keys being overwritten by those from the uri at index 1. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension. From 4ad480cb38a86939c51702b687ed40da6676d272 Mon Sep 17 00:00:00 2001 From: James Milligan Date: Wed, 8 Mar 2023 16:11:45 +0000 Subject: [PATCH 24/24] conflict resolutions Signed-off-by: James Milligan --- go.sum | 2 -- pkg/runtime/from_config.go | 7 +++---- pkg/sync/kubernetes/kubernetes_sync_test.go | 8 +------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/go.sum b/go.sum index 5cd7b0c51..66a29fecd 100644 --- a/go.sum +++ b/go.sum @@ -407,8 +407,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/pkg/runtime/from_config.go b/pkg/runtime/from_config.go index 54164f5ff..2e86b9052 100644 --- a/pkg/runtime/from_config.go +++ b/pkg/runtime/from_config.go @@ -88,9 +88,9 @@ func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error { r.SyncImpl, r.newFile(syncProvider, logger), ) - rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", uri)) - case regCrd.Match(uriB): - k, err := r.newK8s(uri, logger) + rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", syncProvider.URI)) + case syncProviderKubernetes: + k, err := r.newK8s(syncProvider.URI, logger) if err != nil { return err } @@ -154,7 +154,6 @@ func (r *Runtime) newK8s(uri string, logger *logger.Logger) (*kubernetes.Sync, e zap.String("sync", "kubernetes"), ), regCrd.ReplaceAllString(uri, ""), - r.config.ProviderArgs, reader, dynamic, ), nil diff --git a/pkg/sync/kubernetes/kubernetes_sync_test.go b/pkg/sync/kubernetes/kubernetes_sync_test.go index 013b80def..c533ebb95 100644 --- a/pkg/sync/kubernetes/kubernetes_sync_test.go +++ b/pkg/sync/kubernetes/kubernetes_sync_test.go @@ -14,7 +14,7 @@ import ( "github.com/open-feature/flagd/pkg/sync" "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" "go.uber.org/zap/zapcore" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic/fake" @@ -813,14 +813,11 @@ func Test_NewK8sSync(t *testing.T) { } const uri = "myURI" log := logger.NewLogger(l, true) - const key, value = "myKey", "myValue" - args := map[string]string{key: value} rc := newFakeReadClient() dc := fake.NewSimpleDynamicClient(runtime.NewScheme()) k := NewK8sSync( log, uri, - args, rc, dc, ) @@ -833,9 +830,6 @@ func Test_NewK8sSync(t *testing.T) { if k.logger != log { t.Errorf("Object not initialized with the right logger") } - if k.providerArgs[key] != value { - t.Errorf("Object not initialized with the right arguments") - } if k.readClient != rc { t.Errorf("Object not initialized with the right K8s client") }