feat: introduce per-sync configurations

cmd/start.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/open-feature/flagd/pkg/logger"
 	"github.com/open-feature/flagd/pkg/runtime"
+	"github.com/open-feature/flagd/pkg/sync"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
@@ -24,6 +25,7 @@ const (
 	serverCertPathFlagName = "server-cert-path"
 	serverKeyPathFlagName  = "server-key-path"
 	socketPathFlagName     = "socket-path"
+	syncProvidersFlagName  = "sync-providers"
 	syncProviderFlagName   = "sync-provider"
 	uriFlagName            = "uri"
 )
@@ -57,6 +59,10 @@ func init() {
 	flags.StringP(
 		syncProviderFlagName, "y", "", "DEPRECATED: Set a sync provider e.g. filepath or remote",
 	)
+	flags.StringP(
+		syncProvidersFlagName, "s", "", "JSON representation of an array of ProviderConfig objects. This object contains "+
+			"2 required fields, uri (string) and provider (string). Documentation for this object can be found here: ",
+	)
 	flags.StringP(logFormatFlagName, "z", "console", "Set the logging format, e.g. console or json ")
 
 	_ = viper.BindPFlag(bearerTokenFlagName, flags.Lookup(bearerTokenFlagName))
@@ -70,6 +76,7 @@ func init() {
 	_ = viper.BindPFlag(serverKeyPathFlagName, flags.Lookup(serverKeyPathFlagName))
 	_ = viper.BindPFlag(socketPathFlagName, flags.Lookup(socketPathFlagName))
 	_ = viper.BindPFlag(syncProviderFlagName, flags.Lookup(syncProviderFlagName))
+	_ = viper.BindPFlag(syncProvidersFlagName, flags.Lookup(syncProviderFlagName))
 	_ = viper.BindPFlag(uriFlagName, flags.Lookup(uriFlagName))
 }
 
@@ -96,26 +103,38 @@ var startCmd = &cobra.Command{
 
 		rtLogger.Info(fmt.Sprintf("flagd version: %s (%s), built at: %s", Version, Commit, Date))
 
-		if viper.GetString(syncProviderFlagName) != "" {
-			rtLogger.Warn("DEPRECATED: The --sync-provider flag has been deprecated. " +
-				"Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md")
-		}
-
 		if viper.GetString(evaluatorFlagName) != "json" {
 			rtLogger.Warn("DEPRECATED: The --evaluator flag has been deprecated. " +
 				"Docs: https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md")
 		}
+
+		syncProviders, err := runtime.SyncProvidersFromURIs(viper.GetStringSlice(uriFlagName))
+		if err != nil {
+			log.Fatal(err)
+		}
+		syncProviders2 := []sync.ProviderConfig{}
+		if cfgFile == "" && viper.GetString(syncProvidersFlagName) != "" {
+			syncProviders2, err = runtime.SyncProviderArgPass(viper.GetString(syncProvidersFlagName))
+			if err != nil {
+				log.Fatal(err)
+			}
+		} else {
+			err = viper.UnmarshalKey(syncProvidersFlagName, &syncProviders2)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		syncProviders = append(syncProviders, syncProviders2...)
+
 		// Build Runtime -----------------------------------------------------------
 		rt, err := runtime.FromConfig(logger, runtime.Config{
 			CORS:              viper.GetStringSlice(corsFlagName),
 			MetricsPort:       viper.GetInt32(metricsPortFlagName),
-			ProviderArgs:      viper.GetStringMapString(providerArgsFlagName),
 			ServiceCertPath:   viper.GetString(serverCertPathFlagName),
 			ServiceKeyPath:    viper.GetString(serverKeyPathFlagName),
 			ServicePort:       viper.GetInt32(portFlagName),
 			ServiceSocketPath: viper.GetString(socketPathFlagName),
-			SyncBearerToken:   viper.GetString(bearerTokenFlagName),
-			SyncURI:           viper.GetStringSlice(uriFlagName),
+			SyncProviders:     syncProviders,
 		})
 		if err != nil {
 			rtLogger.Fatal(err.Error())

docs/configuration/configuration.md

@@ -10,7 +10,7 @@ Environment variable keys are uppercased, prefixed with `FLAGD_` and all `-` are
 
 Config file expects the keys to have the exact naming as the flags.
 
-### URI patterns
+### <a name="uri-patterns"></a> URI patterns
 
 Any URI passed to flagd via the `--uri` flag must follow one of the 4 following patterns to ensure that it is passed to the correct implementation: 
 
@@ -22,7 +22,6 @@ Any URI passed to flagd via the `--uri` flag must follow one of the 4 following
 | Grpc       | `grpc://flag-source-url`           | `grpc://my-flags-server`        |
 
 
-
 ### Customising sync providers
 
 Custom sync providers can be used to provide flag evaluation logic.
@@ -35,4 +34,29 @@ To use an existing FeatureFlagConfiguration custom resource, start flagD with th
 
 ```shell
 flagd start --uri core.openfeature.dev/default/my_example
+```
+
+### Sync Provider Configuration
+
+While a URI may be passed to flagd via the `--uri` flag, some implementations may require further configurations. In these cases the `--sync-providers` flag should be used.
+The flag takes a string argument, which should be a JSON representation of an array of `ProviderConfig` objects. Alternatively, these configurations should be passed to
+flagd via config file, specified using the `--config` flag.
+
+| Field       | Type  | 
+|------------|------------------------------------|
+| uri | required `string` |  |
+| provider   | required `string` (`file`, `kubernetes`, `http` or `grpc`) |
+| bearerToken     | optional `string`        |
+
+The `uri` field values do not need to follow the [URI patterns](#uri-patterns), the provider type is instead derived from the provider field.
+
+Example start command using a filepath sync provider and the equivalent config file definition:
+```sh
+./flagd start --sync-providers=\[{\"uri\":\"config/samples/example_flags.json\"\,\"provider\":\"file\"}\]
+```
+
+```yaml
+sync-providers:
+- uri: config/samples/example_flags.json
+  provider: file
 ```

docs/configuration/flagd_start.md

@@ -21,6 +21,7 @@ flagd start [flags]
   -d, --socket-path string                  Flagd socket path. With grpc the service will become available on this address. With http(s) the grpc-gateway proxy will use this address internally.
   -y, --sync-provider string                DEPRECATED: Set a sync provider e.g. filepath or remote
   -a, --sync-provider-args stringToString   Sync provider arguments as key values separated by = (default [])
+  -s, --sync-providers string               JSON representation of an array of ProviderConfig objects. This object contains 2 required fields, uri (string) and provider (string). Documentation for this object can be found here: 
   -f, --uri .yaml/.yml/.json                Set a sync provider uri to read data from, this can be a filepath,url (http and grpc) or FeatureFlagConfiguration. Using multiple providers is supported however if flag keys are duplicated across multiple sources it may lead to unexpected behavior. Please note that if you are using filepath, flagd only supports files with .yaml/.yml/.json extension.
 ```
 

pkg/runtime/from_config.go

@@ -1,6 +1,8 @@
 package runtime
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -19,6 +21,13 @@ import (
 	"go.uber.org/zap"
 )
 
+const (
+	syncProviderFile       = "file"
+	syncProviderGrpc       = "grpc"
+	syncProviderKubernetes = "kubernetes"
+	syncProviderHTTP       = "http"
+)
+
 var (
 	regCrd  *regexp.Regexp
 	regURL  *regexp.Regexp
@@ -64,43 +73,43 @@ func (r *Runtime) setService(logger *logger.Logger) {
 
 func (r *Runtime) setSyncImplFromConfig(logger *logger.Logger) error {
 	rtLogger := logger.WithFields(zap.String("component", "runtime"))
-	r.SyncImpl = make([]sync.ISync, 0, len(r.config.SyncURI))
-	for _, uri := range r.config.SyncURI {
-		switch uriB := []byte(uri); {
-		case regFile.Match(uriB):
+	r.SyncImpl = make([]sync.ISync, 0, len(r.config.SyncProviders))
+	for _, syncProvider := range r.config.SyncProviders {
+		switch syncProvider.Provider {
+		case syncProviderFile:
 			r.SyncImpl = append(
 				r.SyncImpl,
-				r.newFile(uri, logger),
+				r.newFile(syncProvider, logger),
 			)
-			rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", uri))
-		case regCrd.Match(uriB):
+			rtLogger.Debug(fmt.Sprintf("using filepath sync-provider for: %q", syncProvider.URI))
+		case syncProviderKubernetes:
 			r.SyncImpl = append(
 				r.SyncImpl,
-				r.newK8s(uri, logger),
+				r.newK8s(syncProvider, logger),
 			)
-			rtLogger.Debug(fmt.Sprintf("using kubernetes sync-provider for: %s", uri))
-		case regURL.Match(uriB):
+			rtLogger.Debug(fmt.Sprintf("using kubernetes sync-provider for: %s", syncProvider.URI))
+		case syncProviderHTTP:
 			r.SyncImpl = append(
 				r.SyncImpl,
-				r.newHTTP(uri, logger),
+				r.newHTTP(syncProvider, logger),
 			)
-			rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %q", uri))
-		case regGRPC.Match(uriB):
+			rtLogger.Debug(fmt.Sprintf("using remote sync-provider for: %q", syncProvider.URI))
+		case syncProviderGrpc:
 			r.SyncImpl = append(
 				r.SyncImpl,
-				r.newGRPC(uri, logger),
+				r.newGRPC(syncProvider, logger),
 			)
 		default:
 			return fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', 'http(s)://', 'grpc://',"+
-				" or 'core.openfeature.dev'", uri)
+				" or 'core.openfeature.dev'", syncProvider.URI)
 		}
 	}
 	return nil
 }
 
-func (r *Runtime) newGRPC(uri string, logger *logger.Logger) *grpc.Sync {
+func (r *Runtime) newGRPC(config sync.ProviderConfig, logger *logger.Logger) *grpc.Sync {
 	return &grpc.Sync{
-		Target: grpc.URLToGRPCTarget(uri),
+		Target: grpc.URLToGRPCTarget(config.URI),
 		Logger: logger.WithFields(
 			zap.String("component", "sync"),
 			zap.String("sync", "grpc"),
@@ -109,41 +118,88 @@ func (r *Runtime) newGRPC(uri string, logger *logger.Logger) *grpc.Sync {
 	}
 }
 
-func (r *Runtime) newHTTP(uri string, logger *logger.Logger) *httpSync.Sync {
+func (r *Runtime) newHTTP(config sync.ProviderConfig, logger *logger.Logger) *httpSync.Sync {
 	return &httpSync.Sync{
-		URI:         uri,
-		BearerToken: r.config.SyncBearerToken,
+		URI: config.URI,
 		Client: &http.Client{
 			Timeout: time.Second * 10,
 		},
 		Logger: logger.WithFields(
 			zap.String("component", "sync"),
 			zap.String("sync", "remote"),
 		),
-		ProviderArgs: r.config.ProviderArgs,
-		Cron:         cron.New(),
+		Config: config,
+		Cron:   cron.New(),
 	}
 }
 
-func (r *Runtime) newK8s(uri string, logger *logger.Logger) *kubernetes.Sync {
+func (r *Runtime) newK8s(config sync.ProviderConfig, logger *logger.Logger) *kubernetes.Sync {
 	return &kubernetes.Sync{
 		Logger: logger.WithFields(
 			zap.String("component", "sync"),
 			zap.String("sync", "kubernetes"),
 		),
-		URI:          regCrd.ReplaceAllString(uri, ""),
-		ProviderArgs: r.config.ProviderArgs,
+		URI:    config.URI,
+		Config: config,
 	}
 }
 
-func (r *Runtime) newFile(uri string, logger *logger.Logger) *file.Sync {
+func (r *Runtime) newFile(config sync.ProviderConfig, logger *logger.Logger) *file.Sync {
 	return &file.Sync{
-		URI: regFile.ReplaceAllString(uri, ""),
+		URI: config.URI,
 		Logger: logger.WithFields(
 			zap.String("component", "sync"),
 			zap.String("sync", "filepath"),
 		),
-		ProviderArgs: r.config.ProviderArgs,
-		Mux:          &msync.RWMutex{},
+		Config: config,
+		Mux:    &msync.RWMutex{},
+	}
+}
+
+func SyncProviderArgPass(syncProviders string) ([]sync.ProviderConfig, error) {
+	syncProvidersParsed := []sync.ProviderConfig{}
+	if err := json.Unmarshal([]byte(syncProviders), &syncProvidersParsed); err != nil {
+		return syncProvidersParsed, fmt.Errorf("unable to parse sync providers: %w", err)
+	}
+	for _, sp := range syncProvidersParsed {
+		if sp.URI == "" {
+			return syncProvidersParsed, errors.New("sync provider argument parse: uri is a required field")
+		}
+		if sp.Provider == "" {
+			return syncProvidersParsed, errors.New("sync provider argument parse: provider is a required field")
+		}
+	}
+	return syncProvidersParsed, nil
+}
+
+func SyncProvidersFromURIs(uris []string) ([]sync.ProviderConfig, error) {
+	syncProvidersParsed := []sync.ProviderConfig{}
+	for _, uri := range uris {
+		switch uriB := []byte(uri); {
+		case regFile.Match(uriB):
+			syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{
+				URI:      regFile.ReplaceAllString(uri, ""),
+				Provider: syncProviderFile,
+			})
+		case regCrd.Match(uriB):
+			syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{
+				URI:      regCrd.ReplaceAllString(uri, ""),
+				Provider: syncProviderKubernetes,
+			})
+		case regURL.Match(uriB):
+			syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{
+				URI:      uri,
+				Provider: syncProviderHTTP,
+			})
+		case regGRPC.Match(uriB):
+			syncProvidersParsed = append(syncProvidersParsed, sync.ProviderConfig{
+				URI:      uri,
+				Provider: syncProviderGrpc,
+			})
+		default:
+			return syncProvidersParsed, fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', "+
+				"'http(s)://', 'grpc://', or 'core.openfeature.dev'", uri)
+		}
 	}
+	return syncProvidersParsed, nil
 }

pkg/runtime/runtime.go

@@ -33,12 +33,8 @@ type Config struct {
 	ServiceCertPath   string
 	ServiceKeyPath    string
 
-	ProviderArgs    sync.ProviderArgs
-	SyncURI         []string
-	RemoteSyncType  string
-	SyncBearerToken string
-
-	CORS []string
+	SyncProviders []sync.ProviderConfig
+	CORS          []string
 }
 
 func (r *Runtime) Start() error {