diff --git a/dnscollector.go b/dnscollector.go index f2a6a628..371fbd70 100644 --- a/dnscollector.go +++ b/dnscollector.go @@ -9,7 +9,6 @@ import ( _ "net/http/pprof" - "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/pkglinker" "github.com/dmachard/go-dnscollector/pkgutils" @@ -95,9 +94,8 @@ func main() { // create logger logger := logger.New(true) - // get DNSMessage flat model - dmRef := dnsutils.GetReferenceDNSMessage() - config, err := pkgutils.LoadConfig(configPath, dmRef) + // load config + config, err := pkgconfig.LoadConfig(configPath) if err != nil { fmt.Printf("config error: %v\n", err) os.Exit(1) @@ -143,7 +141,7 @@ func main() { logger.Info("main - SIGHUP received") // read config - err := pkgutils.ReloadConfig(configPath, config, dmRef) + err := pkgconfig.ReloadConfig(configPath, config) if err != nil { panic(fmt.Sprintf("main - reload config error: %v", err)) } diff --git a/pkgconfig/collectors.go b/pkgconfig/collectors.go index 18b1a100..0da6c18a 100644 --- a/pkgconfig/collectors.go +++ b/pkgconfig/collectors.go @@ -1,6 +1,8 @@ package pkgconfig -import "reflect" +import ( + "reflect" +) type ConfigCollectors struct { DNSMessage struct { @@ -155,23 +157,6 @@ func (c *ConfigCollectors) SetDefault() { c.Tzsp.ChannelBufferSize = 65535 } -func (c *ConfigCollectors) GetTags() (ret []string) { - cl := reflect.TypeOf(*c) - - for i := 0; i < cl.NumField(); i++ { - field := cl.Field(i) - tag := field.Tag.Get("yaml") - ret = append(ret, tag) - } - return ret -} - -func (c *ConfigCollectors) IsValid(name string) bool { - tags := c.GetTags() - for i := range tags { - if name == tags[i] { - return true - } - } - return false +func (c *ConfigCollectors) IsValid(userCfg map[string]interface{}) error { + return CheckConfigWithTags(reflect.ValueOf(*c), userCfg) } diff --git a/pkgconfig/config.go b/pkgconfig/config.go index 0e05c9a6..22bb273f 100644 --- a/pkgconfig/config.go +++ b/pkgconfig/config.go @@ -2,6 +2,9 @@ package pkgconfig import ( "os" + "reflect" + + "github.com/pkg/errors" ) func IsValidMode(mode string) bool { @@ -45,6 +48,42 @@ func (c *Config) SetDefault() { c.OutgoingTransformers.SetDefault() } +func (c *Config) IsValid(userCfg map[string]interface{}) error { + for userKey, userValue := range userCfg { + switch userKey { + case "global": + if kvMap, ok := userValue.(map[string]interface{}); ok { + if err := c.Global.Check(kvMap); err != nil { + return errors.Errorf("global section - %s", err) + } + } else { + return errors.Errorf("unexpected type for global value, got %T", kvMap) + } + + case "multiplexer": + if kvMap, ok := userValue.(map[string]interface{}); ok { + if err := c.Multiplexer.IsValid(kvMap); err != nil { + return errors.Errorf("mutiplexer section - %s", err) + } + } else { + return errors.Errorf("unexpected type for multiplexer value, got %T", kvMap) + } + + case "pipelines": + for i, cv := range userValue.([]interface{}) { + cfg := ConfigPipelines{} + if err := cfg.IsValid(cv.(map[string]interface{})); err != nil { + return errors.Errorf("stanza(index=%d) - %s", i, err) + } + } + + default: + return errors.Errorf("unknown key=%s\n", userKey) + } + } + return nil +} + func (c *Config) GetServerIdentity() string { if len(c.Global.ServerIdentity) > 0 { return c.Global.ServerIdentity @@ -63,3 +102,34 @@ func GetFakeConfig() *Config { config.SetDefault() return config } + +func CheckConfigWithTags(v reflect.Value, userCfg map[string]interface{}) error { + t := v.Type() + for k, kv := range userCfg { + keyExist := false + for i := 0; i < v.NumField(); i++ { + fieldValue := v.Field(i) + fieldType := t.Field(i) + fieldTag := fieldType.Tag.Get("yaml") + + if fieldTag == k { + keyExist = true + } + if fieldValue.Kind() == reflect.Struct && fieldTag == k { + if kvMap, ok := kv.(map[string]interface{}); ok { + err := CheckConfigWithTags(fieldValue, kvMap) + if err != nil { + return errors.Errorf("%s in subkey=`%s`", err, k) + } + } else { + return errors.Errorf("unexpected type for key `%s`, got %T", k, kv) + } + } + } + + if !keyExist { + return errors.Errorf("unknown key=`%s`", k) + } + } + return nil +} diff --git a/pkgconfig/configchecker.go b/pkgconfig/configchecker.go new file mode 100644 index 00000000..a24d5e66 --- /dev/null +++ b/pkgconfig/configchecker.go @@ -0,0 +1,83 @@ +package pkgconfig + +import ( + "io" + "os" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +func ReloadConfig(configPath string, config *Config) error { + // Open config file + configFile, err := os.Open(configPath) + if err != nil { + return nil + } + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configFile); err != nil { + return err + } + + // Init new YAML decode + configFile.Seek(0, 0) + d := yaml.NewDecoder(configFile) + + // Start YAML decoding from file + if err := d.Decode(&config); err != nil { + return err + } + return nil +} + +func LoadConfig(configPath string) (*Config, error) { + // Open config file + configFile, err := os.Open(configPath) + if err != nil { + return nil, err + } + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configFile); err != nil { + return nil, err + } + + // Init new YAML decode + configFile.Seek(0, 0) + d := yaml.NewDecoder(configFile) + + // Start YAML decoding to go + config := &Config{} + config.SetDefault() + + if err := d.Decode(&config); err != nil { + return nil, err + } + + return config, nil +} + +func CheckConfig(configFile *os.File) error { + // Read config file bytes + configBytes, err := io.ReadAll(configFile) + if err != nil { + return errors.Wrap(err, "Error reading configuration file") + } + + // Unmarshal YAML to map + userCfg := make(map[string]interface{}) + err = yaml.Unmarshal(configBytes, &userCfg) + if err != nil { + return errors.Wrap(err, "error parsing YAML file") + } + + // check the user config with the default one + config := &Config{} + config.SetDefault() + + // check if the provided config is valid + return config.IsValid(userCfg) +} diff --git a/pkgconfig/configchecker_test.go b/pkgconfig/configchecker_test.go new file mode 100644 index 00000000..e5bf683f --- /dev/null +++ b/pkgconfig/configchecker_test.go @@ -0,0 +1,232 @@ +package pkgconfig + +import ( + "os" + "testing" +) + +func createTempConfigFile(content string) (string, error) { + tempFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + return "", err + } + defer tempFile.Close() + + if _, err := tempFile.WriteString(content); err != nil { + return "", err + } + + return tempFile.Name(), nil +} + +func TestConfig_CheckConfig(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + }{ + { + name: "Valid multiplexer configuration", + content: ` +global: + trace: + verbose: true + server-identity: "dns-collector" +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + transforms: + normalize: + qname-lowercase: false + loggers: + - name: console + stdout: + mode: text + routes: + - from: [ tap ] + to: [ console ] +`, + wantErr: false, + }, + { + name: "Valid pipeline configuration", + content: ` +global: + trace: + verbose: true + server-identity: "dns-collector" +pipelines: + - name: dnsdist-main + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + routing-policy: + default: [ console ] + + - name: console + stdout: + mode: text +`, + wantErr: false, + }, + { + name: "Invalid key", + content: ` +global: + logger: bad-position +`, + wantErr: true, + }, + { + name: "Invalid multiplexer config format", + content: ` +multiplexer: + - name: block + dnstap: + listen-ip: 0.0.0.0 + transforms: + normalize: + qname-lowercase: true +`, + wantErr: true, + }, + { + name: "Invalid multiplexer logger", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + loggers: + - name: tapOut + dnstap: + listen-ip: 0.0.0.0 + routes: + - from: [ tapIn ] + to: [ tapOut ] +`, + wantErr: true, + }, + { + name: "Invalid pipeline transform", + content: ` +pipelines: + - name: dnsdist-main + dnstap: + listen-ip: 0.0.0.0 + transforms: + normalize: + qname-lowercase: true + routing-policy: + default: [ console ] +`, + wantErr: true, + }, + { + name: "Invalid multiplexer route", + content: ` +multiplexer: + routes: + - from: [test-route] + unknown-key: invalid +`, + wantErr: true, + }, + { + name: "pipeline dynamic keys", + content: ` +pipelines: + - name: match + dnsmessage: + matching: + include: + atags.tags.*: test + atags.tags.2: test + dns.resources-records.*: test +`, + wantErr: false, + }, + { + name: "freeform loki #643", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + loggers: + - name: loki + lokiclient: + server-url: "https://grafana-loki.example.com/loki/api/v1/push" + job-name: "dnscollector" + mode: "flat-json" + tls-insecure: true + tenant-id: fake + relabel-configs: + - source_labels: ["__dns_qtype"] + target_label: "qtype" + replacement: "test" + action: "update" + separator: "," + regex: "test" + routes: + - from: [ tap ] + to: [ loki ] +`, + wantErr: false, + }, + { + name: "freeform scalyr #676", + content: ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + loggers: + - name: scalyr + scalyrclient: + apikey: XXXXX + attrs: + service: dnstap + type: queries + flush-interval: 10 + mode: flat-json + sessioninfo: + cloud_provider: Azure + cloud_region: westeurope + routes: + - from: [ tap ] + to: [ scalyr ] +`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tempFile, err := createTempConfigFile(tt.content) + if err != nil { + t.Fatalf("Error creating temporary file: %v", err) + } + defer os.Remove(tempFile) + configFile, err := os.Open(tempFile) + if err != nil { + t.Fatalf("Read temporary file: %v", err) + } + defer configFile.Close() + + err = CheckConfig(configFile) + if (err != nil) != tt.wantErr { + t.Errorf("CheckConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/pkgconfig/global.go b/pkgconfig/global.go index 28f6dc45..1ea6650c 100644 --- a/pkgconfig/global.go +++ b/pkgconfig/global.go @@ -1,5 +1,9 @@ package pkgconfig +import ( + "reflect" +) + type ConfigGlobal struct { TextFormat string `yaml:"text-format"` TextFormatDelimiter string `yaml:"text-format-delimiter"` @@ -27,3 +31,7 @@ func (c *ConfigGlobal) SetDefault() { c.Trace.MaxBackups = 10 c.ServerIdentity = "" } + +func (c *ConfigGlobal) Check(userCfg map[string]interface{}) error { + return CheckConfigWithTags(reflect.ValueOf(*c), userCfg) +} diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go index 4e4e783c..3da70959 100644 --- a/pkgconfig/loggers.go +++ b/pkgconfig/loggers.go @@ -590,23 +590,6 @@ func (c *ConfigLoggers) SetDefault() { c.ClickhouseClient.Table = "records" } -func (c *ConfigLoggers) GetTags() (ret []string) { - cl := reflect.TypeOf(*c) - - for i := 0; i < cl.NumField(); i++ { - field := cl.Field(i) - tag := field.Tag.Get("yaml") - ret = append(ret, tag) - } - return ret -} - -func (c *ConfigLoggers) IsValid(name string) bool { - tags := c.GetTags() - for i := range tags { - if name == tags[i] { - return true - } - } - return false +func (c *ConfigLoggers) IsValid(userCfg map[string]interface{}) error { + return CheckConfigWithTags(reflect.ValueOf(*c), userCfg) } diff --git a/pkgconfig/multiplexer.go b/pkgconfig/multiplexer.go index f48cc302..8a83b606 100644 --- a/pkgconfig/multiplexer.go +++ b/pkgconfig/multiplexer.go @@ -1,5 +1,9 @@ package pkgconfig +import ( + "github.com/pkg/errors" +) + type ConfigMultiplexer struct { Collectors []MultiplexInOut `yaml:"collectors"` Loggers []MultiplexInOut `yaml:"loggers"` @@ -12,13 +16,84 @@ func (c *ConfigMultiplexer) SetDefault() { c.Routes = []MultiplexRoutes{} } +func (c *ConfigMultiplexer) IsValid(userCfg map[string]interface{}) error { + for k, v := range userCfg { + switch k { + case "collectors": + for i, cv := range v.([]interface{}) { + cfg := MultiplexInOut{IsCollector: true} + if err := cfg.IsValid(cv.(map[string]interface{})); err != nil { + return errors.Errorf("collector(index=%d) - %s", i, err) + } + } + + case "loggers": + for i, cv := range v.([]interface{}) { + cfg := MultiplexInOut{IsCollector: false} + if err := cfg.IsValid(cv.(map[string]interface{})); err != nil { + return errors.Errorf("logger(index=%d) - %s", i, err) + } + } + + case "routes": + for i, cv := range v.([]interface{}) { + cfg := MultiplexRoutes{} + if err := cfg.IsValid(cv.(map[string]interface{})); err != nil { + return errors.Errorf("route(index=%d) - %s", i, err) + } + } + + default: + return errors.Errorf("unknown multiplexer key=%s\n", k) + } + } + return nil +} + type MultiplexInOut struct { - Name string `yaml:"name"` - Transforms map[string]interface{} `yaml:"transforms"` - Params map[string]interface{} `yaml:",inline"` + Name string `yaml:"name"` + Transforms map[string]interface{} `yaml:"transforms"` + Params map[string]interface{} `yaml:",inline"` + IsCollector bool +} + +func (c *MultiplexInOut) IsValid(userCfg map[string]interface{}) error { + if _, ok := userCfg["name"]; !ok { + return errors.Errorf("name key is required") + } + delete(userCfg, "name") + + if _, ok := userCfg["transforms"]; ok { + cfg := ConfigTransformers{} + if err := cfg.IsValid(userCfg["transforms"].(map[string]interface{})); err != nil { + return errors.Errorf("transform - %s", err) + } + delete(userCfg, "transforms") + } + + var err error + if c.IsCollector { + cfg := ConfigCollectors{} + err = cfg.IsValid(userCfg) + } else { + cfg := ConfigLoggers{} + err = cfg.IsValid(userCfg) + } + + return err } type MultiplexRoutes struct { Src []string `yaml:"from,flow"` Dst []string `yaml:"to,flow"` } + +func (c *MultiplexRoutes) IsValid(userCfg map[string]interface{}) error { + if _, ok := userCfg["from"]; !ok { + return errors.Errorf("the key 'from' is required") + } + if _, ok := userCfg["to"]; !ok { + return errors.Errorf("the key 'to' is required") + } + return nil +} diff --git a/pkgconfig/pipelines.go b/pkgconfig/pipelines.go index 126f743a..6219b871 100644 --- a/pkgconfig/pipelines.go +++ b/pkgconfig/pipelines.go @@ -1,5 +1,11 @@ package pkgconfig +import ( + "fmt" + + "github.com/pkg/errors" +) + type ConfigPipelines struct { Name string `yaml:"name"` Transforms map[string]interface{} `yaml:"transforms"` @@ -7,7 +13,50 @@ type ConfigPipelines struct { RoutingPolicy PipelinesRouting `yaml:"routing-policy"` } +func (c *ConfigPipelines) IsValid(userCfg map[string]interface{}) error { + if _, ok := userCfg["name"]; !ok { + return errors.Errorf("name key is required") + } + delete(userCfg, "name") + + if _, ok := userCfg["transforms"]; ok { + cfg := ConfigTransformers{} + if err := cfg.IsValid(userCfg["transforms"].(map[string]interface{})); err != nil { + return errors.Errorf("transform - %s", err) + } + delete(userCfg, "transforms") + } + + if _, ok := userCfg["routing-policy"]; ok { + cfg := PipelinesRouting{} + if err := cfg.IsValid(userCfg["routing-policy"].(map[string]interface{})); err != nil { + return errors.Errorf("routing-policy - %s", err) + } + delete(userCfg, "routing-policy") + } + + a := ConfigCollectors{} + errA := a.IsValid(userCfg) + b := ConfigLoggers{} + errB := b.IsValid(userCfg) + + if errA != nil && errB != nil { + return errors.Errorf("invalid stranza - %s", errA) + } + + return nil +} + type PipelinesRouting struct { Default []string `yaml:"default,flow"` Dropped []string `yaml:"dropped,flow"` } + +func (c *PipelinesRouting) IsValid(userCfg map[string]interface{}) error { + for k := range userCfg { + if k != "default" && k != "dropped" { + return fmt.Errorf("invalid key '%s'", k) + } + } + return nil +} diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index 201d7c19..e3c8c699 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -1,5 +1,7 @@ package pkgconfig +import "reflect" + type RelabelingConfig struct { Regex string `yaml:"regex"` Replacement string `yaml:"replacement"` @@ -150,6 +152,10 @@ func (c *ConfigTransformers) SetDefault() { c.Relabeling.Rename = []RelabelingConfig{} } +func (c *ConfigTransformers) IsValid(userCfg map[string]interface{}) error { + return CheckConfigWithTags(reflect.ValueOf(*c), userCfg) +} + func GetFakeConfigTransformers() *ConfigTransformers { config := &ConfigTransformers{} config.SetDefault() diff --git a/pkglinker/pipelines.go b/pkglinker/pipelines.go index cfb39149..c8c5ffc9 100644 --- a/pkglinker/pipelines.go +++ b/pkglinker/pipelines.go @@ -19,13 +19,6 @@ func GetStanzaConfig(config *pkgconfig.Config, item pkgconfig.ConfigPipelines) * // Enable the provided collector or loggers for k, p := range item.Params { - // is a logger or collector ? - if !config.Loggers.IsValid(k) && !config.Collectors.IsValid(k) { - panic(fmt.Sprintln("main - get stanza config error")) - } - if config.Loggers.IsValid(k) { - section = "loggers" - } if p == nil { item.Params[k] = make(map[string]interface{}) } diff --git a/pkgutils/configchecker.go b/pkgutils/configchecker.go deleted file mode 100644 index 23863063..00000000 --- a/pkgutils/configchecker.go +++ /dev/null @@ -1,443 +0,0 @@ -package pkgutils - -import ( - "fmt" - "io" - "os" - "reflect" - "regexp" - "strings" - - "github.com/dmachard/go-dnscollector/dnsutils" - "github.com/dmachard/go-dnscollector/pkgconfig" - "github.com/pkg/errors" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/relabel" - "gopkg.in/yaml.v3" -) - -func ReloadConfig(configPath string, config *pkgconfig.Config, dmRef dnsutils.DNSMessage) error { - // Open config file - configFile, err := os.Open(configPath) - if err != nil { - return nil - } - defer configFile.Close() - - // Check config to detect unknown keywords - if err := CheckConfig(configPath, dmRef); err != nil { - return err - } - - // Init new YAML decode - d := yaml.NewDecoder(configFile) - - // Start YAML decoding from file - if err := d.Decode(&config); err != nil { - return err - } - return nil -} - -func LoadConfig(configPath string, dmRef dnsutils.DNSMessage) (*pkgconfig.Config, error) { - // Open config file - configFile, err := os.Open(configPath) - if err != nil { - return nil, err - } - defer configFile.Close() - - // Check config to detect unknown keywords - if err := CheckConfig(configPath, dmRef); err != nil { - return nil, err - } - - // Init new YAML decode - d := yaml.NewDecoder(configFile) - - // Start YAML decoding to go - config := &pkgconfig.Config{} - config.SetDefault() - - if err := d.Decode(&config); err != nil { - return nil, err - } - - return config, nil -} - -func CheckConfig(userConfigPath string, dmRef dnsutils.DNSMessage) error { - - flatDmRef, err := dmRef.Flatten() - if err != nil { - return err - } - - // create default config - defaultConfig := &pkgconfig.Config{} - defaultConfig.SetDefault() - - // and simulate items in multiplexer and pipelines mode - defaultConfig.Multiplexer.Routes = append(defaultConfig.Multiplexer.Routes, pkgconfig.MultiplexRoutes{}) - defaultConfig.Multiplexer.Loggers = append(defaultConfig.Multiplexer.Loggers, pkgconfig.MultiplexInOut{}) - defaultConfig.Multiplexer.Collectors = append(defaultConfig.Multiplexer.Collectors, pkgconfig.MultiplexInOut{}) - defaultConfig.Pipelines = append(defaultConfig.Pipelines, pkgconfig.ConfigPipelines{}) - - // add default relabeling examples - defaultConfig.IngoingTransformers.Relabeling.Remove = append(defaultConfig.IngoingTransformers.Relabeling.Remove, pkgconfig.RelabelingConfig{}) - defaultConfig.IngoingTransformers.Relabeling.Rename = append(defaultConfig.IngoingTransformers.Relabeling.Rename, pkgconfig.RelabelingConfig{}) - - // add fake value in relabel.Config because yaml tag is set to omitempty - // avoid this type of exception in future .... - defaultConfig.Loggers.LokiClient.RelabelConfigs = append(defaultConfig.Loggers.LokiClient.RelabelConfigs, - &relabel.Config{ - TargetLabel: "target", - SourceLabels: []model.LabelName{"source"}, - Separator: "separator", - Replacement: "replacement", - Action: relabel.Action("action"), - Regex: relabel.Regexp{}, - }, - ) - - // Convert default config to map - // And get unique YAML keys - defaultConfigMap, err := convertConfigToMap(defaultConfig) - if err != nil { - return errors.Wrap(err, "error converting default config to map") - } - - defaultKeywords := getUniqueKeywords(defaultConfigMap) - - // add DNSMessage default keys - for k := range flatDmRef { - defaultKeywords[k] = true - } - - // Read user configuration file - // And get unique YAML keys from user config - userConfigMap, err := loadUserConfigToMap(userConfigPath) - if err != nil { - return err - } - userKeywords := getUniqueKeywords(userConfigMap) - - // Check for unknown keys in user config - // ignore dynamic keys as atags.tags.*: google - - // Define regular expressions to match dynamic keys - regexPatterns := []string{`\.\*(\.)?`, `\.(\d+)(\.)?`} - - for key := range userKeywords { - // Ignore dynamic keys that contain ".*" or .[digits]. - matched := false - for _, pattern := range regexPatterns { - match, _ := regexp.MatchString(pattern, key) - if match { - matched = true - break - } - } - if matched { - continue - } - - // search in default keywords - if _, ok := defaultKeywords[key]; !ok { - return errors.Errorf("unknown YAML key `%s` in configuration", key) - } - } - - // detect bad keyword position - err = checkKeywordsPosition(userConfigMap, defaultConfigMap, defaultConfigMap, "") - if err != nil { - return err - } - - // check for invalid text directives - // the text-format key is reserved - // search this key on all keys - err = checkTextFormatKey(userConfigMap, dmRef) - if err != nil { - return err - } - - return nil -} - -func checkTextFormatKey(data map[string]interface{}, dmRef dnsutils.DNSMessage) error { - key := "text-format" - for k, v := range data { - if k == key { - if str, ok := v.(string); ok { - _, err := dmRef.ToTextLine(strings.Fields(str), "", "") - if err != nil { - return err - } - } - } - if nestedMap, ok := v.(map[string]interface{}); ok { - err := checkTextFormatKey(nestedMap, dmRef) - if err != nil { - return err - } - } - if nestedSlice, ok := v.([]interface{}); ok { - for _, item := range nestedSlice { - if nestedMap, ok := item.(map[string]interface{}); ok { - err := checkTextFormatKey(nestedMap, dmRef) - if err != nil { - return err - } - } - } - } - } - return nil -} - -func checkKeywordsPosition(nextUserCfg, nextDefCfg map[string]interface{}, defaultConf map[string]interface{}, sectionName string) error { - for k, v := range nextUserCfg { - // Check if the key is present in the default config - if len(nextDefCfg) > 0 { - if _, ok := nextDefCfg[k]; !ok { - if sectionName == "" { - return errors.Errorf("invalid key `%s` at root", k) - } - return errors.Errorf("invalid key `%s` in section `%s`", k, sectionName) - } - } - - // If the value is a map, recursively check for invalid keywords - // Recursive call ? - val := reflect.ValueOf(v) - if val.Kind() == reflect.Map { - nextSectionName := fmt.Sprintf("%s.%s", sectionName, k) - if err := checkKeywordsPosition(v.(map[string]interface{}), nextDefCfg[k].(map[string]interface{}), defaultConf, nextSectionName); err != nil { - return err - } - } - - // If the value is a slice and we are in the multiplexer part - // Multiplixer part is dynamic, we need specific function to check it - if val.Kind() == reflect.Slice && sectionName == ".multiplexer" { - if err := checkMultiplexerConfig(val, nextDefCfg[k].([]interface{}), defaultConf, k); err != nil { - return err - } - } - - // If the value is a slice and we are in the pipelines part - if val.Kind() == reflect.Slice && k == "pipelines" { - if err := checkPipelinesConfig(val, nextDefCfg[k].([]interface{}), defaultConf, k); err != nil { - return err - } - } - } - return nil -} - -func checkPipelinesConfig(currentVal reflect.Value, currentRef []interface{}, defaultConf map[string]interface{}, k string) error { - refLoggers := defaultConf[pkgconfig.KeyLoggers].(map[string]interface{}) - refCollectors := defaultConf[pkgconfig.KeyCollectors].(map[string]interface{}) - refTransforms := defaultConf["collectors-transformers"].(map[string]interface{}) - - for pos, item := range currentVal.Interface().([]interface{}) { - valReflect := reflect.ValueOf(item) - refItem := currentRef[0].(map[string]interface{}) - if valReflect.Kind() == reflect.Map { - for _, key := range valReflect.MapKeys() { - strKey := key.Interface().(string) - mapVal := valReflect.MapIndex(key) - - if _, ok := refItem[strKey]; !ok { - // Check if the key exists in neither loggers nor collectors - loggerExists := refLoggers[strKey] != nil - collectorExists := refCollectors[strKey] != nil - if !loggerExists && !collectorExists { - return errors.Errorf("invalid `%s` in `%s` pipelines at position %d", strKey, k, pos) - } - - // check logger or collectors - if loggerExists || collectorExists { - nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey) - refMap := refLoggers - if collectorExists { - refMap = refCollectors - } - // Type assertion to check if the value is a map - if value, ok := mapVal.Interface().(map[string]interface{}); ok { - if err := checkKeywordsPosition(value, refMap[strKey].(map[string]interface{}), defaultConf, nextSectionName); err != nil { - return err - } - } else { - return errors.Errorf("invalid `%s` value in `%s` pipelines at position %d", strKey, k, pos) - } - } - } - - // Check transforms section - // Type assertion to check if the value is a map - if strKey == "transforms" { - nextSectionName := fmt.Sprintf("%s.%s", k, strKey) - if value, ok := mapVal.Interface().(map[string]interface{}); ok { - if err := checkKeywordsPosition(value, refTransforms, defaultConf, nextSectionName); err != nil { - return err - } - } else { - return errors.Errorf("invalid `%s` value in `%s` pipelines at position %d", strKey, k, pos) - } - } - } - } else { - return errors.Errorf("invalid item type in pipelines list: %s", valReflect.Kind()) - } - } - return nil -} - -func checkMultiplexerConfig(currentVal reflect.Value, currentRef []interface{}, defaultConf map[string]interface{}, k string) error { - refLoggers := defaultConf[pkgconfig.KeyLoggers].(map[string]interface{}) - refCollectors := defaultConf[pkgconfig.KeyCollectors].(map[string]interface{}) - refTransforms := defaultConf["collectors-transformers"].(map[string]interface{}) - - // iter over the slice - for pos, item := range currentVal.Interface().([]interface{}) { - valReflect := reflect.ValueOf(item) - refItem := currentRef[0].(map[string]interface{}) - if valReflect.Kind() == reflect.Map { - for _, key := range valReflect.MapKeys() { - strKey := key.Interface().(string) - mapVal := valReflect.MapIndex(key) - - // First, check in the initial configuration reference. - // If not found, then look in the logger and collector references. - if _, ok := refItem[strKey]; !ok { - // we are in routes section ? - if !(k == pkgconfig.KeyCollectors || k == pkgconfig.KeyLoggers) { - return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos) - } - - // Check if the key exists in neither loggers nor collectors - loggerExists := refLoggers[strKey] != nil - collectorExists := refCollectors[strKey] != nil - if !loggerExists && !collectorExists { - return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos) - } - - // check logger or collectors - if k == pkgconfig.KeyLoggers || k == pkgconfig.KeyCollectors { - nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey) - refMap := refLoggers - if k == pkgconfig.KeyCollectors { - refMap = refCollectors - } - - // Type assertion to check if the value is a map - if value, ok := mapVal.Interface().(map[string]interface{}); ok { - if _, ok := refMap[strKey]; !ok { - return errors.Errorf("invalid logger `%s`", strKey) - } - if err := checkKeywordsPosition(value, refMap[strKey].(map[string]interface{}), defaultConf, nextSectionName); err != nil { - return err - } - } else { - return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos) - } - } - } - - // Check transforms section - // Type assertion to check if the value is a map - if strKey == "transforms" { - nextSectionName := fmt.Sprintf("%s.%s", k, strKey) - if value, ok := mapVal.Interface().(map[string]interface{}); ok { - if err := checkKeywordsPosition(value, refTransforms, defaultConf, nextSectionName); err != nil { - return err - } - } else { - return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos) - } - } - } - } else { - return errors.Errorf("invalid item type in multiplexer list: %s", valReflect.Kind()) - } - } - return nil -} - -func convertConfigToMap(config *pkgconfig.Config) (map[string]interface{}, error) { - // Convert config to YAML - yamlData, err := yaml.Marshal(config) - if err != nil { - return nil, err - } - - // Convert YAML to map - configMap := make(map[string]interface{}) - err = yaml.Unmarshal(yamlData, &configMap) - if err != nil { - return nil, err - } - - return configMap, nil -} - -func loadUserConfigToMap(configPath string) (map[string]interface{}, error) { - // Read user configuration file - configFile, err := os.Open(configPath) - if err != nil { - return nil, err - } - defer configFile.Close() - - // Read config file bytes - configBytes, err := io.ReadAll(configFile) - if err != nil { - return nil, errors.Wrap(err, "Error reading configuration file") - } - - // Unmarshal YAML to map - userConfigMap := make(map[string]interface{}) - err = yaml.Unmarshal(configBytes, &userConfigMap) - if err != nil { - return nil, errors.Wrap(err, "error parsing YAML file") - } - - return userConfigMap, nil -} - -func getUniqueKeywords(s map[string]interface{}) map[string]bool { - keys := extractYamlKeys(s) - uniqueKeys := make(map[string]bool) - for _, key := range keys { - if _, ok := uniqueKeys[key]; ok { - continue - } - uniqueKeys[key] = true - } - return uniqueKeys -} - -func extractYamlKeys(s map[string]interface{}) []string { - keys := []string{} - for k, v := range s { - keys = append(keys, k) - val := reflect.ValueOf(v) - if val.Kind() == reflect.Map { - nextKeys := extractYamlKeys(val.Interface().(map[string]interface{})) - keys = append(keys, nextKeys...) - } - if val.Kind() == reflect.Slice { - for _, v2 := range val.Interface().([]interface{}) { - val2 := reflect.ValueOf(v2) - if val2.Kind() == reflect.Map { - nextKeys := extractYamlKeys(val2.Interface().(map[string]interface{})) - keys = append(keys, nextKeys...) - } - } - } - - } - return keys -} diff --git a/pkgutils/configchecker_test.go b/pkgutils/configchecker_test.go deleted file mode 100644 index 8019e834..00000000 --- a/pkgutils/configchecker_test.go +++ /dev/null @@ -1,404 +0,0 @@ -package pkgutils - -import ( - "os" - "testing" - - "github.com/dmachard/go-dnscollector/dnsutils" - "github.com/pkg/errors" -) - -// Valid minimal user configuration -func TestConfig_CheckConfig_Valid(t *testing.T) { - // Create a temporary file for the user configuration - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - validUserConfigContent := ` -global: - trace: false -multiplexer: - routes: - - from: [test-route] - loggers: - - name: test-logger - collectors: - - name: test-collector -` - err = os.WriteFile(userConfigFile.Name(), []byte(validUserConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err != nil { - t.Errorf("failed: Unexpected error: %v", err) - } -} - -// Invalid user configuration with an unknown key -func TestConfig_CheckConfig_UnknownKeywords(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -global: - trace: false -multiplexer: - routes: - - from: [test-route] - unknown-key: invalid -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - expectedError := errors.Errorf("unknown YAML key `unknown-key` in configuration") - if err := CheckConfig(userConfigFile.Name(), dm); err == nil || err.Error() != expectedError.Error() { - t.Errorf("Expected error %v, but got %v", expectedError, err) - } -} - -// Ignore dynamic keys -func TestConfig_CheckConfig_IgnoreDynamicKeys(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -global: - trace: false -pipelines: - - name: match - dnsmessage: - matching: - include: - atags.tags.*: test - atags.tags.2: test - dns.resources-records.*: test - dns.resources-records.10.rdata: test - dns.resources-records.*.ttl: test -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err != nil { - t.Errorf("Expected no error, but got %v", err) - } -} - -// Keywork exist but not at the good position -func TestConfig_CheckConfig_BadKeywordPosition(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -global: - trace: false - logger: bad-position -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got %v", err) - } -} - -// Valid multiplexer configuration -func TestConfig_CheckMultiplexerConfig_Valid(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - transforms: - normalize: - qname-lowercase: false - loggers: - - name: console - stdout: - mode: text - routes: - - from: [ tap ] - to: [ console ] -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err != nil { - t.Errorf("failed: Unexpected error: %v", err) - } -} - -// Invalid multiplexer configuration -func TestConfig_CheckMultiplexerConfig_Invalid(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -global: - trace: false -multiplexer: -- name: block - dnstap: - listen-ip: 0.0.0.0 - transforms: - normalize: - qname-lowercase: true -` - - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got %v", err) - } -} - -// https://github.com/dmachard/go-dnscollector/issues/565 -func TestConfig_CheckMultiplexerConfig_InvalidLogger(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - // all keywords in this config are valid but the logger dnstap is not valid in this context - userConfigContent := ` -global: - trace: false -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - loggers: - - name: tapOut - dnstap: - listen-ip: 0.0.0.0 - routes: - - from: [ tapIn ] - to: [ tapOut ] -` - - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got %v", err) - } -} - -// Valid pipeline configuration -func TestConfig_CheckPipelinesConfig_Valid(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -pipelines: -- name: dnsdist-main - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - routing-policy: - default: [ console ] - -- name: console - stdout: - mode: text -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err != nil { - t.Errorf("failed: Unexpected error: %v", err) - } -} - -// Invalid pipeline configuration -func TestConfig_CheckPipelinesConfig_Invalid(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -pipelines: -- name: dnsdist-main - dnstap: - listen-ip: 0.0.0.0 - transforms: - normalize: - qname-lowercase: true - routing-policy: - default: [ console ] -` - - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got %v", err) - } -} - -// Invalid directives -func TestConfig_CheckMultiplexer_InvalidTextDirective(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -multiplexer: - loggers: - - name: dnsdist-main - stdout: - text-format: "qtype latency reducer-occurences" -` - - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err = CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got nil") - } -} - -func TestConfig_CheckPipelines_InvalidTextDirective(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -pipelines: -- name: dnsdist-main - stdout: - text-format: "qtype latency reducer-occurences" - routing-policy: - default: [ console ] -` - - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err = CheckConfig(userConfigFile.Name(), dm); err == nil { - t.Errorf("Expected error, but got nil") - } -} - -// test for issue https://github.com/dmachard/go-dnscollector/issues/643 -func TestConfig_CheckConfig_SpecificsItems(t *testing.T) { - userConfigFile, err := os.CreateTemp("", "user-config.yaml") - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(userConfigFile.Name()) - defer userConfigFile.Close() - - userConfigContent := ` -multiplexer: - collectors: - - name: tap - dnstap: - listen-ip: 0.0.0.0 - listen-port: 6000 - loggers: - - name: loki - lokiclient: - server-url: "https://grafana-loki.example.com/loki/api/v1/push" - job-name: "dnscollector" - mode: "flat-json" - tls-insecure: true - tenant-id: fake - relabel-configs: - - source_labels: ["__dns_qtype"] - target_label: "qtype" - replacement: "test" - action: "update" - separator: "," - regex: "test" - routes: - - from: [ tap ] - to: [ loki ] -` - err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) - if err != nil { - t.Fatal("Error writing to user configuration file:", err) - } - - dm := dnsutils.GetReferenceDNSMessage() - if err := CheckConfig(userConfigFile.Name(), dm); err != nil { - t.Errorf("failed: Unexpected error: %v", err) - } -}