more checks to detect invalid configurations (#523)

* new check config  to detect unknown keys
* test to check multiplexer
* check config on reload
* Update README.md
* no more panic on config error

.github/workflows/testing-go.yml

@@ -140,4 +140,10 @@ jobs:
       run: |
         data=$(find . -name "*.go" -exec grep -v "^$" {} \; -exec grep -v "//" {} \; | wc -l)
         echo "Count of Lines: $data"
+        echo "data=$data" >> $GITHUB_OUTPUT
+
+    - id: count_lines_tests
+      run: |
+        data=$(find . -name "*_test.go" -exec grep -v "^$" {} \; -exec grep -v "//" {} \; | wc -l)
+        echo "Count of Lines in tests file: $data"
         echo "data=$data" >> $GITHUB_OUTPUT

README.md

@@ -3,8 +3,8 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/dmachard/go-dns-collector)](https://goreportcard.com/report/dmachard/go-dns-collector)
 ![Go version](https://img.shields.io/badge/go%20version-min%201.20-blue)
-![Go tests](https://img.shields.io/badge/go%20tests-359-green)
-![Go lines](https://img.shields.io/badge/go%20lines-49536-red)
+![Go tests](https://img.shields.io/badge/go%20tests-366-green)
+![Go lines](https://img.shields.io/badge/go%20lines-32797-red)
 ![Go Tests](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-go.yml/badge.svg)
 ![Github Actions](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-dnstap.yml/badge.svg)
 ![Github Actions PDNS](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-powerdns.yml/badge.svg)
@@ -83,7 +83,7 @@ Multiplexer
 
 Download the latest [`release`](https://github.com/dmachard/go-dns-collector/releases) binary and start the DNS-collector with the provided configuration file. The default configuration listens on `tcp/6000` for a DNSTap stream and DNS logs are printed on standard output.
 
-```go
+```bash
 ./go-dnscollector -config config.yml
 ```
 
@@ -95,6 +95,13 @@ The configuration of DNS-collector is done through a file named [`config.yml`](c
 
 See the full [configuration guide](./docs/configuration.md) for more details.
 
+Run the DNS-collector in dry mode to verify the configuration.
+
+```bash
+./go-dnscollector -config config.yml -test-config
+INFO: 2023/12/24 14:43:29.043730 main - config OK!
+```
+
 ## Usage examples
 
 The [`_examples`](./docs/_examples) folder from documentation contains a number of [various configurations](./docs/examples.md) to get you started with the DNS-collector in differentes ways.

config.yml

@@ -1,5 +1,3 @@
-
-
 ################################################
 # global configuration
 ################################################

dnscollector.go

@@ -89,7 +89,8 @@ func main() {
 	// load config
 	config, err := pkgconfig.LoadConfig(configPath)
 	if err != nil {
-		panic(fmt.Sprintf("main - config error:  %v", err))
+		fmt.Printf("config error: %v\n", err)
+		os.Exit(1)
 	}
 
 	// init logger

pkgconfig/config.go

@@ -1,8 +1,12 @@
 package pkgconfig
 
 import (
+	"fmt"
+	"io"
 	"os"
+	"reflect"
 
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
 
@@ -61,14 +65,19 @@ func (c *Config) GetServerIdentity() string {
 
 func ReloadConfig(configPath string, config *Config) error {
 	// Open config file
-	file, err := os.Open(configPath)
+	configFile, err := os.Open(configPath)
 	if err != nil {
 		return nil
 	}
-	defer file.Close()
+	defer configFile.Close()
+
+	// Check config to detect unknown keywords
+	if err := CheckConfig(configPath); err != nil {
+		return err
+	}
 
 	// Init new YAML decode
-	d := yaml.NewDecoder(file)
+	d := yaml.NewDecoder(configFile)
 
 	// Start YAML decoding from file
 	if err := d.Decode(&config); err != nil {
@@ -78,27 +87,246 @@ func ReloadConfig(configPath string, config *Config) error {
 }
 
 func LoadConfig(configPath string) (*Config, error) {
-	config := &Config{}
-	config.SetDefault()
-
 	// Open config file
-	file, err := os.Open(configPath)
+	configFile, err := os.Open(configPath)
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
+	defer configFile.Close()
+
+	// Check config to detect unknown keywords
+	if err := CheckConfig(configPath); err != nil {
+		return nil, err
+	}
 
 	// Init new YAML decode
-	d := yaml.NewDecoder(file)
+	d := yaml.NewDecoder(configFile)
+
+	// Start YAML decoding to go
+	config := &Config{}
+	config.SetDefault()
 
-	// Start YAML decoding from file
 	if err := d.Decode(&config); err != nil {
 		return nil, err
 	}
 
 	return config, nil
 }
 
+func CheckConfig(userConfigPath string) error {
+	// create default config
+	// and simulate one route, one collector and one logger
+	defaultConfig := &Config{}
+	defaultConfig.SetDefault()
+	defaultConfig.Multiplexer.Routes = append(defaultConfig.Multiplexer.Routes, MultiplexRoutes{})
+	defaultConfig.Multiplexer.Loggers = append(defaultConfig.Multiplexer.Loggers, MultiplexInOut{})
+	defaultConfig.Multiplexer.Collectors = append(defaultConfig.Multiplexer.Collectors, MultiplexInOut{})
+
+	// Convert default config to map
+	// And get unique YAML keys
+	defaultConfigMap, err := convertConfigToMap(defaultConfig)
+	if err != nil {
+		return errors.Wrap(err, "error converting default config to map")
+	}
+	defaultKeywords := getUniqueKeywords(defaultConfigMap)
+
+	// Read user configuration file
+	// And get unique YAML keys from user config
+	userConfigMap, err := loadUserConfigToMap(userConfigPath)
+	if err != nil {
+		return err
+	}
+	userKeywords := getUniqueKeywords(userConfigMap)
+
+	// Check for unknown keys in user config
+	for key := range userKeywords {
+		if _, ok := defaultKeywords[key]; !ok {
+			return errors.Errorf("unknown YAML key `%s` in configuration", key)
+		}
+	}
+
+	// detect bad keyword position
+	err = checkKeywordsPosition(userConfigMap, defaultConfigMap, defaultConfigMap, "")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func checkKeywordsPosition(nextUserCfg, nextDefCfg map[string]interface{}, defaultConf map[string]interface{}, sectionName string) error {
+	for k, v := range nextUserCfg {
+		// Check if the key is present in the default config
+		if _, ok := nextDefCfg[k]; !ok {
+			if sectionName == "" {
+				return errors.Errorf("invalid key `%s` at root", k)
+			}
+			return errors.Errorf("invalid key `%s` in section `%s`", k, sectionName)
+		}
+
+		// If the value is a map, recursively check for invalid keywords
+		// Recursive call ?
+		val := reflect.ValueOf(v)
+		if val.Kind() == reflect.Map {
+			nextSectionName := fmt.Sprintf("%s.%s", sectionName, k)
+			if err := checkKeywordsPosition(v.(map[string]interface{}), nextDefCfg[k].(map[string]interface{}), defaultConf, nextSectionName); err != nil {
+				return err
+			}
+		}
+
+		// If the value is a slice and we are in the multiplexer part
+		// Multiplixer part is dynamic, we need specific function to check it
+		if val.Kind() == reflect.Slice && sectionName == ".multiplexer" {
+			if err := checkMultiplexerConfig(val, nextDefCfg[k].([]interface{}), defaultConf, k); err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
+func checkMultiplexerConfig(currentVal reflect.Value, currentRef []interface{}, defaultConf map[string]interface{}, k string) error {
+	refLoggers := defaultConf[KeyLoggers].(map[string]interface{})
+	refCollectors := defaultConf[KeyCollectors].(map[string]interface{})
+	refTransforms := defaultConf["collectors-transformers"].(map[string]interface{})
+
+	// iter over the slice
+	for pos, item := range currentVal.Interface().([]interface{}) {
+		valReflect := reflect.ValueOf(item)
+		refItem := currentRef[0].(map[string]interface{})
+		if valReflect.Kind() == reflect.Map {
+			for _, key := range valReflect.MapKeys() {
+				strKey := key.Interface().(string)
+				mapVal := valReflect.MapIndex(key)
+
+				// First, check in the initial configuration reference.
+				// If not found, then look in the logger and collector references.
+				if _, ok := refItem[strKey]; !ok {
+					// we are in routes section ?
+					if !(k == KeyCollectors || k == KeyLoggers) {
+						return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos)
+					}
+
+					// Check if the key exists in neither loggers nor collectors
+					loggerExists := refLoggers[strKey] != nil
+					collectorExists := refCollectors[strKey] != nil
+					if !loggerExists && !collectorExists {
+						return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos)
+					}
+
+					// check logger
+					if k == KeyLoggers || k == KeyCollectors {
+						nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey)
+						refMap := refLoggers
+						if k == KeyCollectors {
+							refMap = refCollectors
+						}
+
+						// Type assertion to check if the value is a map
+						if value, ok := mapVal.Interface().(map[string]interface{}); ok {
+							if err := checkKeywordsPosition(value, refMap[strKey].(map[string]interface{}), defaultConf, nextSectionName); err != nil {
+								return err
+							}
+						} else {
+							return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos)
+						}
+					}
+				}
+
+				// Check transforms section
+				// Type assertion to check if the value is a map
+				if strKey == "transforms" {
+					nextSectionName := fmt.Sprintf("%s.%s", k, strKey)
+					if value, ok := mapVal.Interface().(map[string]interface{}); ok {
+						if err := checkKeywordsPosition(value, refTransforms, defaultConf, nextSectionName); err != nil {
+							return err
+						}
+					} else {
+						return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func convertConfigToMap(config *Config) (map[string]interface{}, error) {
+	// Convert config to YAML
+	yamlData, err := yaml.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert YAML to map
+	configMap := make(map[string]interface{})
+	err = yaml.Unmarshal(yamlData, &configMap)
+	if err != nil {
+		return nil, err
+	}
+
+	return configMap, nil
+}
+
+func loadUserConfigToMap(configPath string) (map[string]interface{}, error) {
+	// Read user configuration file
+	configFile, err := os.Open(configPath)
+	if err != nil {
+		return nil, err
+	}
+	defer configFile.Close()
+
+	// Read config file bytes
+	configBytes, err := io.ReadAll(configFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading configuration file")
+	}
+
+	// Unmarshal YAML to map
+	userConfigMap := make(map[string]interface{})
+	err = yaml.Unmarshal(configBytes, &userConfigMap)
+	if err != nil {
+		return nil, errors.Wrap(err, "error parsing YAML file")
+	}
+
+	return userConfigMap, nil
+}
+
+func getUniqueKeywords(s map[string]interface{}) map[string]bool {
+	keys := extractYamlKeys(s)
+	uniqueKeys := make(map[string]bool)
+	for _, key := range keys {
+		if _, ok := uniqueKeys[key]; ok {
+			continue
+		}
+		uniqueKeys[key] = true
+	}
+	return uniqueKeys
+}
+
+func extractYamlKeys(s map[string]interface{}) []string {
+	keys := []string{}
+	for k, v := range s {
+		keys = append(keys, k)
+		val := reflect.ValueOf(v)
+		if val.Kind() == reflect.Map {
+			nextKeys := extractYamlKeys(val.Interface().(map[string]interface{}))
+			keys = append(keys, nextKeys...)
+		}
+		if val.Kind() == reflect.Slice {
+			for _, v2 := range val.Interface().([]interface{}) {
+				val2 := reflect.ValueOf(v2)
+				if val2.Kind() == reflect.Map {
+					nextKeys := extractYamlKeys(val2.Interface().(map[string]interface{}))
+					keys = append(keys, nextKeys...)
+				}
+			}
+		}
+
+	}
+	return keys
+}
+
 func GetFakeConfig() *Config {
 	config := &Config{}
 	config.SetDefault()