diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index b0ac4a03..7c9be4ab 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -140,4 +140,10 @@ jobs: run: | data=$(find . -name "*.go" -exec grep -v "^$" {} \; -exec grep -v "//" {} \; | wc -l) echo "Count of Lines: $data" + echo "data=$data" >> $GITHUB_OUTPUT + + - id: count_lines_tests + run: | + data=$(find . -name "*_test.go" -exec grep -v "^$" {} \; -exec grep -v "//" {} \; | wc -l) + echo "Count of Lines in tests file: $data" echo "data=$data" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/README.md b/README.md index 162f42e4..1f6c78b4 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/dmachard/go-dns-collector)](https://goreportcard.com/report/dmachard/go-dns-collector) ![Go version](https://img.shields.io/badge/go%20version-min%201.20-blue) -![Go tests](https://img.shields.io/badge/go%20tests-359-green) -![Go lines](https://img.shields.io/badge/go%20lines-49536-red) +![Go tests](https://img.shields.io/badge/go%20tests-366-green) +![Go lines](https://img.shields.io/badge/go%20lines-32797-red) ![Go Tests](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-go.yml/badge.svg) ![Github Actions](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-dnstap.yml/badge.svg) ![Github Actions PDNS](https://github.com/dmachard/go-dns-collector/actions/workflows/testing-powerdns.yml/badge.svg) @@ -83,7 +83,7 @@ Multiplexer Download the latest [`release`](https://github.com/dmachard/go-dns-collector/releases) binary and start the DNS-collector with the provided configuration file. The default configuration listens on `tcp/6000` for a DNSTap stream and DNS logs are printed on standard output. -```go +```bash ./go-dnscollector -config config.yml ``` @@ -95,6 +95,13 @@ The configuration of DNS-collector is done through a file named [`config.yml`](c See the full [configuration guide](./docs/configuration.md) for more details. +Run the DNS-collector in dry mode to verify the configuration. + +```bash +./go-dnscollector -config config.yml -test-config +INFO: 2023/12/24 14:43:29.043730 main - config OK! +``` + ## Usage examples The [`_examples`](./docs/_examples) folder from documentation contains a number of [various configurations](./docs/examples.md) to get you started with the DNS-collector in differentes ways. diff --git a/config.yml b/config.yml index 0476eef1..b6f684d7 100644 --- a/config.yml +++ b/config.yml @@ -1,5 +1,3 @@ - - ################################################ # global configuration ################################################ diff --git a/dnscollector.go b/dnscollector.go index a657b751..93340959 100644 --- a/dnscollector.go +++ b/dnscollector.go @@ -89,7 +89,8 @@ func main() { // load config config, err := pkgconfig.LoadConfig(configPath) if err != nil { - panic(fmt.Sprintf("main - config error: %v", err)) + fmt.Printf("config error: %v\n", err) + os.Exit(1) } // init logger diff --git a/pkgconfig/config.go b/pkgconfig/config.go index 4cafdd27..b77a2dd3 100644 --- a/pkgconfig/config.go +++ b/pkgconfig/config.go @@ -1,8 +1,12 @@ package pkgconfig import ( + "fmt" + "io" "os" + "reflect" + "github.com/pkg/errors" "gopkg.in/yaml.v3" ) @@ -61,14 +65,19 @@ func (c *Config) GetServerIdentity() string { func ReloadConfig(configPath string, config *Config) error { // Open config file - file, err := os.Open(configPath) + configFile, err := os.Open(configPath) if err != nil { return nil } - defer file.Close() + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configPath); err != nil { + return err + } // Init new YAML decode - d := yaml.NewDecoder(file) + d := yaml.NewDecoder(configFile) // Start YAML decoding from file if err := d.Decode(&config); err != nil { @@ -78,20 +87,25 @@ func ReloadConfig(configPath string, config *Config) error { } func LoadConfig(configPath string) (*Config, error) { - config := &Config{} - config.SetDefault() - // Open config file - file, err := os.Open(configPath) + configFile, err := os.Open(configPath) if err != nil { return nil, err } - defer file.Close() + defer configFile.Close() + + // Check config to detect unknown keywords + if err := CheckConfig(configPath); err != nil { + return nil, err + } // Init new YAML decode - d := yaml.NewDecoder(file) + d := yaml.NewDecoder(configFile) + + // Start YAML decoding to go + config := &Config{} + config.SetDefault() - // Start YAML decoding from file if err := d.Decode(&config); err != nil { return nil, err } @@ -99,6 +113,220 @@ func LoadConfig(configPath string) (*Config, error) { return config, nil } +func CheckConfig(userConfigPath string) error { + // create default config + // and simulate one route, one collector and one logger + defaultConfig := &Config{} + defaultConfig.SetDefault() + defaultConfig.Multiplexer.Routes = append(defaultConfig.Multiplexer.Routes, MultiplexRoutes{}) + defaultConfig.Multiplexer.Loggers = append(defaultConfig.Multiplexer.Loggers, MultiplexInOut{}) + defaultConfig.Multiplexer.Collectors = append(defaultConfig.Multiplexer.Collectors, MultiplexInOut{}) + + // Convert default config to map + // And get unique YAML keys + defaultConfigMap, err := convertConfigToMap(defaultConfig) + if err != nil { + return errors.Wrap(err, "error converting default config to map") + } + defaultKeywords := getUniqueKeywords(defaultConfigMap) + + // Read user configuration file + // And get unique YAML keys from user config + userConfigMap, err := loadUserConfigToMap(userConfigPath) + if err != nil { + return err + } + userKeywords := getUniqueKeywords(userConfigMap) + + // Check for unknown keys in user config + for key := range userKeywords { + if _, ok := defaultKeywords[key]; !ok { + return errors.Errorf("unknown YAML key `%s` in configuration", key) + } + } + + // detect bad keyword position + err = checkKeywordsPosition(userConfigMap, defaultConfigMap, defaultConfigMap, "") + if err != nil { + return err + } + return nil +} + +func checkKeywordsPosition(nextUserCfg, nextDefCfg map[string]interface{}, defaultConf map[string]interface{}, sectionName string) error { + for k, v := range nextUserCfg { + // Check if the key is present in the default config + if _, ok := nextDefCfg[k]; !ok { + if sectionName == "" { + return errors.Errorf("invalid key `%s` at root", k) + } + return errors.Errorf("invalid key `%s` in section `%s`", k, sectionName) + } + + // If the value is a map, recursively check for invalid keywords + // Recursive call ? + val := reflect.ValueOf(v) + if val.Kind() == reflect.Map { + nextSectionName := fmt.Sprintf("%s.%s", sectionName, k) + if err := checkKeywordsPosition(v.(map[string]interface{}), nextDefCfg[k].(map[string]interface{}), defaultConf, nextSectionName); err != nil { + return err + } + } + + // If the value is a slice and we are in the multiplexer part + // Multiplixer part is dynamic, we need specific function to check it + if val.Kind() == reflect.Slice && sectionName == ".multiplexer" { + if err := checkMultiplexerConfig(val, nextDefCfg[k].([]interface{}), defaultConf, k); err != nil { + return err + } + } + + } + return nil +} + +func checkMultiplexerConfig(currentVal reflect.Value, currentRef []interface{}, defaultConf map[string]interface{}, k string) error { + refLoggers := defaultConf[KeyLoggers].(map[string]interface{}) + refCollectors := defaultConf[KeyCollectors].(map[string]interface{}) + refTransforms := defaultConf["collectors-transformers"].(map[string]interface{}) + + // iter over the slice + for pos, item := range currentVal.Interface().([]interface{}) { + valReflect := reflect.ValueOf(item) + refItem := currentRef[0].(map[string]interface{}) + if valReflect.Kind() == reflect.Map { + for _, key := range valReflect.MapKeys() { + strKey := key.Interface().(string) + mapVal := valReflect.MapIndex(key) + + // First, check in the initial configuration reference. + // If not found, then look in the logger and collector references. + if _, ok := refItem[strKey]; !ok { + // we are in routes section ? + if !(k == KeyCollectors || k == KeyLoggers) { + return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos) + } + + // Check if the key exists in neither loggers nor collectors + loggerExists := refLoggers[strKey] != nil + collectorExists := refCollectors[strKey] != nil + if !loggerExists && !collectorExists { + return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos) + } + + // check logger + if k == KeyLoggers || k == KeyCollectors { + nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey) + refMap := refLoggers + if k == KeyCollectors { + refMap = refCollectors + } + + // Type assertion to check if the value is a map + if value, ok := mapVal.Interface().(map[string]interface{}); ok { + if err := checkKeywordsPosition(value, refMap[strKey].(map[string]interface{}), defaultConf, nextSectionName); err != nil { + return err + } + } else { + return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos) + } + } + } + + // Check transforms section + // Type assertion to check if the value is a map + if strKey == "transforms" { + nextSectionName := fmt.Sprintf("%s.%s", k, strKey) + if value, ok := mapVal.Interface().(map[string]interface{}); ok { + if err := checkKeywordsPosition(value, refTransforms, defaultConf, nextSectionName); err != nil { + return err + } + } else { + return errors.Errorf("invalid `%s` value in `%s` list at position %d", strKey, k, pos) + } + } + } + } + } + return nil +} + +func convertConfigToMap(config *Config) (map[string]interface{}, error) { + // Convert config to YAML + yamlData, err := yaml.Marshal(config) + if err != nil { + return nil, err + } + + // Convert YAML to map + configMap := make(map[string]interface{}) + err = yaml.Unmarshal(yamlData, &configMap) + if err != nil { + return nil, err + } + + return configMap, nil +} + +func loadUserConfigToMap(configPath string) (map[string]interface{}, error) { + // Read user configuration file + configFile, err := os.Open(configPath) + if err != nil { + return nil, err + } + defer configFile.Close() + + // Read config file bytes + configBytes, err := io.ReadAll(configFile) + if err != nil { + return nil, errors.Wrap(err, "Error reading configuration file") + } + + // Unmarshal YAML to map + userConfigMap := make(map[string]interface{}) + err = yaml.Unmarshal(configBytes, &userConfigMap) + if err != nil { + return nil, errors.Wrap(err, "error parsing YAML file") + } + + return userConfigMap, nil +} + +func getUniqueKeywords(s map[string]interface{}) map[string]bool { + keys := extractYamlKeys(s) + uniqueKeys := make(map[string]bool) + for _, key := range keys { + if _, ok := uniqueKeys[key]; ok { + continue + } + uniqueKeys[key] = true + } + return uniqueKeys +} + +func extractYamlKeys(s map[string]interface{}) []string { + keys := []string{} + for k, v := range s { + keys = append(keys, k) + val := reflect.ValueOf(v) + if val.Kind() == reflect.Map { + nextKeys := extractYamlKeys(val.Interface().(map[string]interface{})) + keys = append(keys, nextKeys...) + } + if val.Kind() == reflect.Slice { + for _, v2 := range val.Interface().([]interface{}) { + val2 := reflect.ValueOf(v2) + if val2.Kind() == reflect.Map { + nextKeys := extractYamlKeys(val2.Interface().(map[string]interface{})) + keys = append(keys, nextKeys...) + } + } + } + + } + return keys +} + func GetFakeConfig() *Config { config := &Config{} config.SetDefault() diff --git a/pkgconfig/config_test.go b/pkgconfig/config_test.go new file mode 100644 index 00000000..0cb0c8c6 --- /dev/null +++ b/pkgconfig/config_test.go @@ -0,0 +1,184 @@ +package pkgconfig + +import ( + "os" + "testing" + + "github.com/pkg/errors" +) + +// ServerIdentity is set in the config +func TestConfig_GetServerIdentity(t *testing.T) { + config := &Config{ + Global: ConfigGlobal{ + ServerIdentity: "test-server", + }, + } + expected1 := "test-server" + if result1 := config.GetServerIdentity(); result1 != expected1 { + t.Errorf("Expected %s, but got %s", expected1, result1) + } +} + +// ServerIdentity is not set in the config, hostname is available +func TestConfig_GetServerIdentity_Hostname(t *testing.T) { + config := &Config{ + Global: ConfigGlobal{}, + } + hostname, err := os.Hostname() + if err != nil { + t.Fatal("Error getting hostname:", err) + } + expected2 := hostname + if result2 := config.GetServerIdentity(); result2 != expected2 { + t.Errorf("Expected %s, but got %s", expected2, result2) + } +} + +// Valid minimal user configuration +func TestConfig_CheckConfig_Valid(t *testing.T) { + // Create a temporary file for the user configuration + userConfigFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + t.Fatal("Error creating temporary file:", err) + } + defer os.Remove(userConfigFile.Name()) + defer userConfigFile.Close() + + validUserConfigContent := ` +global: + trace: false +multiplexer: + routes: + - from: [test-route] + loggers: + - name: test-logger + collectors: + - name: test-collector +` + err = os.WriteFile(userConfigFile.Name(), []byte(validUserConfigContent), 0644) + if err != nil { + t.Fatal("Error writing to user configuration file:", err) + } + + if err := CheckConfig(userConfigFile.Name()); err != nil { + t.Errorf("failed: Unexpected error: %v", err) + } +} + +// Invalid user configuration with an unknown key +func TestConfig_CheckConfig_UnknownKeywords(t *testing.T) { + userConfigFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + t.Fatal("Error creating temporary file:", err) + } + defer os.Remove(userConfigFile.Name()) + defer userConfigFile.Close() + + userConfigContent := ` +global: + trace: false +multiplexer: + routes: + - from: [test-route] + unknown-key: invalid +` + err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) + if err != nil { + t.Fatal("Error writing to user configuration file:", err) + } + + expectedError := errors.Errorf("unknown YAML key `unknown-key` in configuration") + if err := CheckConfig(userConfigFile.Name()); err == nil || err.Error() != expectedError.Error() { + t.Errorf("Expected error %v, but got %v", expectedError, err) + } +} + +// Keywork exist but not at the good position +func TestConfig_CheckConfig_BadKeywordPosition(t *testing.T) { + userConfigFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + t.Fatal("Error creating temporary file:", err) + } + defer os.Remove(userConfigFile.Name()) + defer userConfigFile.Close() + + userConfigContent := ` +global: + trace: false + logger: bad-position +` + err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) + if err != nil { + t.Fatal("Error writing to user configuration file:", err) + } + if err := CheckConfig(userConfigFile.Name()); err == nil { + t.Errorf("Expected error, but got %v", err) + } +} + +// Valid multiplexer configuration +func TestConfig_CheckMultiplexerConfig_Valid(t *testing.T) { + userConfigFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + t.Fatal("Error creating temporary file:", err) + } + defer os.Remove(userConfigFile.Name()) + defer userConfigFile.Close() + + userConfigContent := ` +multiplexer: + collectors: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + transforms: + normalize: + qname-lowercase: false + loggers: + - name: console + stdout: + mode: text + routes: + - from: [ tap ] + to: [ console ] +` + err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) + if err != nil { + t.Fatal("Error writing to user configuration file:", err) + } + if err := CheckConfig(userConfigFile.Name()); err != nil { + t.Errorf("failed: Unexpected error: %v", err) + } +} + +// Invalid multiplexer configuration +func TestConfig_CheckMultiplexerConfig_Invalid(t *testing.T) { + userConfigFile, err := os.CreateTemp("", "user-config.yaml") + if err != nil { + t.Fatal("Error creating temporary file:", err) + } + defer os.Remove(userConfigFile.Name()) + defer userConfigFile.Close() + + userConfigContent := ` +global: + trace: false +multiplexer: +- name: block + dnstap: + listen-ip: 0.0.0.0 + transforms: + normalize: + qname-lowercase: true +` + + err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644) + if err != nil { + t.Fatal("Error writing to user configuration file:", err) + } + if err := CheckConfig(userConfigFile.Name()); err == nil { + t.Errorf("Expected error, but got %v", err) + } +} diff --git a/pkgconfig/constants.go b/pkgconfig/constants.go index f33841d8..7e8a90f7 100644 --- a/pkgconfig/constants.go +++ b/pkgconfig/constants.go @@ -12,6 +12,9 @@ const ( AnyIP = "0.0.0.0" HTTPOK = "HTTP/1.1 200 OK\r\n\r\n" + KeyCollectors = "collectors" + KeyLoggers = "loggers" + ValidDomain = "dnscollector.dev." BadDomainLabel = "ultramegaverytoolonglabel-ultramegaverytoolonglabel-ultramegaverytoolonglabel.dnscollector.dev." badLongLabel = "ultramegaverytoolonglabel-ultramegaverytoolonglabel-"