Support loading multiple configuration files (prometheus#970)

* Support loading multiple configuration files

Support loading multiple configuration files by by allowing
`--config.file` to be repeateable. As well as supporting glob file
matching for `config.d` style setups.
* Conflicting auth or module keys are treated as errors.

Fixes: prometheus#628
---------

Signed-off-by: SuperQ <[email protected]>
Signed-off-by: Ben Kochie <[email protected]>
Co-authored-by: Richard Hartmann <[email protected]>
Signed-off-by: Stephan Windischmann <[email protected]>

README.md

@@ -117,6 +117,11 @@ by hand. If you need to change it, see
 The default `snmp.yml` file covers a variety of common hardware walking them
 using SNMP v2 GETBULK.
 
+The `--config.file` parameter can be used multiple times to load more than one file.
+It also supports [glob filename matching](https://pkg.go.dev/path/filepath#Glob), e.g. `snmp*.yml`.
+
+Duplicate `module` or `auth` entries are treated as invalid and can not be loaded.
+
 ## Prometheus Configuration
 
 The URL params `target`, `auth`, and `module` can be controlled through relabelling.

config/config.go

@@ -16,22 +16,31 @@ package config
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"time"
 
 	"github.com/gosnmp/gosnmp"
 	"gopkg.in/yaml.v2"
 )
 
-func LoadFile(filename string) (*Config, error) {
-	content, err := os.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
+func LoadFile(paths []string) (*Config, error) {
 	cfg := &Config{}
-	err = yaml.UnmarshalStrict(content, cfg)
-	if err != nil {
-		return nil, err
+	for _, p := range paths {
+		files, err := filepath.Glob(p)
+		if err != nil {
+			return nil, err
+		}
+		for _, f := range files {
+			content, err := os.ReadFile(f)
+			if err != nil {
+				return nil, err
+			}
+			err = yaml.UnmarshalStrict(content, cfg)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 	return cfg, nil
 }

config_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestHideConfigSecrets(t *testing.T) {
 	sc := &SafeConfig{}
-	err := sc.ReloadConfig("testdata/snmp-auth.yml")
+	err := sc.ReloadConfig([]string{"testdata/snmp-auth.yml"})
 	if err != nil {
 		t.Errorf("Error loading config %v: %v", "testdata/snmp-auth.yml", err)
 	}
@@ -41,7 +41,7 @@ func TestHideConfigSecrets(t *testing.T) {
 
 func TestLoadConfigWithOverrides(t *testing.T) {
 	sc := &SafeConfig{}
-	err := sc.ReloadConfig("testdata/snmp-with-overrides.yml")
+	err := sc.ReloadConfig([]string{"testdata/snmp-with-overrides.yml"})
 	if err != nil {
 		t.Errorf("Error loading config %v: %v", "testdata/snmp-with-overrides.yml", err)
 	}
@@ -52,3 +52,18 @@ func TestLoadConfigWithOverrides(t *testing.T) {
 		t.Errorf("Error marshaling config: %v", err)
 	}
 }
+
+func TestLoadMultipleConfigs(t *testing.T) {
+	sc := &SafeConfig{}
+	configs := []string{"testdata/snmp-auth.yml", "testdata/snmp-with-overrides.yml"}
+	err := sc.ReloadConfig(configs)
+	if err != nil {
+		t.Errorf("Error loading configs %v: %v", configs, err)
+	}
+	sc.RLock()
+	_, err = yaml.Marshal(sc.C)
+	sc.RUnlock()
+	if err != nil {
+		t.Errorf("Error marshaling config: %v", err)
+	}
+}

main.go

@@ -41,7 +41,7 @@ import (
 )
 
 var (
-	configFile  = kingpin.Flag("config.file", "Path to configuration file.").Default("snmp.yml").String()
+	configFile  = kingpin.Flag("config.file", "Path to configuration file.").Default("snmp.yml").Strings()
 	dryRun      = kingpin.Flag("dry-run", "Only verify configuration is valid and exit.").Default("false").Bool()
 	concurrency = kingpin.Flag("snmp.module-concurrency", "The number of modules to fetch concurrently per scrape").Default("1").Int()
 	metricsPath = kingpin.Flag(
@@ -152,7 +152,7 @@ type SafeConfig struct {
 	C *config.Config
 }
 
-func (sc *SafeConfig) ReloadConfig(configFile string) (err error) {
+func (sc *SafeConfig) ReloadConfig(configFile []string) (err error) {
 	conf, err := config.LoadFile(configFile)
 	if err != nil {
 		return err