Implement wildcards on the file list. (#180)

* Implement wildcards on the file list.

This is done by searching for the '*' string in the file path, which is
a sub-group of the whole "file glob" capabilities of Golang.

* Raise errors instead of panicing

* Update documentation for wildcard

Wildcard are only evaluated at program start, I added a bit of documentation about that.

Co-authored-by: Martin Helmich <[email protected]>

README.adoc

@@ -540,6 +540,36 @@ relabel "request_uri" {
 }
 ----
 
+### File Globs
+
+You can specify one or more wildcards in the source file names, in which case the wildcards will be resolved to the corresponding list of files at startup of the exporter.
+
+Be aware that the list of matches is only evaluated at the start of the program. If a new file is added with a match of one glob filter, you'll have to restart the program for it to be monitored.
+
+Given a config like this:
+
+[source,hcl]
+---
+namespace "test" {
+  source {
+    files = ["/var/log/nginx/*_access.log"]
+    // ...
+  }
+}
+---
+
+And a folder containing these files:
+
+```bash
+# /var/log/nginx
+main_access.log
+main_error.log
+virtualhost1_access.log
+virtualhost1_error.log
+````
+
+The list of files monitored by this namespace will be `/var/log/nginx/main_access.log,/var/log/nginx/virtualhost1_access.log`.
+
 ### JSON log_format
 
 You can use the JSON parser by setting the `--parser` command line flag or `parser` config file property to `json`.

config/loader.go

@@ -59,6 +59,10 @@ func LoadConfigFromStream(config *Config, stream io.Reader, typ FileFormat) erro
 
 	for i := range config.Namespaces {
 		config.Namespaces[i].ResolveDeprecations()
+
+		if err := config.Namespaces[i].ResolveGlobs(); err != nil {
+			return err
+		}
 	}
 
 	return nil

config/loader_test.go

@@ -31,7 +31,9 @@ consul {
 namespace "nginx" {
   source_files = [
     "test.log",
-    "foo.log"
+    "foo.log",
+    "test/file_pattern*.txt",
+    "test/file_3.txt"
   ]
   format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
 
@@ -83,6 +85,8 @@ namespaces:
     source_files:
       - test.log
       - foo.log
+      - test/file_pattern*.txt
+      - test/file_3.txt
     format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
     labels:
       app: "magicapp"
@@ -117,10 +121,13 @@ func assertConfigContents(t *testing.T, cfg Config) {
 	require.Len(t, cfg.Namespaces, 1)
 
 	n := cfg.Namespaces[0]
+	expectedFileSource := FileSource{"test.log", "foo.log", "test/file_pattern_1.txt", "test/file_pattern_2.txt", "test/file_3.txt"}
+	expectedSourceFiles := []string{"test.log", "foo.log", "test/file_pattern_1.txt", "test/file_pattern_2.txt", "test/file_3.txt"}
+
 	assert.Equal(t, "nginx", n.Name)
 	assert.Equal(t, "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"", n.Format)
-	assert.Equal(t, []string{"test.log", "foo.log"}, n.SourceFiles)
-	assert.Equal(t, FileSource{"test.log", "foo.log"}, n.SourceData.Files)
+	assert.Equal(t, expectedSourceFiles, n.SourceFiles)
+	assert.Equal(t, expectedFileSource, n.SourceData.Files)
 	assert.Equal(t, "magicapp", n.Labels["app"])
 	assert.Nil(t, n.NamespaceLabels)
 	assert.Nil(t, n.MetricsOverride)

config/struct_namespace.go

@@ -2,7 +2,10 @@ package config
 
 import (
 	"errors"
+	"fmt"
+	"path/filepath"
 	"sort"
+	"strings"
 )
 
 // NamespaceConfig is a struct describing single metric namespaces
@@ -77,6 +80,32 @@ func (c *NamespaceConfig) ResolveDeprecations() {
 	}
 }
 
+// ResolveGlobs finds globs in file sources and expand them to the actual
+// list of files
+func (c *NamespaceConfig) ResolveGlobs() error {
+	if len(c.SourceData.Files) > 0 {
+		resolvedFiles := make([]string, 0)
+		for _, sf := range c.SourceData.Files {
+			if strings.Contains(sf, "*") {
+				matches, err := filepath.Glob(sf)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("Resolved globs %v to %v\n", sf, matches)
+				resolvedFiles = append(resolvedFiles, matches...)
+			} else {
+				fmt.Printf("No globs for %v\n", sf)
+				resolvedFiles = append(resolvedFiles, sf)
+			}
+		}
+
+		// update fields with new list of files
+		c.SourceData.Files = resolvedFiles
+		c.SourceFiles = resolvedFiles
+	}
+	return nil
+}
+
 // Compile compiles the configuration (mostly regular expressions that are used
 // in configuration variables) for later use
 func (c *NamespaceConfig) Compile() error {