Backport fix for doublestar globbing

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/antonmedv/expr v1.9.0
 	github.com/aws/aws-sdk-go v1.40.26
 	github.com/bmatcuk/doublestar/v2 v2.0.4
+	github.com/bmatcuk/doublestar/v3 v3.0.0
 	github.com/cenkalti/backoff/v4 v4.1.1
 	github.com/elastic/go-elasticsearch/v7 v7.13.0
 	github.com/golang/protobuf v1.5.2

go.sum

@@ -213,6 +213,8 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
 github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
+github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
+github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=

operator/builtin/input/file/config.go

@@ -40,9 +40,7 @@ func NewInputConfig(operatorID string) *InputConfig {
 // InputConfig is the configuration of a file input operator
 type InputConfig struct {
 	helper.InputConfig `yaml:",inline"`
-
-	Include []string `json:"include,omitempty" yaml:"include,omitempty"`
-	Exclude []string `json:"exclude,omitempty" yaml:"exclude,omitempty"`
+	Finder             `mapstructure:",squash" yaml:",inline"`
 
 	PollInterval            helper.Duration        `json:"poll_interval,omitempty"               yaml:"poll_interval,omitempty"`
 	Multiline               helper.MultilineConfig `json:"multiline,omitempty"                   yaml:"multiline,omitempty"`
@@ -167,8 +165,7 @@ func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator,
 
 	op := &InputOperator{
 		InputOperator:         inputOperator,
-		Include:               c.Include,
-		Exclude:               c.Exclude,
+		finder:                c.Finder,
 		SplitFunc:             splitFunc,
 		PollInterval:          c.PollInterval.Raw(),
 		persist:               helper.NewScopedDBPersister(context.Database, c.ID()),

operator/builtin/input/file/config_test.go

@@ -546,7 +546,7 @@ func TestBuild(t *testing.T) {
 			require.NoError,
 			func(t *testing.T, f *InputOperator) {
 				require.Equal(t, f.OutputOperators[0], fakeOutput)
-				require.Equal(t, f.Include, []string{"/var/log/testpath.*"})
+				require.Equal(t, f.finder.Include, []string{"/var/log/testpath.*"})
 				require.Equal(t, f.FilePathField, entry.NewNilField())
 				require.Equal(t, f.FileNameField, entry.NewLabelField("file_name"))
 				require.Equal(t, f.PollInterval, 10*time.Millisecond)

operator/builtin/input/file/file.go

@@ -7,12 +7,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"regexp"
 	"sync"
 	"time"
 
-	"github.com/bmatcuk/doublestar/v2"
 	"github.com/observiq/stanza/entry"
 	"github.com/observiq/stanza/operator/helper"
 	"go.uber.org/zap"
@@ -22,8 +20,7 @@ import (
 type InputOperator struct {
 	helper.InputOperator
 
-	Include               []string
-	Exclude               []string
+	finder                Finder
 	FilePathField         entry.Field
 	FileNameField         entry.Field
 	FilePathResolvedField entry.Field
@@ -125,9 +122,11 @@ func (f *InputOperator) poll(ctx context.Context) {
 		}
 
 		// Get the list of paths on disk
-		matches = getMatches(f.Include, f.Exclude)
+		matches = f.finder.FindFiles()
 		if f.firstCheck && len(matches) == 0 {
-			f.Warnw("no files match the configured include patterns", "include", f.Include)
+			f.Warnw("no files match the configured include patterns",
+				"include", f.finder.Include,
+				"exclude", f.finder.Exclude)
 		} else if len(matches) > f.maxBatchFiles {
 			matches, f.queuedMatches = matches[:f.maxBatchFiles], matches[f.maxBatchFiles:]
 		}
@@ -187,32 +186,6 @@ OUTER:
 	f.syncLastPollFiles()
 }
 
-// getMatches gets a list of paths given an array of glob patterns to include and exclude
-func getMatches(includes, excludes []string) []string {
-	all := make([]string, 0, len(includes))
-	for _, include := range includes {
-		matches, _ := filepath.Glob(include) // compile error checked in build
-	INCLUDE:
-		for _, match := range matches {
-			for _, exclude := range excludes {
-				if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches {
-					continue INCLUDE
-				}
-			}
-
-			for _, existing := range all {
-				if existing == match {
-					continue INCLUDE
-				}
-			}
-
-			all = append(all, match)
-		}
-	}
-
-	return all
-}
-
 // makeReaders takes a list of paths, then creates readers from each of those paths,
 // discarding any that have a duplicate fingerprint to other files that have already
 // been read this polling interval

operator/builtin/input/file/file_test.go

@@ -742,46 +742,3 @@ func TestEncodings(t *testing.T) {
 		})
 	}
 }
-
-// TestExclude tests that a log file will be excluded if it matches the
-// glob specified in the operator.
-func TestExclude(t *testing.T) {
-	tempDir := testutil.NewTempDir(t)
-	paths := writeTempFiles(tempDir, []string{"include.log", "exclude.log"})
-
-	includes := []string{filepath.Join(tempDir, "*")}
-	excludes := []string{filepath.Join(tempDir, "*exclude.log")}
-
-	matches := getMatches(includes, excludes)
-	require.ElementsMatch(t, matches, paths[:1])
-}
-func TestExcludeEmpty(t *testing.T) {
-	tempDir := testutil.NewTempDir(t)
-	paths := writeTempFiles(tempDir, []string{"include.log", "exclude.log"})
-
-	includes := []string{filepath.Join(tempDir, "*")}
-	excludes := []string{}
-
-	matches := getMatches(includes, excludes)
-	require.ElementsMatch(t, matches, paths)
-}
-func TestExcludeMany(t *testing.T) {
-	tempDir := testutil.NewTempDir(t)
-	paths := writeTempFiles(tempDir, []string{"a1.log", "a2.log", "b1.log", "b2.log"})
-
-	includes := []string{filepath.Join(tempDir, "*")}
-	excludes := []string{filepath.Join(tempDir, "a*.log"), filepath.Join(tempDir, "*2.log")}
-
-	matches := getMatches(includes, excludes)
-	require.ElementsMatch(t, matches, paths[2:3])
-}
-func TestExcludeDuplicates(t *testing.T) {
-	tempDir := testutil.NewTempDir(t)
-	paths := writeTempFiles(tempDir, []string{"a1.log", "a2.log", "b1.log", "b2.log"})
-
-	includes := []string{filepath.Join(tempDir, "*1*"), filepath.Join(tempDir, "a*")}
-	excludes := []string{filepath.Join(tempDir, "a*.log"), filepath.Join(tempDir, "*2.log")}
-
-	matches := getMatches(includes, excludes)
-	require.ElementsMatch(t, matches, paths[2:3])
-}

operator/builtin/input/file/finder.go

@@ -0,0 +1,50 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"github.com/bmatcuk/doublestar/v3"
+)
+
+type Finder struct {
+	Include []string `mapstructure:"include,omitempty" json:"include,omitempty" yaml:"include,omitempty"`
+	Exclude []string `mapstructure:"exclude,omitempty" json:"exclude,omitempty" yaml:"exclude,omitempty"`
+}
+
+// FindFiles gets a list of paths given an array of glob patterns to include and exclude
+func (f Finder) FindFiles() []string {
+	all := make([]string, 0, len(f.Include))
+	for _, include := range f.Include {
+		matches, _ := doublestar.Glob(include) // compile error checked in build
+	INCLUDE:
+		for _, match := range matches {
+			for _, exclude := range f.Exclude {
+				if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches {
+					continue INCLUDE
+				}
+			}
+
+			for _, existing := range all {
+				if existing == match {
+					continue INCLUDE
+				}
+			}
+
+			all = append(all, match)
+		}
+	}
+
+	return all
+}

operator/builtin/input/file/finder_test.go

@@ -0,0 +1,155 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/observiq/stanza/testutil"
+)
+
+func TestFinder(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		name     string
+		files    []string
+		include  []string
+		exclude  []string
+		expected []string
+	}{
+		{
+			name:     "IncludeOne",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"a1.log"},
+			exclude:  []string{},
+			expected: []string{"a1.log"},
+		},
+		{
+			name:     "IncludeNone",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"c*.log"},
+			exclude:  []string{},
+			expected: []string{},
+		},
+		{
+			name:     "IncludeAll",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"*"},
+			exclude:  []string{},
+			expected: []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+		},
+		{
+			name:     "IncludeLogs",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"*.log"},
+			exclude:  []string{},
+			expected: []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+		},
+		{
+			name:     "IncludeA",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"a*.log"},
+			exclude:  []string{},
+			expected: []string{"a1.log", "a2.log"},
+		},
+		{
+			name:     "Include2s",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"*2.log"},
+			exclude:  []string{},
+			expected: []string{"a2.log", "b2.log"},
+		},
+		{
+			name:     "Exclude",
+			files:    []string{"include.log", "exclude.log"},
+			include:  []string{"*"},
+			exclude:  []string{"exclude.log"},
+			expected: []string{"include.log"},
+		},
+		{
+			name:     "ExcludeMany",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"*"},
+			exclude:  []string{"a*.log", "*2.log"},
+			expected: []string{"b1.log"},
+		},
+		{
+			name:     "ExcludeDuplicates",
+			files:    []string{"a1.log", "a2.log", "b1.log", "b2.log"},
+			include:  []string{"*1*", "a*"},
+			exclude:  []string{"a*.log", "*2.log"},
+			expected: []string{"b1.log"},
+		},
+		{
+			name:     "IncludeMultipleDirectories",
+			files:    []string{"a/1.log", "a/2.log", "b/1.log", "b/2.log"},
+			include:  []string{"a/*.log", "b/*.log"},
+			exclude:  []string{},
+			expected: []string{"a/1.log", "a/2.log", "b/1.log", "b/2.log"},
+		},
+		{
+			name:     "IncludeMultipleDirectoriesVaryingDepth",
+			files:    []string{"1.log", "a/1.log", "a/b/1.log", "c/1.log"},
+			include:  []string{"*.log", "a/*.log", "a/b/*.log", "c/*.log"},
+			exclude:  []string{},
+			expected: []string{"1.log", "a/1.log", "a/b/1.log", "c/1.log"},
+		},
+		{
+			name:     "DoubleStarSameDepth",
+			files:    []string{"a/1.log", "b/1.log", "c/1.log"},
+			include:  []string{"**/*.log"},
+			exclude:  []string{},
+			expected: []string{"a/1.log", "b/1.log", "c/1.log"},
+		},
+		{
+			name:     "DoubleStarVaryingDepth",
+			files:    []string{"1.log", "a/1.log", "a/b/1.log", "c/1.log"},
+			include:  []string{"**/*.log"},
+			exclude:  []string{},
+			expected: []string{"1.log", "a/1.log", "a/b/1.log", "c/1.log"},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			tempDir := testutil.NewTempDir(t)
+			files := absPath(tempDir, tc.files)
+			include := absPath(tempDir, tc.include)
+			exclude := absPath(tempDir, tc.exclude)
+			expected := absPath(tempDir, tc.expected)
+
+			for _, f := range files {
+				os.MkdirAll(filepath.Dir(f), 0755)
+				ioutil.WriteFile(f, []byte(filepath.Base(f)), 0755)
+			}
+
+			finder := Finder{include, exclude}
+			require.ElementsMatch(t, finder.FindFiles(), expected)
+		})
+	}
+}
+
+func absPath(tempDir string, files []string) []string {
+	absFiles := make([]string, 0, len(files))
+	for _, f := range files {
+		absFiles = append(absFiles, filepath.Join(tempDir, f))
+	}
+	return absFiles
+}

operator/builtin/input/file/util_test.go

@@ -162,14 +162,3 @@ func expectNoMessagesUntil(t *testing.T, c chan *entry.Entry, d time.Duration) {
 	case <-time.After(d):
 	}
 }
-
-// writes file with the specified file names and returns their full paths in order
-func writeTempFiles(tempDir string, names []string) []string {
-	result := make([]string, 0, len(names))
-	for _, name := range names {
-		path := filepath.Join(tempDir, name)
-		ioutil.WriteFile(path, []byte(name), 0755)
-		result = append(result, path)
-	}
-	return result
-}