Add generic glob filter (trufflesecurity#1858)

* Add generic glob filter

* Make nil filters safe

* Include glob in error

* Use better example for exclude and include test

* Allow user to configure the ambiguous case

* Rename Pass to ShouldInclude and invert logic

* Test default *Filter and Filter have the same behavior of allow

* Add property based tests

* Remove configuration for the not found ambiguous case

go.mod

@@ -81,6 +81,7 @@ require (
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
+	pgregory.net/rapid v1.1.0
 	sigs.k8s.io/yaml v1.3.0
 )
 

go.sum

@@ -1013,6 +1013,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

pkg/common/glob/glob.go

@@ -0,0 +1,120 @@
+package common
+
+import (
+	"fmt"
+
+	"github.com/gobwas/glob"
+)
+
+// Filter is a generic filter for excluding and including globs (limited
+// regular expressions). Exclusion takes precedence if both include and exclude
+// lists are provided.
+type Filter struct {
+	exclude []glob.Glob
+	include []glob.Glob
+}
+
+type globFilterOpt func(*Filter) error
+
+// WithExcludeGlobs adds exclude globs to the filter.
+func WithExcludeGlobs(excludes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, exclude := range excludes {
+			g, err := glob.Compile(exclude)
+			if err != nil {
+				return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
+			}
+			f.exclude = append(f.exclude, g)
+		}
+		return nil
+	}
+}
+
+// WithIncludeGlobs adds include globs to the filter.
+func WithIncludeGlobs(includes ...string) globFilterOpt {
+	return func(f *Filter) error {
+		for _, include := range includes {
+			g, err := glob.Compile(include)
+			if err != nil {
+				return fmt.Errorf("invalid include glob %q: %w", include, err)
+			}
+			f.include = append(f.include, g)
+		}
+		return nil
+	}
+}
+
+// NewGlobFilter creates a new Filter with the provided options.
+func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
+	filter := &Filter{}
+	for _, opt := range opts {
+		if err := opt(filter); err != nil {
+			return nil, err
+		}
+	}
+	return filter, nil
+}
+
+// ShouldInclude returns whether the object is in the include list or not in
+// the exclude list (exclude taking precedence).
+func (f *Filter) ShouldInclude(object string) bool {
+	if f == nil {
+		return true
+	}
+	exclude, include := len(f.exclude), len(f.include)
+	if exclude == 0 && include == 0 {
+		return true
+	} else if exclude > 0 && include == 0 {
+		return f.shouldIncludeFromExclude(object)
+	} else if exclude == 0 && include > 0 {
+		return f.shouldIncludeFromInclude(object)
+	} else {
+		if ok, err := f.shouldIncludeFromBoth(object); err == nil {
+			return ok
+		}
+		// Ambiguous case.
+		return false
+	}
+}
+
+// shouldIncludeFromExclude checks for explicitly excluded paths. This should
+// only be called when the include list is empty.
+func (f *Filter) shouldIncludeFromExclude(object string) bool {
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false
+		}
+	}
+	return true
+}
+
+// shouldIncludeFromInclude checks for explicitly included paths. This should
+// only be called when the exclude list is empty.
+func (f *Filter) shouldIncludeFromInclude(object string) bool {
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true
+		}
+	}
+	return false
+}
+
+// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
+// takes precedence. If neither list contains the object, true is returned.
+func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
+	// Exclude takes precedence. If we find the object in the exclude list,
+	// we should not match.
+	for _, glob := range f.exclude {
+		if glob.Match(object) {
+			return false, nil
+		}
+	}
+	// If we find the object in the include list, we should match.
+	for _, glob := range f.include {
+		if glob.Match(object) {
+			return true, nil
+		}
+	}
+	// If we find it in neither, return an error to let the caller decide.
+	return false, fmt.Errorf("ambiguous match")
+}

pkg/common/glob/glob_test.go

@@ -0,0 +1,138 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"pgregory.net/rapid"
+)
+
+type globTest struct {
+	input         string
+	shouldInclude bool
+}
+
+func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			// Invert because mentally it's easier to say whether an
+			// input should be included.
+			assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
+		})
+	}
+}
+
+func TestGlobFilterExclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"bar", false},
+		globTest{"bara", false},
+		globTest{"barb", false},
+		globTest{"barbosa", false},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", false},
+		globTest{"food", false},
+		globTest{"anything else", false},
+	)
+}
+
+func TestGlobFilterEmpty(t *testing.T) {
+	filter, err := NewGlobFilter()
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", true},
+		globTest{"bar", true},
+		globTest{"bara", true},
+		globTest{"barb", true},
+		globTest{"barbosa", true},
+		globTest{"foobar", true},
+		globTest{"food", true},
+		globTest{"anything else", true},
+	)
+}
+
+func TestGlobFilterExcludeInclude(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"/foo/a", true},
+		globTest{"/foo/b", true},
+		globTest{"/foo/c/d/e", true},
+		globTest{"/foo/bar/a", false},
+		globTest{"/foo/bar/b", false},
+		globTest{"/foo/bar/c/d/e", false},
+		globTest{"/any/other/path", false},
+	)
+}
+
+func TestGlobFilterExcludePrecedence(t *testing.T) {
+	filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
+	assert.NoError(t, err)
+
+	testGlobs(t, filter,
+		globTest{"foo", false},
+		globTest{"foobar", true},
+	)
+}
+
+func TestGlobErrorContainsGlob(t *testing.T) {
+	invalidGlob := "[this is invalid because it doesn't close the capture group"
+	_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), invalidGlob)
+}
+
+// The filters in this test should be mutually exclusive because one includes
+// and the other excludes the same glob.
+func TestGlobInverse(t *testing.T) {
+	for _, glob := range []string{
+		"a",
+		"a*",
+		"a**",
+		"*a",
+		"**a",
+		"*",
+	} {
+		include, err := NewGlobFilter(WithIncludeGlobs(glob))
+		assert.NoError(t, err)
+		exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
+		assert.NoError(t, err)
+		rapid.Check(t, func(t *rapid.T) {
+			input := rapid.String().Draw(t, "input")
+			a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
+			if a == b {
+				t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
+			}
+		})
+	}
+}
+
+func TestGlobDefaultFilters(t *testing.T) {
+	for _, filter := range []*Filter{nil, {}} {
+		rapid.Check(t, func(t *rapid.T) {
+			if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
+				t.Fatalf("filter %#v did not include input", filter)
+			}
+		})
+	}
+}