Feature: unified include/exclude file (#224)

* refactor: Simplify config/flag/default behavior.

The new `pickFirst()` function returns the first non-empty string, making the
code simpler more readable.

* feat(pattern_file): Add the `pattern_file` config option and flag.

Issue: #223

* test(pattern_file): Add integration test for the `pattern_file` option.

* chore(pattern_file): Remove the `pattern_file` option and use `include` instead.

* fix: Allow the `doublestar` option to overwrite the default match type.

* refactor: Rename `PatternFile` to `PatternFileCollector`.

* refactor: Use pointer receiver for `PatternFileCollector`.

* fix: Copy loop variable for Go 1.18.

* docs(match_type): Document the `match_type` option and the `gitignore` mode.

* docs: Improve markup.

Co-authored-by: Braydon Kains <[email protected]>

* chore: Regenerate integration test outputs.

```shell
make integrationtest_update
```

---------

Co-authored-by: Braydon Kains <[email protected]>

cmd/yamlfmt/config.go

@@ -27,6 +27,7 @@ import (
 	"github.com/braydonk/yaml"
 	"github.com/google/yamlfmt"
 	"github.com/google/yamlfmt/command"
+	"github.com/google/yamlfmt/engine"
 	"github.com/google/yamlfmt/internal/collections"
 	"github.com/google/yamlfmt/internal/logger"
 	"github.com/mitchellh/mapstructure"
@@ -278,12 +279,14 @@ func makeCommandConfigFromData(configData map[string]any) (*command.Config, erro
 	if !config.GitignoreExcludes {
 		config.GitignoreExcludes = *flagGitignoreExcludes
 	}
-	if config.GitignorePath == "" {
-		config.GitignorePath = *flagGitignorePath
-	}
-	if config.OutputFormat == "" {
-		config.OutputFormat = getOutputFormatFromFlag()
+	config.GitignorePath = pickFirst(config.GitignorePath, *flagGitignorePath)
+	config.OutputFormat = pickFirst(config.OutputFormat, getOutputFormatFromFlag(), engine.EngineOutputDefault)
+
+	defaultMatchType := yamlfmt.MatchTypeStandard
+	if config.Doublestar {
+		defaultMatchType = yamlfmt.MatchTypeDoublestar
 	}
+	config.MatchType = pickFirst(config.MatchType, yamlfmt.MatchType(*flagMatchType), defaultMatchType)
 
 	// Overwrite config if includes are provided through args
 	if len(flag.Args()) > 0 {
@@ -297,6 +300,17 @@ func makeCommandConfigFromData(configData map[string]any) (*command.Config, erro
 	return &config, nil
 }
 
+// pickFirst returns the first string in ss that is not empty.
+func pickFirst[T ~string](ss ...T) T {
+	for _, s := range ss {
+		if s != "" {
+			return s
+		}
+	}
+
+	return ""
+}
+
 func parseFormatterConfigFlag(flagValues []string) (map[string]any, error) {
 	formatterValues := map[string]any{}
 	flagErrors := collections.Errors{}

cmd/yamlfmt/flags.go

@@ -41,6 +41,7 @@ operation without performing it.`)
 	flagGitignoreExcludes *bool   = flag.Bool("gitignore_excludes", false, "Use a gitignore file for excludes")
 	flagGitignorePath     *string = flag.String("gitignore_path", ".gitignore", "Path to gitignore file to use")
 	flagOutputFormat      *string = flag.String("output_format", "default", "The engine output format")
+	flagMatchType         *string = flag.String("match_type", "", "The file discovery method to use. Valid values: standard, doublestar, gitignore")
 	flagExclude                   = arrayFlag{}
 	flagFormatter                 = arrayFlag{}
 	flagExtensions                = arrayFlag{}

command/command.go

@@ -40,6 +40,7 @@ func NewFormatterConfig() *FormatterConfig {
 
 type Config struct {
 	Extensions        []string                  `mapstructure:"extensions"`
+	MatchType         yamlfmt.MatchType         `mapstructure:"match_type"`
 	Include           []string                  `mapstructure:"include"`
 	Exclude           []string                  `mapstructure:"exclude"`
 	RegexExclude      []string                  `mapstructure:"regex_exclude"`
@@ -190,7 +191,11 @@ func (c *Command) getFormatter() (yamlfmt.Formatter, error) {
 }
 
 func (c *Command) collectPaths() ([]string, error) {
-	collector := c.makePathCollector()
+	collector, err := c.makePathCollector()
+	if err != nil {
+		return nil, err
+	}
+
 	return collector.CollectPaths()
 }
 
@@ -203,17 +208,31 @@ func (c *Command) analyzePaths(paths []string) ([]string, error) {
 	return includePaths, err
 }
 
-func (c *Command) makePathCollector() yamlfmt.PathCollector {
-	if c.Config.Doublestar {
+func (c *Command) makePathCollector() (yamlfmt.PathCollector, error) {
+	switch c.Config.MatchType {
+	case yamlfmt.MatchTypeDoublestar:
 		return &yamlfmt.DoublestarCollector{
 			Include: c.Config.Include,
 			Exclude: c.Config.Exclude,
+		}, nil
+	case yamlfmt.MatchTypeGitignore:
+		files := c.Config.Include
+		if len(files) == 0 {
+			files = []string{yamlfmt.DefaultPatternFile}
 		}
-	}
-	return &yamlfmt.FilepathCollector{
-		Include:    c.Config.Include,
-		Exclude:    c.Config.Exclude,
-		Extensions: c.Config.Extensions,
+
+		patternFile, err := yamlfmt.NewPatternFileCollector(files...)
+		if err != nil {
+			return nil, fmt.Errorf("NewPatternFile(%q): %w", files, err)
+		}
+
+		return patternFile, nil
+	default:
+		return &yamlfmt.FilepathCollector{
+			Include:    c.Config.Include,
+			Exclude:    c.Config.Exclude,
+			Extensions: c.Config.Extensions,
+		}, nil
 	}
 }
 

docs/command-usage.md

@@ -81,6 +81,7 @@ The string array flags can be a bit confusing. See the [String Array Flags](#str
 | Global Config         | `-global_conf`        | bool              | `yamlfmt -global_conf`                                    | Force yamlfmt to use the configuration file from the system config directory. |
 | Disable Global Config | `-no_global_conf`     | bool              | `yamlfmt -no_global_conf`                                 | Disable looking for the configuration file from the system config directory. |
 | Doublestar            | `-dstar`              | bool              | `yamlfmt -dstar "**/*.yaml"`                              | Enable [Doublestar](./paths.md#doublestar) path collection mode. Note that doublestar patterns should be specified with quotes in bash to prevent shell expansion. |
+| Match type            | `-match_type`         | string            | `yamlfmt -match_type standard`                            | Controls how `include` and `exclude` are interpreted. See [Specifying Paths](./paths.md) for more details. |
 | Exclude               | `-exclude`            | []string          | `yamlfmt -exclude ./not/,these_paths.yaml`                | Patterns to exclude from path collection. These are in addition to the exclude patterns specified in the [config file](./config-file.md) |
 | Gitignore Excludes    | `-gitignore_excludes` | bool              | `yamlfmt -gitignore_excludes`                             | Use a gitignore file to exclude paths. This is in addition to otherwise specified exclude patterns. |
 | Gitignore Path        | `-gitignore_path`     | string            | `yamlfmt -gitignore_path .special_gitignore`              | Specify a path to a gitignore file to use. Defaults to `.gitignore` (in working directory). |

docs/config-file.md

@@ -39,6 +39,7 @@ The command package defines the main command engine that `cmd/yamlfmt` uses. It
 | `line_ending`            | `lf` or `crlf` | `crlf` on Windows, `lf` otherwise | Parse and write the file with "lf" or "crlf" line endings. This global setting will override any formatter `line_ending` options. |
 | `doublestar`             | bool                | false         | Use [doublestar](https://github.com/bmatcuk/doublestar) for include and exclude paths. (This was the default before 0.7.0) |
 | `continue_on_error`      | bool                | false         | Continue formatting and don't exit with code 1 when there is an invalid yaml file found. |
+| `match_type`             | string              | `standard`    | Controls how `include` and `exclude` are interpreted. See [Specifying Paths][] for more details. |
 | `include`                | []string            | []            | The paths for the command to include for formatting. See [Specifying Paths][] for more details. |
 | `exclude`                | []string            | []            | The paths for the command to exclude from formatting. See [Specifying Paths][] for more details. |
 | `gitignore_excludes`     | bool                | false         | Use gitignore files for exclude paths. This is in addition to the patterns from the `exclude` option. |

docs/paths.md

@@ -1,6 +1,6 @@
 # Paths
 
-`yamlfmt` can collect paths in two modes: Standard, or Doublestar.
+`yamlfmt` can collect paths in three modes: Standard, Doublestar, and Gitignore. The `match_type` option allows you to select between the modes, using the values `standard`, `doublestar`, and `gitignore`.
 
 ## Standard (default)
 
@@ -12,14 +12,41 @@ This mode does *not* support wildcards, aka. globbing. That means with `*.yaml`
 
 In Doublestar mode, paths are specified using wildcard patterns explained in the [doublestar](https://github.com/bmatcuk/doublestar) package. It is almost identical to bash and git's style of glob pattern specification.
 
-To enable the doublestar mode, set `doublestar: true` in the config file or use the `-dstar` command line flag.
+To enable the doublestar mode, set `match_type: doublestar` in the config file or use the `-match_type doublestar` command line flag.
+
+## Gitignore
+
+This mode allows you to use a file (or files) using the [gitignore](https://git-scm.com/docs/gitignore) syntax to determine which files to include and exclude.
+
+Despite having "ignore" in the name, yamlfmt will format all files that match the patterns listed in the file, unless the patterns are negated. For example, the following file will format all `*.yaml` and `*.yml` files, while ignoring other files such as `README.md` as well as files in `testdata` directories:
+
+```gitignore
+# Include
+*.yaml
+*.yml
+
+# Exclude
+!testdata/
+```
+
+Please read the [gitignore manpage](https://git-scm.com/docs/gitignore) for the full syntax description and further examples.
+
+To use the `gitignore` mode, set `match_type: gitignore` in the config file or use the `-match_type gitignore` command line flag.
+In this mode, positional arguments on the command line and files listed in the `include` config option are considered to be files using the gitignore syntax.
+If no file is specified, yamlfmt will look for a file called `yamlfmt.patterns` in the working directory.
+
+The `exclude` option is ignored in this mode.
+
 
 ## Include and Exclude
 
-In both modes, `yamlfmt` will allow you to configure include and exclude paths. These can be paths to files in Standard or Doublestar modes, paths to directories in Standard mode, and valid doublestar patterns in Doublestar mode. These paths should be specified **relative to the working directory of `yamlfmt`**. They will work as absolute paths if both the includes and excludes are specified as absolute paths or if both are relative paths, however it will not work as expected if they are mixed together. It usually easier to reason about includes and excludes when always specifying both as relative paths from the directory `yamlfmt` is going to be run in.
+In the `standard` and `doublestar` modes, `yamlfmt` will allow you to configure include and exclude paths. These can be paths to files in Standard or Doublestar modes, paths to directories in Standard mode, and valid doublestar patterns in Doublestar mode. These paths should be specified **relative to the working directory of `yamlfmt`**. They will work as absolute paths if both the includes and excludes are specified as absolute paths or if both are relative paths, however it will not work as expected if they are mixed together. It usually easier to reason about includes and excludes when always specifying both as relative paths from the directory `yamlfmt` is going to be run in.
+
+In the `gitignore` mode, files specified on the command line or via the `include` config option are expected to contain patterns following the [gitignore](https://git-scm.com/docs/gitignore) syntax.
 
 Exclude paths can be specified on the command line using the `-exclude` flag.
 Paths excluded from the command line are **added* to excluded paths from the config file.
+The `gitignore` mode ignores the `exclude` option.
 
 Include paths can be specified on the command line via the positional arguments, i.e. there is no flag for it.
 Paths from the command line take precedence over and **replace** any paths configured in the config file.

integrationtest/command/command_test.go

@@ -155,3 +155,11 @@ func TestGitLabOutput(t *testing.T) {
 		Update:  *updateFlag,
 	}.Run(t)
 }
+
+func TestPatternFile(t *testing.T) {
+	TestCase{
+		Dir:     "pattern_file",
+		Command: yamlfmtWithArgs("-match_type gitignore yamlfmt.patterns"),
+		Update:  *updateFlag,
+	}.Run(t)
+}

integrationtest/command/testdata/pattern_file/after/a.yaml

@@ -0,0 +1 @@
+formatted: true

integrationtest/command/testdata/pattern_file/after/b.yml

@@ -0,0 +1 @@
+formatted: true

integrationtest/command/testdata/pattern_file/after/excluded_dir/a.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/after/excluded_file.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/after/included_dir/a.yaml

@@ -0,0 +1 @@
+formatted: true

integrationtest/command/testdata/pattern_file/after/yamlfmt.patterns

@@ -0,0 +1,7 @@
+# vim: syntax=gitignore
+
+*.yaml
+*.yml
+
+!excluded_file.yaml
+!excluded_dir/

integrationtest/command/testdata/pattern_file/before/a.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/before/b.yml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/before/excluded_dir/a.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/before/excluded_file.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/before/included_dir/a.yaml

@@ -0,0 +1 @@
+formatted:  true

integrationtest/command/testdata/pattern_file/before/yamlfmt.patterns

@@ -0,0 +1,7 @@
+# vim: syntax=gitignore
+
+*.yaml
+*.yml
+
+!excluded_file.yaml
+!excluded_dir/

integrationtest/command/testdata/print_conf_file/stdout/stdout.txt

@@ -9,6 +9,7 @@ gitignore_excludes: false
 gitignore_path: .my_gitignore
 include: []
 line_ending: crlf
+match_type: doublestar
 output_format: default
 regex_exclude: []
 formatter:

integrationtest/command/testdata/print_conf_flags/stdout/stdout.txt

@@ -8,6 +8,7 @@ gitignore_excludes: false
 gitignore_path: .gitignore
 include: []
 line_ending: lf
+match_type: standard
 output_format: default
 regex_exclude: []
 formatter:

integrationtest/command/testdata/print_conf_flags_and_file/stdout/stdout.txt

@@ -9,6 +9,7 @@ gitignore_excludes: false
 gitignore_path: .my_gitignore
 include: []
 line_ending: crlf
+match_type: doublestar
 output_format: default
 regex_exclude: []
 formatter:

internal/tempfile/golden.go

@@ -43,7 +43,7 @@ func (g GoldenCtx) CompareGoldenFile(path string, gotContent []byte) error {
 	// If we are not updating, check that the content is the same.
 	expectedContent, err := os.ReadFile(g.goldenPath(path))
 	if err != nil {
-		return err
+		return fmt.Errorf("os.ReadFile(%q): %w", g.goldenPath(path), err)
 	}
 	// Edge case for empty stdout.
 	if gotContent == nil {
@@ -84,11 +84,11 @@ func (g GoldenCtx) CompareDirectory(resultPath string) error {
 	for path := range resultPaths {
 		gotContent, err := os.ReadFile(filepath.Join(resultPath, path))
 		if err != nil {
-			return fmt.Errorf("%s: %w", path, err)
+			return fmt.Errorf("os.ReadFile(%q): %w", path, err)
 		}
 		err = g.CompareGoldenFile(path, gotContent)
 		if err != nil {
-			return fmt.Errorf("%s: %w", path, err)
+			return fmt.Errorf("CompareGoldenFile(%q): %w", path, err)
 		}
 	}
 	// If there are no errors this will be nil, otherwise will be a
@@ -118,7 +118,7 @@ func (g GoldenCtx) updateGoldenDirectory(resultPath string) error {
 func readAllPaths(dirPath string) (collections.Set[string], error) {
 	paths := collections.Set[string]{}
 	allNamesButCurrentDirectory := func(path string, d fs.DirEntry, err error) error {
-		if path == dirPath {
+		if d.IsDir() {
 			return nil
 		}
 		paths.Add(d.Name())