diff --git a/filter.go b/filter.go new file mode 100644 index 0000000..6b90f1f --- /dev/null +++ b/filter.go @@ -0,0 +1,49 @@ +package files + +import ( + "os" + + ignore "github.com/crackcomm/go-gitignore" +) + +// Filter represents a set of rules for determining if a file should be included or excluded. +// A rule follows the syntax for patterns used in .gitgnore files for specifying untracked files. +// Examples: +// foo.txt +// *.app +// bar/ +// **/baz +// fizz/** +type Filter struct { + // IncludeHidden - Include hidden files + IncludeHidden bool + // Rules - File filter rules + Rules *ignore.GitIgnore +} + +// NewFilter creates a new file filter from a .gitignore file and/or a list of ignore rules. +// An ignoreFile is a path to a file with .gitignore-style patterns to exclude, one per line +// rules is an array of strings representing .gitignore-style patterns +// For reference on ignore rule syntax, see https://git-scm.com/docs/gitignore +func NewFilter(ignoreFile string, rules []string, includeHidden bool) (*Filter, error) { + var ignoreRules *ignore.GitIgnore + var err error + if ignoreFile == "" { + ignoreRules, err = ignore.CompileIgnoreLines(rules...) + } else { + ignoreRules, err = ignore.CompileIgnoreFileAndLines(ignoreFile, rules...) + } + if err != nil { + return nil, err + } + return &Filter{IncludeHidden: includeHidden, Rules: ignoreRules}, nil +} + +// ShouldExclude takes an os.FileInfo object and applies rules to determine if its target should be excluded. +func (filter *Filter) ShouldExclude(fileInfo os.FileInfo) (result bool) { + path := fileInfo.Name() + if !filter.IncludeHidden && isHidden(fileInfo) { + return true + } + return filter.Rules.MatchesPath(path) +} diff --git a/filter_test.go b/filter_test.go new file mode 100644 index 0000000..d33b114 --- /dev/null +++ b/filter_test.go @@ -0,0 +1,50 @@ +package files + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +type mockFileInfo struct { + os.FileInfo + name string +} + +func (m *mockFileInfo) Name() string { + return m.name +} + +var _ os.FileInfo = &mockFileInfo{} + +func TestFileFilter(t *testing.T) { + includeHidden := true + filter, err := NewFilter("", nil, includeHidden) + if err != nil { + t.Errorf("failed to create filter with empty rules") + } + if filter.IncludeHidden != includeHidden { + t.Errorf("new filter should include hidden files") + } + _, err = NewFilter("ignoreFileThatDoesNotExist", nil, false) + if err == nil { + t.Errorf("creating a filter without an invalid ignore file path should have failed") + } + tmppath, err := ioutil.TempDir("", "filter-test") + if err != nil { + t.Fatal(err) + } + ignoreFilePath := filepath.Join(tmppath, "ignoreFile") + ignoreFileContents := []byte("a.txt") + if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil { + t.Fatal(err) + } + filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false) + if err != nil { + t.Errorf("failed to create filter with ignore file") + } + if !filterWithIgnoreFile.ShouldExclude(&mockFileInfo{name: "a.txt"}) { + t.Errorf("filter should've excluded expected file from ignoreFile: %s", "a.txt") + } +} diff --git a/go.mod b/go.mod index a8fd75e..3fd9331 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,8 @@ module github.com/ipfs/go-ipfs-files -require golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 +require ( + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 + golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 +) go 1.12 diff --git a/go.sum b/go.sum index 55a8c3a..b177827 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsGMrAv2Q2jyCpi7DmfpQ= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/serialfile.go b/serialfile.go index cd60160..f56fcc6 100644 --- a/serialfile.go +++ b/serialfile.go @@ -11,16 +11,16 @@ import ( // serialFile implements Node, and reads from a path on the OS filesystem. // No more than one file will be opened at a time. type serialFile struct { - path string - files []os.FileInfo - stat os.FileInfo - handleHiddenFiles bool + path string + files []os.FileInfo + stat os.FileInfo + filter *Filter } type serialIterator struct { - files []os.FileInfo - handleHiddenFiles bool - path string + files []os.FileInfo + path string + filter *Filter curName string curFile Node @@ -28,8 +28,20 @@ type serialIterator struct { err error } -// TODO: test/document limitations -func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { +// NewSerialFile takes a filepath, a bool specifying if hidden files should be included, +// and a fileInfo and returns a Node representing file, directory or special file. +func NewSerialFile(path string, includeHidden bool, stat os.FileInfo) (Node, error) { + filter, err := NewFilter("", nil, includeHidden) + if err != nil { + return nil, err + } + return NewSerialFileWithFilter(path, filter, stat) +} + +// NewSerialFileWith takes a filepath, a filter for determining which files should be +// operated upon if the filepath is a directory, and a fileInfo and returns a +// Node representing file, directory or special file. +func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Node, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) @@ -44,7 +56,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { if err != nil { return nil, err } - return &serialFile{path, contents, stat, hidden}, nil + return &serialFile{path, contents, stat, filter}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { @@ -72,7 +84,7 @@ func (it *serialIterator) Next() bool { stat := it.files[0] it.files = it.files[1:] - for !it.handleHiddenFiles && isHidden(stat) { + for it.filter.ShouldExclude(stat) { if len(it.files) == 0 { return false } @@ -87,7 +99,7 @@ func (it *serialIterator) Next() bool { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat) + sf, err := NewSerialFileWithFilter(filePath, it.filter, stat) if err != nil { it.err = err return false @@ -104,9 +116,9 @@ func (it *serialIterator) Err() error { func (f *serialFile) Entries() DirIterator { return &serialIterator{ - path: f.path, - files: f.files, - handleHiddenFiles: f.handleHiddenFiles, + path: f.path, + files: f.files, + filter: f.filter, } } diff --git a/serialfile_test.go b/serialfile_test.go index 748ba16..edd5bb9 100644 --- a/serialfile_test.go +++ b/serialfile_test.go @@ -5,27 +5,30 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strings" "testing" ) -func isPathHidden(p string) bool { +func isFullPathHidden(p string) bool { return strings.HasPrefix(p, ".") || strings.Contains(p, "/.") } func TestSerialFile(t *testing.T) { - t.Run("Hidden", func(t *testing.T) { testSerialFile(t, true) }) - t.Run("NotHidden", func(t *testing.T) { testSerialFile(t, false) }) + t.Run("Hidden/NoFilter", func(t *testing.T) { testSerialFile(t, true, false) }) + t.Run("Hidden/Filter", func(t *testing.T) { testSerialFile(t, true, true) }) + t.Run("NotHidden/NoFilter", func(t *testing.T) { testSerialFile(t, false, false) }) + t.Run("NotHidden/Filter", func(t *testing.T) { testSerialFile(t, false, true) }) } -func testSerialFile(t *testing.T, hidden bool) { +func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) { tmppath, err := ioutil.TempDir("", "files-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmppath) - expected := map[string]string{ + testInputs := map[string]string{ "1": "Some text!\n", "2": "beep", "3": "", @@ -38,8 +41,18 @@ func testSerialFile(t *testing.T, hidden bool) { ".8": "", ".8/foo": "bla", } + fileFilter, err := NewFilter("", []string{"9", "10"}, hidden) + if err != nil { + t.Fatal(err) + } + if withIgnoreRules { + testInputs["9"] = "" + testInputs["9/b"] = "bebop" + testInputs["10"] = "" + testInputs["10/.c"] = "doowop" + } - for p, c := range expected { + for p, c := range testInputs { path := filepath.Join(tmppath, p) if c != "" { continue @@ -49,7 +62,7 @@ func testSerialFile(t *testing.T, hidden bool) { } } - for p, c := range expected { + for p, c := range testInputs { path := filepath.Join(tmppath, p) if c == "" { continue @@ -58,6 +71,22 @@ func testSerialFile(t *testing.T, hidden bool) { t.Fatal(err) } } + expectedHiddenPaths := make([]string, 0, 4) + expectedRegularPaths := make([]string, 0, 6) + for p := range testInputs { + path := filepath.Join(tmppath, p) + stat, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if !fileFilter.ShouldExclude(stat) { + if isFullPathHidden(path) { + expectedHiddenPaths = append(expectedHiddenPaths, p) + } else { + expectedRegularPaths = append(expectedRegularPaths, p) + } + } + } stat, err := os.Stat(tmppath) if err != nil { @@ -65,12 +94,17 @@ func testSerialFile(t *testing.T, hidden bool) { } sf, err := NewSerialFile(tmppath, hidden, stat) + if withIgnoreRules { + sf, err = NewSerialFileWithFilter(tmppath, fileFilter, stat) + } if err != nil { t.Fatal(err) } defer sf.Close() rootFound := false + actualRegularPaths := make([]string, 0, len(expectedRegularPaths)) + actualHiddenPaths := make([]string, 0, len(expectedHiddenPaths)) err = Walk(sf, func(path string, nd Node) error { defer nd.Close() @@ -85,16 +119,23 @@ func testSerialFile(t *testing.T, hidden bool) { rootFound = true return nil } - - if !hidden && isPathHidden(path) { + if isFullPathHidden(path) { + actualHiddenPaths = append(actualHiddenPaths, path) + } else { + actualRegularPaths = append(actualRegularPaths, path) + } + if !hidden && isFullPathHidden(path) { return fmt.Errorf("found a hidden file") } + if fileFilter.Rules.MatchesPath(path) { + return fmt.Errorf("found a file that should be excluded") + } - data, ok := expected[path] + data, ok := testInputs[path] if !ok { return fmt.Errorf("expected something at %q", path) } - delete(expected, path) + delete(testInputs, path) switch nd := nd.(type) { case *Symlink: @@ -117,10 +158,16 @@ func testSerialFile(t *testing.T, hidden bool) { if !rootFound { t.Fatal("didn't find the root") } - for p := range expected { - if !hidden && isPathHidden(p) { - continue + for _, regular := range expectedRegularPaths { + if idx := sort.SearchStrings(actualRegularPaths, regular); idx < 0 { + t.Errorf("missed regular path %q", regular) + } + } + if hidden && len(actualHiddenPaths) != len(expectedHiddenPaths) { + for _, missing := range expectedHiddenPaths { + if idx := sort.SearchStrings(actualHiddenPaths, missing); idx < 0 { + t.Errorf("missed hidden path %q", missing) + } } - t.Errorf("missed %q", p) } }