Skip to content

Commit

Permalink
Fix filecount plugin size tests (#6038)
Browse files Browse the repository at this point in the history
  • Loading branch information
nicgrobler authored and danielnelson committed Jun 24, 2019
1 parent aa84011 commit bd9ddd8
Show file tree
Hide file tree
Showing 4 changed files with 236 additions and 22 deletions.
11 changes: 7 additions & 4 deletions plugins/inputs/filecount/filecount.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ type FileCount struct {
MTime internal.Duration `toml:"mtime"`
fileFilters []fileFilterFunc
globPaths []globpath.GlobPath
Fs fileSystem
}

func (_ *FileCount) Description() string {
Expand Down Expand Up @@ -159,7 +160,7 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa
if err == nil && rel == "." {
return nil
}
file, err := os.Stat(path)
file, err := fc.Fs.Stat(path)
if err != nil {
if os.IsNotExist(err) {
return nil
Expand Down Expand Up @@ -244,18 +245,18 @@ func (fc *FileCount) Gather(acc telegraf.Accumulator) error {
}

for _, glob := range fc.globPaths {
for _, dir := range onlyDirectories(glob.GetRoots()) {
for _, dir := range fc.onlyDirectories(glob.GetRoots()) {
fc.count(acc, dir, glob)
}
}

return nil
}

func onlyDirectories(directories []string) []string {
func (fc *FileCount) onlyDirectories(directories []string) []string {
out := make([]string, 0)
for _, path := range directories {
info, err := os.Stat(path)
info, err := fc.Fs.Stat(path)
if err == nil && info.IsDir() {
out = append(out, path)
}
Expand Down Expand Up @@ -286,6 +287,7 @@ func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) {
fc.globPaths = append(fc.globPaths, *glob)
}
}

}

func NewFileCount() *FileCount {
Expand All @@ -298,6 +300,7 @@ func NewFileCount() *FileCount {
Size: internal.Size{Size: 0},
MTime: internal.Duration{Duration: 0},
fileFilters: nil,
Fs: osFS{},
}
}

Expand Down
84 changes: 66 additions & 18 deletions plugins/inputs/filecount/filecount_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package filecount

import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
Expand All @@ -18,7 +17,7 @@ func TestNoFilters(t *testing.T) {
matches := []string{"foo", "bar", "baz", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"subdir/nested2", "subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 9084)
fileCountEquals(t, fc, len(matches), 5096)
}

func TestNoFiltersOnChildDir(t *testing.T) {
Expand All @@ -30,9 +29,8 @@ func TestNoFiltersOnChildDir(t *testing.T) {
tags := map[string]string{"directory": getTestdataDir() + "/subdir"}
acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4542)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(600)))
}

func TestNoRecursiveButSuperMeta(t *testing.T) {
Expand All @@ -46,7 +44,7 @@ func TestNoRecursiveButSuperMeta(t *testing.T) {
acc.GatherError(fc.Gather)

require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(4096)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(200)))
}

func TestNameFilter(t *testing.T) {
Expand All @@ -60,20 +58,22 @@ func TestNonRecursive(t *testing.T) {
fc := getNoFilterFileCount()
fc.Recursive = false
matches := []string{"foo", "bar", "baz", "qux", "subdir"}
fileCountEquals(t, fc, len(matches), 4542)

fileCountEquals(t, fc, len(matches), 4496)
}

func TestDoubleAndSimpleStar(t *testing.T) {
fc := getNoFilterFileCount()
fc.Directories = []string{getTestdataDir() + "/**/*"}
matches := []string{"qux"}

tags := map[string]string{"directory": getTestdataDir() + "/subdir/nested2"}

acc := testutil.Accumulator{}
acc.GatherError(fc.Gather)

require.True(t, acc.HasPoint("filecount", tags, "count", int64(len(matches))))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(446)))
require.True(t, acc.HasPoint("filecount", tags, "size_bytes", int64(400)))
}

func TestRegularOnlyFilter(t *testing.T) {
Expand All @@ -82,7 +82,8 @@ func TestRegularOnlyFilter(t *testing.T) {
matches := []string{
"foo", "bar", "baz", "qux", "subdir/quux", "subdir/quuz",
"subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 892)

fileCountEquals(t, fc, len(matches), 800)
}

func TestSizeFilter(t *testing.T) {
Expand All @@ -94,23 +95,22 @@ func TestSizeFilter(t *testing.T) {

fc.Size = internal.Size{Size: 100}
matches = []string{"qux", "subdir/nested2//qux"}
fileCountEquals(t, fc, len(matches), 892)

fileCountEquals(t, fc, len(matches), 800)
}

func TestMTimeFilter(t *testing.T) {
oldFile := filepath.Join(getTestdataDir(), "baz")
mtime := time.Date(1979, time.December, 14, 18, 25, 5, 0, time.UTC)
if err := os.Chtimes(oldFile, mtime, mtime); err != nil {
t.Skip("skipping mtime filter test.")
}

mtime := time.Date(2011, time.December, 14, 18, 25, 5, 0, time.UTC)
fileAge := time.Since(mtime) - (60 * time.Second)

fc := getNoFilterFileCount()
fc.MTime = internal.Duration{Duration: -fileAge}
matches := []string{"foo", "bar", "qux",
"subdir/", "subdir/quux", "subdir/quuz",
"sbudir/nested2", "subdir/nested2/qux"}
fileCountEquals(t, fc, len(matches), 9084)
"subdir/nested2", "subdir/nested2/qux"}

fileCountEquals(t, fc, len(matches), 5096)

fc.MTime = internal.Duration{Duration: fileAge}
matches = []string{"baz"}
Expand All @@ -126,12 +126,60 @@ func getNoFilterFileCount() FileCount {
Size: internal.Size{Size: 0},
MTime: internal.Duration{Duration: 0},
fileFilters: nil,
Fs: getFakeFileSystem(getTestdataDir()),
}
}

func getTestdataDir() string {
_, filename, _, _ := runtime.Caller(1)
return strings.Replace(filename, "filecount_test.go", "testdata", 1)
dir, err := os.Getwd()
if err != nil {
// if we cannot even establish the test directory, further progress is meaningless
panic(err)
}

var chunks []string
var testDirectory string

if runtime.GOOS == "windows" {
chunks = strings.Split(dir, "\\")
testDirectory = strings.Join(chunks[:], "\\") + "\\testdata"
} else {
chunks = strings.Split(dir, "/")
testDirectory = strings.Join(chunks[:], "/") + "/testdata"
}
return testDirectory
}

func getFakeFileSystem(basePath string) fakeFileSystem {
// create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested

mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)
olderMtime := time.Date(2010, time.December, 14, 18, 25, 5, 0, time.UTC)

// set file permisions
var fmask uint32 = 0666
var dmask uint32 = 0666

// set directory bit
dmask |= (1 << uint(32-1))

// create a lookup map for getting "files" from the "filesystem"
fileList := map[string]fakeFileInfo{
basePath: {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime},
basePath + "/bar": {name: "bar", filemode: uint32(fmask), modtime: mtime},
basePath + "/baz": {name: "baz", filemode: uint32(fmask), modtime: olderMtime},
basePath + "/qux": {name: "qux", size: int64(400), filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir": {name: "subdir", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/subdir/quux": {name: "quux", filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir/quuz": {name: "quuz", filemode: uint32(fmask), modtime: mtime},
basePath + "/subdir/nested2": {name: "nested2", size: int64(200), filemode: uint32(dmask), modtime: mtime, isdir: true},
basePath + "/subdir/nested2/qux": {name: "qux", filemode: uint32(fmask), modtime: mtime, size: int64(400)},
}

fs := fakeFileSystem{files: fileList}
return fs

}

func fileCountEquals(t *testing.T, fc FileCount, expectedCount int, expectedSize int) {
Expand Down
73 changes: 73 additions & 0 deletions plugins/inputs/filecount/filesystem_helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package filecount

import (
"errors"
"io"
"os"
"time"
)

/*
The code below is lifted from numerous articles and originates from Andrew Gerrand's 10 things you (probably) don't know about Go.
it allows for mocking a filesystem; this allows for consistent testing of this code across platforms (directory sizes reported
differently by different platforms, for example), while preserving the rest of the functionality as-is, without modification.
*/

type fileSystem interface {
Open(name string) (file, error)
Stat(name string) (os.FileInfo, error)
}

type file interface {
io.Closer
io.Reader
io.ReaderAt
io.Seeker
Stat() (os.FileInfo, error)
}

// osFS implements fileSystem using the local disk
type osFS struct{}

func (osFS) Open(name string) (file, error) { return os.Open(name) }
func (osFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) }

/*
The following are for mocking the filesystem - this allows us to mock Stat() files. This means that we can set file attributes, and know that they
will be the same regardless of the platform sitting underneath our tests (directory sizes vary see https://github.com/influxdata/telegraf/issues/6011)
NOTE: still need the on-disk file structure to mirror this because the 3rd party library ("github.com/karrick/godirwalk") uses its own
walk functions, that we cannot mock from here.
*/

type fakeFileSystem struct {
files map[string]fakeFileInfo
}

type fakeFileInfo struct {
name string
size int64
filemode uint32
modtime time.Time
isdir bool
sys interface{}
}

func (f fakeFileInfo) Name() string { return f.name }
func (f fakeFileInfo) Size() int64 { return f.size }
func (f fakeFileInfo) Mode() os.FileMode { return os.FileMode(f.filemode) }
func (f fakeFileInfo) ModTime() time.Time { return f.modtime }
func (f fakeFileInfo) IsDir() bool { return f.isdir }
func (f fakeFileInfo) Sys() interface{} { return f.sys }

func (f fakeFileSystem) Open(name string) (file, error) {
return nil, &os.PathError{Op: "Open", Path: name, Err: errors.New("Not implemented by fake filesystem")}
}

func (f fakeFileSystem) Stat(name string) (os.FileInfo, error) {
if fakeInfo, found := f.files[name]; found {
return fakeInfo, nil
}
return nil, &os.PathError{Op: "Stat", Path: name, Err: errors.New("No such file or directory")}

}
90 changes: 90 additions & 0 deletions plugins/inputs/filecount/filesystem_helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package filecount

import (
"testing"
"time"

"github.com/stretchr/testify/require"
)

func TestMTime(t *testing.T) {
//this is the time our foo file should have
mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)

fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata/foo")
require.Nil(t, err)
require.Equal(t, mtime, fileInfo.ModTime())
}

func TestSize(t *testing.T) {
//this is the time our foo file should have
size := int64(4096)
fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata")
require.Nil(t, err)
require.Equal(t, size, fileInfo.Size())
}

func TestIsDir(t *testing.T) {
//this is the time our foo file should have
dir := true
fs := getTestFileSystem()
fileInfo, err := fs.Stat("/testdata")
require.Nil(t, err)
require.Equal(t, dir, fileInfo.IsDir())
}

func TestRealFS(t *testing.T) {
//test that the default (non-test) empty FS causes expected behaviour
var fs fileSystem = osFS{}
//the following file exists on disk - and not in our fake fs
fileInfo, err := fs.Stat(getTestdataDir() + "/qux")
require.Nil(t, err)
require.Equal(t, false, fileInfo.IsDir())
require.Equal(t, int64(446), fileInfo.Size())

// now swap out real, for fake filesystem
fs = getTestFileSystem()
// now, the same test as above will return an error as the file doesn't exist in our fake fs
expectedError := "Stat " + getTestdataDir() + "/qux: No such file or directory"
fileInfo, err = fs.Stat(getTestdataDir() + "/qux")
require.Equal(t, expectedError, err.Error())
// and verify that what we DO expect to find, we do
fileInfo, err = fs.Stat("/testdata/foo")
require.Nil(t, err)
}

func getTestFileSystem() fakeFileSystem {
/*
create our desired "filesystem" object, complete with an internal map allowing our funcs to return meta data as requested
type FileInfo interface {
Name() string // base name of the file
Size() int64 // length in bytes of file
Mode() FileMode // file mode bits
ModTime() time.Time // modification time
IsDir() bool // returns bool indicating if a Dir or not
Sys() interface{} // underlying data source. always nil (in this case)
}
*/

mtime := time.Date(2015, time.December, 14, 18, 25, 5, 0, time.UTC)

// set file permisions
var fmask uint32 = 0666
var dmask uint32 = 0666

// set directory bit
dmask |= (1 << uint(32-1))

fileList := map[string]fakeFileInfo{
"/testdata": {name: "testdata", size: int64(4096), filemode: uint32(dmask), modtime: mtime, isdir: true},
"/testdata/foo": {name: "foo", filemode: uint32(fmask), modtime: mtime},
}

fs := fakeFileSystem{files: fileList}
return fs

}

0 comments on commit bd9ddd8

Please sign in to comment.