Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for virtual files in mapfs #3654

Merged
merged 1 commit into from
Feb 20, 2023
Merged

feat: add support for virtual files in mapfs #3654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 96 additions & 1 deletion pkg/mapfs/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package mapfs

import (
"io"
"io/fs"
"os"
"path/filepath"
Expand All @@ -15,17 +16,27 @@ import (

var separator = "/"

// file represents one of them:
// - an actual file
// - a virtual file
// - a virtual dir
type file struct {
path string // underlying file path
data []byte // virtual file, only either of 'path' or 'data' has a value.
stat fileStat
files syncx.Map[string, *file]
}

func (f *file) isVirtual() bool {
return len(f.data) != 0 || f.stat.IsDir()
}

func (f *file) Open(name string) (fs.File, error) {
if name == "" || name == "." {
return f.open()
}

// TODO: support directory
if sub, err := f.getFile(name); err == nil && !sub.stat.IsDir() {
return sub.open()
}
Expand All @@ -38,6 +49,15 @@ func (f *file) Open(name string) (fs.File, error) {
}

func (f *file) open() (fs.File, error) {
// virtual file
if len(f.data) != 0 {
return &openMapFile{
path: f.stat.name,
file: f,
offset: 0,
}, nil
}
// real file
return os.Open(f.path)
}

Expand Down Expand Up @@ -107,7 +127,7 @@ func (f *file) ReadDir(name string) ([]fs.DirEntry, error) {
var entries []fs.DirEntry
var err error
f.files.Range(func(name string, value *file) bool {
if value.stat.IsDir() {
if value.isVirtual() {
entries = append(entries, &value.stat)
} else {
var fi os.FileInfo
Expand Down Expand Up @@ -192,6 +212,30 @@ func (f *file) WriteFile(path, underlyingPath string) error {
return dir.WriteFile(strings.Join(parts[1:], separator), underlyingPath)
}

func (f *file) WriteVirtualFile(path string, data []byte, mode fs.FileMode) error {
parts := strings.Split(path, separator)

if len(parts) == 1 {
f.files.Store(parts[0], &file{
data: data,
stat: fileStat{
name: parts[0],
size: int64(len(data)),
mode: mode,
modTime: time.Now(),
},
})
return nil
}

dir, ok := f.files.Load(parts[0])
if !ok || !dir.stat.IsDir() {
return fs.ErrNotExist
}

return dir.WriteVirtualFile(strings.Join(parts[1:], separator), data, mode)
}

func (f *file) glob(pattern string) ([]string, error) {
var entries []string
parts := strings.Split(pattern, separator)
Expand Down Expand Up @@ -225,3 +269,54 @@ func (f *file) glob(pattern string) ([]string, error) {
sort.Strings(entries)
return entries, nil
}

// An openMapFile is a regular (non-directory) fs.File open for reading.
// ported from https://github.com/golang/go/blob/99bc53f5e819c2d2d49f2a56c488898085be3982/src/testing/fstest/mapfs.go
type openMapFile struct {
path string
*file
offset int64
}

func (f *openMapFile) Stat() (fs.FileInfo, error) { return &f.file.stat, nil }

func (f *openMapFile) Close() error { return nil }

func (f *openMapFile) Read(b []byte) (int, error) {
if f.offset >= int64(len(f.file.data)) {
return 0, io.EOF
}
if f.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid}
}
n := copy(b, f.file.data[f.offset:])
f.offset += int64(n)
return n, nil
}

func (f *openMapFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case 0:
// offset += 0
case 1:
offset += f.offset
case 2:
offset += int64(len(f.file.data))
}
if offset < 0 || offset > int64(len(f.file.data)) {
return 0, &fs.PathError{Op: "seek", Path: f.path, Err: fs.ErrInvalid}
}
f.offset = offset
return offset, nil
}

func (f *openMapFile) ReadAt(b []byte, offset int64) (int, error) {
if offset < 0 || offset > int64(len(f.file.data)) {
return 0, &fs.PathError{Op: "read", Path: f.path, Err: fs.ErrInvalid}
}
n := copy(b, f.file.data[offset:])
if n < len(b) {
return n, io.EOF
}
return n, nil
}
9 changes: 7 additions & 2 deletions pkg/mapfs/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (m *FS) Stat(name string) (fs.FileInfo, error) {
Err: err,
}
}
if f.stat.IsDir() {
if f.isVirtual() {
return &f.stat, nil
}
return os.Stat(f.path)
Expand All @@ -105,11 +105,16 @@ func (m *FS) Open(name string) (fs.File, error) {
return m.root.Open(cleanPath(name))
}

// WriteFile writes the specified bytes to the named file. If the file exists, it will be overwritten.
// WriteFile creates a mapping between path and underlyingPath.
func (m *FS) WriteFile(path, underlyingPath string) error {
return m.root.WriteFile(cleanPath(path), underlyingPath)
}

// WriteVirtualFile writes the specified bytes to the named file. If the file exists, it will be overwritten.
func (m *FS) WriteVirtualFile(path string, data []byte, mode fs.FileMode) error {
return m.root.WriteVirtualFile(cleanPath(path), data, mode)
}

// MkdirAll creates a directory named path,
// along with any necessary parents, and returns nil,
// or else returns an error.
Expand Down
75 changes: 59 additions & 16 deletions pkg/mapfs/fs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,6 @@ import (
"github.com/aquasecurity/trivy/pkg/mapfs"
)

func initFS(t *testing.T) *mapfs.FS {
fsys := mapfs.New()
require.NoError(t, fsys.MkdirAll("a/b/c", 0700))
require.NoError(t, fsys.MkdirAll("a/b/empty", 0700))
require.NoError(t, fsys.WriteFile("hello.txt", "testdata/hello.txt"))
require.NoError(t, fsys.WriteFile("a/b/b.txt", "testdata/b.txt"))
require.NoError(t, fsys.WriteFile("a/b/c/c.txt", "testdata/c.txt"))
require.NoError(t, fsys.WriteFile("a/b/c/.dotfile", "testdata/dotfile"))
return fsys
}

type fileInfo struct {
name string
fileMode fs.FileMode
Expand All @@ -45,6 +34,12 @@ var (
isDir: false,
size: 3,
}
virtualFileInfo = fileInfo{
name: "virtual.txt",
fileMode: 0600,
isDir: false,
size: 7,
}
cdirFileInfo = fileInfo{
name: "c",
fileMode: fs.FileMode(0700) | fs.ModeDir,
Expand All @@ -53,6 +48,18 @@ var (
}
)

func initFS(t *testing.T) *mapfs.FS {
fsys := mapfs.New()
require.NoError(t, fsys.MkdirAll("a/b/c", 0700))
require.NoError(t, fsys.MkdirAll("a/b/empty", 0700))
require.NoError(t, fsys.WriteFile("hello.txt", "testdata/hello.txt"))
require.NoError(t, fsys.WriteFile("a/b/b.txt", "testdata/b.txt"))
require.NoError(t, fsys.WriteFile("a/b/c/c.txt", "testdata/c.txt"))
require.NoError(t, fsys.WriteFile("a/b/c/.dotfile", "testdata/dotfile"))
require.NoError(t, fsys.WriteVirtualFile("a/b/c/virtual.txt", []byte("virtual"), 0600))
return fsys
}

func assertFileInfo(t *testing.T, want fileInfo, got fs.FileInfo) {
if got == nil {
return
Expand Down Expand Up @@ -95,7 +102,7 @@ func TestFS_Stat(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{
name: "regular file",
name: "ordinary file",
filePath: "hello.txt",
want: helloFileInfo,
wantErr: assert.NoError,
Expand All @@ -106,6 +113,12 @@ func TestFS_Stat(t *testing.T) {
want: btxtFileInfo,
wantErr: assert.NoError,
},
{
name: "virtual file",
filePath: "a/b/c/virtual.txt",
want: virtualFileInfo,
wantErr: assert.NoError,
},
{
name: "dir",
filePath: "a/b/c",
Expand Down Expand Up @@ -198,6 +211,13 @@ func TestFS_ReadDir(t *testing.T) {
size: 0,
},
},
{
name: "virtual.txt",
fileMode: 0600,
isDir: false,
size: 0,
fileInfo: virtualFileInfo,
},
},
wantErr: assert.NoError,
},
Expand Down Expand Up @@ -241,14 +261,23 @@ func TestFS_Open(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{
name: "regular file",
name: "ordinary file",
filePath: "hello.txt",
want: file{
fileInfo: helloFileInfo,
body: "hello world",
},
wantErr: assert.NoError,
},
{
name: "virtual file",
filePath: "a/b/c/virtual.txt",
want: file{
fileInfo: virtualFileInfo,
body: "virtual",
},
wantErr: assert.NoError,
},
{
name: "dir",
filePath: "a/b/c",
Expand Down Expand Up @@ -292,11 +321,17 @@ func TestFS_ReadFile(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{
name: "regular file",
name: "ordinary file",
filePath: "hello.txt",
want: "hello world",
wantErr: assert.NoError,
},
{
name: "virtual file",
filePath: "a/b/c/virtual.txt",
want: "virtual",
wantErr: assert.NoError,
},
{
name: "no such file",
filePath: "nosuch.txt",
Expand Down Expand Up @@ -345,6 +380,7 @@ func TestFS_Glob(t *testing.T) {
pattern: "*/b/c/*.txt",
want: []string{
"a/b/c/c.txt",
"a/b/c/virtual.txt",
},
wantErr: assert.NoError,
},
Expand Down Expand Up @@ -372,7 +408,7 @@ func TestFS_Remove(t *testing.T) {
wantErr assert.ErrorAssertionFunc
}{
{
name: "regular file",
name: "ordinary file",
path: "hello.txt",
wantErr: assert.NoError,
},
Expand All @@ -381,6 +417,11 @@ func TestFS_Remove(t *testing.T) {
path: "a/b/b.txt",
wantErr: assert.NoError,
},
{
name: "virtual file",
path: "a/b/c/virtual.txt",
wantErr: assert.NoError,
},
{
name: "empty dir",
path: "a/b/empty",
Expand Down Expand Up @@ -415,7 +456,7 @@ func TestFS_Remove(t *testing.T) {

func TestFS_RemoveAll(t *testing.T) {
fsys := initFS(t)
t.Run("regular file", func(t *testing.T) {
t.Run("ordinary file", func(t *testing.T) {
err := fsys.RemoveAll("hello.txt")
require.NoError(t, err)
_, err = fsys.Stat("hello.txt")
Expand All @@ -428,5 +469,7 @@ func TestFS_RemoveAll(t *testing.T) {
require.ErrorIs(t, err, fs.ErrNotExist)
_, err = fsys.Stat("a/b/c/.dotfile")
require.ErrorIs(t, err, fs.ErrNotExist)
_, err = fsys.Stat("a/b/c/virtual.txt")
require.ErrorIs(t, err, fs.ErrNotExist)
})
}