Skip to content

Commit

Permalink
Added file lazy loading
Browse files Browse the repository at this point in the history
  • Loading branch information
terricain committed May 11, 2022
1 parent b556d85 commit 4a3acbe
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 23 deletions.
56 changes: 37 additions & 19 deletions pkg/document/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,31 @@ import (
"strings"

"github.com/gobwas/glob"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)

// Near identical to https://github.com/helm/helm/blob/main/pkg/engine/files.go as to preserve the interface.

type files map[string][]byte
type fileEntry struct {
Path string
data []byte
}

func (f *fileEntry) GetData() []byte {
if f.data == nil {
data, err := ioutil.ReadFile(f.Path)
if err != nil {
log.Warnf("Error reading file contents for %s: %s", f.Path, err.Error())
return []byte{}
}
f.data = data
}

return f.data
}

type files map[string]*fileEntry

func getFiles(dir string) (files, error) {
result := make(files)
Expand All @@ -28,25 +47,19 @@ func getFiles(dir string) (files, error) {
return nil
}

data, err := ioutil.ReadFile(path)
if err != nil {
return err
}

result[path] = data

result[path] = &fileEntry{Path: path}
return nil
})
if err != nil {
return map[string][]byte{}, err
return map[string]*fileEntry{}, err
}

return result, nil
}

func (f files) GetBytes(name string) []byte {
if v, ok := f[name]; ok {
return v
return v.GetData()
}
return []byte{}
}
Expand All @@ -56,15 +69,16 @@ func (f files) Get(name string) string {
}

func (f files) Glob(pattern string) files {
result := make(files)
g, err := glob.Compile(pattern, '/')
if err != nil {
g, _ = glob.Compile("**")
log.Warnf("Error compiling Glob patten %s: %s", pattern, err.Error())
return result
}

result := make(files)
for name, contents := range f {
if g.Match(name) {
result[name] = contents
for filePath, entry := range f {
if g.Match(filePath) {
result[filePath] = entry
}
}

Expand All @@ -80,7 +94,7 @@ func (f files) AsConfig() string {

// Explicitly convert to strings, and file names
for k, v := range f {
m[path.Base(k)] = string(v)
m[path.Base(k)] = string(v.GetData())
}

return toYAML(m)
Expand All @@ -94,18 +108,22 @@ func (f files) AsSecrets() string {
m := make(map[string]string)

for k, v := range f {
m[path.Base(k)] = base64.StdEncoding.EncodeToString(v)
m[path.Base(k)] = base64.StdEncoding.EncodeToString(v.GetData())
}

return toYAML(m)
}

func (f files) Lines(path string) []string {
if f == nil || f[path] == nil {
if f == nil {
return []string{}
}
entry, exists := f[path]
if !exists {
return []string{}
}

return strings.Split(string(f[path]), "\n")
return strings.Split(string(entry.GetData()), "\n")
}

func toYAML(v interface{}) string {
Expand Down
15 changes: 11 additions & 4 deletions pkg/document/files_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,17 @@ var cases = []struct {
func getTestFiles() files {
a := make(files, len(cases))
for _, c := range cases {
a[c.path] = []byte(c.data)
a[c.path] = &fileEntry{
Path: c.path,
data: []byte(c.data),
}
}
return a
}

func TestNewFiles(t *testing.T) {
files := getTestFiles()

if len(files) != len(cases) {
t.Errorf("Expected len() = %d, got %d", len(cases), len(files))
}
Expand Down Expand Up @@ -97,14 +101,15 @@ func TestGetFiles(t *testing.T) {
})

testFiles := getTestFiles()
for filePath, fileData := range testFiles {
for filePath, entry := range testFiles {
fullPath := path.Join(chartDir, filePath)
baseDir := path.Dir(fullPath)
if err = os.MkdirAll(baseDir, 0o755); err != nil {
t.Fatal(err)
}
data := entry.GetData()

if err = os.WriteFile(fullPath, fileData, 0o644); err != nil {
if err = os.WriteFile(fullPath, data, 0o644); err != nil {
t.Fatal(err)
}
}
Expand All @@ -119,7 +124,9 @@ func TestGetFiles(t *testing.T) {
}

// Sanity check the files have been read
for filePath, data := range chartFiles {
for filePath, entry := range chartFiles {
data := entry.GetData()

if len(data) == 0 {
t.Errorf("%s: expected file contents, got 0 bytes", filePath)
}
Expand Down

0 comments on commit 4a3acbe

Please sign in to comment.