forked from operator-framework/catalogd
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
closes operator-framework#113
Showing
6 changed files
with
387 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
package storage | ||
|
||
import ( | ||
"bytes" | ||
"compress/gzip" | ||
"context" | ||
"errors" | ||
"fmt" | ||
"io" | ||
"io/fs" | ||
"net/http" | ||
"net/url" | ||
"os" | ||
"path/filepath" | ||
|
||
"github.com/nlepage/go-tarfs" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
"github.com/operator-framework/catalogd/pkg/util" | ||
rukpak_storage "github.com/operator-framework/rukpak/pkg/storage" | ||
) | ||
|
||
var _ rukpak_storage.Storage = &LocalDirectory{} | ||
|
||
const DefaultFBCCacheDir = "/var/cache/catalogs" | ||
|
||
type LocalDirectory struct { | ||
RootDirectory string | ||
URL url.URL | ||
} | ||
|
||
func (s *LocalDirectory) Load(_ context.Context, owner client.Object) (fs.FS, error) { | ||
fbc, err := os.Open(s.fbcPath(owner.GetName())) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer fbc.Close() | ||
tarReader, err := gzip.NewReader(fbc) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return tarfs.New(tarReader) | ||
} | ||
|
||
func (s *LocalDirectory) Store(_ context.Context, owner client.Object, fbc fs.FS) error { | ||
buf := &bytes.Buffer{} | ||
if err := util.FSToTarGZ(buf, fbc); err != nil { | ||
return fmt.Errorf("convert fbc %q to tar.gz: %v", owner.GetName(), err) | ||
} | ||
|
||
fbcFile, err := os.Create(s.fbcPath(owner.GetName())) | ||
if err != nil { | ||
return err | ||
} | ||
defer fbcFile.Close() | ||
|
||
if _, err := io.Copy(fbcFile, buf); err != nil { | ||
return err | ||
} | ||
return nil | ||
} | ||
|
||
func (s *LocalDirectory) Delete(_ context.Context, owner client.Object) error { | ||
return ignoreNotExist(os.Remove(s.fbcPath(owner.GetName()))) | ||
} | ||
|
||
func (s *LocalDirectory) ServeHTTP(resp http.ResponseWriter, req *http.Request) { | ||
fsys := &util.FilesOnlyFilesystem{FS: os.DirFS(s.RootDirectory)} | ||
http.StripPrefix(s.URL.Path, http.FileServer(http.FS(fsys))).ServeHTTP(resp, req) | ||
} | ||
|
||
func (s *LocalDirectory) URLFor(_ context.Context, owner client.Object) (string, error) { | ||
return fmt.Sprintf("%s%s", s.URL.String(), localDirectoryCatalogFile(owner.GetName())), nil | ||
} | ||
|
||
func (s *LocalDirectory) fbcPath(catalogName string) string { | ||
return filepath.Join(s.RootDirectory, localDirectoryCatalogFile(catalogName)) | ||
} | ||
|
||
func localDirectoryCatalogFile(catalogName string) string { | ||
return fmt.Sprintf("%s.tgz", catalogName) | ||
} | ||
|
||
func ignoreNotExist(err error) error { | ||
if errors.Is(err, os.ErrNotExist) { | ||
return nil | ||
} | ||
return err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
package storage | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
"io/fs" | ||
"os" | ||
"path/filepath" | ||
"reflect" | ||
"testing/fstest" | ||
"time" | ||
|
||
. "github.com/onsi/ginkgo/v2" | ||
. "github.com/onsi/gomega" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/types" | ||
"k8s.io/apimachinery/pkg/util/rand" | ||
|
||
"github.com/operator-framework/catalogd/api/core/v1alpha1" | ||
) | ||
|
||
var _ = Describe("LocalDirectory Storage", func() { | ||
var ( | ||
ctx context.Context | ||
owner *v1alpha1.Catalog | ||
store LocalDirectory | ||
testFS fs.FS | ||
) | ||
|
||
BeforeEach(func() { | ||
ctx = context.Background() | ||
owner = &v1alpha1.Catalog{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: fmt.Sprintf("test-catalog-%s", rand.String(5)), | ||
UID: types.UID(rand.String(8)), | ||
}, | ||
} | ||
store = LocalDirectory{RootDirectory: GinkgoT().TempDir()} | ||
testFS = generateFS() | ||
}) | ||
When("a catalog is not stored", func() { | ||
Describe("Store", func() { | ||
It("should store a catalog FS", func() { | ||
Expect(store.Store(ctx, owner, testFS)).To(Succeed()) | ||
_, err := os.Stat(filepath.Join(store.RootDirectory, fmt.Sprintf("%s.tgz", owner.GetName()))) | ||
Expect(err).NotTo(HaveOccurred()) | ||
}) | ||
}) | ||
|
||
Describe("Load", func() { | ||
It("should fail due to file not existing", func() { | ||
_, err := store.Load(ctx, owner) | ||
Expect(err).To(WithTransform(func(err error) bool { return errors.Is(err, os.ErrNotExist) }, BeTrue())) | ||
}) | ||
}) | ||
|
||
Describe("Delete", func() { | ||
It("should succeed despite file not existing", func() { | ||
Expect(store.Delete(ctx, owner)).To(Succeed()) | ||
}) | ||
}) | ||
}) | ||
When("a catalog is stored", func() { | ||
BeforeEach(func() { | ||
Expect(store.Store(ctx, owner, testFS)).To(Succeed()) | ||
}) | ||
Describe("Store", func() { | ||
It("should re-store a catalog FS", func() { | ||
Expect(store.Store(ctx, owner, testFS)).To(Succeed()) | ||
}) | ||
}) | ||
|
||
Describe("Load", func() { | ||
It("should load the catalog", func() { | ||
loadedTestFS, err := store.Load(ctx, owner) | ||
Expect(err).NotTo(HaveOccurred()) | ||
Expect(fsEqual(testFS, loadedTestFS)).To(BeTrue()) | ||
}) | ||
}) | ||
|
||
Describe("Delete", func() { | ||
It("should delete the catalog", func() { | ||
Expect(store.Delete(ctx, owner)).To(Succeed()) | ||
_, err := os.Stat(filepath.Join(store.RootDirectory, fmt.Sprintf("%s.tgz", owner.GetName()))) | ||
Expect(err).To(WithTransform(func(err error) bool { return errors.Is(err, os.ErrNotExist) }, BeTrue())) | ||
}) | ||
}) | ||
}) | ||
}) | ||
|
||
func generateFS() fs.FS { | ||
gen := fstest.MapFS{} | ||
|
||
numFiles := rand.IntnRange(10, 20) | ||
for i := 0; i < numFiles; i++ { | ||
pathLength := rand.IntnRange(30, 60) | ||
filePath := "" | ||
for j := 0; j < pathLength; j += rand.IntnRange(5, 10) { | ||
filePath = filepath.Join(filePath, rand.String(rand.IntnRange(5, 10))) | ||
} | ||
gen[filePath] = &fstest.MapFile{ | ||
Data: []byte(rand.String(rand.IntnRange(1, 400))), | ||
Mode: fs.FileMode(rand.IntnRange(0600, 0777)), | ||
// Need to do some rounding and location shenanigans here to align with nuances of the tar implementation. | ||
ModTime: time.Now().Round(time.Second).Add(time.Duration(-rand.IntnRange(0, 100000)) * time.Second).In(&time.Location{}), | ||
} | ||
} | ||
return &gen | ||
} | ||
|
||
func fsEqual(a, b fs.FS) (bool, error) { | ||
aMap := fstest.MapFS{} | ||
bMap := fstest.MapFS{} | ||
|
||
walkFunc := func(f fs.FS, m fstest.MapFS) fs.WalkDirFunc { | ||
return func(path string, d fs.DirEntry, err error) error { | ||
if err != nil { | ||
return err | ||
} | ||
if d.IsDir() { | ||
return nil | ||
} | ||
data, err := fs.ReadFile(f, path) | ||
if err != nil { | ||
return err | ||
} | ||
info, err := d.Info() | ||
if err != nil { | ||
return err | ||
} | ||
m[path] = &fstest.MapFile{ | ||
Data: data, | ||
Mode: d.Type(), | ||
ModTime: info.ModTime().UTC(), | ||
} | ||
return nil | ||
} | ||
} | ||
if err := fs.WalkDir(a, ".", walkFunc(a, aMap)); err != nil { | ||
return false, err | ||
} | ||
if err := fs.WalkDir(b, ".", walkFunc(b, bMap)); err != nil { | ||
return false, err | ||
} | ||
return reflect.DeepEqual(aMap, bMap), nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
package util | ||
|
||
import ( | ||
"io/fs" | ||
"os" | ||
) | ||
|
||
type FilesOnlyFilesystem struct { | ||
FS fs.FS | ||
} | ||
|
||
func (f *FilesOnlyFilesystem) Open(name string) (fs.File, error) { | ||
file, err := f.FS.Open(name) | ||
if err != nil { | ||
return nil, err | ||
} | ||
stat, err := file.Stat() | ||
if err != nil { | ||
return nil, err | ||
} | ||
if !stat.Mode().IsRegular() { | ||
return nil, os.ErrNotExist | ||
} | ||
return file, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
package util | ||
|
||
import ( | ||
"archive/tar" | ||
"compress/gzip" | ||
"fmt" | ||
"io" | ||
"io/fs" | ||
"os" | ||
) | ||
|
||
// FSToTarGZ writes the filesystem represented by fsys to w as a gzipped tar archive. | ||
// This function unsets user and group information in the tar archive so that readers | ||
// of archives produced by this function do not need to account for differences in | ||
// permissions between source and destination filesystems. | ||
func FSToTarGZ(w io.Writer, fsys fs.FS) error { | ||
gzw := gzip.NewWriter(w) | ||
tw := tar.NewWriter(gzw) | ||
if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if d.Type()&os.ModeSymlink != 0 { | ||
return nil | ||
} | ||
info, err := d.Info() | ||
if err != nil { | ||
return fmt.Errorf("get file info for %q: %v", path, err) | ||
} | ||
|
||
h, err := tar.FileInfoHeader(info, "") | ||
if err != nil { | ||
return fmt.Errorf("build tar file info header for %q: %v", path, err) | ||
} | ||
h.Uid = 0 | ||
h.Gid = 0 | ||
h.Uname = "" | ||
h.Gname = "" | ||
h.Name = path | ||
|
||
if err := tw.WriteHeader(h); err != nil { | ||
return fmt.Errorf("write tar header for %q: %v", path, err) | ||
} | ||
if d.IsDir() { | ||
return nil | ||
} | ||
f, err := fsys.Open(path) | ||
if err != nil { | ||
return fmt.Errorf("open file %q: %v", path, err) | ||
} | ||
if _, err := io.Copy(tw, f); err != nil { | ||
return fmt.Errorf("write tar data for %q: %v", path, err) | ||
} | ||
return nil | ||
}); err != nil { | ||
return fmt.Errorf("generate tar.gz from FS: %v", err) | ||
} | ||
if err := tw.Close(); err != nil { | ||
return err | ||
} | ||
return gzw.Close() | ||
} |