feat: report file clean up (#410)

* add cronjob worker to schedule clean up job
* add list filemetas API

apis/meta/v1alpha1/listmeta_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"net/url"
 	"regexp"
 
 	"github.com/katanomi/pkg/common"
@@ -74,7 +75,7 @@ type ListOptions struct {
 
 	// Custom search options
 	// +optional
-	Search map[string][]string `json:",inline"`
+	Search url.Values `json:",inline"`
 
 	// Subresoures for listing
 	// will only work for lists that support this feature
@@ -86,6 +87,13 @@ type ListOptions struct {
 	Sort []SortOptions `json:"sort"`
 }
 
+func (opt *ListOptions) SearchSet(key, value string) {
+	if opt.Search == nil {
+		opt.Search = make(url.Values)
+	}
+	opt.Search.Set(key, value)
+}
+
 // GetSearchFirstElement get first element by key that in search map
 func (opt *ListOptions) GetSearchFirstElement(key string) (value string) {
 	if valueList, ok := opt.Search[key]; ok {
@@ -195,14 +203,9 @@ type UserOptions struct {
 // GetSearchValue get search value from option
 // use `searchValue` instead of `name`
 func GetSearchValue(option ListOptions) string {
-	if value := option.GetSearchFirstElement(SearchValueKey); value != "" {
-		return value
-	}
-
-	// Deprecated
-	if value := option.GetSearchFirstElement("name"); value != "" {
-		return value
+	if option.Search.Has(SearchValueKey) {
+		return option.Search.Get(SearchValueKey)
 	}
 
-	return ""
+	return option.Search.Get("name")
 }

apis/storage/v1alpha1/filemeta_funcs.go

@@ -20,6 +20,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"strings"
+	"time"
 )
 
 // ParseFileKey returns pluginName and fileObjectName
@@ -40,6 +41,20 @@ func (in *FileMeta) Encode() string {
 	return base64.StdEncoding.EncodeToString(marshaledMeta)
 }
 
+// LastModified return lastModifiedTime from storage server or zero time if no creation info
+func (in *FileMeta) LastModified() time.Time {
+	if lastModifiedAnno, ok := in.Annotations[LastModifiedAnnotation]; ok {
+		parsedTime, _ := time.Parse(time.RFC3339, lastModifiedAnno)
+		return parsedTime
+	}
+	return time.Time{}
+}
+
+// Key returns file key
+func (in *FileMeta) Key() string {
+	return in.Spec.Key
+}
+
 // DecodeAsFileMeta decodes encoded string to a pointer FileMeta
 func DecodeAsFileMeta(encoded string) (*FileMeta, error) {
 	decodedBytes, err := base64.StdEncoding.DecodeString(encoded)

apis/storage/v1alpha1/filemeta_types.go

@@ -41,7 +41,7 @@ type FileMetaSpec struct {
 	// +optional
 	Entry string `json:"entry,omitempty"`
 
-	// Key for file key of file object, following format:  {storage-plugin-name}:{file-object-name}
+	// Key for file key of file object, following format:  {storage-plugin-name}/-/{file-object-name}
 	// +optional
 	Key string `json:"key,omitempty"`
 
@@ -56,6 +56,14 @@ type FileMetaSpec struct {
 	FileType string `json:"fileType,omitempty"`
 }
 
+// FileMetaListOptions for list option of FileMeta
+type FileMetaListOptions struct {
+	Prefix     string `json:"prefix,omitempty"`
+	Recursive  bool   `json:"recursive,omitempty"`
+	StartAfter string `json:"startAfter,omitempty"`
+	Limit      int    `json:"limit,omitempty"`
+}
+
 // FileMetaList list of FileMetas
 type FileMetaList struct {
 	metav1.TypeMeta `json:",inline"`

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/cobra v1.6.0
 	github.com/spf13/pflag v1.0.5
 	github.com/tektoncd/pipeline v0.41.0

go.sum

@@ -432,6 +432,8 @@ github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf
 github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ=
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=

plugin/storage/capabilities/capabilities_test.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"testing"
 
-	metav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1"
 	"github.com/katanomi/pkg/apis/storage/v1alpha1"
 
 	archivev1alpha1 "github.com/katanomi/pkg/apis/archive/v1alpha1"
@@ -31,7 +30,7 @@ import (
 
 type fakeFileStoreImp struct{}
 
-func (f fakeFileStoreImp) ListFileMetas(ctx context.Context, opt metav1alpha1.ListOptions) ([]v1alpha1.FileMeta, error) {
+func (f fakeFileStoreImp) ListFileMetas(ctx context.Context, opt v1alpha1.FileMetaListOptions) ([]v1alpha1.FileMeta, error) {
 	// TODO implement me
 	panic("implement me")
 }

plugin/storage/capabilities/filestore/v1alpha1/interface.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"io"
 
-	kmetav1alpha1 "github.com/katanomi/pkg/apis/meta/v1alpha1"
 	"github.com/katanomi/pkg/apis/storage/v1alpha1"
 )
 
@@ -47,7 +46,7 @@ type FileObjectInterface interface {
 // FileMetaInterface for file meta
 type FileMetaInterface interface {
 	// ListFileMetas for list file metas
-	ListFileMetas(ctx context.Context, opt kmetav1alpha1.ListOptions) ([]v1alpha1.FileMeta, error)
+	ListFileMetas(ctx context.Context, opt v1alpha1.FileMetaListOptions) ([]v1alpha1.FileMeta, error)
 	// GetFileMeta for get file meta
 	GetFileMeta(ctx context.Context, objectName string) (*v1alpha1.FileMeta, error)
 }

plugin/storage/client/versioned/filestore/v1alpha1/filemeta.go

@@ -19,18 +19,21 @@ package v1alpha1
 import (
 	"context"
 	"fmt"
+	"strconv"
 
+	v1alpha12 "github.com/katanomi/pkg/apis/meta/v1alpha1"
 	"github.com/katanomi/pkg/apis/storage/v1alpha1"
-	client2 "github.com/katanomi/pkg/plugin/client"
+	pclient "github.com/katanomi/pkg/plugin/client"
 	"github.com/katanomi/pkg/plugin/storage/client"
+	"knative.dev/pkg/logging"
 )
 
 //go:generate ../../../../../../bin/mockgen -source=filemeta.go -destination=../../../../../../testing/mock/github.com/katanomi/pkg/plugin/storage/client/versioned/filestore/v1alpha1/filemeta.go -package=v1alpha1 FileMetaInterface
 
 // FileMetaInterface for file meta restful resource methods
 type FileMetaInterface interface {
 	GET(ctx context.Context, key string) (*v1alpha1.FileMeta, error)
-	// TODO: Add List methods
+	List(ctx context.Context, opts v1alpha1.FileMetaListOptions) ([]v1alpha1.FileMeta, error)
 }
 
 type fileMetas struct {
@@ -49,9 +52,37 @@ func newFileMetas(c *FileStoreV1alpha1Client, pluginName string) *fileMetas {
 func (f *fileMetas) GET(ctx context.Context, key string) (*v1alpha1.FileMeta, error) {
 	path := fmt.Sprintf("storageplugins/%s/filemetas/%s", f.pluginName, key)
 	fileMeta := v1alpha1.FileMeta{}
-	err := f.client.Get(ctx, path, client2.ResultOpts(&fileMeta))
+	err := f.client.Get(ctx, path, pclient.ResultOpts(&fileMeta))
 	if err != nil {
 		return nil, err
 	}
 	return &fileMeta, nil
 }
+
+func (f *fileMetas) List(ctx context.Context, opts v1alpha1.FileMetaListOptions) ([]v1alpha1.FileMeta, error) {
+	logger := logging.FromContext(ctx)
+	path := fmt.Sprintf("storageplugins/%s/filemetas", f.pluginName)
+	listOpt := v1alpha12.ListOptions{}
+	// default recursive is true
+	if opts.Recursive == false {
+		listOpt.SearchSet("recursive", "false")
+	}
+	if opts.Prefix != "" {
+		listOpt.SearchSet("prefix", opts.Prefix)
+	}
+	if opts.StartAfter != "" {
+		listOpt.SearchSet("startAfter", opts.StartAfter)
+	}
+	if opts.Limit > 0 {
+		listOpt.SearchSet("limit", strconv.Itoa(opts.Limit))
+	}
+
+	logger.Debugw("set list meta options for client", "opts", listOpt)
+	fileMetas := make([]v1alpha1.FileMeta, 0)
+
+	err := f.client.Get(ctx, path, pclient.ResultOpts(&fileMetas), pclient.ListOpts(listOpt))
+	if err != nil {
+		return nil, err
+	}
+	return fileMetas, nil
+}

plugin/storage/client/versioned/filestore/v1alpha1/filemeta_test.go

@@ -57,4 +57,19 @@ var _ = Describe("Test.Fileobject", func() {
 		})
 	})
 
+	Context("Test.FileMeta.List", func() {
+		It("returns filemetas", func() {
+
+			expectedFileMetas := []v1alpha1.FileMeta{}
+			testing.MustLoadJSON("testdata/filemetas.normal.golden.json", &expectedFileMetas)
+
+			mockStoragePluginClient.EXPECT().
+				Get(ctx, "storageplugins/foo/filemetas", gomock.Any()).
+				Return(nil)
+
+			_, err := fileMeta.List(ctx, v1alpha1.FileMetaListOptions{})
+			Expect(err).ShouldNot(HaveOccurred())
+		})
+	})
+
 })

plugin/storage/client/versioned/filestore/v1alpha1/filestore_client.go

@@ -63,6 +63,6 @@ func (c *FileStoreV1alpha1Client) FileMeta(pluginName string) FileMetaInterface
 	return newFileMetas(c, pluginName)
 }
 
-func NewForClient(pClient client.Interface) *FileStoreV1alpha1Client {
+func NewForClient(pClient client.Interface) FileStoreV1alpha1Interface {
 	return &FileStoreV1alpha1Client{restClient: pClient.ForGroupVersion(&filestorev1alpha1.FileStoreV1alpha1GV)}
 }

plugin/storage/client/versioned/filestore/v1alpha1/testdata/filemetas.normal.golden.json

@@ -0,0 +1,16 @@
+[
+  {
+    "metadata": {
+      "creationTimestamp": null,
+      "annotations": {
+        "storage.katanomi.dev/fileObject.lastModified": "2023-02-08 10:03:32 +0000 UTC"
+      }
+    },
+    "spec": {
+      "key": "minio.default/-/kubernetes/-/apis/builds.katanomi.dev/v1alpha1/namespaces/default/buildrun/core.k-1/x.xml",
+      "contentType": "application/vnd.katanomi.junit+xml",
+      "contentLength": 11957884,
+      "fileType": "unitTest"
+    }
+  }
+]

plugin/storage/client/versioned/filestore/v1alpha1/v1alpha1_suite_test.go

@@ -33,7 +33,7 @@ var (
 	mockSuiteCtrl *gomock.Controller
 
 	mockStoragePluginClient *v1alpha1.MockInterface
-	mockFileStoreClient     *v1alpha13.FileStoreV1alpha1Client
+	mockFileStoreClient     v1alpha13.FileStoreV1alpha1Interface
 )
 
 var _ = BeforeSuite(func() {
@@ -50,10 +50,6 @@ var _ = BeforeSuite(func() {
 	Expect(mockFileStoreClient.RESTClient().APIVersion()).To(Equal(&filestorev1alpha1.FileStoreV1alpha1GV))
 })
 
-var _ = AfterSuite(func() {
-	mockSuiteCtrl.Finish()
-})
-
 func TestV1alpha1(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "V1alpha1 Suite")

plugin/storage/clientset.go

@@ -27,6 +27,8 @@ import (
 	v1 "knative.dev/pkg/apis/duck/v1"
 )
 
+//go:generate ../../bin/mockgen -source=clientset.go -destination=../../testing/mock/github.com/katanomi/pkg/plugin/storage/clientset.go -package=storage Interface
+
 type Interface interface {
 	CoreV1alpha1() corev1alpha1.CoreV1alpha1Interface
 	FileStoreV1alpha1() v1alpha1.FileStoreV1alpha1Interface

plugin/storage/context.go

@@ -20,15 +20,32 @@ import "context"
 
 type storagePluginNameKey struct{}
 
+type clientSetKey struct{}
+
 // CtxWithPluginName return context.Context with plugin name
 func CtxWithPluginName(ctx context.Context, pluginName string) context.Context {
 	return context.WithValue(ctx, storagePluginNameKey{}, pluginName)
 }
 
+// PluginNameFromCtx returns plugin name from context
 func PluginNameFromCtx(ctx context.Context) string {
 	val := ctx.Value(storagePluginNameKey{})
 	if val == nil {
 		return ""
 	}
 	return val.(string)
 }
+
+// WithClientSet return context with client set
+func WithClientSet(ctx context.Context, clientSet Interface) context.Context {
+	return context.WithValue(ctx, clientSetKey{}, clientSet)
+}
+
+// ClientSetFromCtx return clientset from context
+func ClientSetFromCtx(ctx context.Context) Interface {
+	val := ctx.Value(clientSetKey{})
+	if val == nil {
+		return nil
+	}
+	return val.(Interface)
+}