feat: search data from search storage layer

cmd/app/options/search_storage.go

@@ -0,0 +1,61 @@
+// Copyright The Karbour Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"github.com/KusionStack/karbour/pkg/apiserver"
+	"github.com/spf13/pflag"
+)
+
+type ElasticSearchConfig struct {
+	Addresses []string
+	UserName  string
+	Password  string
+}
+
+type SearchStorageOptions struct {
+	SearchStorageType      string
+	ElasticSearchAddresses []string
+	ElasticSearchName      string
+	ElasticSearchPassword  string
+}
+
+func NewSearchStorageOptions() *SearchStorageOptions {
+	return &SearchStorageOptions{}
+}
+
+func (o *SearchStorageOptions) Validate() []error {
+	return nil
+}
+
+func (o *SearchStorageOptions) ApplyTo(config *apiserver.ExtraConfig) error {
+	config.SearchStorageType = o.SearchStorageType
+	config.ElasticSearchAddresses = o.ElasticSearchAddresses
+	config.ElasticSearchName = o.ElasticSearchName
+	config.ElasticSearchPassword = o.ElasticSearchPassword
+	return nil
+}
+
+// AddFlags adds flags for a specific Option to the specified FlagSet
+func (o *SearchStorageOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.SearchStorageType, "search-storage-type", "", "The search storage type")
+	fs.StringSliceVar(&o.ElasticSearchAddresses, "elastic-search-addresses", nil, "The elastic search address")
+	fs.StringVar(&o.ElasticSearchName, "elastic-search-username", "", "The elastic search username")
+	fs.StringVar(&o.ElasticSearchPassword, "elastic-search-password", "", "The elastic search password")
+}

cmd/app/server.go

@@ -25,11 +25,13 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/KusionStack/karbour/cmd/app/options"
 	"github.com/KusionStack/karbour/pkg/apiserver"
 	clientset "github.com/KusionStack/karbour/pkg/generated/clientset/versioned"
 	informers "github.com/KusionStack/karbour/pkg/generated/informers/externalversions"
 	karbouropenapi "github.com/KusionStack/karbour/pkg/generated/openapi"
 	"github.com/KusionStack/karbour/pkg/scheme"
+	filtersutil "github.com/KusionStack/karbour/pkg/util/filters"
 	proxyutil "github.com/KusionStack/karbour/pkg/util/proxy"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -49,8 +51,9 @@ const defaultEtcdPathPrefix = "/registry/karbour"
 
 // Options contains state for master/api server
 type Options struct {
-	ServerRunOptions   *genericoptions.ServerRunOptions
-	RecommendedOptions *genericoptions.RecommendedOptions
+	ServerRunOptions     *genericoptions.ServerRunOptions
+	RecommendedOptions   *genericoptions.RecommendedOptions
+	SearchStorageOptions *options.SearchStorageOptions
 
 	SharedInformerFactory informers.SharedInformerFactory
 	StdOut                io.Writer
@@ -67,9 +70,9 @@ func NewOptions(out, errOut io.Writer) *Options {
 			defaultEtcdPathPrefix,
 			scheme.Codecs.LegacyCodec(scheme.Versions...),
 		),
-
-		StdOut: out,
-		StdErr: errOut,
+		SearchStorageOptions: options.NewSearchStorageOptions(),
+		StdOut:               out,
+		StdErr:               errOut,
 	}
 	o.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = schema.GroupVersions(scheme.Versions)
 	return o
@@ -104,13 +107,15 @@ func NewApiserverCommand(stopCh <-chan struct{}) *cobra.Command {
 func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	o.ServerRunOptions.AddUniversalFlags(fs)
 	o.RecommendedOptions.AddFlags(fs)
+	o.SearchStorageOptions.AddFlags(fs)
 }
 
 // Validate validates Options
 func (o *Options) Validate(args []string) error {
 	errors := []error{}
 	errors = append(errors, o.ServerRunOptions.Validate()...)
 	errors = append(errors, o.RecommendedOptions.Validate()...)
+	errors = append(errors, o.SearchStorageOptions.Validate()...)
 	return utilerrors.NewAggregate(errors)
 }
 
@@ -147,6 +152,11 @@ func (o *Options) Config() (*apiserver.Config, error) {
 		return nil, err
 	}
 
+	extraConfig := &apiserver.ExtraConfig{}
+	if err := o.SearchStorageOptions.ApplyTo(extraConfig); err != nil {
+		return nil, err
+	}
+
 	serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(karbouropenapi.GetOpenAPIDefinitions, openapi.NewDefinitionNamer(scheme.Scheme))
 	serverConfig.OpenAPIConfig.Info.Title = "Karbour"
 	serverConfig.OpenAPIConfig.Info.Version = "0.1"
@@ -160,12 +170,15 @@ func (o *Options) Config() (*apiserver.Config, error) {
 		serverConfig.OpenAPIV3Config.Info.Version = "0.1"
 	}
 	serverConfig.BuildHandlerChainFunc = func(handler http.Handler, c *genericapiserver.Config) http.Handler {
-		return proxyutil.WithProxyByCluster(genericapiserver.DefaultBuildHandlerChain(handler, c))
+		handler = genericapiserver.DefaultBuildHandlerChain(handler, c)
+		handler = proxyutil.WithProxyByCluster(handler)
+		handler = filtersutil.SearchFilter(handler)
+		return handler
 	}
 
 	config := &apiserver.Config{
 		GenericConfig: serverConfig,
-		ExtraConfig:   apiserver.ExtraConfig{},
+		ExtraConfig:   extraConfig,
 	}
 	return config, nil
 }

go.mod

@@ -25,6 +25,7 @@ require (
 )
 
 require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
@@ -59,6 +60,7 @@ require (
 	github.com/huandu/xstrings v1.3.1 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jinzhu/configor v1.2.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect

go.sum

@@ -241,6 +241,8 @@ github.com/jgroeneveld/schema v1.0.0 h1:J0E10CrOkiSEsw6dfb1IfrDJD14pf6QLVJ3tRPl/
 github.com/jgroeneveld/schema v1.0.0/go.mod h1:M14lv7sNMtGvo3ops1MwslaSYgDYxrSmbzWIQ0Mr5rs=
 github.com/jgroeneveld/trial v2.0.0+incompatible h1:d59ctdgor+VqdZCAiUfVN8K13s0ALDioG5DWwZNtRuQ=
 github.com/jgroeneveld/trial v2.0.0+incompatible/go.mod h1:I6INLW96EN8WysNBXUFI3M4RIC8ePg9ntAc/Wy+U/+M=
+github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko=
+github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc=
 github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=

pkg/apiserver/apiserver.go

@@ -30,13 +30,16 @@ import (
 
 // ExtraConfig holds custom apiserver config
 type ExtraConfig struct {
-	// Place you custom config here.
+	SearchStorageType      string
+	ElasticSearchAddresses []string
+	ElasticSearchName      string
+	ElasticSearchPassword  string
 }
 
 // Config defines the config for the apiserver
 type Config struct {
 	GenericConfig *genericapiserver.RecommendedConfig
-	ExtraConfig   ExtraConfig
+	ExtraConfig   *ExtraConfig
 }
 
 // APIServer contains state for a Kubernetes cluster master/api server.
@@ -58,7 +61,7 @@ type CompletedConfig struct {
 func (cfg *Config) Complete() CompletedConfig {
 	c := completedConfig{
 		cfg.GenericConfig.Complete(),
-		&cfg.ExtraConfig,
+		cfg.ExtraConfig,
 	}
 
 	c.GenericConfig.Version = &version.Info{
@@ -82,7 +85,12 @@ func (c completedConfig) New() (*APIServer, error) {
 
 	restStorageProviders := []registry.RESTStorageProvider{
 		clusterstorage.RESTStorageProvider{},
-		searchstorage.RESTStorageProvider{},
+		searchstorage.RESTStorageProvider{
+			SearchStorageType:      c.ExtraConfig.SearchStorageType,
+			ElasticSearchAddresses: c.ExtraConfig.ElasticSearchAddresses,
+			ElasticSearchName:      c.ExtraConfig.ElasticSearchName,
+			ElasticSearchPassword:  c.ExtraConfig.ElasticSearchPassword,
+		},
 	}
 
 	for _, restStorageProvider := range restStorageProviders {

pkg/registry/search/storage_provider.go

@@ -17,26 +17,62 @@ limitations under the License.
 package search
 
 import (
+	"fmt"
+
 	"github.com/KusionStack/karbour/pkg/apis/search"
 	"github.com/KusionStack/karbour/pkg/registry"
 	"github.com/KusionStack/karbour/pkg/scheme"
+	"github.com/KusionStack/karbour/pkg/search/storage"
+	"github.com/KusionStack/karbour/pkg/search/storage/elasticsearch"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 )
 
 var _ registry.RESTStorageProvider = &RESTStorageProvider{}
 
-type RESTStorageProvider struct{}
+type RESTStorageProvider struct {
+	SearchStorageType      string
+	ElasticSearchAddresses []string
+	ElasticSearchName      string
+	ElasticSearchPassword  string
+}
 
 func (p RESTStorageProvider) GroupName() string {
 	return search.GroupName
 }
 
 func (p RESTStorageProvider) NewRESTStorage(restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(search.GroupName, scheme.Scheme, scheme.ParameterCodec, scheme.Codecs)
-	v1beta1 := map[string]rest.Storage{}
-	apiGroupInfo.VersionedResourcesStorageMap["v1beta1"] = v1beta1
-	v1beta1["uniresources"] = NewUniResourceREST()
+	searchStorageGetter, err := p.SearchStorageGetter()
+	if err != nil {
+		return genericapiserver.APIGroupInfo{}, err
+	}
+
+	storageMap, err := p.v1beta1Storage(restOptionsGetter, searchStorageGetter)
+	if err != nil {
+		return genericapiserver.APIGroupInfo{}, err
+	}
+
+	apiGroupInfo.VersionedResourcesStorageMap["v1beta1"] = storageMap
 	return apiGroupInfo, nil
 }
+
+func (p RESTStorageProvider) v1beta1Storage(restOptionsGetter generic.RESTOptionsGetter, searchStorageGetter storage.SearchStorageGetter) (map[string]rest.Storage, error) {
+	v1beta1Storage := map[string]rest.Storage{}
+	uniResourceStorage, err := NewUniResourceREST(searchStorageGetter)
+	if err != nil {
+		return v1beta1Storage, err
+	}
+	v1beta1Storage["uniresources"] = uniResourceStorage
+	return v1beta1Storage, nil
+}
+
+func (p RESTStorageProvider) SearchStorageGetter() (storage.SearchStorageGetter, error) {
+	switch p.SearchStorageType {
+	case "elasticsearch":
+		return elasticsearch.NewSearchStorageGetter(p.ElasticSearchAddresses, p.ElasticSearchName, p.ElasticSearchPassword), nil
+	default:
+		return nil, fmt.Errorf("invalid search storage type %s", p.SearchStorageType)
+	}
+}

pkg/registry/search/uniresource_storage.go

@@ -16,10 +16,14 @@ package search
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/KusionStack/karbour/pkg/apis/search"
+	"github.com/KusionStack/karbour/pkg/search/storage"
+	filtersutil "github.com/KusionStack/karbour/pkg/util/filters"
 	"k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/rest"
 )
@@ -31,10 +35,19 @@ var (
 	_ rest.ShortNamesProvider = &UniResourceREST{}
 )
 
-type UniResourceREST struct{}
+type UniResourceREST struct {
+	SearchStorage storage.SearchStorage
+}
+
+func NewUniResourceREST(searchStorageGetter storage.SearchStorageGetter) (rest.Storage, error) {
+	searchStorage, err := searchStorageGetter.GetSearchStorage()
+	if err != nil {
+		return nil, err
+	}
 
-func NewUniResourceREST() rest.Storage {
-	return &UniResourceREST{}
+	return &UniResourceREST{
+		SearchStorage: searchStorage,
+	}, nil
 }
 
 func (r *UniResourceREST) New() runtime.Object {
@@ -53,8 +66,23 @@ func (r *UniResourceREST) NewList() runtime.Object {
 }
 
 func (r *UniResourceREST) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
-	// TODO: add real logic of list when the storage layer is implemented
-	return &search.UniResourceList{}, nil
+	queryString, ok := filtersutil.SearchQueryFrom(ctx)
+	if !ok {
+		return nil, fmt.Errorf("query can't be empty")
+	}
+
+	res, err := r.SearchStorage.SearchByString(ctx, queryString)
+	if err != nil {
+		return nil, err
+	}
+
+	rt := &search.UniResourceList{}
+	for _, resource := range res.Resources {
+		unObj := &unstructured.Unstructured{}
+		unObj.SetUnstructuredContent(resource.Object)
+		rt.Items = append(rt.Items, unObj)
+	}
+	return rt, nil
 }
 
 func (r *UniResourceREST) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {

pkg/search/storage/elasticsearch/client.go

@@ -29,8 +29,8 @@ const (
 )
 
 var (
-	_ storage.Storage  = &ESClient{}
-	_ storage.Searcher = &ESClient{}
+	_ storage.Storage       = &ESClient{}
+	_ storage.SearchStorage = &ESClient{}
 )
 
 type ESClient struct {