From 003fe0083e4da4d7475d2e914192f4da315c6b3b Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Fri, 12 Apr 2024 11:58:12 +0800 Subject: [PATCH] feat: add ReferrerStoreManager interface to wrap operations on namespaced stores [multi-tenancy PR 4] (#1380) --- pkg/controllers/resource_map.go | 4 + pkg/controllers/store_controller.go | 18 +-- pkg/controllers/store_controller_test.go | 31 +++--- pkg/customresources/referrerstores/api.go | 38 +++++++ pkg/customresources/referrerstores/stores.go | 89 +++++++++++++++ .../referrerstores/stores_test.go | 104 ++++++++++++++++++ pkg/manager/manager.go | 11 +- 7 files changed, 258 insertions(+), 37 deletions(-) create mode 100644 pkg/customresources/referrerstores/api.go create mode 100644 pkg/customresources/referrerstores/stores.go create mode 100644 pkg/customresources/referrerstores/stores_test.go diff --git a/pkg/controllers/resource_map.go b/pkg/controllers/resource_map.go index 6a496e6a6..5352a1462 100644 --- a/pkg/controllers/resource_map.go +++ b/pkg/controllers/resource_map.go @@ -15,6 +15,7 @@ package controllers import ( "github.com/deislabs/ratify/pkg/customresources/policies" + rs "github.com/deislabs/ratify/pkg/customresources/referrerstores" "github.com/deislabs/ratify/pkg/customresources/verifiers" ) @@ -24,4 +25,7 @@ var ( // ActivePolicy is the active policy generated from CRD. There would be exactly // one active policy belonging to a namespace at any given time. ActivePolicies = policies.NewActivePolicies() + + // a map to track active stores + StoreMap = rs.NewActiveStores() ) diff --git a/pkg/controllers/store_controller.go b/pkg/controllers/store_controller.go index b4421a6ef..5371e7c4c 100644 --- a/pkg/controllers/store_controller.go +++ b/pkg/controllers/store_controller.go @@ -27,7 +27,7 @@ import ( configv1beta1 "github.com/deislabs/ratify/api/v1beta1" "github.com/deislabs/ratify/config" - "github.com/deislabs/ratify/pkg/referrerstore" + "github.com/deislabs/ratify/internal/constants" rc "github.com/deislabs/ratify/pkg/referrerstore/config" sf "github.com/deislabs/ratify/pkg/referrerstore/factory" "github.com/deislabs/ratify/pkg/referrerstore/types" @@ -40,11 +40,6 @@ type StoreReconciler struct { Scheme *runtime.Scheme } -var ( - // a map to track active stores - StoreMap = map[string]referrerstore.ReferrerStore{} -) - //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=stores,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=stores/status,verbs=get;update;patch //+kubebuilder:rbac:groups=config.ratify.deislabs.io,resources=stores/finalizers,verbs=update @@ -64,7 +59,8 @@ func (r *StoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl if err := r.Get(ctx, req.NamespacedName, &store); err != nil { if apierrors.IsNotFound(err) { storeLogger.Infof("deletion detected, removing store %v", req.Name) - storeRemove(resource) + // TODO: pass the actual namespace once multi-tenancy is supported. + StoreMap.DeleteStore(constants.EmptyNamespace, resource) } else { storeLogger.Error(err, "unable to fetch store") } @@ -115,17 +111,13 @@ func storeAddOrReplace(spec configv1beta1.StoreSpec, fullname string) error { return fmt.Errorf("store factory failed to create store from store config, err: %w", err) } - StoreMap[fullname] = storeReference + // TODO: pass the actual namespace once multi-tenancy is supported. + StoreMap.AddStore(constants.EmptyNamespace, fullname, storeReference) logrus.Infof("store '%v' added to store map", storeReference.Name()) return nil } -// Remove store from map -func storeRemove(resourceName string) { - delete(StoreMap, resourceName) -} - // Returns a store reference from spec func specToStoreConfig(storeSpec configv1beta1.StoreSpec) (rc.StorePluginConfig, error) { storeConfig := rc.StorePluginConfig{} diff --git a/pkg/controllers/store_controller_test.go b/pkg/controllers/store_controller_test.go index 8838b9c6d..1897d5711 100644 --- a/pkg/controllers/store_controller_test.go +++ b/pkg/controllers/store_controller_test.go @@ -22,7 +22,8 @@ import ( "testing" configv1beta1 "github.com/deislabs/ratify/api/v1beta1" - "github.com/deislabs/ratify/pkg/referrerstore" + "github.com/deislabs/ratify/internal/constants" + rs "github.com/deislabs/ratify/pkg/customresources/referrerstores" "github.com/deislabs/ratify/pkg/utils" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" @@ -47,15 +48,15 @@ func TestStoreAdd_EmptyParameter(t *testing.T) { if err := storeAddOrReplace(testStoreSpec, "oras"); err != nil { t.Fatalf("storeAddOrReplace() expected no error, actual %v", err) } - if len(StoreMap) != 1 { - t.Fatalf("Store map expected size 1, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 1 { + t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount()) } } func TestStoreAdd_WithParameters(t *testing.T) { resetStoreMap() - if len(StoreMap) != 0 { - t.Fatalf("Store map expected size 0, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 0 { + t.Fatalf("Store map expected size 0, actual %v", StoreMap.GetStoreCount()) } dirPath, err := utils.CreatePlugin(sampleName) if err != nil { @@ -68,8 +69,8 @@ func TestStoreAdd_WithParameters(t *testing.T) { if err := storeAddOrReplace(testStoreSpec, "testObject"); err != nil { t.Fatalf("storeAddOrReplace() expected no error, actual %v", err) } - if len(StoreMap) != 1 { - t.Fatalf("Store map expected size 1, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 1 { + t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount()) } } @@ -137,8 +138,8 @@ func TestStore_UpdateAndDelete(t *testing.T) { if err := storeAddOrReplace(testStoreSpec, sampleName); err != nil { t.Fatalf("storeAddOrReplace() expected no error, actual %v", err) } - if len(StoreMap) != 1 { - t.Fatalf("Store map expected size 1, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 1 { + t.Fatalf("Store map expected size 1, actual %v", StoreMap.GetStoreCount()) } // modify the Store @@ -152,19 +153,19 @@ func TestStore_UpdateAndDelete(t *testing.T) { } // validate no Store has been added - if len(StoreMap) != 1 { - t.Fatalf("Store map should be 1 after replacement, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 1 { + t.Fatalf("Store map should be 1 after replacement, actual %v", StoreMap.GetStoreCount()) } - storeRemove(sampleName) + StoreMap.DeleteStore(constants.EmptyNamespace, sampleName) - if len(StoreMap) != 0 { - t.Fatalf("Store map should be 0 after deletion, actual %v", len(StoreMap)) + if StoreMap.GetStoreCount() != 0 { + t.Fatalf("Store map should be 0 after deletion, actual %v", StoreMap.GetStoreCount()) } } func resetStoreMap() { - StoreMap = map[string]referrerstore.ReferrerStore{} + StoreMap = rs.NewActiveStores() } func getOrasStoreSpec(pluginName, pluginPath string) configv1beta1.StoreSpec { diff --git a/pkg/customresources/referrerstores/api.go b/pkg/customresources/referrerstores/api.go new file mode 100644 index 000000000..6b435c7e9 --- /dev/null +++ b/pkg/customresources/referrerstores/api.go @@ -0,0 +1,38 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package referrerstores + +import ( + "github.com/deislabs/ratify/pkg/referrerstore" +) + +// ReferrerStoreManager is an interface that defines the methods for managing referrer stores across different scopes. +type ReferrerStoreManager interface { + // Stores returns the list of referrer stores for the given scope. + GetStores(scope string) []referrerstore.ReferrerStore + + // AddStore adds the given store under the given scope. + AddStore(scope, storeName string, store referrerstore.ReferrerStore) + + // DeleteStore deletes the policy from the given scope. + DeleteStore(scope, storeName string) + + // IsEmpty returns true if there are no stores. + IsEmpty() bool + + // GetStoreCount returns the number of stores in all scopes. + GetStoreCount() int +} diff --git a/pkg/customresources/referrerstores/stores.go b/pkg/customresources/referrerstores/stores.go new file mode 100644 index 000000000..f60f83b0e --- /dev/null +++ b/pkg/customresources/referrerstores/stores.go @@ -0,0 +1,89 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package referrerstores + +import ( + "github.com/deislabs/ratify/pkg/referrerstore" +) + +// ActiveStores implements the ReferrerStoreManager interface. +type ActiveStores struct { + // TODO: Implement concurrent safety using sync.Map + // The structure of the map is as follows: + // The first level maps from scope to stores + // The second level maps from store name to store + // Example: + // { + // "namespace1": { + // "store1": store1, + // "store2": store2 + // } + // } + // Note: Scope is utilized for organizing and isolating stores. In a Kubernetes (K8s) environment, the scope can be either a namespace or an empty string ("") for cluster-wide stores. + ScopedStores map[string]map[string]referrerstore.ReferrerStore +} + +func NewActiveStores() ReferrerStoreManager { + return &ActiveStores{ + ScopedStores: make(map[string]map[string]referrerstore.ReferrerStore), + } +} + +// GetStores fulfills the ReferrerStoreManager interface. +// It returns all the stores in the ActiveStores for the given scope. If no stores are found for the given scope, it returns cluster-wide stores. +// TODO: Current implementation fetches stores for all namespaces including cluster-wide ones. Will support actual namespace based stores in future. +func (s *ActiveStores) GetStores(_ string) []referrerstore.ReferrerStore { + stores := []referrerstore.ReferrerStore{} + for _, scopedStores := range s.ScopedStores { + for _, store := range scopedStores { + stores = append(stores, store) + } + } + return stores +} + +// AddStore fulfills the ReferrerStoreManager interface. +// It adds the given store under the given scope. +func (s *ActiveStores) AddStore(scope, storeName string, store referrerstore.ReferrerStore) { + if _, ok := s.ScopedStores[scope]; !ok { + s.ScopedStores[scope] = make(map[string]referrerstore.ReferrerStore) + } + s.ScopedStores[scope][storeName] = store +} + +// DeleteStore fulfills the ReferrerStoreManager interface. +// It deletes the store with the given name under the given scope. +func (s *ActiveStores) DeleteStore(scope, storeName string) { + if stores, ok := s.ScopedStores[scope]; ok { + delete(stores, storeName) + } +} + +// IsEmpty fulfills the ReferrerStoreManager interface. +// It returns true if there are no stores in the ActiveStores. +func (s *ActiveStores) IsEmpty() bool { + return s.GetStoreCount() == 0 +} + +// GetStore fulfills the ReferrerStoreManager interface. +// GetStoreCount returns the total number of stores in the ActiveStores. +func (s *ActiveStores) GetStoreCount() int { + count := 0 + for _, stores := range s.ScopedStores { + count += len(stores) + } + return count +} diff --git a/pkg/customresources/referrerstores/stores_test.go b/pkg/customresources/referrerstores/stores_test.go new file mode 100644 index 000000000..46a375801 --- /dev/null +++ b/pkg/customresources/referrerstores/stores_test.go @@ -0,0 +1,104 @@ +/* +Copyright The Ratify Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package referrerstores + +import ( + "context" + "testing" + + "github.com/deislabs/ratify/internal/constants" + "github.com/deislabs/ratify/pkg/common" + "github.com/deislabs/ratify/pkg/ocispecs" + rs "github.com/deislabs/ratify/pkg/referrerstore" + "github.com/deislabs/ratify/pkg/referrerstore/config" + "github.com/opencontainers/go-digest" +) + +type mockStore struct { + name string +} + +func (s mockStore) Name() string { + return s.name +} + +func (s mockStore) ListReferrers(_ context.Context, _ common.Reference, _ []string, _ string, _ *ocispecs.SubjectDescriptor) (rs.ListReferrersResult, error) { + return rs.ListReferrersResult{}, nil +} + +func (s mockStore) GetBlobContent(_ context.Context, _ common.Reference, _ digest.Digest) ([]byte, error) { + return nil, nil +} + +func (s mockStore) GetReferenceManifest(_ context.Context, _ common.Reference, _ ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error) { + return ocispecs.ReferenceManifest{}, nil +} + +func (s mockStore) GetConfig() *config.StoreConfig { + return nil +} + +func (s mockStore) GetSubjectDescriptor(_ context.Context, _ common.Reference) (*ocispecs.SubjectDescriptor, error) { + return nil, nil +} + +const ( + namespace1 = constants.EmptyNamespace + namespace2 = "namespace2" + name1 = "name1" + name2 = "name2" +) + +var ( + store1 = mockStore{name: name1} + store2 = mockStore{name: name2} +) + +func TestStoresOperations(t *testing.T) { + stores := NewActiveStores() + stores.AddStore(namespace1, store1.Name(), store1) + stores.AddStore(namespace1, store2.Name(), store2) + stores.AddStore(namespace2, store1.Name(), store1) + stores.AddStore(namespace2, store2.Name(), store2) + + if stores.GetStoreCount() != 4 { + t.Fatalf("Expected 4 namespaces, got %d", stores.GetStoreCount()) + } + + stores.DeleteStore(namespace2, store1.Name()) + if len(stores.GetStores(namespace2)) != 3 { + t.Fatalf("Expected 3 store in namespace %s, got %d", namespace2, len(stores.GetStores(namespace2))) + } + + stores.DeleteStore(namespace2, store2.Name()) + if len(stores.GetStores(namespace2)) != 2 { + t.Fatalf("Expected 2 stores in namespace %s, got %d", namespace2, len(stores.GetStores(namespace2))) + } + + stores.DeleteStore(namespace1, store1.Name()) + if len(stores.GetStores(namespace1)) != 1 { + t.Fatalf("Expected 1 store in namespace %s, got %d", namespace1, len(stores.GetStores(namespace1))) + } + + stores.DeleteStore(namespace1, store2.Name()) + if len(stores.GetStores(namespace1)) != 0 { + t.Fatalf("Expected 0 stores in namespace %s, got %d", namespace1, len(stores.GetStores(namespace1))) + } + + if !stores.IsEmpty() { + t.Fatalf("Expected stores to be empty") + } +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index bb9dfebec..2527f43ce 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -51,7 +51,6 @@ import ( ctxUtils "github.com/deislabs/ratify/internal/context" "github.com/deislabs/ratify/pkg/controllers" ef "github.com/deislabs/ratify/pkg/executor/core" - "github.com/deislabs/ratify/pkg/referrerstore" //+kubebuilder:scaffold:imports ) @@ -84,17 +83,11 @@ func StartServer(httpServerAddress, configFilePath, certDirectory, caCertFile st // initialize server server, err := httpserver.NewServer(context.Background(), httpServerAddress, func(ctx context.Context) *ef.Executor { - var activeStores []referrerstore.ReferrerStore namespace := ctxUtils.GetNamespace(ctx) + activeVerifiers := controllers.VerifierMap.GetVerifiers(namespace) activePolicyEnforcer := controllers.ActivePolicies.GetPolicy(namespace) - - // check if there are active stores from crd controller - if len(controllers.StoreMap) > 0 { - for _, value := range controllers.StoreMap { - activeStores = append(activeStores, value) - } - } + activeStores := controllers.StoreMap.GetStores(namespace) // return executor with latest configuration executor := ef.Executor{