diff --git a/controllers/lvmcluster_controller.go b/controllers/lvmcluster_controller.go index d0ca0a15e..e4ba67f38 100644 --- a/controllers/lvmcluster_controller.go +++ b/controllers/lvmcluster_controller.go @@ -46,14 +46,12 @@ import ( type EventReasonInfo string type EventReasonError string -const EventReasonErrorDeletionPending EventReasonError = "DeletionPending" -const EventReasonErrorResourceReconciliationFailed EventReasonError = "ResourceReconciliationFailed" -const EventReasonResourceReconciliationSuccess EventReasonInfo = "ResourceReconciliationSuccess" - -var lvmClusterFinalizer = "lvmcluster.topolvm.io" - const ( - ControllerName = "lvmcluster-controller" + EventReasonErrorDeletionPending EventReasonError = "DeletionPending" + EventReasonErrorResourceReconciliationFailed EventReasonError = "ResourceReconciliationFailed" + EventReasonResourceReconciliationSuccess EventReasonInfo = "ResourceReconciliationSuccess" + + lvmClusterFinalizer = "lvmcluster.topolvm.io" ) // NOTE: when updating this, please also update docs/design/lvm-operator-manager.md @@ -73,10 +71,11 @@ type resourceManager interface { type LVMClusterReconciler struct { client.Client record.EventRecorder - Scheme *runtime.Scheme - ClusterType cluster.Type - Namespace string - ImageName string + Scheme *runtime.Scheme + ClusterType cluster.Type + EnableSnapshotting bool + Namespace string + ImageName string // TopoLVMLeaderElectionPassthrough uses the given leaderElection when initializing TopoLVM to synchronize // leader election configuration @@ -184,13 +183,16 @@ func (r *LVMClusterReconciler) reconcile(ctx context.Context, instance *lvmv1alp &vgManager{}, &lvmVG{}, &topolvmStorageClass{}, - &topolvmVolumeSnapshotClass{}, } if r.ClusterType == cluster.TypeOCP { resources = append(resources, openshiftSccs{}) } + if r.EnableSnapshotting { + resources = append(resources, &topolvmVolumeSnapshotClass{}) + } + resourceSyncStart := time.Now() results := make(chan error, len(resources)) create := func(i int) { @@ -393,6 +395,10 @@ func (r *LVMClusterReconciler) processDelete(ctx context.Context, instance *lvmv resourceDeletionList = append(resourceDeletionList, openshiftSccs{}) } + if r.EnableSnapshotting { + resourceDeletionList = append(resourceDeletionList, &topolvmVolumeSnapshotClass{}) + } + for _, unit := range resourceDeletionList { if err := unit.ensureDeleted(r, ctx, instance); err != nil { err := fmt.Errorf("failed cleaning up %s: %w", unit.getName(), err) diff --git a/controllers/lvmcluster_controller_watches.go b/controllers/lvmcluster_controller_watches.go index 937d9a7c9..de6a09d3e 100644 --- a/controllers/lvmcluster_controller_watches.go +++ b/controllers/lvmcluster_controller_watches.go @@ -19,17 +19,23 @@ package controllers import ( "context" + snapapiv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + corev1helper "k8s.io/component-helpers/scheduling/corev1" + secv1 "github.com/openshift/api/security/v1" + lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" "github.com/openshift/lvm-operator/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/openshift/lvm-operator/pkg/labels" appsv1 "k8s.io/api/apps/v1" - storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -38,46 +44,111 @@ func (r *LVMClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { builder := ctrl.NewControllerManagedBy(mgr). For(&lvmv1alpha1.LVMCluster{}). Owns(&appsv1.DaemonSet{}). + Owns(&appsv1.Deployment{}). Owns(&lvmv1alpha1.LVMVolumeGroup{}). - Owns(&lvmv1alpha1.LVMVolumeGroupNodeStatus{}). Watches( &lvmv1alpha1.LVMVolumeGroupNodeStatus{}, - handler.EnqueueRequestsFromMapFunc(r.getLVMClusterObjsForReconcile), + handler.EnqueueRequestsFromMapFunc(r.getLVMClusterObjsByNameFittingNodeSelector), ). Watches( &storagev1.StorageClass{}, - handler.EnqueueRequestsFromMapFunc(r.getLVMClusterObjsForReconcile), + handler.EnqueueRequestsFromMapFunc(r.getManagedLabelObjsForReconcile), ) if r.ClusterType == cluster.TypeOCP { builder = builder.Watches( &secv1.SecurityContextConstraints{}, - handler.EnqueueRequestsFromMapFunc(r.getLVMClusterObjsForReconcile), + handler.EnqueueRequestsFromMapFunc(r.getManagedLabelObjsForReconcile), + ) + } + if r.EnableSnapshotting { + builder = builder.Watches( + &snapapiv1.VolumeSnapshotClass{}, + handler.EnqueueRequestsFromMapFunc(r.getManagedLabelObjsForReconcile), ) } return builder.Complete(r) } -func (r *LVMClusterReconciler) getLVMClusterObjsForReconcile(ctx context.Context, obj client.Object) []reconcile.Request { - +// getManagedLabelObjsForReconcile reconciles the object anytime the given object has all management labels +// set to the available lvmclusters. This can be used especially if owner references are not a valid option (e.g. +// the namespaced LVMCluster needs to "own" a cluster-scoped resource, in which case owner references are invalid). +// This should generally only be used for cluster-scoped resources. Also it should be noted that deletion logic must +// be handled manually as garbage collection is not handled automatically like for owner references. +func (r *LVMClusterReconciler) getManagedLabelObjsForReconcile(ctx context.Context, obj client.Object) []reconcile.Request { foundLVMClusterList := &lvmv1alpha1.LVMClusterList{} listOps := &client.ListOptions{ Namespace: obj.GetNamespace(), } if err := r.Client.List(ctx, foundLVMClusterList, listOps); err != nil { - log.FromContext(ctx).Error(err, "getLVMClusterObjsForReconcile: Failed to get LVMCluster objs") + log.FromContext(ctx).Error(err, "getManagedLabelObjsForReconcile: Failed to get LVMCluster objs") return []reconcile.Request{} } - requests := make([]reconcile.Request, len(foundLVMClusterList.Items)) - for i, item := range foundLVMClusterList.Items { - requests[i] = reconcile.Request{ + var requests []reconcile.Request + for _, lvmCluster := range foundLVMClusterList.Items { + if !labels.MatchesManagedLabels(r.Scheme, obj, &lvmCluster) { + continue + } + requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ - Name: item.GetName(), - Namespace: item.GetNamespace(), + Name: lvmCluster.GetName(), + Namespace: lvmCluster.GetNamespace(), }, + }) + } + return requests +} + +// getLVMClusterObjsByNameFittingNodeSelector enqueues the cluster in case the object name fits the node name. +// this means that as if the obj name fits to a given node on the cluster and that node is part of the node selector, +// then the lvm cluster will get updated as well. Should only be used in conjunction with LVMVolumeGroupNodeStatus +// as other objects do not use the node name as resource name. +func (r *LVMClusterReconciler) getLVMClusterObjsByNameFittingNodeSelector(ctx context.Context, obj client.Object) []reconcile.Request { + foundLVMClusterList := &lvmv1alpha1.LVMClusterList{} + listOps := &client.ListOptions{ + Namespace: obj.GetNamespace(), + } + + if err := r.Client.List(ctx, foundLVMClusterList, listOps); err != nil { + log.FromContext(ctx).Error(err, "getLVMClusterObjsByNameFittingNodeSelector: Failed to get LVMCluster objs") + return []reconcile.Request{} + } + + node := &v1.Node{} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(obj), node); err != nil { + log.FromContext(ctx).Error(err, "getLVMClusterObjsByNameFittingNodeSelector: Failed to get Node") + return []reconcile.Request{} + } + + var requests []reconcile.Request + for _, lvmCluster := range foundLVMClusterList.Items { + selector, _ := extractNodeSelectorAndTolerations(&lvmCluster) + // if the selector is nil then the default behavior is to match all nodes + if selector == nil { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: lvmCluster.GetName(), + Namespace: lvmCluster.GetNamespace(), + }, + }) + continue + } + + match, err := corev1helper.MatchNodeSelectorTerms(node, selector) + if err != nil { + log.FromContext(ctx).Error(err, "getLVMClusterObjsByNameFittingNodeSelector: node selector matching in event handler failed") + return []reconcile.Request{} + } + if match { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: lvmCluster.GetName(), + Namespace: lvmCluster.GetNamespace(), + }, + }) } } return requests diff --git a/controllers/scc.go b/controllers/scc.go index 9b923ad27..66f6e7914 100644 --- a/controllers/scc.go +++ b/controllers/scc.go @@ -21,7 +21,10 @@ import ( "fmt" secv1 "github.com/openshift/api/security/v1" + lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "github.com/openshift/lvm-operator/pkg/labels" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,7 +46,7 @@ func (c openshiftSccs) getName() string { return sccName } -func (c openshiftSccs) ensureCreated(r *LVMClusterReconciler, ctx context.Context, _ *lvmv1alpha1.LVMCluster) error { +func (c openshiftSccs) ensureCreated(r *LVMClusterReconciler, ctx context.Context, cluster *lvmv1alpha1.LVMCluster) error { logger := log.FromContext(ctx).WithValues("resourceManager", c.getName()) sccs := getAllSCCs(r.Namespace) for _, scc := range sccs { @@ -58,6 +61,7 @@ func (c openshiftSccs) ensureCreated(r *LVMClusterReconciler, ctx context.Contex return fmt.Errorf("something went wrong when checking for SecurityContextConstraint: %w", err) } + labels.SetManagedLabels(r.Scheme, scc, cluster) if err := r.Create(ctx, scc); err != nil { return fmt.Errorf("%s failed to reconcile: %w", c.getName(), err) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index a6b99c672..69e1b37a6 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "errors" "path/filepath" "testing" @@ -27,6 +28,7 @@ import ( secv1 "github.com/openshift/api/security/v1" "github.com/openshift/lvm-operator/pkg/cluster" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -121,13 +123,24 @@ var _ = BeforeSuite(func() { clusterType, err := cluster.NewTypeResolver(k8sClient).GetType(ctx) Expect(err).ToNot(HaveOccurred()) + enableSnapshotting := true + vsc := &snapapi.VolumeSnapshotClassList{} + if err := k8sClient.List(ctx, vsc, &client.ListOptions{Limit: 1}); err != nil { + // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + logger.Info("VolumeSnapshotClasses do not exist on the cluster, ignoring") + enableSnapshotting = false + } + } + err = (&LVMClusterReconciler{ - Client: k8sManager.GetClient(), - EventRecorder: k8sManager.GetEventRecorderFor(ControllerName), - Scheme: k8sManager.GetScheme(), - ClusterType: clusterType, - Namespace: testLvmClusterNamespace, - ImageName: testImageName, + Client: k8sManager.GetClient(), + EventRecorder: k8sManager.GetEventRecorderFor("LVMClusterReconciler"), + EnableSnapshotting: enableSnapshotting, + Scheme: k8sManager.GetScheme(), + ClusterType: clusterType, + Namespace: testLvmClusterNamespace, + ImageName: testImageName, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/controllers/topolvm_csi_driver.go b/controllers/topolvm_csi_driver.go index 59ba40029..03b13568e 100644 --- a/controllers/topolvm_csi_driver.go +++ b/controllers/topolvm_csi_driver.go @@ -21,6 +21,8 @@ import ( "fmt" lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "github.com/openshift/lvm-operator/pkg/labels" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -44,11 +46,12 @@ func (c csiDriver) getName() string { //+kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers,verbs=get;create;delete;watch;list -func (c csiDriver) ensureCreated(r *LVMClusterReconciler, ctx context.Context, _ *lvmv1alpha1.LVMCluster) error { +func (c csiDriver) ensureCreated(r *LVMClusterReconciler, ctx context.Context, cluster *lvmv1alpha1.LVMCluster) error { logger := log.FromContext(ctx).WithValues("resourceManager", c.getName()) csiDriverResource := getCSIDriverResource() result, err := cutil.CreateOrUpdate(ctx, r.Client, csiDriverResource, func() error { + labels.SetManagedLabels(r.Scheme, csiDriverResource, cluster) // no need to mutate any field return nil }) diff --git a/controllers/topolvm_snapshotclass.go b/controllers/topolvm_snapshotclass.go index 3e7d11a8b..7b065f744 100644 --- a/controllers/topolvm_snapshotclass.go +++ b/controllers/topolvm_snapshotclass.go @@ -18,14 +18,15 @@ package controllers import ( "context" - "errors" "fmt" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "github.com/openshift/lvm-operator/pkg/labels" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" cutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" @@ -46,19 +47,17 @@ func (s topolvmVolumeSnapshotClass) getName() string { //+kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotclasses,verbs=get;create;delete;watch;list -func (s topolvmVolumeSnapshotClass) ensureCreated(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error { +func (s topolvmVolumeSnapshotClass) ensureCreated(r *LVMClusterReconciler, ctx context.Context, cluster *lvmv1alpha1.LVMCluster) error { logger := log.FromContext(ctx).WithValues("resourceManager", s.getName()) // one volume snapshot class for every deviceClass based on CR is created - topolvmSnapshotClasses := getTopolvmSnapshotClasses(lvmCluster) + topolvmSnapshotClasses := getTopolvmSnapshotClasses(cluster) for _, vsc := range topolvmSnapshotClasses { // we anticipate no edits to volume snapshot class - result, err := cutil.CreateOrUpdate(ctx, r.Client, vsc, func() error { return nil }) + result, err := cutil.CreateOrUpdate(ctx, r.Client, vsc, func() error { + labels.SetManagedLabels(r.Scheme, vsc, cluster) + return nil + }) if err != nil { - // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local - if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { - logger.Info("volume snapshot class CRDs do not exist on the cluster, ignoring", "VolumeSnapshotClass", vscName) - return nil - } return fmt.Errorf("%s failed to reconcile: %w", s.getName(), err) } logger.Info("VolumeSnapshotClass applied to cluster", "operation", result, "name", vsc.Name) @@ -77,11 +76,6 @@ func (s topolvmVolumeSnapshotClass) ensureDeleted(r *LVMClusterReconciler, ctx c vsc := &snapapi.VolumeSnapshotClass{} if err := r.Client.Get(ctx, types.NamespacedName{Name: vscName}, vsc); err != nil { - // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local - if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { - logger.Info("VolumeSnapshotClasses do not exist on the cluster, ignoring") - return nil - } return client.IgnoreNotFound(err) } diff --git a/controllers/topolvm_storageclass.go b/controllers/topolvm_storageclass.go index 9605e021b..29eedd68a 100644 --- a/controllers/topolvm_storageclass.go +++ b/controllers/topolvm_storageclass.go @@ -21,6 +21,8 @@ import ( "fmt" lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "github.com/openshift/lvm-operator/pkg/labels" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -44,14 +46,17 @@ func (s topolvmStorageClass) getName() string { //+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;create;delete;watch;list -func (s topolvmStorageClass) ensureCreated(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error { +func (s topolvmStorageClass) ensureCreated(r *LVMClusterReconciler, ctx context.Context, cluster *lvmv1alpha1.LVMCluster) error { logger := log.FromContext(ctx).WithValues("resourceManager", s.getName()) // one storage class for every deviceClass based on CR is created - topolvmStorageClasses := s.getTopolvmStorageClasses(r, ctx, lvmCluster) + topolvmStorageClasses := s.getTopolvmStorageClasses(r, ctx, cluster) for _, sc := range topolvmStorageClasses { // we anticipate no edits to storage class - result, err := cutil.CreateOrUpdate(ctx, r.Client, sc, func() error { return nil }) + result, err := cutil.CreateOrUpdate(ctx, r.Client, sc, func() error { + labels.SetManagedLabels(r.Scheme, sc, cluster) + return nil + }) if err != nil { return fmt.Errorf("%s failed to reconcile: %w", s.getName(), err) } diff --git a/controllers/vgmanager.go b/controllers/vgmanager.go index 714b723e7..8ecc88129 100644 --- a/controllers/vgmanager.go +++ b/controllers/vgmanager.go @@ -21,6 +21,8 @@ import ( "fmt" lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "github.com/openshift/lvm-operator/pkg/labels" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -56,6 +58,7 @@ func (v vgManager) ensureCreated(r *LVMClusterReconciler, ctx context.Context, l // the anonymous mutate function modifies the daemonset object after fetching it. // if the daemonset does not already exist, it creates it, otherwise, it updates it result, err := ctrl.CreateOrUpdate(ctx, r.Client, ds, func() error { + labels.SetManagedLabels(r.Scheme, ds, lvmCluster) if err := ctrl.SetControllerReference(lvmCluster, ds, r.Scheme); err != nil { return fmt.Errorf("failed to set controller reference on vgManager daemonset %q. %v", dsTemplate.Name, err) } diff --git a/main.go b/main.go index 659dcc0a1..c596be5ac 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "errors" "flag" "fmt" "os" @@ -25,6 +26,7 @@ import ( configv1 "github.com/openshift/api/config/v1" secv1 "github.com/openshift/api/security/v1" "github.com/openshift/lvm-operator/controllers/node" + "k8s.io/client-go/discovery" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -124,6 +126,16 @@ func main() { os.Exit(1) } + enableSnapshotting := true + vsc := &snapapi.VolumeSnapshotClassList{} + if err := setupClient.List(context.Background(), vsc, &client.ListOptions{Limit: 1}); err != nil { + // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + setupLog.Info("VolumeSnapshotClasses do not exist on the cluster, ignoring") + enableSnapshotting = false + } + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, @@ -141,10 +153,11 @@ func main() { if err = (&controllers.LVMClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor(controllers.ControllerName), + EventRecorder: mgr.GetEventRecorderFor("LVMClusterReconciler"), ClusterType: clusterType, Namespace: operatorNamespace, TopoLVMLeaderElectionPassthrough: leaderElectionConfig, + EnableSnapshotting: enableSnapshotting, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LVMCluster") os.Exit(1) diff --git a/pkg/labels/managed.go b/pkg/labels/managed.go new file mode 100644 index 000000000..03f1478d0 --- /dev/null +++ b/pkg/labels/managed.go @@ -0,0 +1,58 @@ +package labels + +import ( + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +// orients itself on https://sdk.operatorframework.io/docs/building-operators/ansible/reference/retroactively-owned-resources/#for-objects-which-are-not-in-the-same-namespace-as-the-owner-cr +// but uses labels and split apiVersion,kind and namespace,name to make label searches and filters possible. +const ( + OwnedByPrefix = "owned-by.topolvm.io" + OwnedByName = OwnedByPrefix + "/name" + OwnedByNamespace = OwnedByPrefix + "/namespace" + OwnedByUID = OwnedByPrefix + "/uid" + OwnedByGroup = OwnedByPrefix + "/group" + OwnedByVersion = OwnedByPrefix + "/version" + OwnedByKind = OwnedByPrefix + "/kind" +) + +func SetManagedLabels(scheme *runtime.Scheme, obj client.Object, owner client.Object) { + lbls := obj.GetLabels() + if lbls == nil { + lbls = make(map[string]string) + } + lbls[OwnedByName] = owner.GetName() + lbls[OwnedByNamespace] = owner.GetNamespace() + lbls[OwnedByUID] = string(owner.GetUID()) + if runtimeObj, ok := owner.(runtime.Object); ok { + if gvk, err := apiutil.GVKForObject(runtimeObj, scheme); err == nil { + lbls[OwnedByGroup] = gvk.Group + lbls[OwnedByVersion] = gvk.Version + lbls[OwnedByKind] = gvk.Kind + } + } + obj.SetLabels(lbls) +} + +func MatchesManagedLabels(scheme *runtime.Scheme, obj client.Object, owner client.Object) bool { + if lbls := obj.GetLabels(); lbls != nil { + baseMatch := lbls[OwnedByName] == owner.GetName() && + lbls[OwnedByNamespace] == owner.GetNamespace() && + lbls[OwnedByUID] == string(owner.GetUID()) + + if !baseMatch { + return false + } + + if ownerRuntimeObj, ok := owner.(runtime.Object); ok { + if gvk, err := apiutil.GVKForObject(ownerRuntimeObj, scheme); err == nil { + return lbls[OwnedByGroup] == gvk.Group && + lbls[OwnedByVersion] == gvk.Version && + lbls[OwnedByKind] == gvk.Kind + } + } + } + return false +} diff --git a/pkg/vgmanager/status.go b/pkg/vgmanager/status.go index c4076f3e6..6caeddd81 100644 --- a/pkg/vgmanager/status.go +++ b/pkg/vgmanager/status.go @@ -21,6 +21,7 @@ import ( "fmt" lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -func (r *VGReconciler) setVolumeGroupProgressingStatus(ctx context.Context, vgName string) error { +func (r *VGReconciler) setVolumeGroupProgressingStatus(ctx context.Context, vg *lvmv1alpha1.LVMVolumeGroup) error { status := &lvmv1alpha1.VGStatus{ - Name: vgName, + Name: vg.GetName(), Status: lvmv1alpha1.VGStatusProgressing, } @@ -39,12 +40,12 @@ func (r *VGReconciler) setVolumeGroupProgressingStatus(ctx context.Context, vgNa return err } - return r.setVolumeGroupStatus(ctx, status) + return r.setVolumeGroupStatus(ctx, vg, status) } -func (r *VGReconciler) setVolumeGroupReadyStatus(ctx context.Context, vgName string) error { +func (r *VGReconciler) setVolumeGroupReadyStatus(ctx context.Context, vg *lvmv1alpha1.LVMVolumeGroup) error { status := &lvmv1alpha1.VGStatus{ - Name: vgName, + Name: vg.GetName(), Status: lvmv1alpha1.VGStatusReady, } @@ -53,12 +54,12 @@ func (r *VGReconciler) setVolumeGroupReadyStatus(ctx context.Context, vgName str return err } - return r.setVolumeGroupStatus(ctx, status) + return r.setVolumeGroupStatus(ctx, vg, status) } -func (r *VGReconciler) setVolumeGroupFailedStatus(ctx context.Context, vgName string, err error) error { +func (r *VGReconciler) setVolumeGroupFailedStatus(ctx context.Context, vg *lvmv1alpha1.LVMVolumeGroup, err error) error { status := &lvmv1alpha1.VGStatus{ - Name: vgName, + Name: vg.GetName(), Status: lvmv1alpha1.VGStatusFailed, Reason: err.Error(), } @@ -71,16 +72,21 @@ func (r *VGReconciler) setVolumeGroupFailedStatus(ctx context.Context, vgName st status.Status = lvmv1alpha1.VGStatusDegraded } - return r.setVolumeGroupStatus(ctx, status) + return r.setVolumeGroupStatus(ctx, vg, status) } -func (r *VGReconciler) setVolumeGroupStatus(ctx context.Context, status *lvmv1alpha1.VGStatus) error { - logger := log.FromContext(ctx) +func (r *VGReconciler) setVolumeGroupStatus(ctx context.Context, vg *lvmv1alpha1.LVMVolumeGroup, status *lvmv1alpha1.VGStatus) error { + logger := log.FromContext(ctx).WithValues("VolumeGroup", client.ObjectKeyFromObject(vg)) // Get LVMVolumeGroupNodeStatus and set the relevant VGStatus nodeStatus := r.getLVMVolumeGroupNodeStatus() result, err := ctrl.CreateOrUpdate(ctx, r.Client, nodeStatus, func() error { + // set an owner instead of a controller reference, as there can be multiple volume groups. + if err := controllerutil.SetOwnerReference(nodeStatus, vg, r.Scheme); err != nil { + logger.Error(err, "failed to set owner-reference when updating volume-group status") + } + exists := false for i, existingVGStatus := range nodeStatus.Spec.LVMVGStatus { if existingVGStatus.Name == status.Name { @@ -108,7 +114,7 @@ func (r *VGReconciler) setVolumeGroupStatus(ctx context.Context, status *lvmv1al return nil } -func (r *VGReconciler) removeVolumeGroupStatus(ctx context.Context, vgName string) error { +func (r *VGReconciler) removeVolumeGroupStatus(ctx context.Context, vg *lvmv1alpha1.LVMVolumeGroup) error { logger := log.FromContext(ctx) // Get LVMVolumeGroupNodeStatus and remove the relevant VGStatus @@ -123,7 +129,7 @@ func (r *VGReconciler) removeVolumeGroupStatus(ctx context.Context, vgName strin index := 0 result, err := ctrl.CreateOrUpdate(ctx, r.Client, nodeStatus, func() error { for i, existingVGStatus := range nodeStatus.Spec.LVMVGStatus { - if existingVGStatus.Name == vgName { + if existingVGStatus.Name == vg.GetName() { exist = true index = i } @@ -131,6 +137,16 @@ func (r *VGReconciler) removeVolumeGroupStatus(ctx context.Context, vgName strin if exist { nodeStatus.Spec.LVMVGStatus = append(nodeStatus.Spec.LVMVGStatus[:index], nodeStatus.Spec.LVMVGStatus[index+1:]...) + // if we remove the vgstatus, we also remove the owner reference + for ownerRefIndex, ownerRef := range nodeStatus.GetOwnerReferences() { + if ownerRef.UID == vg.GetUID() { + nodeStatus.SetOwnerReferences(append( + nodeStatus.GetOwnerReferences()[:ownerRefIndex], + nodeStatus.GetOwnerReferences()[ownerRefIndex+1:]...), + ) + break + } + } } return nil diff --git a/pkg/vgmanager/vgmanager_controller.go b/pkg/vgmanager/vgmanager_controller.go index e1ddc184c..633ab9757 100644 --- a/pkg/vgmanager/vgmanager_controller.go +++ b/pkg/vgmanager/vgmanager_controller.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/tools/record" corev1helper "k8s.io/component-helpers/scheduling/corev1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" @@ -71,6 +72,7 @@ var ( func (r *VGReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&lvmv1alpha1.LVMVolumeGroup{}). + Owns(&lvmv1alpha1.LVMVolumeGroupNodeStatus{}, builder.MatchEveryOwner). Complete(r) } @@ -130,7 +132,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L lvmdConfig, err := loadLVMDConfig() if err != nil { err = fmt.Errorf("failed to read the lvmd config file: %w", err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -180,7 +182,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if !volumeGroupExists { err := fmt.Errorf("the volume group %s does not exist and there were no available devices to create it", volumeGroup.GetName()) r.WarningEvent(ctx, volumeGroup, EventReasonErrorNoAvailableDevicesForVG, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -190,7 +192,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if err := r.validateLVs(ctx, volumeGroup); err != nil { err := fmt.Errorf("error while validating logical volumes in existing volume group: %w", err) r.WarningEvent(ctx, volumeGroup, EventReasonErrorInconsistentLVs, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -198,7 +200,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L msg := "all the available devices are attached to the volume group" logger.Info(msg) - if err := r.setVolumeGroupReadyStatus(ctx, volumeGroup.Name); err != nil { + if err := r.setVolumeGroupReadyStatus(ctx, volumeGroup); err != nil { return ctrl.Result{}, fmt.Errorf("failed to set status for volume group %s to ready: %w", volumeGroup.Name, err) } r.NormalEvent(ctx, volumeGroup, EventReasonVolumeGroupReady, msg) @@ -210,7 +212,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if err = r.addDevicesToVG(ctx, vgs, volumeGroup.Name, availableDevices); err != nil { err = fmt.Errorf("failed to create/extend volume group %s: %w", volumeGroup.Name, err) r.WarningEvent(ctx, volumeGroup, EventReasonErrorVGCreateOrExtendFailed, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -220,7 +222,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if err = r.addThinPoolToVG(ctx, volumeGroup.Name, volumeGroup.Spec.ThinPoolConfig); err != nil { err := fmt.Errorf("failed to create thin pool %s for volume group %s: %w", volumeGroup.Spec.ThinPoolConfig.Name, volumeGroup.Name, err) r.WarningEvent(ctx, volumeGroup, EventReasonErrorThinPoolCreateOrExtendFailed, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -230,7 +232,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if err := r.validateLVs(ctx, volumeGroup); err != nil { err := fmt.Errorf("error while validating logical volumes in existing volume group: %w", err) r.WarningEvent(ctx, volumeGroup, EventReasonErrorInconsistentLVs, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -264,7 +266,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L if !cmp.Equal(existingLvmdConfig, lvmdConfig) { if err := saveLVMDConfig(lvmdConfig); err != nil { err := fmt.Errorf("failed to update lvmd config file to update volume group %s: %w", volumeGroup.GetName(), err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed") } return ctrl.Result{}, err @@ -274,7 +276,7 @@ func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.L r.NormalEvent(ctx, volumeGroup, EventReasonLVMDConfigUpdated, msg) } - if err := r.setVolumeGroupReadyStatus(ctx, volumeGroup.Name); err != nil { + if err := r.setVolumeGroupReadyStatus(ctx, volumeGroup); err != nil { return ctrl.Result{}, fmt.Errorf("failed to set status for volume group %s to ready: %w", volumeGroup.Name, err) } msg := "all the available devices are attached to the volume group" @@ -296,7 +298,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph logger.Info("lvmd config file does not exist, assuming deleted state and removing Volume Group Status for Node") // Remove the VG entry in the LVMVolumeGroupNodeStatus that was added to indicate the failures to the user. // This allows the LVMCluster to get deleted and not stuck/wait forever as LVMCluster looks for the LVMVolumeGroupNodeStatus before deleting. - if err := r.removeVolumeGroupStatus(ctx, volumeGroup.Name); err != nil { + if err := r.removeVolumeGroupStatus(ctx, volumeGroup); err != nil { return fmt.Errorf("failed to remove status for volume group %s: %w", volumeGroup.Name, err) } return nil @@ -310,7 +312,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph if !found { // Nothing to do here. logger.Info("could not find volume group in lvmd deviceclasses list, assuming deleted state and removing Volume Group Status for Node") - if err := r.removeVolumeGroupStatus(ctx, volumeGroup.Name); err != nil { + if err := r.removeVolumeGroupStatus(ctx, volumeGroup); err != nil { return fmt.Errorf("failed to remove status for volume group %s: %w", volumeGroup.Name, err) } return nil @@ -336,7 +338,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph if lvExists { if err := DeleteLV(r.executor, thinPoolName, volumeGroup.Name); err != nil { err := fmt.Errorf("failed to delete thin pool %s in volume group %s: %w", thinPoolName, volumeGroup.Name, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed", "VGName", volumeGroup.GetName()) } return err @@ -349,7 +351,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph if err = vg.Delete(r.executor); err != nil { err := fmt.Errorf("failed to delete volume group %s: %w", volumeGroup.Name, err) - if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, err); err != nil { + if err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, err); err != nil { logger.Error(err, "failed to set status to failed", "VGName", volumeGroup.GetName()) } return err @@ -374,7 +376,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph r.NormalEvent(ctx, volumeGroup, EventReasonLVMDConfigDeleted, msg) } - if err := r.removeVolumeGroupStatus(ctx, volumeGroup.Name); err != nil { + if err := r.removeVolumeGroupStatus(ctx, volumeGroup); err != nil { return fmt.Errorf("failed to remove status for volume group %s: %w", volumeGroup.Name, err) }