feat: save lvmd config to a configmap instead of a file

Signed-off-by: Suleyman Akbas <[email protected]>
openshift · Nov 9, 2023 · faed224 · faed224
1 parent 7499d08
commit faed224
Show file tree

Hide file tree

Showing 9 changed files with 256 additions and 125 deletions.
diff --git a/bundle/manifests/lvms-operator.clusterserviceversion.yaml b/bundle/manifests/lvms-operator.clusterserviceversion.yaml
@@ -853,6 +853,18 @@ spec:
           - create
           - patch
           - update
+        - apiGroups:
+          - ""
+          resources:
+          - configmaps
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - patch
+          - update
+          - watch
         serviceAccountName: vg-manager
     strategy: deployment
   installModes:

diff --git a/cmd/vgmanager/vgmanager.go b/cmd/vgmanager/vgmanager.go
@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/go-logr/logr"
+	"github.com/openshift/lvm-operator/internal/cluster"
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager"
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/dmsetup"
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/filter"
@@ -84,14 +85,18 @@ func NewCmd(opts *Options) *cobra.Command {
 func run(cmd *cobra.Command, _ []string, opts *Options) error {
 	lvm := lvm.NewDefaultHostLVM()
 	nodeName := os.Getenv("NODE_NAME")
-	namespace := os.Getenv("POD_NAMESPACE")
+
+	operatorNamespace, err := cluster.GetOperatorNamespace()
+	if err != nil {
+		return fmt.Errorf("unable to get operatorNamespace: %w", err)
+	}
 
 	setupClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: opts.Scheme})
 	if err != nil {
 		return fmt.Errorf("unable to initialize setup client for pre-manager startup checks: %w", err)
 	}
 
-	if err := tagging.AddTagToVGs(cmd.Context(), setupClient, lvm, nodeName, namespace); err != nil {
+	if err := tagging.AddTagToVGs(cmd.Context(), setupClient, lvm, nodeName, operatorNamespace); err != nil {
 		opts.SetupLog.Error(err, "failed to tag existing volume groups")
 	}
 
@@ -109,7 +114,7 @@ func run(cmd *cobra.Command, _ []string, opts *Options) error {
 		LeaderElection:         false,
 		Cache: cache.Options{
 			DefaultNamespaces: map[string]cache.Config{
-				os.Getenv("POD_NAMESPACE"): {},
+				operatorNamespace: {},
 			},
 		},
 	})
@@ -120,14 +125,14 @@ func run(cmd *cobra.Command, _ []string, opts *Options) error {
 	if err = (&vgmanager.Reconciler{
 		Client:        mgr.GetClient(),
 		EventRecorder: mgr.GetEventRecorderFor(vgmanager.ControllerName),
-		LVMD:          lvmd.DefaultConfigurator(),
+		LVMD:          lvmd.NewFileConfigurator(mgr.GetClient(), operatorNamespace),
 		Scheme:        mgr.GetScheme(),
 		LSBLK:         lsblk.NewDefaultHostLSBLK(),
 		Wipefs:        wipefs.NewDefaultHostWipefs(),
 		Dmsetup:       dmsetup.NewDefaultHostDmsetup(),
 		LVM:           lvm,
 		NodeName:      nodeName,
-		Namespace:     namespace,
+		Namespace:     operatorNamespace,
 		Filters:       filter.DefaultFilters,
 	}).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create controller VGManager: %w", err)

diff --git a/config/rbac/vg_manager_role.yaml b/config/rbac/vg_manager_role.yaml
@@ -66,3 +66,15 @@ rules:
     - create
     - patch
     - update
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go
@@ -61,9 +61,6 @@ const (
 	LivenessProbeMemRequest = "15Mi"
 	LivenessProbeCPURequest = "1m"
 
-	FileCheckerMemRequest = "10Mi"
-	FileCheckerCPURequest = "1m"
-
 	// topoLVM Node
 	TopolvmNodeServiceAccount       = "topolvm-node"
 	TopolvmNodeDaemonsetName        = "topolvm-node"
@@ -74,6 +71,10 @@ const (
 	DefaultCSISocket = "/run/topolvm/csi-topolvm.sock"
 	DeviceClassKey   = "topolvm.io/device-class"
 
+	LVMDConfigMapName         = "lvmd-config"
+	LVMDDefaultConfigDir      = "/etc/topolvm"
+	LVMDDefaultFileConfigPath = "/etc/topolvm/lvmd.yaml"
+
 	// name of the lvm-operator container
 	LVMOperatorContainerName = "manager"
 

diff --git a/internal/controllers/lvmcluster/resource/topolvm_node.go b/internal/controllers/lvmcluster/resource/topolvm_node.go
@@ -25,7 +25,6 @@ import (
 	lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1"
 	"github.com/openshift/lvm-operator/internal/controllers/constants"
 	"github.com/openshift/lvm-operator/internal/controllers/lvmcluster/selector"
-	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvmd"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -59,7 +58,6 @@ func (n topolvmNode) EnsureCreated(r Reconciler, ctx context.Context, lvmCluster
 	// get desired daemonSet spec
 	dsTemplate := getNodeDaemonSet(lvmCluster,
 		r.GetNamespace(),
-		r.GetImageName(),
 		r.GetLogPassthroughOptions().TopoLVMNode.AsArgs(),
 		r.GetLogPassthroughOptions().CSISideCar.AsArgs(),
 	)
@@ -83,14 +81,11 @@ func (n topolvmNode) EnsureCreated(r Reconciler, ctx context.Context, lvmCluster
 			return nil
 		}
 		// if update, update only mutable fields
-		// For topolvm Node, we have containers, initcontainers, node selector and toleration terms
+		// For topolvm Node, we have containers, node selector and toleration terms
 
 		// containers
 		ds.Spec.Template.Spec.Containers = dsTemplate.Spec.Template.Spec.Containers
 
-		// initcontainers
-		ds.Spec.Template.Spec.InitContainers = dsTemplate.Spec.Template.Spec.InitContainers
-
 		// tolerations
 		ds.Spec.Template.Spec.Tolerations = dsTemplate.Spec.Template.Spec.Tolerations
 
@@ -121,7 +116,7 @@ func (n topolvmNode) EnsureDeleted(_ Reconciler, _ context.Context, _ *lvmv1alph
 	return nil
 }
 
-func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, initImage string, args, csiArgs []string) *appsv1.DaemonSet {
+func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, args, csiArgs []string) *appsv1.DaemonSet {
 
 	hostPathDirectory := corev1.HostPathDirectory
 	hostPathDirectoryOrCreateType := corev1.HostPathDirectoryOrCreate
@@ -149,9 +144,12 @@ func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, init
 					Type: &hostPathDirectoryOrCreateType}}},
 		{Name: "lvmd-config-dir",
 			VolumeSource: corev1.VolumeSource{
-				HostPath: &corev1.HostPathVolumeSource{
-					Path: filepath.Dir(lvmd.DefaultFileConfigPath),
-					Type: &hostPathDirectory}}},
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: constants.LVMDConfigMapName,
+					},
+				},
+			}},
 		{Name: "metrics-cert",
 			VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
@@ -161,7 +159,6 @@ func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, init
 			}},
 	}
 
-	initContainers := []corev1.Container{*getNodeInitContainer(initImage)}
 	containers := []corev1.Container{
 		*getNodeContainer(args),
 		*getRBACProxyContainer(),
@@ -204,7 +201,6 @@ func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, init
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: constants.TopolvmNodeServiceAccount,
-					InitContainers:     initContainers,
 					Containers:         containers,
 					Volumes:            volumes,
 					HostPID:            true,
@@ -220,38 +216,11 @@ func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster, namespace string, init
 	return nodeDaemonSet
 }
 
-func getNodeInitContainer(initImage string) *corev1.Container {
-	command := []string{
-		"/usr/bin/bash",
-		"-c",
-		fmt.Sprintf("until [ -f %s ]; do echo waiting for lvmd config file; sleep 5; done", lvmd.DefaultFileConfigPath),
-	}
-
-	volumeMounts := []corev1.VolumeMount{
-		{Name: "lvmd-config-dir", MountPath: filepath.Dir(lvmd.DefaultFileConfigPath)},
-	}
-
-	fileChecker := &corev1.Container{
-		Name:         "file-checker",
-		Image:        initImage,
-		Command:      command,
-		VolumeMounts: volumeMounts,
-		Resources: corev1.ResourceRequirements{
-			Requests: corev1.ResourceList{
-				corev1.ResourceCPU:    resource.MustParse(constants.FileCheckerCPURequest),
-				corev1.ResourceMemory: resource.MustParse(constants.FileCheckerMemRequest),
-			},
-		},
-	}
-
-	return fileChecker
-}
-
 func getNodeContainer(args []string) *corev1.Container {
 	command := []string{
 		"/topolvm-node",
 		"--embed-lvmd",
-		fmt.Sprintf("--config=%s", lvmd.DefaultFileConfigPath),
+		fmt.Sprintf("--config=%s", constants.LVMDDefaultFileConfigPath),
 	}
 
 	command = append(command, args...)
@@ -267,7 +236,7 @@ func getNodeContainer(args []string) *corev1.Container {
 
 	volumeMounts := []corev1.VolumeMount{
 		{Name: "node-plugin-dir", MountPath: filepath.Dir(constants.DefaultCSISocket)},
-		{Name: "lvmd-config-dir", MountPath: filepath.Dir(lvmd.DefaultFileConfigPath)},
+		{Name: "lvmd-config-dir", MountPath: constants.LVMDDefaultConfigDir},
 		{Name: "pod-volumes-dir",
 			MountPath:        fmt.Sprintf("%spods", getAbsoluteKubeletPath(constants.CSIKubeletRootDir)),
 			MountPropagation: &mountPropagationMode},

diff --git a/internal/controllers/vgmanager/controller.go b/internal/controllers/vgmanager/controller.go
@@ -31,6 +31,7 @@ import (
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvm"
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/lvmd"
 	"github.com/openshift/lvm-operator/internal/controllers/vgmanager/wipefs"
+
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,8 +44,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 const (
@@ -79,9 +82,38 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&lvmv1alpha1.LVMVolumeGroup{}).
 		Owns(&lvmv1alpha1.LVMVolumeGroupNodeStatus{}, builder.MatchEveryOwner, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.getObjsInNamespaceForReconcile)).
 		Complete(r)
 }
 
+// getObjsInNamespaceForReconcile reconciles the object anytime the given object is in the same namespace
+// as the available LVMVolumeGroups.
+func (r *Reconciler) getObjsInNamespaceForReconcile(ctx context.Context, obj client.Object) []reconcile.Request {
+	foundLVMVolumeGroupList := &lvmv1alpha1.LVMVolumeGroupList{}
+	listOps := &client.ListOptions{
+		Namespace: obj.GetNamespace(),
+	}
+
+	if err := r.List(ctx, foundLVMVolumeGroupList, listOps); err != nil {
+		log.FromContext(ctx).Error(err, "getObjsInNamespaceForReconcile: Failed to get LVMVolumeGroup objs")
+		return []reconcile.Request{}
+	}
+	if len(foundLVMVolumeGroupList.Items) < 1 {
+		return []reconcile.Request{}
+	}
+
+	var requests []reconcile.Request
+	for _, lvmVG := range foundLVMVolumeGroupList.Items {
+		requests = append(requests, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Name:      lvmVG.GetName(),
+				Namespace: lvmVG.GetNamespace(),
+			},
+		})
+	}
+	return requests
+}
+
 type Reconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
@@ -288,7 +320,7 @@ func (r *Reconciler) applyLVMDConfig(ctx context.Context, volumeGroup *lvmv1alph
 	logger := log.FromContext(ctx).WithValues("VGName", volumeGroup.Name)
 
 	// Read the lvmd config file
-	lvmdConfig, err := r.LVMD.Load()
+	lvmdConfig, err := r.LVMD.Load(ctx)
 	if err != nil {
 		err = fmt.Errorf("failed to read the lvmd config file: %w", err)
 		if _, err := r.setVolumeGroupFailedStatus(ctx, volumeGroup, nil, err); err != nil {
@@ -354,7 +386,7 @@ func (r *Reconciler) updateLVMDConfigAfterReconcile(
 			r.NormalEvent(ctx, volumeGroup, EventReasonLVMDConfigMissing, msg)
 		}
 
-		if err := r.LVMD.Save(newCFG); err != nil {
+		if err := r.LVMD.Save(ctx, newCFG); err != nil {
 			return fmt.Errorf("failed to update lvmd config file to update volume group %s: %w", volumeGroup.GetName(), err)
 		}
 		msg := "updated lvmd config with new deviceClasses"
@@ -369,7 +401,7 @@ func (r *Reconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alpha1
 	logger.Info("deleting")
 
 	// Read the lvmd config file
-	lvmdConfig, err := r.LVMD.Load()
+	lvmdConfig, err := r.LVMD.Load(ctx)
 	if err != nil {
 		// Failed to read lvmdconfig file. Reconcile again
 		return fmt.Errorf("failed to read the lvmd config file: %w", err)
@@ -437,14 +469,14 @@ func (r *Reconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alpha1
 	// if there was no config file in the first place, nothing has to be removed.
 	if lvmdConfig != nil {
 		if len(lvmdConfig.DeviceClasses) > 0 {
-			if err = r.LVMD.Save(lvmdConfig); err != nil {
+			if err = r.LVMD.Save(ctx, lvmdConfig); err != nil {
 				return fmt.Errorf("failed to update lvmd.conf file for volume group %s: %w", volumeGroup.GetName(), err)
 			}
 			msg := "updated lvmd config after deviceClass was removed"
 			logger.Info(msg)
 			r.NormalEvent(ctx, volumeGroup, EventReasonLVMDConfigUpdated, msg)
 		} else {
-			if err = r.LVMD.Delete(); err != nil {
+			if err = r.LVMD.Delete(ctx); err != nil {
 				return fmt.Errorf("failed to delete lvmd.conf file for volume group %s: %w", volumeGroup.GetName(), err)
 			}
 			msg := "removed lvmd config after last deviceClass was removed"