From b06ab526c49a65dd13dd75f315dcf6ef862ca58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Miguel=20Olmo=20Mart=C3=ADnez?= Date: Thu, 9 Dec 2021 19:54:50 +0100 Subject: [PATCH] WIP: CSI Node daemonset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Miguel Olmo Martínez --- controllers/defaults.go | 16 ++ controllers/lvmcluster_controller.go | 4 +- controllers/topolvm_node.go | 276 +++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 controllers/topolvm_node.go diff --git a/controllers/defaults.go b/controllers/defaults.go index 8ceff8ac9..d193ab7e5 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -38,6 +38,22 @@ var ( CsiProvisionerImage = GetEnvOrDefault("CSI_PROVISIONER_IMAGE") CsiLivenessProbeImage = GetEnvOrDefault("CSI_LIVENESSPROBE_IMAGE") CsiResizerImage = GetEnvOrDefault("CSI_RESIZER_IMAGE") + + TopolvmControllerServiceAccount = "topolvm-controller" + AppAttr = "app.kubernetes.io/name" + + // topoLVM Node + TopolvmNodeDaemonsetName = "topolvm-node" + CSIKubeletRootDir = "/var/lib/kubelet/" + lvmdConfigCM = "lvmd_config" + NodeContainerName = "topolvm-node" + TopolvmNodeContainerHealthzName = "healthz" + + // topoLVM Node resource requests/limits + TopolvmNodeMemRequest = "250Mi" + TopolvmNodeMemLimit = "250Mi" + TopolvmNodeCPURequest = "250m" + TopolvmNodeCPULimit = "250m" ) func GetEnvOrDefault(env string) string { diff --git a/controllers/lvmcluster_controller.go b/controllers/lvmcluster_controller.go index 6b04ecafa..a143aaeaa 100644 --- a/controllers/lvmcluster_controller.go +++ b/controllers/lvmcluster_controller.go @@ -95,7 +95,9 @@ func (r *LVMClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) // errors returned by this will be updated in the reconcileSucceeded condition of the LVMCluster func (r *LVMClusterReconciler) reconcile(ctx context.Context, instance *lvmv1alpha1.LVMCluster) (ctrl.Result, error) { - resourceList := []resourceManager{} + resourceList := []resourceManager{ + &topolvmNode{}, + } //The resource was deleted if !instance.DeletionTimestamp.IsZero() { diff --git a/controllers/topolvm_node.go b/controllers/topolvm_node.go new file mode 100644 index 000000000..d23627a99 --- /dev/null +++ b/controllers/topolvm_node.go @@ -0,0 +1,276 @@ +package controllers + +import ( + "context" + "fmt" + lvmv1alpha1 "github.com/red-hat-storage/lvm-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + cutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "strings" +) + +const ( + topolvmNodeName = "topolvm-node" +) + +type topolvmNode struct{} + +func (n topolvmNode) getName() string { + return topolvmNodeName +} + +func (n topolvmNode) ensureCreated(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error { + nodeDaemonSet := getNodeDaemonSet(lvmCluster) + result, err := cutil.CreateOrUpdate(ctx, r.Client, nodeDaemonSet, func() error { + // make sure LVMCluster CR garbage collects this daemonset and also block owner removal + return cutil.SetControllerReference(lvmCluster, nodeDaemonSet, r.Scheme) + }) + + switch result { + case cutil.OperationResultCreated: + r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name) + case cutil.OperationResultUpdated: + r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name) + case cutil.OperationResultNone: + r.Log.Info(topolvmNodeName, "operation", result, "name", nodeDaemonSet.Name) + default: + r.Log.Error(err, fmt.Sprintf("%s reconcile failure", topolvmNodeName), "name", nodeDaemonSet.Name) + return err + } + + return nil +} + +// ensureDeleted should wait for the resources to be cleaned up +func (n topolvmNode) ensureDeleted(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error { + NodeDaemonSet := &appsv1.DaemonSet{} + err := r.Client.Get(ctx, types.NamespacedName{Name: TopolvmNodeDaemonsetName, Namespace: lvmCluster.Namespace}, NodeDaemonSet) + + if err != nil { + if errors.IsNotFound(err) { + r.Log.Info("topolvm node deleted", "TopolvmNode", NodeDaemonSet.Name) + return nil + } + r.Log.Error(err, "unable to retrieve topolvm node daemonset", "TopolvmNode", NodeDaemonSet.Name) + return err + } + return nil +} + +// updateStatus should optionally update the CR's status about the health of the managed resource +// each unit will have updateStatus called individually so +// avoid status fields like lastHeartbeatTime and have a +// status that changes only when the operands change. +func (n topolvmNode) updateStatus(r *LVMClusterReconciler, ctx context.Context, lvmCluster *lvmv1alpha1.LVMCluster) error { + return nil +} + +func getNodeDaemonSet(lvmCluster *lvmv1alpha1.LVMCluster) *v1.DaemonSet { + hostPathDirectory := corev1.HostPathDirectory + hostPathDirectoryOrCreateType := corev1.HostPathDirectoryOrCreate + storageMedium := corev1.StorageMediumMemory + + volumes := []corev1.Volume{ + {Name: "registration-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: fmt.Sprintf("%splugins_registry/", getAbsoluteKubeletPath(CSIKubeletRootDir)), Type: &hostPathDirectory}}}, + {Name: "node-plugin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: fmt.Sprintf("%splugins/topolvm.cybozu.com/node", getAbsoluteKubeletPath(CSIKubeletRootDir)), Type: &hostPathDirectoryOrCreateType}}}, + {Name: "csi-plugin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: fmt.Sprintf("%splugins/kubernetes.io/csi", getAbsoluteKubeletPath(CSIKubeletRootDir)), Type: &hostPathDirectoryOrCreateType}}}, + {Name: "pod-volumes-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: fmt.Sprintf("%spods/", getAbsoluteKubeletPath(CSIKubeletRootDir)), Type: &hostPathDirectoryOrCreateType}}}, + {Name: "lvmd-config-dir", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: lvmdConfigCM}}}}, + {Name: "lvmd-socket-dir", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: storageMedium}}}, + } + + containers := []corev1.Container{*getLvmdContainer(), *getNodeContainer(), *getCsiRegistrarContainer(), *getLivenessProbeContainer()} + + // TODO: Add the same node selector we will have in the lvmcluster CRD + nodeDaemonset := &v1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: TopolvmNodeDaemonsetName, + Namespace: lvmCluster.Namespace, + }, + Spec: v1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + AppAttr: lvmCluster.Name, + }, + }, + UpdateStrategy: v1.DaemonSetUpdateStrategy{ + Type: v1.RollingUpdateDaemonSetStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: lvmCluster.Name, + Labels: map[string]string{ + AppAttr: lvmCluster.Name, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: TopolvmControllerServiceAccount, + Containers: containers, + Volumes: volumes, + HostPID: true, + Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, + }, + }, + }, + } + + return nodeDaemonset +} + +func getLvmdContainer() *corev1.Container { + command := []string{ + "/lvmd", + "--config=/etc/topolvm/lvmd.yaml", + "--container=true", + } + + resourceRequirements := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(TopolvmNodeCPULimit), + corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemLimit), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(TopolvmNodeCPURequest), + corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemRequest), + }, + } + + volumeMounts := []corev1.VolumeMount{ + {Name: "lvmd-socket-dir", MountPath: "/run/topolvm"}, + {Name: "lvmd-config-dir", MountPath: "/etc/topolvm"}} + + env := []corev1.EnvVar{ + {Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}, + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + } + + privileged := true + runAsUser := int64(0) + lvmd := &corev1.Container{ + Name: "lvmd", + Image: TopolvmCsiImage, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + }, + Command: command, + Resources: resourceRequirements, + VolumeMounts: volumeMounts, + Env: env, + } + return lvmd +} + +func getNodeContainer() *corev1.Container { + privileged := true + runAsUser := int64(0) + + command := []string{ + "/topolvm-node", + "--lvmd-socket=/run/lvmd/lvmd.sock", + } + + requirements := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(TopolvmNodeCPULimit), + corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemLimit), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(TopolvmNodeCPURequest), + corev1.ResourceMemory: resource.MustParse(TopolvmNodeMemRequest), + }, + } + + mountPropagationMode := corev1.MountPropagationBidirectional + + volumeMounts := []corev1.VolumeMount{ + {Name: "node-plugin-dir", MountPath: "/run/topolvm"}, + {Name: "lvmd-socket-dir", MountPath: "/run/lvmd"}, + {Name: "pod-volumes-dir", MountPath: fmt.Sprintf("%spods", getAbsoluteKubeletPath(CSIKubeletRootDir)), MountPropagation: &mountPropagationMode}, + {Name: "csi-plugin-dir", MountPath: fmt.Sprintf("%splugins/kubernetes.io/csi", getAbsoluteKubeletPath(CSIKubeletRootDir)), MountPropagation: &mountPropagationMode}, + } + + env := []corev1.EnvVar{ + {Name: "NODE_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}}, + } + + node := &corev1.Container{ + Name: NodeContainerName, + Image: TopolvmCsiImage, + Command: command, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + }, + Ports: []corev1.ContainerPort{{Name: TopolvmNodeContainerHealthzName, ContainerPort: 9808, Protocol: corev1.ProtocolTCP}}, + LivenessProbe: &corev1.Probe{Handler: corev1.Handler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromString(TopolvmNodeContainerHealthzName)}}, + FailureThreshold: 3, InitialDelaySeconds: 10, TimeoutSeconds: 3, PeriodSeconds: 60}, + Resources: requirements, + Env: env, + VolumeMounts: volumeMounts, + } + return node +} + +func getCsiRegistrarContainer() *corev1.Container { + command := []string{ + "/csi-node-driver-registrar", + "--csi-address=/run/topolvm/csi-topolvm.sock", + fmt.Sprintf("--kubelet-registration-path=%splugins/topolvm.cybozu.com/node/csi-topolvm.sock", getAbsoluteKubeletPath(CSIKubeletRootDir)), + } + + volumeMounts := []corev1.VolumeMount{ + {Name: "node-plugin-dir", MountPath: "/run/topolvm"}, + {Name: "registration-dir", MountPath: "/registration"}, + } + + preStopCmd := []string{ + "/bin/sh", + "-c", + "rm -rf /registration/topolvm.cybozu.com /registration/topolvm.cybozu.com-reg.sock", + } + + csiRegistrar := &corev1.Container{ + Name: "csi-registrar", + Image: CsiRegistrarImage, + Command: command, + Lifecycle: &corev1.Lifecycle{PreStop: &corev1.Handler{Exec: &corev1.ExecAction{Command: preStopCmd}}}, + VolumeMounts: volumeMounts, + } + return csiRegistrar +} + +func getLivenessProbeContainer() *corev1.Container { + command := []string{ + "/livenessprobe", + "--csi-address=/run/topolvm/csi-topolvm.sock", + } + + volumeMounts := []corev1.VolumeMount{ + {Name: "node-plugin-dir", MountPath: "/run/topolvm"}, + } + + liveness := &corev1.Container{ + Name: "liveness-probe", + Image: CsiLivenessProbeImage, + Command: command, + VolumeMounts: volumeMounts, + } + return liveness +} + +func getAbsoluteKubeletPath(name string) string { + if strings.HasSuffix(name, "/") { + return name + } else { + return name + "/" + } +}