diff --git a/cmd/vgmanager/main.go b/cmd/vgmanager/main.go index 3dd435970..61be8ede4 100644 --- a/cmd/vgmanager/main.go +++ b/cmd/vgmanager/main.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Red Hat Openshift Data Foundation. +Copyright © 2023 Red Hat, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,28 +17,25 @@ limitations under the License. package main import ( - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - "flag" "os" - _ "k8s.io/client-go/plugin/pkg/client/auth" - lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" "github.com/openshift/lvm-operator/pkg/vgmanager" + "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() ) func init() { @@ -46,21 +43,22 @@ func init() { utilruntime.Must(lvmv1alpha1.AddToScheme(scheme)) } -var metricsAddr string -var probeAddr string -var developmentMode bool - func main() { + var metricsAddr string + var probeAddr string + var developmentMode bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&developmentMode, "development", false, "enable to enable development") + flag.BoolVar(&developmentMode, "development", false, "The switch to enable development mode.") + flag.Parse() + opts := zap.Options{} opts.BindFlags(flag.CommandLine) - flag.Parse() opts.Development = developmentMode - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + setupLog := ctrl.Log.WithName("setup") + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, @@ -80,7 +78,7 @@ func main() { NodeName: os.Getenv("NODE_NAME"), Namespace: os.Getenv("POD_NAMESPACE"), }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "LVMCluster") + setupLog.Error(err, "unable to create controller", "controller", "VGManager") os.Exit(1) } diff --git a/pkg/vgmanager/vgmanager_controller.go b/pkg/vgmanager/vgmanager_controller.go index 06e7e73f1..81d9c75d5 100644 --- a/pkg/vgmanager/vgmanager_controller.go +++ b/pkg/vgmanager/vgmanager_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Red Hat Openshift Data Foundation. +Copyright © 2023 Red Hat, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ import ( "github.com/openshift/lvm-operator/pkg/internal" "github.com/topolvm/topolvm/lvmd" lvmdCMD "github.com/topolvm/topolvm/pkg/lvmd/cmd" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,8 +47,13 @@ import ( ) const ( - ControllerName = "vg-manager" - DefaultChunkSize = "128" + ControllerName = "vg-manager" + DefaultChunkSize = "128" + reconcileInterval = 1 * time.Minute +) + +var ( + reconcileAgain = ctrl.Result{Requeue: true, RequeueAfter: reconcileInterval} ) // SetupWithManager sets up the controller with the Manager. @@ -89,30 +95,23 @@ func (r *VGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re r.Log.Error(err, "failed to match nodeSelector to node labels", "VGName", volumeGroup.Name) return ctrl.Result{}, err } - if !nodeMatches { // Nothing to be done on this node for the VG. r.Log.Info("node labels do not match the selector", "VGName", volumeGroup.Name) return ctrl.Result{}, nil } + r.executor = &internal.CommandExecutor{} - res, err := r.reconcile(ctx, req, volumeGroup) + res, err := r.reconcile(ctx, volumeGroup) if err != nil { - r.Log.Error(err, "reconcile error", "lvmvolumegroup", req.Name) + r.Log.Error(err, "reconcile error", "lvmvolumegroup", volumeGroup.Name) } r.Log.Info("reconcile complete", "result", res) return res, err - } -var reconcileInterval = time.Minute * 1 -var reconcileAgain = ctrl.Result{Requeue: true, RequeueAfter: reconcileInterval} - -//TODO: Refactor this function to move the ctrl result to a single place - -func (r *VGReconciler) reconcile(ctx context.Context, req ctrl.Request, volumeGroup *lvmv1alpha1.LVMVolumeGroup) (ctrl.Result, error) { - - // The LVMVolumeGroup resource was deleted +func (r *VGReconciler) reconcile(ctx context.Context, volumeGroup *lvmv1alpha1.LVMVolumeGroup) (ctrl.Result, error) { + // Check if the LVMVolumeGroup resource is deleted if !volumeGroup.DeletionTimestamp.IsZero() { err := r.processDelete(ctx, volumeGroup) return ctrl.Result{}, err @@ -121,11 +120,12 @@ func (r *VGReconciler) reconcile(ctx context.Context, req ctrl.Request, volumeGr // Read the lvmd config file lvmdConfig, err := loadLVMDConfig() if err != nil { - // Failed to read lvmdconfig file. Reconcile again r.Log.Error(err, "failed to read the lvmd config file") + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, fmt.Sprintf("failed to read the lvmd config file: %v", err.Error())); statuserr != nil { + r.Log.Error(statuserr, "failed to update status", "name", volumeGroup.Name) + } return reconcileAgain, err } - if lvmdConfig == nil { // The lvmdconfig file does not exist and will be created. r.Log.Info("lvmd config file doesn't exist, will create") @@ -135,26 +135,12 @@ func (r *VGReconciler) reconcile(ctx context.Context, req ctrl.Request, volumeGr } existingLvmdConfig := *lvmdConfig - // To avoid having to iterate through device classes multiple times, map from name to config index - deviceClassMap := make(map[string]int) - for i, deviceClass := range lvmdConfig.DeviceClasses { - deviceClassMap[deviceClass.Name] = i - } - - status := &lvmv1alpha1.VGStatus{ - Name: req.Name, - } - _, found := deviceClassMap[volumeGroup.Name] - //Get the block devices that can be used for this volumegroup matchingDevices, delayedDevices, err := r.getMatchingDevicesForVG(volumeGroup) if err != nil { r.Log.Error(err, "failed to get block devices for volumegroup", "name", volumeGroup.Name) - - status.Reason = err.Error() - if statuserr := r.updateStatus(ctx, status); statuserr != nil { + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, fmt.Sprintf("failed to get block devices for volumegroup %s: %v", volumeGroup.Name, err.Error())); statuserr != nil { r.Log.Error(statuserr, "failed to update status", "name", volumeGroup.Name) - return reconcileAgain, nil } // Failed to get devices for this vg. Reconcile again. @@ -167,29 +153,42 @@ func (r *VGReconciler) reconcile(ctx context.Context, req ctrl.Request, volumeGr return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil //30 seconds to make sure delayed devices become available } - if found { - // Update the status again just to be safe. - if statuserr := r.updateStatus(ctx, nil); statuserr != nil { - r.Log.Error(statuserr, "failed to update status", "name", volumeGroup.Name) - return reconcileAgain, nil - } + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, "no matching devices found for volume group"); statuserr != nil { + r.Log.Error(statuserr, "failed to update status", "name", volumeGroup.Name) + return reconcileAgain, nil } + return ctrl.Result{}, nil } - // create/extend VG and update lvmd config + // Create/extend VG err = r.addDevicesToVG(volumeGroup.Name, matchingDevices) if err != nil { - status.Reason = err.Error() - r.Log.Error(err, "failed to create/extend volume group", "VGName", volumeGroup.Name) - - if statuserr := r.updateStatus(ctx, status); statuserr != nil { + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, fmt.Sprintf("failed to create/extend volume group %s: %v", volumeGroup.Name, err.Error())); statuserr != nil { r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) } return reconcileAgain, err } + // Create thin pool + err = r.addThinPoolToVG(volumeGroup.Name, volumeGroup.Spec.ThinPoolConfig) + if err != nil { + r.Log.Error(err, "failed to create thin pool", "VGName", "ThinPool", volumeGroup.Name, volumeGroup.Spec.ThinPoolConfig.Name) + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, + fmt.Sprintf("failed to create thin pool %s for volume group %s: %v", volumeGroup.Spec.ThinPoolConfig.Name, volumeGroup.Name, err.Error())); statuserr != nil { + r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) + return reconcileAgain, nil + } + } + + // Add the volume group to device classes if not exists + found := false + for _, deviceClass := range lvmdConfig.DeviceClasses { + if deviceClass.Name == volumeGroup.Name { + found = true + } + } if !found { dc := &lvmd.DeviceClass{ Name: volumeGroup.Name, @@ -207,30 +206,22 @@ func (r *VGReconciler) reconcile(ctx context.Context, req ctrl.Request, volumeGr lvmdConfig.DeviceClasses = append(lvmdConfig.DeviceClasses, dc) } - // Create thin pool - err = r.addThinPoolToVG(volumeGroup.Name, volumeGroup.Spec.ThinPoolConfig) - if err != nil { - r.Log.Error(err, "failed to create thin pool", "VGName", "ThinPool", volumeGroup.Name, volumeGroup.Spec.ThinPoolConfig.Name) - } - - // apply and save lvmconfig - // pass config to configChannel only if config has changed + // Apply and save lvmd config if !cmp.Equal(existingLvmdConfig, lvmdConfig) { err := saveLVMDConfig(lvmdConfig) if err != nil { - r.Log.Error(err, "failed to update lvmd.conf file", "VGName", volumeGroup.Name) + r.Log.Error(err, "failed to update lvmd.yaml file", "VGName", volumeGroup.Name) + if statuserr := r.setVolumeGroupFailedStatus(ctx, volumeGroup.Name, fmt.Sprintf("failed to update lvmd.yaml file: %v", err.Error())); statuserr != nil { + r.Log.Error(statuserr, "failed to update status", "name", volumeGroup.Name) + } return reconcileAgain, err } r.Log.Info("updated lvmd config", "VGName", volumeGroup.Name) } - if err == nil { - if statuserr := r.updateStatus(ctx, nil); statuserr != nil { - r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) - return reconcileAgain, nil - } - } else { - r.Log.Error(err, "failed to get volume group from the host", "name", volumeGroup.Name) + if statuserr := r.setVolumeGroupReadyStatus(ctx, volumeGroup.Name); statuserr != nil { + r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) + return reconcileAgain, nil } // requeue faster if some devices are too recently observed to consume @@ -326,7 +317,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph r.Log.Info("lvmd config file does not exist") // Remove the VG entry in the lvmvolumegroupnodestatus that was added to indicate the failures to the user. // This allows the lvmcluster to get deleted and not stuck/wait forever as lvmcluster looks for the lvmvolumegroupnodestatus before deleting. - if statuserr := r.updateStatus(ctx, nil); statuserr != nil { + if statuserr := r.removeVolumeGroupStatus(ctx, volumeGroup.Name); statuserr != nil { r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) return statuserr } @@ -396,7 +387,7 @@ func (r *VGReconciler) processDelete(ctx context.Context, volumeGroup *lvmv1alph } } - if statuserr := r.updateStatus(ctx, nil); statuserr != nil { + if statuserr := r.removeVolumeGroupStatus(ctx, volumeGroup.Name); statuserr != nil { r.Log.Error(statuserr, "failed to update status", "VGName", volumeGroup.Name) return statuserr } @@ -430,6 +421,7 @@ func (r *VGReconciler) addDevicesToVG(vgName string, devices []internal.BlockDev } } + // TODO: Check if we can use functions from lvm.go here var cmd string if vgFound { r.Log.Info("extending an existing volume group", "VGName", vgName) @@ -589,56 +581,94 @@ func (r *VGReconciler) getMatchingDevicesForVG(volumeGroup *lvmv1alpha1.LVMVolum return matchingDevices, delayedDevices, nil } -func (r *VGReconciler) generateVolumeGroupNodeStatus(vgStatus *lvmv1alpha1.VGStatus) (*lvmv1alpha1.LVMVolumeGroupNodeStatus, error) { +func (r *VGReconciler) setVolumeGroupReadyStatus(ctx context.Context, vgName string) error { + status := &lvmv1alpha1.VGStatus{ + Name: vgName, + Status: lvmv1alpha1.VGStatusReady, + } - vgs, err := ListVolumeGroups(r.executor) - if err != nil { - return nil, fmt.Errorf("failed to list volume groups. %v", err) + // Set devices for the VGStatus. + if _, err := r.setDevices(status); err != nil { + return err } - //lvmvgstatus := vgNodeStatus.Spec.LVMVGStatus - var statusarr []lvmv1alpha1.VGStatus + return r.setVolumeGroupStatus(ctx, status) +} - var vgExists bool +func (r *VGReconciler) setVolumeGroupFailedStatus(ctx context.Context, vgName string, reason string) error { + status := &lvmv1alpha1.VGStatus{ + Name: vgName, + Status: lvmv1alpha1.VGStatusFailed, + Reason: reason, + } - for _, vg := range vgs { - newStatus := &lvmv1alpha1.VGStatus{ - Name: vg.Name, - Devices: vg.PVs, - Status: lvmv1alpha1.VGStatusReady, - } + // Set devices for the VGStatus. + // If there is backing volume group, then set as degraded + devicesExist, err := r.setDevices(status) + if err != nil { + return err + } + if devicesExist { + status.Status = lvmv1alpha1.VGStatusDegraded + } - if vgStatus != nil && vgStatus.Name == vg.Name { - vgExists = true - newStatus.Status = lvmv1alpha1.VGStatusDegraded - newStatus.Reason = vgStatus.Reason - } + return r.setVolumeGroupStatus(ctx, status) +} - statusarr = append(statusarr, *newStatus) +func (r *VGReconciler) setDevices(status *lvmv1alpha1.VGStatus) (bool, error) { + vgs, err := ListVolumeGroups(r.executor) + if err != nil { + return false, fmt.Errorf("failed to list volume groups. %v", err) } - if vgStatus != nil && !vgExists { - vgStatus.Status = lvmv1alpha1.VGStatusFailed - statusarr = append(statusarr, *vgStatus) + devicesExist := false + for _, vg := range vgs { + if status.Name == vg.Name { + devicesExist = true + status.Devices = vg.PVs + } } - vgNodeStatus := &lvmv1alpha1.LVMVolumeGroupNodeStatus{ + return devicesExist, nil +} + +func (r *VGReconciler) setVolumeGroupStatus(ctx context.Context, status *lvmv1alpha1.VGStatus) error { + // Get LVMVolumeGroupNodeStatus and set the relevant VGStatus + nodeStatus := &lvmv1alpha1.LVMVolumeGroupNodeStatus{ ObjectMeta: metav1.ObjectMeta{ Name: r.NodeName, Namespace: r.Namespace, }, - Spec: lvmv1alpha1.LVMVolumeGroupNodeStatusSpec{ - LVMVGStatus: statusarr, - }, } - return vgNodeStatus, nil -} + result, err := ctrl.CreateOrUpdate(ctx, r.Client, nodeStatus, func() error { + exists := false + for _, existingVGStatus := range nodeStatus.Spec.LVMVGStatus { + if existingVGStatus.Name == status.Name { + exists = true + existingVGStatus = *status + } + } + if !exists { + nodeStatus.Spec.LVMVGStatus = append(nodeStatus.Spec.LVMVGStatus, *status) + } -func (r *VGReconciler) updateStatus(ctx context.Context, vgStatus *lvmv1alpha1.VGStatus) error { + return nil + }) + if err != nil { + r.Log.Error(err, "failed to create or update lvmvolumegroupnodestatus", "name", nodeStatus.Name) + return err + } else if result != controllerutil.OperationResultNone { + r.Log.Info("lvmvolumegroupnodestatus modified", "operation", result, "name", nodeStatus.Name) + } else { + r.Log.Info("lvmvolumegroupnodestatus unchanged") + } - vgNodeStatus, err := r.generateVolumeGroupNodeStatus(vgStatus) + return nil +} +func (r *VGReconciler) removeVolumeGroupStatus(ctx context.Context, vgName string) error { + // Get LVMVolumeGroupNodeStatus and remove the relevant VGStatus nodeStatus := &lvmv1alpha1.LVMVolumeGroupNodeStatus{ ObjectMeta: metav1.ObjectMeta{ Name: r.NodeName, @@ -646,25 +676,32 @@ func (r *VGReconciler) updateStatus(ctx context.Context, vgStatus *lvmv1alpha1.V }, } - if err != nil { - r.Log.Error(err, "failed to generate nodeStatus") - return err - } - + exist := false + index := 0 result, err := ctrl.CreateOrUpdate(ctx, r.Client, nodeStatus, func() error { - nodeStatus.Spec.LVMVGStatus = vgNodeStatus.Spec.LVMVGStatus + for i, existingVGStatus := range nodeStatus.Spec.LVMVGStatus { + if existingVGStatus.Name == vgName { + exist = true + index = i + } + } + + if exist { + nodeStatus.Spec.LVMVGStatus = append(nodeStatus.Spec.LVMVGStatus[:index], nodeStatus.Spec.LVMVGStatus[index+1:]...) + } + return nil }) - if err != nil { - r.Log.Error(err, "failed to create or update lvmvolumegroupnodestatus", "name", vgNodeStatus.Name) + r.Log.Error(err, "failed to create or update lvmvolumegroupnodestatus", "name", nodeStatus.Name) return err } else if result != controllerutil.OperationResultNone { - r.Log.Info("lvmvolumegroupnodestatus modified", "operation", result, "name", vgNodeStatus.Name) + r.Log.Info("lvmvolumegroupnodestatus modified", "operation", result, "name", nodeStatus.Name) } else { r.Log.Info("lvmvolumegroupnodestatus unchanged") } - return err + + return nil } // filterAvailableDevices returns: