Store drain status in SriovNetworkNodeState

api/v1/sriovnetworknodestate_types.go

@@ -85,6 +85,7 @@ type VirtualFunction struct {
 // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
 type SriovNetworkNodeStateStatus struct {
 	Interfaces    InterfaceExts `json:"interfaces,omitempty"`
+	DrainStatus   string        `json:"drainStatus,omitempty"`
 	SyncStatus    string        `json:"syncStatus,omitempty"`
 	LastSyncError string        `json:"lastSyncError,omitempty"`
 }

config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -88,6 +88,8 @@ spec:
             description: SriovNetworkNodeStateStatus defines the observed state of
               SriovNetworkNodeState
             properties:
+              drainStatus:
+                type: string
               interfaces:
                 items:
                   properties:

config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml

@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

controllers/drain_controller.go

@@ -76,7 +76,7 @@ func (dr *DrainReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 
 	drainingNodes := 0
 	for _, node := range nodeList.Items {
-		if utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.AnnoDraining) || utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.AnnoMcpPaused) {
+		if utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.Draining) || utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.DrainMcpPaused) {
 			drainingNodes++
 		}
 	}
@@ -88,12 +88,12 @@ func (dr *DrainReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 	}
 
 	for _, node := range nodeList.Items {
-		if !utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.AnnoDrainRequired) {
+		if !utils.NodeHasAnnotation(node, constants.NodeDrainAnnotation, constants.DrainRequired) {
 			continue
 		}
 		if config.Spec.MaxParallelNodeConfiguration == 0 || drainingNodes < config.Spec.MaxParallelNodeConfiguration {
 			reqLogger.Info("Start draining node", "node", node.Name)
-			patch := []byte(fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"}}}`, constants.NodeDrainAnnotation, constants.AnnoDraining))
+			patch := []byte(fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"}}}`, constants.NodeDrainAnnotation, constants.Draining))
 			err = dr.Client.Patch(context.TODO(), &node, client.RawPatch(types.StrategicMergePatchType, patch))
 			if err != nil {
 				reqLogger.Error(err, "Failed to patch node annotations")

controllers/migration_controller.go

@@ -0,0 +1,80 @@
+package controllers
+
+import (
+	"context"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	v1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned"
+	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
+)
+
+type MigrationReconciler struct {
+	client.Client
+	Scheme    *runtime.Scheme
+	ClientSet snclientset.Interface
+}
+
+//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch
+
+// Reconcile is part of the main kubernetes reconciliation loop which aims to
+// move the current state of the cluster closer to the desired state.
+// For more details, check Reconcile and its Result here:
+// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
+func (mr *MigrationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	req.Namespace = namespace
+	reqLogger := log.FromContext(ctx).WithValues("drain", req.NamespacedName)
+	reqLogger.Info("Reconciling annotation migration")
+
+	node := &corev1.Node{}
+	err := mr.Get(ctx, types.NamespacedName{
+		Name: req.Name}, node)
+	if err != nil {
+		reqLogger.Error(err, "Error occurred on GET SriovOperatorConfig request from API server.")
+		return reconcile.Result{}, err
+	}
+
+	if anno, ok := node.Annotations[constants.NodeDrainAnnotation]; ok {
+		nodeState := &v1.SriovNetworkNodeState{}
+		err := mr.Get(ctx, types.NamespacedName{
+			Name: node.Name, Namespace: namespace}, nodeState)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return reconcile.Result{}, nil
+			}
+			reqLogger.Error(err, "Error occurred on GET SriovNetworkNodeState request from API server.")
+			return reconcile.Result{}, err
+		}
+		nodeState.Status.DrainStatus = anno
+		mr.ClientSet.SriovnetworkV1().SriovNetworkNodeStates(namespace).UpdateStatus(context.TODO(), nodeState, metav1.UpdateOptions{})
+		delete(node.Annotations, constants.NodeDrainAnnotation)
+		mr.Update(ctx, node)
+		//patch := []byte(fmt.Sprintf(`{"status":{"drainStatus":"%s"}}`, anno))
+		//err = mr.Client.Patch(context.TODO(), nodeState, client.RawPatch(types.StrategicMergePatchType, patch))
+		//if err != nil {
+		//	reqLogger.Error(err, "Error occurred on SriovNetworkNodeState update.")
+		//	return reconcile.Result{}, err
+		//}
+
+	}
+
+	return reconcile.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (mr *MigrationReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	nodePredicates := builder.WithPredicates(DrainAnnotationPredicate{})
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Node{}, nodePredicates).
+		Complete(mr)
+}

controllers/sriovnetworknodepolicy_controller.go

@@ -20,10 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
-	"sort"
-	"strings"
-
+	dptypes "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	errs "github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -35,13 +32,18 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	kscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/workqueue"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-
-	dptypes "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+	"sort"
+	"strings"
 
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/apply"
@@ -152,8 +154,23 @@ func (r *SriovNetworkNodePolicyReconciler) Reconcile(ctx context.Context, req ct
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	qHandler := func(q workqueue.RateLimitingInterface) {
+		q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
+			Namespace: namespace,
+			Name:      "node-policy-reconcile-name",
+		}})
+	}
+	createUpdateEnqueue := handler.Funcs{
+		CreateFunc: func(e event.CreateEvent, q workqueue.RateLimitingInterface) {
+			qHandler(q)
+		},
+		UpdateFunc: func(e event.UpdateEvent, q workqueue.RateLimitingInterface) {
+			qHandler(q)
+		},
+	}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&sriovnetworkv1.SriovNetworkNodePolicy{}).
+		Watches(&source.Kind{Type: &sriovnetworkv1.SriovNetworkNodeState{}}, createUpdateEnqueue).
 		Complete(r)
 }
 
@@ -245,6 +262,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con
 				}
 			}
 			if !found {
+				logger.Info("Remove SriovNetworkNodeState custom resource for unselected node", "node", ns.Name)
 				err := r.Delete(ctx, &ns, &client.DeleteOptions{})
 				if err != nil {
 					logger.Info("Fail to Delete", "SriovNetworkNodeState CR:", ns.GetName())
@@ -285,6 +303,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context
 		}
 
 		logger.Info("SriovNetworkNodeState already exists, updating")
+		logger.Info("SriovNetworkNodeState drainStatus", "drainStatus", found.Status.DrainStatus)
 		newVersion := found.DeepCopy()
 		newVersion.Spec = ns.Spec
 
@@ -311,6 +330,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context
 				ppp = p.Spec.Priority
 			}
 		}
+		logger.Info("SriovNetworkNodeState new drainStatus", "drainStatus", newVersion.Status.DrainStatus)
 		newVersion.Spec.DpConfigVersion = cksum
 		if equality.Semantic.DeepDerivative(newVersion.Spec, found.Spec) {
 			logger.Info("SriovNetworkNodeState did not change, not updating")

deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml

@@ -88,6 +88,8 @@ spec:
             description: SriovNetworkNodeStateStatus defines the observed state of
               SriovNetworkNodeState
             properties:
+              drainStatus:
+                type: string
               interfaces:
                 items:
                   properties:

deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml

@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

main.go

@@ -41,13 +41,16 @@ import (
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/controllers"
+	snclientset "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned"
 	constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/leaderelection"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
@@ -134,6 +137,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	var config *rest.Config
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig != "" {
+		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	} else {
+		// creates the in-cluster config
+		config, err = rest.InClusterConfig()
+	}
+
+	setupLog.Info("###config: ", "config", config)
+	snclient := snclientset.NewForConfigOrDie(config)
+
 	if err = (&controllers.SriovNetworkReconciler{
 		Client: mgrGlobal.GetClient(),
 		Scheme: mgrGlobal.GetScheme(),
@@ -196,6 +211,14 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "DrainReconciler")
 		os.Exit(1)
 	}
+	if err = (&controllers.MigrationReconciler{
+		Client:    mgr.GetClient(),
+		Scheme:    mgr.GetScheme(),
+		ClientSet: snclient,
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "MigrationReconciler")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	// Create a default SriovNetworkNodePolicy

pkg/consts/constants.go

@@ -24,11 +24,11 @@ const (
 	OVSHWOLMachineConfigNameSuffix     = "ovs-hw-offload"
 
 	NodeDrainAnnotation = "sriovnetwork.openshift.io/state"
-	AnnoIdle            = "Idle"
-	AnnoDrainRequired   = "Drain_Required"
-	AnnoMcpPaused       = "Draining_MCP_Paused"
-	AnnoDraining        = "Draining"
-	AnnoDrainComplete   = "DrainComplete"
+	DrainIdle           = "Idle"
+	DrainRequired       = "Drain_Required"
+	DrainMcpPaused      = "Draining_MCP_Paused"
+	Draining            = "Draining"
+	DrainComplete       = "DrainComplete"
 
 	LinkTypeEthernet   = "ether"
 	LinkTypeInfiniband = "infiniband"