reconcile network interfaces in Linstor clusters

In some cases, such as a k8s node restart, a pod can change IP address, which
requires updating the net-interface of the node in Linstor.

Signed-off-by: Moritz "WanzenBug" Wanzenböck <[email protected]>

CHANGELOG.md

@@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Set node name variable for Controller Pods, enabling [k8s-await-election] to correctly set up the endpoint for
   hairpin mode.
 
+### Fixed
+
+* Update the network address of controller pods if they diverged between Linstor and kubernetes. This can happen after
+  a node restart, where a pod is recreated with the same name but different IP address.
+
 [k8s-await-election]: https://github.com/LINBIT/k8s-await-election/commit/60748fcec722e4c32b8881c4c84957cc50543db2
 
 ## [v1.3.1] - 2021-01-14

pkg/controller/linstorcontroller/linstorcontroller_controller.go

@@ -290,6 +290,7 @@ func (r *ReconcileLinstorController) reconcileControllers(ctx context.Context, c
 				{
 					Name:                    "default",
 					Address:                 pod.Status.PodIP,
+					IsActive:                true,
 					SatellitePort:           controllerResource.Spec.SslConfig.Port(),
 					SatelliteEncryptionType: controllerResource.Spec.SslConfig.Type(),
 				},

pkg/controller/linstorsatelliteset/linstorsatelliteset_controller.go

@@ -436,6 +436,7 @@ func (r *ReconcileLinstorSatelliteSet) reconcileSingleNodeRegistration(ctx conte
 			{
 				Name:                    "default",
 				Address:                 pod.Status.HostIP,
+				IsActive:                true,
 				SatellitePort:           satelliteSet.Spec.SslConfig.Port(),
 				SatelliteEncryptionType: satelliteSet.Spec.SslConfig.Type(),
 			},

pkg/linstor/client/client.go

@@ -151,53 +151,52 @@ func NewHighLevelClient(options ...lapi.Option) (*HighLevelClient, error) {
 }
 
 // GetNodeOrCreate gets a linstor node, creating it if it is not already present.
-func (c *HighLevelClient) GetNodeOrCreate(ctx context.Context, node lapi.Node) (lapi.Node, error) {
-	n, err := c.Nodes.Get(ctx, node.Name)
+func (c *HighLevelClient) GetNodeOrCreate(ctx context.Context, node lapi.Node) (*lapi.Node, error) {
+	existingNode, err := c.Nodes.Get(ctx, node.Name)
 	if err != nil {
 		// For 404
 		if err != lapi.NotFoundError {
-			return n, fmt.Errorf("unable to get node %s: %v", node.Name, err)
-		}
-
-		if len(node.NetInterfaces) != 1 {
-			return n, fmt.Errorf("only able to create a new node with a single interface")
+			return nil, fmt.Errorf("unable to get node %s: %w", node.Name, err)
 		}
 
 		// Node doesn't exist, create it.
 		if err := c.Nodes.Create(ctx, node); err != nil {
-			return n, fmt.Errorf("unable to create node %s: %v", node.Name, err)
+			return nil, fmt.Errorf("unable to create node %s: %w", node.Name, err)
 		}
 
 		newNode, err := c.Nodes.Get(ctx, node.Name)
 		if err != nil {
-			return newNode, fmt.Errorf("unable to get newly created node %s: %v", node.Name, err)
+			return nil, fmt.Errorf("unable to get newly created node %s: %w", node.Name, err)
 		}
 
-		return newNode, c.ensureWantedInterface(ctx, newNode, node.NetInterfaces[0])
+		existingNode = newNode
+	}
+
+	for _, nic := range node.NetInterfaces {
+		err = c.ensureWantedInterface(ctx, existingNode, nic)
+		if err != nil {
+			return nil, fmt.Errorf("failed to update network interface: %w", err)
+		}
 	}
 
-	return n, nil
+	return &existingNode, nil
 }
 
 func (c *HighLevelClient) ensureWantedInterface(ctx context.Context, node lapi.Node, wanted lapi.NetInterface) error {
-	// Make sure default network interface is to spec.
 	for _, nodeIf := range node.NetInterfaces {
-		if nodeIf.Name == wanted.Name {
-
-			// TODO: Maybe we should error out here.
-			if nodeIf.Address != wanted.Address {
-				if err := c.Nodes.ModifyNetInterface(ctx, node.Name, wanted.Name, wanted); err != nil {
-					return fmt.Errorf("unable to modify default network interface on %s: %v", node.Name, err)
-				}
-			}
-			break
+		if nodeIf.Name != wanted.Name {
+			continue
 		}
 
-		if err := c.Nodes.CreateNetInterface(ctx, node.Name, wanted); err != nil {
-			return fmt.Errorf("unable to create default network interface on %s: %v", node.Name, err)
+		if nodeIf.Address == wanted.Address {
+			return nil
 		}
+
+		return c.Nodes.ModifyNetInterface(ctx, node.Name, wanted.Name, wanted)
 	}
-	return nil
+
+	// Interface was not found, creating it now
+	return c.Nodes.CreateNetInterface(ctx, node.Name, wanted)
 }
 
 // GetAllResourcesOnNode returns a list of all resources on the specified node.