From 94ec73ffc773f8f29b95db2495c750b1447aa66f Mon Sep 17 00:00:00 2001 From: Alessandro Olivero Date: Tue, 28 Nov 2023 11:48:51 +0100 Subject: [PATCH] internal fabric operator --- .../v1alpha1/internalfabric_types.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 6 +- cmd/liqo-controller-manager/main.go | 7 + .../networking.liqo.io_internalfabrics.yaml | 8 +- .../liqo-controller-manager-ClusterRole.yaml | 20 ++ .../internalfabric-controller/doc.go | 16 ++ .../internalfabric_controller.go | 183 ++++++++++++++++++ .../internal-network/ipam/node.go | 59 ++++++ pkg/utils/getters/k8sGetters.go | 17 ++ 9 files changed, 311 insertions(+), 7 deletions(-) create mode 100644 pkg/liqo-controller-manager/internal-network/internalfabric-controller/doc.go create mode 100644 pkg/liqo-controller-manager/internal-network/internalfabric-controller/internalfabric_controller.go create mode 100644 pkg/liqo-controller-manager/internal-network/ipam/node.go diff --git a/apis/networking/v1alpha1/internalfabric_types.go b/apis/networking/v1alpha1/internalfabric_types.go index d28e2b1196..be41da3a70 100644 --- a/apis/networking/v1alpha1/internalfabric_types.go +++ b/apis/networking/v1alpha1/internalfabric_types.go @@ -59,7 +59,7 @@ type InternalFabricSpec struct { // InternalFabricStatus defines the observed state of InternalFabric. type InternalFabricStatus struct { // AssignedIPs is the list of IP addresses assigned to interfaces in the nodes. - AssignedIPs []IP `json:"assignedIPs,omitempty"` + AssignedIPs map[string]IP `json:"assignedIPs,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/networking/v1alpha1/zz_generated.deepcopy.go b/apis/networking/v1alpha1/zz_generated.deepcopy.go index fff342dac9..e560ff876d 100644 --- a/apis/networking/v1alpha1/zz_generated.deepcopy.go +++ b/apis/networking/v1alpha1/zz_generated.deepcopy.go @@ -864,8 +864,10 @@ func (in *InternalFabricStatus) DeepCopyInto(out *InternalFabricStatus) { *out = *in if in.AssignedIPs != nil { in, out := &in.AssignedIPs, &out.AssignedIPs - *out = make([]IP, len(*in)) - copy(*out, *in) + *out = make(map[string]IP, len(*in)) + for key, val := range *in { + (*out)[key] = val + } } } diff --git a/cmd/liqo-controller-manager/main.go b/cmd/liqo-controller-manager/main.go index a0ed7f3457..dcbf50aa72 100644 --- a/cmd/liqo-controller-manager/main.go +++ b/cmd/liqo-controller-manager/main.go @@ -73,6 +73,7 @@ import ( wggatewaycontrollers "github.com/liqotech/liqo/pkg/liqo-controller-manager/external-network/wireguard" foreignclusteroperator "github.com/liqotech/liqo/pkg/liqo-controller-manager/foreign-cluster-operator" internalclientcontroller "github.com/liqotech/liqo/pkg/liqo-controller-manager/internal-network/client-controller" + internalfabriccontroller "github.com/liqotech/liqo/pkg/liqo-controller-manager/internal-network/internalfabric-controller" internalservercontroller "github.com/liqotech/liqo/pkg/liqo-controller-manager/internal-network/server-controller" ipctrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/ip-controller" mapsctrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/namespacemap-controller" @@ -725,6 +726,12 @@ func main() { klog.Error(err) os.Exit(1) } + + internalFabricReconciler := internalfabriccontroller.NewInternalFabricReconciler(mgr.GetClient(), mgr.GetScheme()) + if err := internalFabricReconciler.SetupWithManager(mgr); err != nil { + klog.Error(err) + os.Exit(1) + } } klog.Info("starting manager as controller manager") diff --git a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml index e2ffb6afc4..c17b4cb30b 100644 --- a/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml +++ b/deployments/liqo/charts/liqo-crds/crds/networking.liqo.io_internalfabrics.yaml @@ -88,13 +88,13 @@ spec: description: InternalFabricStatus defines the observed state of InternalFabric. properties: assignedIPs: - description: AssignedIPs is the list of IP addresses assigned to interfaces - in the nodes. - items: + additionalProperties: description: IP defines a syntax validated IP. pattern: ^(([1-9]{0,1}[0-9]{0,2}|2[0-4][0-9]|25[0-5])\.){3}([1-9]{0,1}[0-9]{0,2}|2[0-4][0-9]|25[0-5])$ type: string - type: array + description: AssignedIPs is the list of IP addresses assigned to interfaces + in the nodes. + type: object type: object type: object served: true diff --git a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml index fc356b925a..fa8e5bd050 100644 --- a/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml +++ b/deployments/liqo/files/liqo-controller-manager-ClusterRole.yaml @@ -505,6 +505,26 @@ rules: - patch - update - watch +- apiGroups: + - networking.liqo.io + resources: + - internalfabrics/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.liqo.io + resources: + - internalnodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - networking.liqo.io resources: diff --git a/pkg/liqo-controller-manager/internal-network/internalfabric-controller/doc.go b/pkg/liqo-controller-manager/internal-network/internalfabric-controller/doc.go new file mode 100644 index 0000000000..6eafd8ae41 --- /dev/null +++ b/pkg/liqo-controller-manager/internal-network/internalfabric-controller/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internalfabriccontroller implements the logic of the InternalFabric Controller to create InternalNodes. +package internalfabriccontroller diff --git a/pkg/liqo-controller-manager/internal-network/internalfabric-controller/internalfabric_controller.go b/pkg/liqo-controller-manager/internal-network/internalfabric-controller/internalfabric_controller.go new file mode 100644 index 0000000000..c3f89d8d8c --- /dev/null +++ b/pkg/liqo-controller-manager/internal-network/internalfabric-controller/internalfabric_controller.go @@ -0,0 +1,183 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internalfabriccontroller + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" + "github.com/liqotech/liqo/pkg/liqo-controller-manager/internal-network/ipam" + "github.com/liqotech/liqo/pkg/utils/getters" +) + +const ( + internalFabricLabelKey = "liqo.io/internal-fabric" +) + +// InternalFabricReconciler manage InternalFabric lifecycle. +type InternalFabricReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// NewInternalFabricReconciler returns a new InternalFabricReconciler. +func NewInternalFabricReconciler(cl client.Client, s *runtime.Scheme) *InternalFabricReconciler { + return &InternalFabricReconciler{ + Client: cl, + Scheme: s, + } +} + +// cluster-role +// +kubebuilder:rbac:groups=networking.liqo.io,resources=internalfabrics,verbs=get;list;watch;delete;create;update;patch +// +kubebuilder:rbac:groups=networking.liqo.io,resources=internalfabrics/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.liqo.io,resources=internalnodes,verbs=get;list;watch;delete;create;update;patch +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch + +// Reconcile manage InternalFabric lifecycle. +func (r *InternalFabricReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) { + internalFabric := &networkingv1alpha1.InternalFabric{} + if err = r.Get(ctx, req.NamespacedName, internalFabric); err != nil { + if apierrors.IsNotFound(err) { + klog.Infof("InternalFabric %q not found", req.NamespacedName) + return ctrl.Result{}, nil + } + klog.Errorf("Unable to get the InternalFabric %q: %s", req.NamespacedName, err) + return ctrl.Result{}, err + } + + nodes, err := getters.ListPhysicalNodes(ctx, r.Client) + if err != nil { + klog.Errorf("Unable to list physical nodes: %s", err) + return ctrl.Result{}, err + } + + for i := range nodes.Items { + node := &nodes.Items[i] + if err = r.reconcileNode(ctx, internalFabric, node); err != nil { + klog.Errorf("Unable to reconcile node %q: %s", node.Name, err) + return ctrl.Result{}, err + } + } + + // ensure status + + var internalNodes networkingv1alpha1.InternalNodeList + if err = r.List(ctx, &internalNodes, client.InNamespace(internalFabric.Namespace), + client.MatchingLabels{internalFabricLabelKey: req.Name}); err != nil { + klog.Errorf("Unable to list InternalNodes: %s", err) + return ctrl.Result{}, err + } + + internalFabric.Status.AssignedIPs = make(map[string]networkingv1alpha1.IP) + for i := range internalNodes.Items { + internalNode := &internalNodes.Items[i] + internalFabric.Status.AssignedIPs[internalNode.Name] = internalNode.Spec.IP + } + + if err = r.Status().Update(ctx, internalFabric); err != nil { + klog.Errorf("Unable to update InternalFabric status: %s", err) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *InternalFabricReconciler) reconcileNode(ctx context.Context, + internalFabric *networkingv1alpha1.InternalFabric, node *corev1.Node) error { + internalNode := &networkingv1alpha1.InternalNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + Namespace: internalFabric.Namespace, + }, + } + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, internalNode, func() error { + if internalNode.Labels == nil { + internalNode.Labels = make(map[string]string) + } + internalNode.Labels[internalFabricLabelKey] = internalFabric.Name + + internalNode.Spec.FabricRef = &corev1.ObjectReference{ + Name: internalFabric.Name, + Namespace: internalFabric.Namespace, + UID: internalFabric.UID, + } + + intIPAM, err := ipam.GetNodeIpam(ctx, r.Client) + if err != nil { + return err + } + + ip, err := intIPAM.Allocate(fmt.Sprintf("%s/%s/%s", + internalFabric.Namespace, internalFabric.Name, internalNode.Name)) + if err != nil { + return err + } + + internalNode.Spec.IP = networkingv1alpha1.IP(ip.String()) + internalNode.Spec.IsGateway = node.Name == internalFabric.Spec.NodeName + + if err := controllerutil.SetControllerReference(internalFabric, internalNode, r.Scheme); err != nil { + return err + } + return controllerutil.SetOwnerReference(node, internalNode, r.Scheme) + }); err != nil { + klog.Errorf("Unable to create or update InternalNode %q: %s", internalNode.Name, err) + return err + } + return nil +} + +// SetupWithManager register the InternalFabricReconciler to the manager. +func (r *InternalFabricReconciler) SetupWithManager(mgr ctrl.Manager) error { + nodeEnqueuer := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { + var internalFabrics networkingv1alpha1.InternalFabricList + if err := r.List(ctx, &internalFabrics); err != nil { + klog.Errorf("Unable to list InternalFabrics: %s", err) + return nil + } + + var requests = make([]reconcile.Request, len(internalFabrics.Items)) + for i := range internalFabrics.Items { + internalFabric := &internalFabrics.Items[i] + requests[i] = reconcile.Request{ + NamespacedName: client.ObjectKey{ + Name: internalFabric.Name, + Namespace: internalFabric.Namespace, + }, + } + } + return requests + }) + + return ctrl.NewControllerManagedBy(mgr). + Watches(&corev1.Node{}, nodeEnqueuer). + Owns(&networkingv1alpha1.InternalNode{}). + For(&networkingv1alpha1.InternalFabric{}). + Complete(r) +} diff --git a/pkg/liqo-controller-manager/internal-network/ipam/node.go b/pkg/liqo-controller-manager/internal-network/ipam/node.go new file mode 100644 index 0000000000..b930f33435 --- /dev/null +++ b/pkg/liqo-controller-manager/internal-network/ipam/node.go @@ -0,0 +1,59 @@ +// Copyright 2019-2023 The Liqo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipam + +import ( + "context" + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + networkingv1alpha1 "github.com/liqotech/liqo/apis/networking/v1alpha1" +) + +var nodeIpam *IPAM +var nodeIpamOnce sync.Once + +var nodeNetwork string + +// GetNodeIpam returns the IPAM for the internal nodes or creates it if not exists. It is a singleton. +func GetNodeIpam(ctx context.Context, cl client.Client) (*IPAM, error) { + if nodeNetwork == "" { + // TODO: get from network CRD + nodeNetwork = "10.201.0.0/16" + } + + nodeIpamOnce.Do(func() { + var err error + nodeIpam, err = New(nodeNetwork) + runtime.Must(err) + + var internalFabrics networkingv1alpha1.InternalFabricList + err = cl.List(ctx, &internalFabrics) + runtime.Must(err) + + for i := range internalFabrics.Items { + intFab := &internalFabrics.Items[i] + for k, v := range intFab.Status.AssignedIPs { + err = nodeIpam.Configure(fmt.Sprintf("%s/%s/%s", intFab.Namespace, intFab.Name, k), v.String()) + runtime.Must(err) + } + } + }) + + return nodeIpam, nil +} diff --git a/pkg/utils/getters/k8sGetters.go b/pkg/utils/getters/k8sGetters.go index fa43d30338..e692910381 100644 --- a/pkg/utils/getters/k8sGetters.go +++ b/pkg/utils/getters/k8sGetters.go @@ -23,6 +23,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -427,3 +428,19 @@ func GetGatewayClientByClusterID(ctx context.Context, cl client.Client, return nil, fmt.Errorf("multiple GatewayClients found for ForeignCluster %s", clusterID) } } + +// ListPhysicalNodes returns the list of physical nodes. (i.e. nodes not created by Liqo). +func ListPhysicalNodes(ctx context.Context, cl client.Client) (*corev1.NodeList, error) { + req, err := labels.NewRequirement(consts.TypeLabel, selection.DoesNotExist, nil) + if err != nil { + return nil, err + } + + lSelector := labels.NewSelector().Add(*req) + + list := new(corev1.NodeList) + if err := cl.List(ctx, list, &client.ListOptions{LabelSelector: lSelector}); err != nil { + return nil, err + } + return list, nil +}