diff --git a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml index ab63cacb5..afe3cdcf0 100644 --- a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml +++ b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml @@ -59,6 +59,9 @@ spec: default: false type: boolean type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf accessMode: description: Access mode of Subnet, accessible only from within VPC or from outside VPC. @@ -89,6 +92,15 @@ spec: - message: Value is immutable rule: self == oldSelf type: object + x-kubernetes-validations: + - message: DHCPConfig is required once set + rule: '!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)' + - message: ipv4SubnetSize is required once set + rule: '!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)' + - message: accessMode is required once set + rule: '!has(oldSelf.accessMode) || has(self.accessMode)' + - message: ipAddresses is required once set + rule: '!has(oldSelf.ipAddresses) || has(self.ipAddresses)' status: description: SubnetStatus defines the observed state of Subnet. properties: diff --git a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml index a2803e167..662aa2e7f 100644 --- a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml +++ b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml @@ -59,6 +59,9 @@ spec: default: false type: boolean type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf accessMode: description: Access mode of Subnet, accessible only from within VPC or from outside VPC. @@ -79,6 +82,13 @@ spec: - message: Value is immutable rule: self == oldSelf type: object + x-kubernetes-validations: + - message: DHCPConfig is required once set + rule: '!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)' + - message: accessMode is required once set + rule: '!has(oldSelf.accessMode) || has(self.accessMode)' + - message: ipv4SubnetSize is required once set + rule: '!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)' status: description: SubnetSetStatus defines the observed state of SubnetSet. properties: diff --git a/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml b/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml index 1368d2e6a..06752bcc6 100644 --- a/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml +++ b/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml @@ -2,8 +2,6 @@ apiVersion: crd.nsx.vmware.com/v1alpha1 kind: NetworkInfo metadata: creationTimestamp: "2024-05-14T02:14:18Z" - finalizers: - - networkinfo.crd.nsx.vmware.com/finalizer generation: 2 name: kube-system namespace: kube-system diff --git a/build/yaml/webhook/certificate.yaml b/build/yaml/webhook/certificate.yaml index 4f584df41..f6a796072 100644 --- a/build/yaml/webhook/certificate.yaml +++ b/build/yaml/webhook/certificate.yaml @@ -31,8 +31,8 @@ metadata: spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize dnsNames: - - subnetset.vmware-system-nsx.svc - - subnetset.vmware-system-nsx.svc.cluster.local + - vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc + - vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer diff --git a/build/yaml/webhook/manifests.yaml b/build/yaml/webhook/manifests.yaml index e21e5a0d9..95ded4a1d 100644 --- a/build/yaml/webhook/manifests.yaml +++ b/build/yaml/webhook/manifests.yaml @@ -9,7 +9,7 @@ webhooks: - v1 clientConfig: service: - name: subnetset + name: vmware-system-nsx-operator-webhook-service namespace: vmware-system-nsx # kubebuilder webhookpath. path: /validate-crd-nsx-vmware-com-v1alpha1-subnetset @@ -30,3 +30,23 @@ webhooks: resources: - subnetsets sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: vmware-system-nsx-operator-webhook-service + namespace: vmware-system-nsx + path: /validate-crd-nsx-vmware-com-v1alpha1-addressbinding + failurePolicy: Fail + name: addressbinding.validating.crd.nsx.vmware.com + rules: + - apiGroups: + - crd.nsx.vmware.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - addressbindings + sideEffects: None diff --git a/build/yaml/webhook/service.yaml b/build/yaml/webhook/service.yaml index 2a02eb376..5d71bb2fc 100644 --- a/build/yaml/webhook/service.yaml +++ b/build/yaml/webhook/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/created-by: nsx-operator app.kubernetes.io/part-of: nsx-operator app.kubernetes.io/managed-by: kustomize - name: subnetset + name: vmware-system-nsx-operator-webhook-service namespace: vmware-system-nsx spec: ports: diff --git a/cmd/main.go b/cmd/main.go index cb72b6190..428f05015 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" crdv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -213,18 +214,26 @@ func startServiceController(mgr manager.Manager, nsxClient *nsx.Client) { if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil { os.Exit(1) } - enableWebhook := true + var hookServer webhook.Server if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) { log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir) - enableWebhook = false + } else { + hookServer = webhook.NewServer(webhook.Options{ + Port: config.WebhookServerPort, + CertDir: config.WebhookCertDir, + }) + if err := mgr.Add(hookServer); err != nil { + log.Error(err, "failed to add hook server") + os.Exit(1) + } } - if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil { + if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, hookServer); err != nil { os.Exit(1) } node.StartNodeController(mgr, nodeService) staticroutecontroller.StartStaticRouteController(mgr, staticRouteService) - subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService) + subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService, hookServer) pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService) StartIPAddressAllocationController(mgr, ipAddressAllocationService, vpcService) networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService) diff --git a/cmd/webhookcert/main.go b/cmd/webhookcert/main.go index e2ac603c8..d550cdbc2 100644 --- a/cmd/webhookcert/main.go +++ b/cmd/webhookcert/main.go @@ -98,8 +98,8 @@ func generateWebhookCerts() error { Bytes: caBytes, }) - dnsNames := []string{"subnetset", "subnetset.vmware-system-nsx", "subnetset.vmware-system-nsx.svc"} - commonName := "subnetset.vmware-system-nsx.svc" + dnsNames := []string{"vmware-system-nsx-operator-webhook-service", "vmware-system-nsx-operator-webhook-service.vmware-system-nsx", "vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc"} + commonName := "vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc" serialNumber, err = rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) if err != nil { diff --git a/go.mod b/go.mod index d5e23ebc1..c011d2ad6 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ replace ( github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1 => ./pkg/apis/vpc/v1alpha1 github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client - github.com/vmware/vsphere-automation-sdk-go/lib => github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a + github.com/vmware/vsphere-automation-sdk-go/lib => github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962 ) require ( diff --git a/go.sum b/go.sum index 99645ef7d..dedc810d7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962 h1:+nbR8Qgl96Ba0/gMnCGQasrUb7F7XRj+OxdfLyl8r7w= +github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962/go.mod h1:ADkX8BkdnvT1Kc9ZfqHaV4qzaaD+9L8Ok2+pxK4xoD8= +github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962 h1:WjS+j/Y6wWU74RKx6c0IBTNZ3g3fah5X8z/c4KXA2us= +github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962 h1:b5AQfl/tH4rr0OCg/hD9ARka5vJwvC2l26zAQ4dGXMA= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962/go.mod h1:NSjO9WqelbsTEDb3pVxpYYz4zjgX0XPp43dKNT4Y+9k= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962 h1:KIFv8/EpZd9DJPBeLcUjPfDNoF2EU72nJ7GbHei1XGg= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962/go.mod h1:ugk9I4YM62SSAox57l5NAVBCRIkPQ1RNLb3URxyTADc= github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw= github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw= @@ -143,14 +151,6 @@ github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wf github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I= github.com/vmware/govmomi v0.27.4/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a h1:nF3PigKL+lN4ECHkgVJIZgLbpLrV6U6wkKHnIHOU9kA= -github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:ysW7/EqFugBY2TcbvlDeRGaYIoG7Cs0i4l4WsMI/RmQ= -github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a h1:b08LCEgSR6GSsvQzx2fxVbEXSKRnaGcMUqKjlgwR6xM= -github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a h1:XEgprSLuSKIxr7OPEzBWrlo39ra7pDaWFwAIjK0VV7s= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:aJtyfDKvGyuP1ieRHCLoYjo2XtNZ401XfS7lCd43Bqs= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a h1:4FmesihC1B7udmKl7B2giLydxibViMdyldSforV5qbU= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:FX8UiCgNEOxweA73VZsyKZvMLPFfc70GBc1d4dj0nXI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/pkg/apis/vpc/v1alpha1/subnet_types.go b/pkg/apis/vpc/v1alpha1/subnet_types.go index b7d00c6e8..eeae36548 100644 --- a/pkg/apis/vpc/v1alpha1/subnet_types.go +++ b/pkg/apis/vpc/v1alpha1/subnet_types.go @@ -16,6 +16,10 @@ const ( ) // SubnetSpec defines the desired state of Subnet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)", message="DHCPConfig is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipAddresses) || has(self.ipAddresses)", message="ipAddresses is required once set" type SubnetSpec struct { // Size of Subnet based upon estimated workload count. // +kubebuilder:validation:Maximum:=65536 @@ -32,6 +36,7 @@ type SubnetSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" IPAddresses []string `json:"ipAddresses,omitempty"` // DHCPConfig DHCP configuration. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" DHCPConfig DHCPConfig `json:"DHCPConfig,omitempty"` } diff --git a/pkg/apis/vpc/v1alpha1/subnetset_types.go b/pkg/apis/vpc/v1alpha1/subnetset_types.go index c486dc603..1154411e4 100644 --- a/pkg/apis/vpc/v1alpha1/subnetset_types.go +++ b/pkg/apis/vpc/v1alpha1/subnetset_types.go @@ -8,6 +8,9 @@ import ( ) // SubnetSetSpec defines the desired state of SubnetSet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)", message="DHCPConfig is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" type SubnetSetSpec struct { // Size of Subnet based upon estimated workload count. // +kubebuilder:validation:Maximum:=65536 @@ -19,6 +22,7 @@ type SubnetSetSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" AccessMode AccessMode `json:"accessMode,omitempty"` // DHCPConfig DHCP configuration. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" DHCPConfig DHCPConfig `json:"DHCPConfig,omitempty"` } diff --git a/pkg/controllers/common/utils.go b/pkg/controllers/common/utils.go index b71f0b2a3..35c3e0f65 100644 --- a/pkg/controllers/common/utils.go +++ b/pkg/controllers/common/utils.go @@ -42,7 +42,7 @@ func AllocateSubnetFromSubnetSet(subnetSet *v1alpha1.SubnetSet, vpcService servi return *nsxSubnet.Path, nil } } - tags := subnetService.GenerateSubnetNSTags(subnetSet, subnetSet.Namespace) + tags := subnetService.GenerateSubnetNSTags(subnetSet) if tags == nil { return "", errors.New("failed to generate subnet tags") } diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go index a9c0a5a73..a52eb7fbb 100644 --- a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go @@ -70,13 +70,13 @@ func updateFail(r *IPAddressAllocationReconciler, c context.Context, o *v1alpha1 func (r *IPAddressAllocationReconciler) setReadyStatusFalse(ctx context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time, err *error) { conditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX IPAddressAllocation could not be created or updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the IPAddressAllocation CR. Error: %v", *err, ), + Reason: "IPAddressAllocationNotReady", LastTransitionTime: transitionTime, }, } @@ -93,7 +93,7 @@ func (r *IPAddressAllocationReconciler) setReadyStatusTrue(ctx context.Context, Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX IPAddressAllocation has been successfully created/updated", - Reason: "", + Reason: "IPAddressAllocationReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go index 2eee4de45..2699943cd 100644 --- a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go @@ -51,7 +51,7 @@ func TestIPAddressAllocationController_setReadyStatusTrue(t *testing.T) { Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX IPAddressAllocation has been successfully created/updated", - Reason: "", + Reason: "IPAddressAllocationReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index d3f856362..b90207da9 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -7,8 +7,11 @@ import ( "context" "errors" "fmt" + "time" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -17,7 +20,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" @@ -44,225 +46,225 @@ type NetworkInfoReconciler struct { Recorder record.EventRecorder } +func (r *NetworkInfoReconciler) GetVpcConnectivityProfilePathByVpcPath(vpcPath string) (string, error) { + // TODO, if needs to add a cache for it + VPCResourceInfo, err := commonservice.ParseVPCResourcePath(vpcPath) + if err != nil { + log.Error(err, "failed to parse VPC path", "VPC Path", vpcPath) + return "", err + } + // pre created VPC may have more than one attachment, list all the attachment and select the first one + vpcAttachmentsListResult, err := r.Service.NSXClient.VpcAttachmentClient.List(VPCResourceInfo.OrgID, VPCResourceInfo.ProjectID, VPCResourceInfo.VPCID, nil, nil, nil, nil, nil, nil) + if err != nil { + log.Error(err, "failed to list VPC attachment", "VPC Path", vpcPath) + return "", err + } + vpcAttachments := vpcAttachmentsListResult.Results + if len(vpcAttachments) > 0 { + log.V(1).Info("found VPC attachment", "VPC Path", vpcPath, "VPC connectivity profile", vpcAttachments[0].VpcConnectivityProfile) + return *vpcAttachments[0].VpcConnectivityProfile, nil + } else { + err := fmt.Errorf("no VPC attachment found") + log.Error(err, "list VPC attachment", "VPC Path", vpcPath) + return "", err + } +} func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.NetworkInfo{} - log.Info("reconciling NetworkInfo CR", "NetworkInfo", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling NetworkInfo", "NetworkInfo", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNetworkInfo) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch NetworkInfo CR", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) - } - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeNetworkInfo) - if !controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { - controllerutil.AddFinalizer(obj, commonservice.NetworkInfoFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "NetworkInfo", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) + networkInfoCR := &v1alpha1.NetworkInfo{} + if err := r.Client.Get(ctx, req.NamespacedName, networkInfoCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteVPCsByName(ctx, req.Namespace); err != nil { + log.Error(err, "Failed to delete stale NSX VPC", "NetworkInfo", req.NamespacedName) return common.ResultRequeue, err } - log.V(1).Info("added finalizer on NetworkInfo CR", "NetworkInfo", req.NamespacedName) - } - // TODO: - // 1. check whether the logic to get VPC network config can be replaced by GetVPCNetworkConfigByNamespace - // 2. sometimes the variable nc points to a VPCNetworkInfo, sometimes it's a VPCNetworkConfiguration, we need to distinguish between them. - ncName, err := r.Service.GetNetworkconfigNameFromNS(obj.Namespace) - if err != nil { - log.Error(err, "failed to get network config name for VPC when creating NSX VPC", "VPC", obj.Name) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - nc, _exist := r.Service.GetVPCNetworkConfig(ncName) - if !_exist { - message := fmt.Sprintf("failed to read network config %s when creating NSX VPC", ncName) - log.Info(message) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, errors.New(message) - } - log.Info("got network config from store", "NetworkConfig", ncName) - vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} - err = r.Client.Get(ctx, types.NamespacedName{Name: commonservice.SystemVPCNetworkConfigurationName}, vpcNetworkConfiguration) - if err != nil { - log.Error(err, "failed to get system VPCNetworkConfiguration") - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - gatewayConnectionReady, _, err := getGatewayConnectionStatus(ctx, vpcNetworkConfiguration) - if err != nil { - log.Error(err, "failed to get the gateway connection status", "req", req.NamespacedName) - return common.ResultRequeueAfter10sec, err + return common.ResultNormal, nil } + log.Error(err, "Unable to fetch NetworkInfo CR", "NetworkInfo", req.NamespacedName) + return common.ResultRequeue, err + } - gatewayConnectionReason := "" - if !gatewayConnectionReady { - if ncName == commonservice.SystemVPCNetworkConfigurationName { - gatewayConnectionReady, gatewayConnectionReason, err = r.Service.ValidateGatewayConnectionStatus(&nc) - log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "gatewayConnectionReason", gatewayConnectionReason) - if err != nil { - log.Error(err, "failed to validate the edge and gateway connection", "org", nc.Org, "project", nc.NSXProject) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, gatewayConnectionReason) - } else { - log.Info("skipping reconciling the network info because the system gateway connection is not ready", "NetworkInfo", req.NamespacedName) - return common.ResultRequeueAfter60sec, nil - } - } - lbProvider := r.Service.GetLBProvider() - createdVpc, err := r.Service.CreateOrUpdateVPC(obj, &nc, lbProvider) - if err != nil { - log.Error(err, "create vpc failed, would retry exponentially", "VPC", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err + // Check if the CR is marked for deletion + if !networkInfoCR.ObjectMeta.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) + if err := r.deleteVPCsByID(ctx, networkInfoCR.GetNamespace(), string(networkInfoCR.UID)); err != nil { + deleteFail(r, ctx, networkInfoCR, &err, r.Client) + log.Error(err, "Failed to delete stale NSX VPC, retrying", "NetworkInfo", req.NamespacedName) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, networkInfoCR) + return common.ResultNormal, nil + } + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeNetworkInfo) + // TODO: + // 1. check whether the logic to get VPC network config can be replaced by GetVPCNetworkConfigByNamespace + // 2. sometimes the variable nc points to a VPCNetworkInfo, sometimes it's a VPCNetworkConfiguration, we need to distinguish between them. + ncName, err := r.Service.GetNetworkconfigNameFromNS(networkInfoCR.Namespace) + if err != nil { + log.Error(err, "Failed to get network config name for VPC when creating NSX VPC", "NetworkInfo", networkInfoCR.Name) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + nc, _exist := r.Service.GetVPCNetworkConfig(ncName) + if !_exist { + message := fmt.Sprintf("Failed to read network config %s when creating NSX VPC", ncName) + log.Info(message) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, errors.New(message) + } + log.Info("Fetched network config from store", "NetworkConfig", ncName) + vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} + err = r.Client.Get(ctx, types.NamespacedName{Name: commonservice.SystemVPCNetworkConfigurationName}, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get system VPCNetworkConfiguration") + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + gatewayConnectionReady, _, err := getGatewayConnectionStatus(ctx, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get the gateway connection status", "NetworkInfo", req.NamespacedName) + return common.ResultRequeueAfter10sec, err + } - var privateIPs []string - var vpcConnectivityProfilePath string - var nsxLBSPath string - isPreCreatedVPC := vpc.IsPreCreatedVPC(nc) - if isPreCreatedVPC { - privateIPs = createdVpc.PrivateIps - vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile - // Retrieve NSX lbs path if Avi is not used with the pre-created VPC. - if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil || - !*createdVpc.LoadBalancerVpcEndpoint.Enabled { - nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(*createdVpc.Path) - if err != nil { - log.Error(err, "failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } + gatewayConnectionReason := "" + if !gatewayConnectionReady { + if ncName == commonservice.SystemVPCNetworkConfigurationName { + gatewayConnectionReady, gatewayConnectionReason, err = r.Service.ValidateGatewayConnectionStatus(&nc) + log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "gatewayConnectionReason", gatewayConnectionReason) + if err != nil { + log.Error(err, "Failed to validate the edge and gateway connection", "Org", nc.Org, "Project", nc.NSXProject) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err } + setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, gatewayConnectionReason) } else { - privateIPs = nc.PrivateIPs - vpcConnectivityProfilePath = nc.VPCConnectivityProfile + log.Info("Skipping reconciliation due to unready system gateway connection", "NetworkInfo", req.NamespacedName) + return common.ResultRequeueAfter60sec, nil } + } + lbProvider := r.Service.GetLBProvider() + createdVpc, err := r.Service.CreateOrUpdateVPC(networkInfoCR, &nc, lbProvider) + if err != nil { + log.Error(err, "Failed to create or update VPC", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } - snatIP, path, cidr := "", "", "" - - vpcConnectivityProfile, err := r.Service.GetVpcConnectivityProfile(&nc, vpcConnectivityProfilePath) + var privateIPs []string + var vpcConnectivityProfilePath string + var nsxLBSPath string + isPreCreatedVPC := vpc.IsPreCreatedVPC(nc) + if isPreCreatedVPC { + privateIPs = createdVpc.PrivateIps + vpcPath := *createdVpc.Path + vpcConnectivityProfilePath, err = r.GetVpcConnectivityProfilePathByVpcPath(vpcPath) if err != nil { - log.Error(err, "get VpcConnectivityProfile failed, would retry exponentially", "VPC", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) + log.Error(err, "failed to get VPC connectivity profile path", "path", vpcPath) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) return common.ResultRequeueAfter10sec, err } - hasExternalIPs := true - if ncName == commonservice.SystemVPCNetworkConfigurationName { - if len(vpcConnectivityProfile.ExternalIpBlocks) == 0 { - hasExternalIPs = false - log.Error(err, "there is no ExternalIPBlock in VPC ConnectivityProfile", "VPC", req.NamespacedName) - } - setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, hasExternalIPs) - } - // currently, auto snat is not exposed, and use default value True - // checking autosnat to support future extension in vpc configuration - autoSnatEnabled := r.Service.IsEnableAutoSNAT(vpcConnectivityProfile) - if autoSnatEnabled { - snatIP, err = r.Service.GetDefaultSNATIP(*createdVpc) - if err != nil { - log.Error(err, "failed to read default SNAT ip from VPC", "VPC", createdVpc.Id) - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: "", - LoadBalancerIPAddresses: "", - PrivateIPs: privateIPs, - } - updateFail(r, ctx, obj, &err, r.Client, state) - return common.ResultRequeueAfter10sec, err - } - } - if ncName == commonservice.SystemVPCNetworkConfigurationName { - vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} - err := r.Client.Get(ctx, types.NamespacedName{Name: ncName}, vpcNetworkConfiguration) + // Retrieve NSX lbs path if Avi is not used with the pre-created VPC. + if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil || + !*createdVpc.LoadBalancerVpcEndpoint.Enabled { + nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(vpcPath) if err != nil { - log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName) - updateFail(r, ctx, obj, &err, r.Client, nil) + log.Error(err, "failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) return common.ResultRequeueAfter10sec, err } - log.Info("got the AutoSnat status", "autoSnatEnabled", autoSnatEnabled, "req", req.NamespacedName) - setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, autoSnatEnabled) } + } else { + privateIPs = nc.PrivateIPs + vpcConnectivityProfilePath = nc.VPCConnectivityProfile + } - // if lb vpc enabled, read avi subnet path and cidr - // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back, - // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. - if lbProvider == vpc.AVILB && createdVpc.LoadBalancerVpcEndpoint != nil && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { - path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) - if err != nil { - log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id) - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: snatIP, - LoadBalancerIPAddresses: "", - PrivateIPs: privateIPs, - } - updateFail(r, ctx, obj, &err, r.Client, state) - return common.ResultRequeueAfter10sec, err - } - } + snatIP, path, cidr := "", "", "" - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: snatIP, - LoadBalancerIPAddresses: cidr, - PrivateIPs: privateIPs, - VPCPath: *createdVpc.Path, + vpcConnectivityProfile, err := r.Service.GetVpcConnectivityProfile(&nc, vpcConnectivityProfilePath) + if err != nil { + log.Error(err, "Failed to get VPC connectivity profile", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + hasExternalIPs := true + if ncName == commonservice.SystemVPCNetworkConfigurationName { + if len(vpcConnectivityProfile.ExternalIpBlocks) == 0 { + hasExternalIPs = false + log.Error(err, "There is no ExternalIPBlock in VPC ConnectivityProfile", "NetworkInfo", req.NamespacedName) } - - if !isPreCreatedVPC { - nsxLBSPath = r.Service.GetDefaultNSXLBSPathByVPC(*createdVpc.Id) + setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, hasExternalIPs) + } + // currently, auto snat is not exposed, and use default value True + // checking autosnat to support future extension in VPC configuration + autoSnatEnabled := r.Service.IsEnableAutoSNAT(vpcConnectivityProfile) + if autoSnatEnabled { + snatIP, err = r.Service.GetDefaultSNATIP(*createdVpc) + if err != nil { + log.Error(err, "Failed to read default SNAT IP from VPC", "VPC", createdVpc.Id) + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: "", + LoadBalancerIPAddresses: "", + PrivateIPs: privateIPs, + } + updateFail(r, ctx, networkInfoCR, &err, r.Client, state) + return common.ResultRequeueAfter10sec, err } - // AKO needs to know the AVI subnet path created by NSX - setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, nsxLBSPath, *createdVpc.Path) - updateSuccess(r, ctx, obj, r.Client, state, nc.Name, path) - if ncName == commonservice.SystemVPCNetworkConfigurationName && (!gatewayConnectionReady || !autoSnatEnabled || !hasExternalIPs) { - log.Info("requeuing the NetworkInfo CR because VPCNetworkConfiguration system is not ready", "gatewayConnectionReason", gatewayConnectionReason, "autoSnatEnabled", autoSnatEnabled, "hasExternalIPs", hasExternalIPs, "req", req) - return common.ResultRequeueAfter60sec, nil + } + if ncName == commonservice.SystemVPCNetworkConfigurationName { + vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} + err := r.Client.Get(ctx, types.NamespacedName{Name: ncName}, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get VPCNetworkConfiguration", "Name", ncName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err } - } else { - if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) - isShared, err := r.Service.IsSharedVPCNamespaceByNS(obj.GetNamespace()) - if err != nil { - log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) - return common.ResultRequeue, err - } - vpcs := r.Service.GetVPCsByNamespace(obj.GetNamespace()) - // if nsx resource do not exist, continue to remove finalizer, or the crd can not be removed - if len(vpcs) == 0 { - // when nsx vpc not found in vpc store, skip deleting NSX VPC - log.Info("can not find VPC in store, skip deleting NSX VPC, remove finalizer from NetworkInfo CR") - } else if !isShared { - for _, vpc := range vpcs { - // first delete vpc and then ipblock or else it will fail arguing it is being referenced by other objects - if err := r.Service.DeleteVPC(*vpc.Path); err != nil { - log.Error(err, "failed to delete nsx VPC, would retry exponentially", "NetworkInfo", req.NamespacedName) - deleteFail(r, ctx, obj, &err, r.Client) - return common.ResultRequeueAfter10sec, err - } - } - } + log.Info("Got the AutoSnat status", "autoSnatEnabled", autoSnatEnabled, "NetworkInfo", req.NamespacedName) + setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, autoSnatEnabled) + } - controllerutil.RemoveFinalizer(obj, commonservice.NetworkInfoFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - deleteFail(r, ctx, obj, &err, r.Client) - return common.ResultRequeue, err - } - ncName, err := r.Service.GetNetworkconfigNameFromNS(obj.Namespace) - if err != nil { - log.Error(err, "failed to get network config name for VPC when deleting NetworkInfo CR", "NetworkInfo", obj.Name) - return common.ResultRequeueAfter10sec, err + // if lb VPC enabled, read avi subnet path and cidr + // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this VPC back, + // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. + if lbProvider == vpc.AVILB && createdVpc.LoadBalancerVpcEndpoint != nil && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { + path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) + if err != nil { + log.Error(err, "Failed to read LB Subnet path and CIDR", "VPC", createdVpc.Id) + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: snatIP, + LoadBalancerIPAddresses: "", + PrivateIPs: privateIPs, } - log.V(1).Info("removed finalizer", "NetworkInfo", req.NamespacedName) - deleteVPCNetworkConfigurationStatus(ctx, r.Client, ncName, vpcs, r.Service.ListVPC()) - deleteSuccess(r, ctx, obj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, state) + return common.ResultRequeueAfter10sec, err } } + + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: snatIP, + LoadBalancerIPAddresses: cidr, + PrivateIPs: privateIPs, + VPCPath: *createdVpc.Path, + } + + if !isPreCreatedVPC { + nsxLBSPath = r.Service.GetDefaultNSXLBSPathByVPC(*createdVpc.Id) + } + // AKO needs to know the AVI subnet path created by NSX + setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, nsxLBSPath, *createdVpc.Path) + updateSuccess(r, ctx, networkInfoCR, r.Client, state, nc.Name, path) + if ncName == commonservice.SystemVPCNetworkConfigurationName && (!gatewayConnectionReady || !autoSnatEnabled || !hasExternalIPs) { + log.Info("Requeue NetworkInfo CR because VPCNetworkConfiguration system is not ready", "gatewayConnectionReason", gatewayConnectionReason, "autoSnatEnabled", autoSnatEnabled, "hasExternalIPs", hasExternalIPs, "req", req) + return common.ResultRequeueAfter60sec, nil + } + return common.ResultNormal, nil } @@ -274,10 +276,10 @@ func (r *NetworkInfoReconciler) setupWithManager(mgr ctrl.Manager) error { MaxConcurrentReconciles: common.NumReconcile(), }). Watches( - // For created/removed network config, add/remove from vpc network config cache, + // For created/removed network config, add/remove from VPC network config cache, // and update IPBlocksInfo. // For modified network config, currently only support appending ips to public ip blocks, - // update network config in cache and update nsx vpc object. + // update network config in cache and update nsx VPC object. &v1alpha1.VPCNetworkConfiguration{}, &VPCNetworkConfigurationHandler{ Client: mgr.GetClient(), @@ -297,45 +299,145 @@ func (r *NetworkInfoReconciler) Start(mgr ctrl.Manager) error { return nil } -// CollectGarbage logic for nsx-vpc is that: +func (r *NetworkInfoReconciler) listNamespaceCRsNameIDSet(ctx context.Context) (sets.Set[string], sets.Set[string], error) { + // read all Namespaces from K8s + namespaces := &corev1.NamespaceList{} + err := r.Client.List(ctx, namespaces) + if err != nil { + return nil, nil, err + } + nsSet := sets.Set[string]{} + idSet := sets.Set[string]{} + for _, ns := range namespaces.Items { + nsSet.Insert(ns.Name) + idSet.Insert(string(ns.UID)) + } + return nsSet, idSet, nil +} + +// CollectGarbage logic for NSX VPC is that: // 1. list all current existing namespace in kubernetes -// 2. list all the nsx-vpc in vpcStore -// 3. loop all the nsx-vpc to get its namespace, check if the namespace still exist -// 4. if ns do not exist anymore, delete the nsx-vpc resource +// 2. list all the NSX VPC in vpcStore +// 3. loop all the NSX VPC to get its namespace, check if the namespace still exist +// 4. if ns do not exist anymore, delete the NSX VPC resource // it implements the interface GarbageCollector method. func (r *NetworkInfoReconciler) CollectGarbage(ctx context.Context) { - log.Info("VPC garbage collector started") - // read all nsx-vpc from vpc store + startTime := time.Now() + defer func() { + log.Info("VPC garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + // read all NSX VPC from VPC store nsxVPCList := r.Service.ListVPC() if len(nsxVPCList) == 0 { + log.Info("No NSX VPCs found in the store, skipping garbage collection") return } - // read all namespaces from k8s - namespaces := &corev1.NamespaceList{} - err := r.Client.List(ctx, namespaces) + _, idSet, err := r.listNamespaceCRsNameIDSet(ctx) if err != nil { - log.Error(err, "failed to list k8s namespaces") + log.Error(err, "Failed to list Kubernetes Namespaces for VPC garbage collection") return } - nsSet := sets.NewString() - for _, ns := range namespaces.Items { - nsSet.Insert(ns.Name) - } - for i := len(nsxVPCList) - 1; i >= 0; i-- { - nsxVPCNamespace := getNamespaceFromNSXVPC(&nsxVPCList[i]) - if nsSet.Has(nsxVPCNamespace) { + for i, nsxVPC := range nsxVPCList { + nsxVPCNamespaceName := filterTagFromNSXVPC(&nsxVPCList[i], commonservice.TagScopeNamespace) + nsxVPCNamespaceID := filterTagFromNSXVPC(&nsxVPCList[i], commonservice.TagScopeNamespaceUID) + if idSet.Has(nsxVPCNamespaceID) { continue } - elem := nsxVPCList[i] - log.Info("GC collected nsx VPC object", "ID", elem.Id, "Namespace", nsxVPCNamespace) + log.Info("Garbage collecting NSX VPC object", "VPC", nsxVPC.Id, "Namespace", nsxVPCNamespaceName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) - err = r.Service.DeleteVPC(*elem.Path) - if err != nil { + + if err = r.Service.DeleteVPC(*nsxVPC.Path); err != nil { + log.Error(err, "Failed to delete NSX VPC", "VPC", nsxVPC.Id, "Namespace", nsxVPCNamespaceName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeNetworkInfo) - } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeNetworkInfo) + continue } + + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeNetworkInfo) + log.Info("Successfully deleted NSX VPC", "VPC", nsxVPC.Id) } } + +func (r *NetworkInfoReconciler) fetchStaleVPCsByNamespace(ctx context.Context, ns string) ([]*model.Vpc, error) { + isShared, err := r.Service.IsSharedVPCNamespaceByNS(ns) + if err != nil { + return nil, fmt.Errorf("failed to check if Namespace is shared for NS %s: %w", ns, err) + } + if isShared { + log.Info("Shared Namespace, skipping deletion of NSX VPC", "Namespace", ns) + return nil, nil + } + + return r.Service.GetVPCsByNamespace(ns), nil +} + +func (r *NetworkInfoReconciler) deleteVPCsByName(ctx context.Context, ns string) error { + _, idSet, err := r.listNamespaceCRsNameIDSet(ctx) + if err != nil { + log.Error(err, "Failed to list Kubernetes Namespaces") + return fmt.Errorf("failed to list Kubernetes Namespaces while deleting VPCs: %v", err) + } + + staleVPCs, err := r.fetchStaleVPCsByNamespace(ctx, ns) + if err != nil { + return err + } + + var vpcToDelete []*model.Vpc + for _, nsxVPC := range staleVPCs { + namespaceIDofVPC := filterTagFromNSXVPC(nsxVPC, commonservice.TagScopeNamespaceUID) + if idSet.Has(namespaceIDofVPC) { + log.Info("Skipping deletion, Namespace still exists in K8s", "Namespace", ns) + continue + } + vpcToDelete = append(vpcToDelete, nsxVPC) + } + return r.deleteVPCs(ctx, vpcToDelete, ns) +} + +func (r *NetworkInfoReconciler) deleteVPCsByID(ctx context.Context, ns, id string) error { + staleVPCs, err := r.fetchStaleVPCsByNamespace(ctx, ns) + if err != nil { + return err + } + + var vpcToDelete []*model.Vpc + for _, nsxVPC := range staleVPCs { + namespaceIDofVPC := filterTagFromNSXVPC(nsxVPC, commonservice.TagScopeNamespaceUID) + if namespaceIDofVPC == id { + vpcToDelete = append(vpcToDelete, nsxVPC) + } + } + return r.deleteVPCs(ctx, vpcToDelete, ns) +} + +func (r *NetworkInfoReconciler) deleteVPCs(ctx context.Context, staleVPCs []*model.Vpc, ns string) error { + if len(staleVPCs) == 0 { + log.Info("There is no VPCs found in store, skipping deletion of NSX VPC", "Namespace", ns) + return nil + } + var deleteErrs []error + for _, nsxVPC := range staleVPCs { + if nsxVPC.Path == nil { + log.Error(nil, "VPC path is nil, skipping", "VPC", nsxVPC) + continue + } + if err := r.Service.DeleteVPC(*nsxVPC.Path); err != nil { + log.Error(err, "Failed to delete VPC in NSX", "VPC", nsxVPC.Path) + deleteErrs = append(deleteErrs, fmt.Errorf("failed to delete VPC %s: %w", *nsxVPC.Path, err)) + } + } + if len(deleteErrs) > 0 { + return fmt.Errorf("multiple errors occurred while deleting VPCs: %v", deleteErrs) + } + + // Update the VPCNetworkConfiguration Status + ncName, err := r.Service.GetNetworkconfigNameFromNS(ns) + if err != nil { + return fmt.Errorf("failed to get VPCNetworkConfiguration for Namespace when deleting stale VPCs %s: %w", ns, err) + } + deleteVPCNetworkConfigurationStatus(ctx, r.Client, ncName, staleVPCs, r.Service.ListVPC()) + return nil +} diff --git a/pkg/controllers/networkinfo/networkinfo_controller_test.go b/pkg/controllers/networkinfo/networkinfo_controller_test.go index f89cb51c5..2302e4b46 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller_test.go +++ b/pkg/controllers/networkinfo/networkinfo_controller_test.go @@ -5,11 +5,14 @@ package networkinfo import ( "context" + "errors" + "fmt" "reflect" "testing" "github.com/agiledragon/gomonkey" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -21,7 +24,6 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" - servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) @@ -48,12 +50,32 @@ func (c *fakeVPCConnectivityProfilesClient) Update(orgIdParam string, projectIdP return model.VpcConnectivityProfile{}, nil } +type fakeVpcAttachmentClient struct{} + +func (c *fakeVpcAttachmentClient) List(orgIdParam string, projectIdParam string, vpcIdParam string, cursorParam *string, includeMarkForDeleteObjectsParam *bool, includedFieldsParam *string, pageSizeParam *int64, sortAscendingParam *bool, sortByParam *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{}, nil +} + +func (c *fakeVpcAttachmentClient) Get(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string) (model.VpcAttachment, error) { + return model.VpcAttachment{}, nil +} + +func (c *fakeVpcAttachmentClient) Patch(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string, vpcAttachmentParam model.VpcAttachment) error { + return nil +} +func (c *fakeVpcAttachmentClient) Update(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string, vpcAttachmentParam model.VpcAttachment) (model.VpcAttachment, error) { + return model.VpcAttachment{}, nil +} + +var fakeAttachmentClient = &fakeVpcAttachmentClient{} + func createNetworkInfoReconciler() *NetworkInfoReconciler { service := &vpc.VPCService{ Service: servicecommon.Service{ Client: nil, NSXClient: &nsx.Client{ VPCConnectivityProfilesClient: &fakeVPCConnectivityProfilesClient{}, + VpcAttachmentClient: fakeAttachmentClient, }, NSXConfig: &config.NSXOperatorConfig{ @@ -93,11 +115,28 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { wantErr bool }{ { - name: "Empty", - prepareFunc: nil, - args: requestArgs, - want: common.ResultNormal, - wantErr: false, + name: "Empty", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) (patches *gomonkey.Patches) { + patches = gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetNetworkconfigNameFromNS", func(_ *vpc.VPCService, _ string) (string, error) { + return "", nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + patches.ApplyFunc(deleteVPCNetworkConfigurationStatus, func(ctx context.Context, client client.Client, ncName string, staleVPCs []*model.Vpc, aliveVPCs []model.Vpc) { + return + }) + return patches + }, + args: requestArgs, + want: common.ResultNormal, + wantErr: false, }, { name: "GatewayConnectionReadyInSystemVPC", @@ -245,6 +284,10 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { return "non-system", nil }) + patches.ApplyMethod(reflect.TypeOf(r), "GetVpcConnectivityProfilePathByVpcPath", func(_ *NetworkInfoReconciler, _ string) (string, error) { + return "connectivity_profile", nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCNetworkConfig", func(_ *vpc.VPCService, _ string) (servicecommon.VPCNetworkConfigInfo, bool) { return servicecommon.VPCNetworkConfigInfo{ VPCConnectivityProfile: "/orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default", @@ -659,6 +702,9 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { return "pre-vpc-nc", nil }) + patches.ApplyMethod(reflect.TypeOf(r), "GetVpcConnectivityProfilePathByVpcPath", func(_ *NetworkInfoReconciler, _ string) (string, error) { + return "connectivity_profile", nil + }) patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCNetworkConfig", func(_ *vpc.VPCService, _ string) (servicecommon.VPCNetworkConfigInfo, bool) { return servicecommon.VPCNetworkConfigInfo{ Org: "default", @@ -722,3 +768,203 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { }) } } + +func TestNetworkInfoReconciler_deleteStaleVPCs(t *testing.T) { + r := createNetworkInfoReconciler() + + ctx := context.TODO() + namespace := "test-ns" + + t.Run("shared namespace, skip deletion", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return true, nil + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) + + t.Run("non-shared namespace, no VPCs found", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + return nil + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) + + t.Run("failed to delete VPC", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + vpcPath := "/vpc/1" + return []*model.Vpc{{Path: &vpcPath}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return fmt.Errorf("delete failed") + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + assert.Error(t, err) + assert.Contains(t, err.Error(), "delete failed") + }) + + t.Run("successful deletion of VPCs", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []*model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetNetworkconfigNameFromNS", func(_ *vpc.VPCService, _ string) (string, error) { + return "", nil + }) + patches.ApplyFunc(deleteVPCNetworkConfigurationStatus, func(ctx context.Context, client client.Client, ncName string, staleVPCs []*model.Vpc, aliveVPCs []model.Vpc) { + return + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) +} + +func TestNetworkInfoReconciler_CollectGarbage(t *testing.T) { + r := createNetworkInfoReconciler() + + ctx := context.TODO() + + t.Run("no VPCs found in the store", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + defer patches.Reset() + + r.CollectGarbage(ctx) + // No errors expected + }) + + t.Run("successful garbage collection", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return nil + }) + defer patches.Reset() + + r.CollectGarbage(ctx) + }) + + t.Run("failed to delete VPC", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return errors.New("deletion error") + }) + defer patches.Reset() + r.CollectGarbage(ctx) + }) +} +func TestNetworkInfoReconciler_GetVpcConnectivityProfilePathByVpcPath(t *testing.T) { + tests := []struct { + name string + vpcPath string + prepareFunc func(*testing.T, *NetworkInfoReconciler, context.Context) *gomonkey.Patches + want string + wantErr bool + }{ + { + name: "Invalid VPC Path", + vpcPath: "/invalid/path", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + return nil + }, + want: "", + wantErr: true, + }, + { + name: "Failed to list VPC attachment", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service.NSXClient.VpcAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{}, fmt.Errorf("list error") + }) + return patches + }, + want: "", + wantErr: true, + }, + { + name: "No VPC attachment found", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service.NSXClient.VpcAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{Results: []model.VpcAttachment{}}, nil + }) + return patches + }, + want: "", + wantErr: true, + }, + { + name: "Successful VPC attachment retrieval", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(fakeAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{ + Results: []model.VpcAttachment{ + {VpcConnectivityProfile: servicecommon.String("/orgs/default/projects/project-quality/vpc-connectivity-profiles/default"), + ParentPath: servicecommon.String("/orgs/default/projects/project-quality/vpcs/fake-vpc"), + Path: servicecommon.String("/orgs/default/projects/project-quality/vpcs/fake-vpc/attachments/default")}, + }, + }, nil + }) + return patches + }, + want: "/orgs/default/projects/project-quality/vpc-connectivity-profiles/default", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := createNetworkInfoReconciler() + ctx := context.TODO() + if tt.prepareFunc != nil { + patches := tt.prepareFunc(t, r, ctx) + if patches != nil { + defer patches.Reset() + } + } + got, err := r.GetVpcConnectivityProfilePathByVpcPath(tt.vpcPath) + if (err != nil) != tt.wantErr { + t.Errorf("GetVpcConnectivityProfilePathByVpcPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetVpcConnectivityProfilePathByVpcPath() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 2a033ed3e..4933f657f 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -237,10 +237,10 @@ func deleteVPCNetworkConfigurationStatus(ctx context.Context, client client.Clie log.Info("Deleted stale VPCNetworkConfiguration status", "Name", ncName, "nc.Status.VPCs", nc.Status.VPCs, "staleVPCs", staleVPCNames) } -func getNamespaceFromNSXVPC(nsxVPC *model.Vpc) string { +func filterTagFromNSXVPC(nsxVPC *model.Vpc, tagName string) string { tags := nsxVPC.Tags for _, tag := range tags { - if *tag.Scope == svccommon.TagScopeNamespace { + if *tag.Scope == tagName { return *tag.Tag } } diff --git a/pkg/controllers/networkpolicy/networkpolicy_controller.go b/pkg/controllers/networkpolicy/networkpolicy_controller.go index 66097ee98..ac332d138 100644 --- a/pkg/controllers/networkpolicy/networkpolicy_controller.go +++ b/pkg/controllers/networkpolicy/networkpolicy_controller.go @@ -18,7 +18,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -100,21 +99,21 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) if err := r.Client.Get(ctx, req.NamespacedName, networkPolicy); err != nil { - log.Error(err, "unable to fetch network policy", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + // IgnoreNotFound returns nil on NotFound errors. + if client.IgnoreNotFound(err) == nil { + if err := r.deleteNetworkPolicyByName(req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NetworkPolicy", "networkpolicy", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + // In case that client is unable to check CR + log.Error(err, "client is unable to fetch NetworkPolicy CR", "req", req.NamespacedName) + return ResultRequeue, err } if networkPolicy.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) - if !controllerutil.ContainsFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) { - controllerutil.AddFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) - if err := r.Client.Update(ctx, networkPolicy); err != nil { - log.Error(err, "add finalizer", "networkpolicy", req.NamespacedName) - updateFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on networkpolicy", "networkpolicy", req.NamespacedName) - } if err := r.Service.CreateOrUpdateSecurityPolicy(networkPolicy); err != nil { if errors.As(err, &nsxutil.RestrictionError{}) { @@ -135,25 +134,14 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques updateSuccess(r, ctx, networkPolicy) cleanNetworkPolicyErrorAnnotation(ctx, networkPolicy, r.Client) } else { - if controllerutil.ContainsFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - if err := r.Service.DeleteSecurityPolicy(networkPolicy, false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) - deleteFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) - if err := r.Client.Update(ctx, networkPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) - deleteFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "networkpolicy", req.NamespacedName) - deleteSuccess(r, ctx, networkPolicy) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "networkpolicy", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + + if err := r.Service.DeleteSecurityPolicy(networkPolicy, false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) + deleteFail(r, ctx, networkPolicy, &err) + return ResultRequeue, err } + deleteSuccess(r, ctx, networkPolicy) } return ResultNormal, nil @@ -186,19 +174,12 @@ func (r *NetworkPolicyReconciler) CollectGarbage(ctx context.Context) { if len(nsxPolicySet) == 0 { return } - policyList := &networkingv1.NetworkPolicyList{} - err := r.Client.List(ctx, policyList) + + CRPolicySet, err := r.listNetworkPolciyCRIDs() if err != nil { - log.Error(err, "failed to list NetworkPolicy") return } - CRPolicySet := sets.New[string]() - for _, policy := range policyList.Items { - CRPolicySet.Insert(r.Service.BuildNetworkPolicyAllowPolicyID(string(policy.UID))) - CRPolicySet.Insert(r.Service.BuildNetworkPolicyIsolationPolicyID(string(policy.UID))) - } - diffSet := nsxPolicySet.Difference(CRPolicySet) for elem := range diffSet { log.V(1).Info("GC collected NetworkPolicy", "ID", elem) @@ -212,6 +193,46 @@ func (r *NetworkPolicyReconciler) CollectGarbage(ctx context.Context) { } } +func (r *NetworkPolicyReconciler) deleteNetworkPolicyByName(ns, name string) error { + nsxSecurityPolicies := r.Service.ListNetworkPolicyByName(ns, name) + + CRPolicySet, err := r.listNetworkPolciyCRIDs() + if err != nil { + return err + } + for _, item := range nsxSecurityPolicies { + uid := nsxutil.FindTag(item.Tags, servicecommon.TagScopeNetworkPolicyUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, NetworkPolicy CR still exists in K8s", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + continue + } + + log.Info("deleting NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + if err := r.Service.DeleteSecurityPolicy(types.UID(uid), false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { + log.Error(err, "failed to delete NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + return err + } + log.Info("successfully deleted NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + } + return nil +} + +func (r *NetworkPolicyReconciler) listNetworkPolciyCRIDs() (sets.Set[string], error) { + networkPolicyList := &networkingv1.NetworkPolicyList{} + err := r.Client.List(context.Background(), networkPolicyList) + if err != nil { + log.Error(err, "failed to list NetworkPolicy CRs") + return nil, err + } + + CRPolicySet := sets.New[string]() + for _, policy := range networkPolicyList.Items { + CRPolicySet.Insert(r.Service.BuildNetworkPolicyAllowPolicyID(string(policy.UID))) + CRPolicySet.Insert(r.Service.BuildNetworkPolicyIsolationPolicyID(string(policy.UID))) + } + return CRPolicySet, nil +} + func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider) { networkPolicyReconcile := NetworkPolicyReconciler{ Client: mgr.GetClient(), diff --git a/pkg/controllers/node/node_controller.go b/pkg/controllers/node/node_controller.go index e1b25a7fd..c0197cb74 100644 --- a/pkg/controllers/node/node_controller.go +++ b/pkg/controllers/node/node_controller.go @@ -97,7 +97,6 @@ func (r *NodeReconciler) Start(mgr ctrl.Manager) error { return nil } -// PredicateFuncsNode filters out events where only resourceVersion, lastHeartbeatTime, or lastTransitionTime have changed var PredicateFuncsNode = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldNode, okOld := e.ObjectOld.(*v1.Node) @@ -106,13 +105,12 @@ var PredicateFuncsNode = predicate.Funcs{ return true } - // If only the heartbeat time, transition time and resource version has changed, and other properties unchanged, ignore the update + // If only the condition, resource version, allocatable, capacity have changed, and other properties unchanged, ignore the update if len(newNode.Status.Conditions) > 0 && len(oldNode.Status.Conditions) > 0 { - if newNode.ResourceVersion != oldNode.ResourceVersion && - newNode.Status.Conditions[0].LastHeartbeatTime != oldNode.Status.Conditions[0].LastHeartbeatTime && - newNode.Status.Conditions[0].LastTransitionTime != oldNode.Status.Conditions[0].LastTransitionTime { - oldNode.Status.Conditions[0].LastHeartbeatTime = newNode.Status.Conditions[0].LastHeartbeatTime - oldNode.Status.Conditions[0].LastTransitionTime = newNode.Status.Conditions[0].LastTransitionTime + if newNode.ResourceVersion != oldNode.ResourceVersion { + oldNode.Status.Allocatable = newNode.Status.Allocatable + oldNode.Status.Capacity = newNode.Status.Capacity + oldNode.Status.Conditions = newNode.Status.Conditions return !reflect.DeepEqual(oldNode.Status, newNode.Status) } } diff --git a/pkg/controllers/pod/pod_controller.go b/pkg/controllers/pod/pod_controller.go index c13959a71..36a23a484 100644 --- a/pkg/controllers/pod/pod_controller.go +++ b/pkg/controllers/pod/pod_controller.go @@ -7,16 +7,17 @@ import ( "context" "fmt" "os" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -45,18 +46,25 @@ type PodReconciler struct { } func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - pod := &v1.Pod{} log.Info("reconciling pod", "pod", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("finished reconciling Pod", "Pod", req.NamespacedName, "duration", time.Since(startTime)) + }() metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypePod) + pod := &v1.Pod{} if err := r.Client.Get(ctx, req.NamespacedName, pod); err != nil { - log.Error(err, "unable to fetch pod", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) - } - if pod.Spec.HostNetwork { - log.Info("skipping handling hostnetwork pod", "pod", req.NamespacedName) - return common.ResultNormal, nil + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetPortByPodName(ctx, req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NSX SubnetPort", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err + } + return common.ResultNormal, nil + } + log.Error(err, "unable to fetch Pod", "Pod", req.NamespacedName) + return common.ResultRequeue, err } if len(pod.Spec.NodeName) == 0 { log.Info("pod is not scheduled on node yet, skipping", "pod", req.NamespacedName) @@ -65,16 +73,6 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R if !podIsDeleted(pod) { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypePod) - if !controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { - controllerutil.AddFinalizer(pod, servicecommon.PodFinalizerName) - if err := r.Client.Update(ctx, pod); err != nil { - log.Error(err, "add finalizer", "pod", req.NamespacedName) - updateFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - log.Info("added finalizer on pod", "pod", req.NamespacedName) - } - nsxSubnetPath, err := r.GetSubnetPathForPod(ctx, pod) if err != nil { log.Error(err, "failed to get NSX resource path from subnet", "pod.Name", pod.Name, "pod.UID", pod.UID) @@ -105,25 +103,13 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } updateSuccess(r, ctx, pod) } else { - if controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { - metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&pod.ObjectMeta) - if err := r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) - deleteFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - controllerutil.RemoveFinalizer(pod, servicecommon.PodFinalizerName) - if err := r.Client.Update(ctx, pod); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) - deleteFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - log.Info("removed finalizer", "pod", req.NamespacedName) - deleteSuccess(r, ctx, pod) - } else { - log.Info("finalizers cannot be recognized", "pod", req.NamespacedName) + subnetPortID := r.SubnetPortService.BuildSubnetPortId(&pod.ObjectMeta) + if err := r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) + deleteFail(r, ctx, pod, &err) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, pod) } return ctrl.Result{}, nil } @@ -147,14 +133,7 @@ func (r *PodReconciler) GetNodeByName(nodeName string) (*model.HostTransportNode func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1.Pod{}). - WithEventFilter( - predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }, - ). + WithEventFilter(PredicateFuncsPod). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -212,7 +191,7 @@ func (r *PodReconciler) CollectGarbage(ctx context.Context) { for elem := range diffSet { log.V(1).Info("GC collected Pod", "NSXSubnetPortID", elem) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - err = r.SubnetPortService.DeleteSubnetPort(elem) + err = r.SubnetPortService.DeleteSubnetPortById(elem) if err != nil { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } else { @@ -221,17 +200,17 @@ func (r *PodReconciler) CollectGarbage(ctx context.Context) { } } -func updateFail(r *PodReconciler, c context.Context, o *v1.Pod, e *error) { +func updateFail(r *PodReconciler, _ context.Context, o *v1.Pod, e *error) { r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailUpdate, fmt.Sprintf("%v", *e)) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypePod) } -func deleteFail(r *PodReconciler, c context.Context, o *v1.Pod, e *error) { +func deleteFail(r *PodReconciler, _ context.Context, o *v1.Pod, e *error) { r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailDelete, fmt.Sprintf("%v", *e)) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } -func updateSuccess(r *PodReconciler, c context.Context, o *v1.Pod) { +func updateSuccess(r *PodReconciler, _ context.Context, o *v1.Pod) { r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "Pod CR has been successfully updated") metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypePod) } @@ -264,3 +243,57 @@ func (r *PodReconciler) GetSubnetPathForPod(ctx context.Context, pod *v1.Pod) (s func podIsDeleted(pod *v1.Pod) bool { return !pod.ObjectMeta.DeletionTimestamp.IsZero() || pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed" } + +func (r *PodReconciler) deleteSubnetPortByPodName(ctx context.Context, ns string, name string) error { + // When deleting SubnetPort by Name and Namespace, skip the SubnetPort belonging to the existed SubnetPort CR + nsxSubnetPorts := r.SubnetPortService.ListSubnetPortByPodName(ns, name) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) + if err != nil { + log.Error(err, "failed to list SubnetPort CRs") + return err + } + + for _, nsxSubnetPort := range nsxSubnetPorts { + if crSubnetPortIDsSet.Has(*nsxSubnetPort.Id) { + log.Info("skipping deletion, Pod CR still exists in K8s", "ID", *nsxSubnetPort.Id) + continue + } + if err := r.SubnetPortService.DeleteSubnetPort(nsxSubnetPort); err != nil { + return err + } + } + log.Info("successfully deleted nsxSubnetPort for Pod", "namespace", ns, "name", name) + return nil +} + +// PredicateFuncsPod filters out events where pod.Spec.HostNetwork is true +var PredicateFuncsPod = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldPod, okOld := e.ObjectOld.(*v1.Pod) + newPod, okNew := e.ObjectNew.(*v1.Pod) + if !okOld || !okNew { + return true + } + + if oldPod.Spec.HostNetwork && newPod.Spec.HostNetwork { + return false + } + return true + }, + CreateFunc: func(e event.CreateEvent) bool { + pod, ok := e.Object.(*v1.Pod) + if !ok { + return true + } + + if pod.Spec.HostNetwork { + return false + } + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + GenericFunc: func(e event.GenericEvent) bool { + return true + }, +} diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller.go b/pkg/controllers/securitypolicy/securitypolicy_controller.go index 830dec59d..1220eac0a 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller.go @@ -137,12 +137,21 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque obj = &v1alpha1.SecurityPolicy{} } - log.Info("reconciling securitypolicy CR", "securitypolicy", req.NamespacedName) + log.Info("reconciling SecurityPolicy CR", "securitypolicy", req.NamespacedName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch security policy CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + // IgnoreNotFound returns nil on NotFound errors. + if client.IgnoreNotFound(err) == nil { + if err := r.deleteSecurityPolicyByName(req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete SecurityPolicy", "securitypolicy", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + // In case that client is unable to check CR + log.Error(err, "client is unable to fetch SecurityPolicy CR", "req", req.NamespacedName) + return ResultRequeue, err } isZero := false @@ -152,7 +161,6 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque case *crdv1alpha1.SecurityPolicy: o := obj.(*crdv1alpha1.SecurityPolicy) isZero = o.ObjectMeta.DeletionTimestamp.IsZero() - finalizerName = servicecommon.SecurityPolicyFinalizerName realObj = securitypolicy.VPCToT1(o) case *v1alpha1.SecurityPolicy: realObj = obj.(*v1alpha1.SecurityPolicy) @@ -170,15 +178,6 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque if isZero { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) - if !controllerutil.ContainsFinalizer(obj, finalizerName) { - controllerutil.AddFinalizer(obj, finalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "securitypolicy", req.NamespacedName) - updateFail(r, ctx, realObj, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on securitypolicy CR", "securitypolicy", req.NamespacedName) - } if isCRInSysNs, err := util.IsSystemNamespace(r.Client, req.Namespace, nil); err != nil { err = errors.New("fetch namespace associated with security policy CR failed") @@ -213,25 +212,24 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque cleanSecurityPolicyErrorAnnotation(ctx, realObj, securitypolicy.IsVPCEnabled(r.Service), r.Client) } else { log.Info("reconciling CR to delete securitypolicy", "securitypolicy", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + + // For T1 upgrade, the upgraded CRs still has finalizer if controllerutil.ContainsFinalizer(obj, finalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - if err := r.Service.DeleteSecurityPolicy(realObj.UID, false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) - deleteFail(r, ctx, realObj, &err) - return ResultRequeue, err - } controllerutil.RemoveFinalizer(obj, finalizerName) if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) + log.Error(err, "finalizer remove failed, would retry exponentially", "securitypolicy", req.NamespacedName) deleteFail(r, ctx, realObj, &err) return ResultRequeue, err } log.V(1).Info("removed finalizer", "securitypolicy", req.NamespacedName) - deleteSuccess(r, ctx, realObj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "securitypolicy", req.NamespacedName) } + if err := r.Service.DeleteSecurityPolicy(realObj.UID, false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) + deleteFail(r, ctx, realObj, &err) + return ResultRequeue, err + } + deleteSuccess(r, ctx, realObj) } return ResultNormal, nil @@ -243,7 +241,7 @@ func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusTrue(ctx context. Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Security Policy has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "SecurityPolicyReady", LastTransitionTime: transitionTime, }, } @@ -253,13 +251,13 @@ func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusTrue(ctx context. func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusFalse(ctx context.Context, secPolicy *v1alpha1.SecurityPolicy, transitionTime metav1.Time, err *error) { newConditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX Security Policy could not be created/updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the SecurityPolicy CR. Error: %v", *err, ), + Reason: "SecurityPolicyNotReady", LastTransitionTime: transitionTime, }, } @@ -361,16 +359,59 @@ func (r *SecurityPolicyReconciler) CollectGarbage(ctx context.Context) { return } + CRPolicySet, err := r.listSecurityPolciyCRIDs() + if err != nil { + return + } + + diffSet := nsxPolicySet.Difference(CRPolicySet) + for elem := range diffSet { + log.V(1).Info("GC collected SecurityPolicy CR", "securityPolicyUID", elem) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + err = r.Service.DeleteSecurityPolicy(types.UID(elem), true, false, servicecommon.ResourceTypeSecurityPolicy) + if err != nil { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) + } else { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) + } + } +} + +func (r *SecurityPolicyReconciler) deleteSecurityPolicyByName(ns, name string) error { + nsxSecurityPolicies := r.Service.ListSecurityPolicyByName(ns, name) + + CRPolicySet, err := r.listSecurityPolciyCRIDs() + if err != nil { + return err + } + for _, item := range nsxSecurityPolicies { + uid := nsxutil.FindTag(item.Tags, servicecommon.TagValueScopeSecurityPolicyUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, SecurityPolicy CR still exists in K8s", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + continue + } + + log.Info("deleting SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + if err := r.Service.DeleteSecurityPolicy(types.UID(uid), false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { + log.Error(err, "failed to delete SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + return err + } + log.Info("successfully deleted SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + } + return nil +} + +func (r *SecurityPolicyReconciler) listSecurityPolciyCRIDs() (sets.Set[string], error) { var objectList client.ObjectList if securitypolicy.IsVPCEnabled(r.Service) { objectList = &crdv1alpha1.SecurityPolicyList{} } else { objectList = &v1alpha1.SecurityPolicyList{} } - err := r.Client.List(ctx, objectList) + err := r.Client.List(context.Background(), objectList) if err != nil { log.Error(err, "failed to list SecurityPolicy CR") - return + return nil, err } CRPolicySet := sets.New[string]() @@ -387,17 +428,7 @@ func (r *SecurityPolicyReconciler) CollectGarbage(ctx context.Context) { } } - diffSet := nsxPolicySet.Difference(CRPolicySet) - for elem := range diffSet { - log.V(1).Info("GC collected SecurityPolicy CR", "securityPolicyUID", elem) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - err = r.Service.DeleteSecurityPolicy(types.UID(elem), true, false, servicecommon.ResourceTypeSecurityPolicy) - if err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) - } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) - } - } + return CRPolicySet, nil } // It is triggered by associated controller like pod, namespace, etc. diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller_test.go b/pkg/controllers/securitypolicy/securitypolicy_controller_test.go index 49f70a42c..c245d01bc 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller_test.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller_test.go @@ -36,6 +36,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/securitypolicy" + "github.com/vmware-tanzu/nsx-operator/pkg/util" ) func fakeService() *securitypolicy.SecurityPolicyService { @@ -193,8 +194,13 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { // not found errNotFound := errors.New("not found") k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) - _, err := r.Reconcile(ctx, req) + deleteSecurityPolicyByNamePatch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(r), "deleteSecurityPolicyByName", func(_ *SecurityPolicyReconciler, name, ns string) error { + return nil + }) + defer deleteSecurityPolicyByNamePatch.Reset() + result, err := r.Reconcile(ctx, req) assert.Equal(t, err, errNotFound) + assert.Equal(t, ResultRequeue, result) // NSX version check failed case sp := &v1alpha1.SecurityPolicy{} @@ -202,10 +208,10 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { return false }) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - patches := gomonkey.ApplyFunc(updateFail, + updateFailPatch := gomonkey.ApplyFunc(updateFail, func(r *SecurityPolicyReconciler, c context.Context, o *v1alpha1.SecurityPolicy, e *error) { }) - defer patches.Reset() + defer updateFailPatch.Reset() result, ret := r.Reconcile(ctx, req) resultRequeueAfter5mins := controllerruntime.Result{Requeue: true, RequeueAfter: 5 * time.Minute} assert.Equal(t, nil, ret) @@ -217,29 +223,60 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { }) defer checkNsxVersionPatch.Reset() - // DeletionTimestamp.IsZero = ture, client update failed + // DeletionTimestamp.IsZero = ture, create security policy in SystemNamespace k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) - _, ret = r.Reconcile(ctx, req) + err = errors.New("fetch namespace associated with security policy CR failed") + IsSystemNamespacePatch := gomonkey.ApplyFunc(util.IsSystemNamespace, func(client client.Client, ns string, obj *v1.Namespace, + ) (bool, error) { + return true, errors.New("fetch namespace associated with security policy CR failed") + }) + + result, ret = r.Reconcile(ctx, req) + IsSystemNamespacePatch.Reset() assert.Equal(t, err, ret) + assert.Equal(t, ResultRequeue, result) - // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.SecurityPolicyFinalizerName + // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName and update fails k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SecurityPolicy) time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.T1SecurityPolicyFinalizerName} return nil }) + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteSecurityPolicy", func(_ *securitypolicy.SecurityPolicyService, UID interface{}, isGc bool, isVPCCleanup bool) error { assert.FailNow(t, "should not be called") return nil }) - _, ret = r.Reconcile(ctx, req) + + err = errors.New("finalizer remove failed, would retry exponentially") + k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) + deleteFailPatch := gomonkey.ApplyFunc(deleteFail, + func(r *SecurityPolicyReconciler, c context.Context, o *v1alpha1.SecurityPolicy, e *error) { + }) + defer deleteFailPatch.Reset() + result, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, err) + assert.Equal(t, ResultRequeue, result) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.SecurityPolicyFinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.SecurityPolicy) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteSecurityPolicy", func(_ *securitypolicy.SecurityPolicyService, UID interface{}, isGc bool, isVPCCleanup bool) error { + return nil + }) + result, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) + assert.Equal(t, ResultNormal, result) patch.Reset() - // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName + // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SecurityPolicy) time := metav1.Now() @@ -251,8 +288,9 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { return nil }) k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) - _, ret = r.Reconcile(ctx, req) + result, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) + assert.Equal(t, ResultNormal, result) patch.Reset() } diff --git a/pkg/controllers/staticroute/staticroute_controller.go b/pkg/controllers/staticroute/staticroute_controller.go index 9e8e1769b..b1855d5e8 100644 --- a/pkg/controllers/staticroute/staticroute_controller.go +++ b/pkg/controllers/staticroute/staticroute_controller.go @@ -11,6 +11,7 @@ import ( "strings" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -18,9 +19,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -72,28 +70,64 @@ func deleteSuccess(r *StaticRouteReconciler, _ context.Context, o *v1alpha1.Stat metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeStaticRoute) } +func (r *StaticRouteReconciler) listStaticRouteCRIDs() (sets.Set[string], error) { + staticRouteList := &v1alpha1.StaticRouteList{} + err := r.Client.List(context.Background(), staticRouteList) + if err != nil { + log.Error(err, "failed to list StaticRoute CRs") + return nil, err + } + + CRStaticRouteSet := sets.New[string]() + for _, staticroute := range staticRouteList.Items { + CRStaticRouteSet.Insert(string(staticroute.UID)) + } + return CRStaticRouteSet, nil +} + +func (r *StaticRouteReconciler) deleteStaticRouteByName(ns, name string) error { + CRPolicySet, err := r.listStaticRouteCRIDs() + if err != nil { + return err + } + nsxStaticRoutes := r.Service.ListStaticRouteByName(ns, name) + for _, item := range nsxStaticRoutes { + uid := util.FindTag(item.Tags, commonservice.TagScopeStaticRouteCRUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, StaticRoute CR still exists in K8s", "staticrouteUID", uid, "nsxStatciRouteId", *item.Id) + continue + } + + log.Info("deleting StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + path := strings.Split(*item.Path, "/") + if err := r.Service.DeleteStaticRouteByPath(path[2], path[4], path[6], *item.Id); err != nil { + log.Error(err, "failed to delete StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + return err + } + log.Info("successfully deleted StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + } + return nil +} + func (r *StaticRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { obj := &v1alpha1.StaticRoute{} log.Info("reconciling staticroute CR", "staticroute", req.NamespacedName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeStaticRoute) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteStaticRouteByName(req.Namespace, req.Name); err != nil { + return ResultRequeue, err + } else { + return ResultNormal, nil + } + } log.Error(err, "unable to fetch static route CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + return ResultRequeue, err } if obj.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeStaticRoute) - if !controllerutil.ContainsFinalizer(obj, commonservice.StaticRouteFinalizerName) { - controllerutil.AddFinalizer(obj, commonservice.StaticRouteFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "staticroute", req.NamespacedName) - updateFail(r, ctx, obj, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on staticroute CR", "staticroute", req.NamespacedName) - } - if err := r.Service.CreateOrUpdateStaticRoute(req.Namespace, obj); err != nil { updateFail(r, ctx, obj, &err) // TODO: if error is not retriable, not requeue @@ -105,27 +139,14 @@ func (r *StaticRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) } updateSuccess(r, ctx, obj) } else { - if controllerutil.ContainsFinalizer(obj, commonservice.StaticRouteFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeStaticRoute) - // TODO, update the value from 'default' to actual value， get OrgID, ProjectID, VPCID depending on obj.Namespace from vpc store - if err := r.Service.DeleteStaticRoute(obj); err != nil { - log.Error(err, "delete failed, would retry exponentially", "staticroute", req.NamespacedName) - deleteFail(r, ctx, obj, &err) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, commonservice.StaticRouteFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - deleteFail(r, ctx, obj, &err) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "staticroute", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "staticroute", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeStaticRoute) + if err := r.Service.DeleteStaticRoute(obj); err != nil { + log.Error(err, "delete failed, would retry exponentially", "staticroute", req.NamespacedName) + deleteFail(r, ctx, obj, &err) + return ResultRequeue, err } + deleteSuccess(r, ctx, obj) } - return ResultNormal, nil } @@ -135,7 +156,7 @@ func (r *StaticRouteReconciler) setStaticRouteReadyStatusTrue(ctx context.Contex Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Static Route has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "StaticRouteReady", LastTransitionTime: transitionTime, }, } @@ -147,8 +168,8 @@ func (r *StaticRouteReconciler) setStaticRouteReadyStatusFalse(ctx context.Conte { Type: v1alpha1.Ready, Status: v1.ConditionFalse, - Message: "NSX Static Route could not be created/updated/deleted", - Reason: fmt.Sprintf("Error occurred while processing the Static Route CR. Please check the config and try again. Error: %v", *err), + Message: fmt.Sprintf("Error occurred while processing the Static Route CR. Please check the config and try again. Error: %v", *err), + Reason: "StaticRouteNotReady", LastTransitionTime: transitionTime, }, } @@ -198,12 +219,6 @@ func getExistingConditionOfType(conditionType v1alpha1.StaticRouteStatusConditio func (r *StaticRouteReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.StaticRoute{}). - WithEventFilter(predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), diff --git a/pkg/controllers/staticroute/staticroute_controller_test.go b/pkg/controllers/staticroute/staticroute_controller_test.go index 695f03ed4..110fa9762 100644 --- a/pkg/controllers/staticroute/staticroute_controller_test.go +++ b/pkg/controllers/staticroute/staticroute_controller_test.go @@ -18,6 +18,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -175,23 +176,14 @@ func TestStaticRouteReconciler_Reconcile(t *testing.T) { ctx := context.Background() req := controllerruntime.Request{NamespacedName: types.NamespacedName{Namespace: "dummy", Name: "dummy"}} - // not found - errNotFound := errors.New("not found") - k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + // get error + errUnknowError := errors.New("unknown error") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errUnknowError) _, err := r.Reconcile(ctx, req) - assert.Equal(t, err, errNotFound) + assert.Equal(t, err, errUnknowError) - // DeletionTimestamp.IsZero = ture, client update failed sp := &v1alpha1.StaticRoute{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) fakewriter := fakeStatusWriter{} - k8sClient.EXPECT().Status().Return(fakewriter) - _, ret := r.Reconcile(ctx, req) - assert.Equal(t, err, ret) - - // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.FinalizerName k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) time := metav1.Now() @@ -200,74 +192,54 @@ func TestStaticRouteReconciler_Reconcile(t *testing.T) { }) patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { - assert.FailNow(t, "should not be called") return nil }) - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, ret, nil) - patch.Reset() - - // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.StaticRoute) - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} - return nil - }) - patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { - return nil - }) - _, ret = r.Reconcile(ctx, req) + _, ret := r.Reconcile(ctx, req) assert.Equal(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName, DeleteStaticRoute fail + // DeletionTimestamp.IsZero = false, DeleteStaticRoute fail k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { return errors.New("delete failed") }) - - k8sClient.EXPECT().Status().Times(2).Return(fakewriter) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.NotEqual(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateStaticRoute fail + // DeletionTimestamp.IsZero = true, CreateorUpdateStaticRoute fail k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) v1sp.ObjectMeta.DeletionTimestamp = nil - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateStaticRoute", func(_ *staticroute.StaticRouteService, namespace string, obj *v1alpha1.StaticRoute) error { return errors.New("create failed") }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.NotEqual(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateStaticRoute succ + // DeletionTimestamp.IsZero = true, CreateorUpdateStaticRoute succ k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) v1sp.ObjectMeta.DeletionTimestamp = nil - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateStaticRoute", func(_ *staticroute.StaticRouteService, namespace string, obj *v1alpha1.StaticRoute) error { return nil }) - k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) patch.Reset() @@ -366,3 +338,113 @@ func TestStaticRouteReconciler_Start(t *testing.T) { err := r.Start(mgr) assert.NotEqual(t, err, nil) } + +func TestStaticRouteReconciler_listStaticRouteCRIDs(t *testing.T) { + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + r := &StaticRouteReconciler{ + Client: k8sClient, + Scheme: nil, + } + + ctx := context.Background() + + // list returns an error + errList := errors.New("list error") + k8sClient.EXPECT().List(ctx, gomock.Any()).Return(errList) + _, err := r.listStaticRouteCRIDs() + assert.Equal(t, err, errList) + + // list returns no error, but no items + k8sClient.EXPECT().List(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + staticRouteList := list.(*v1alpha1.StaticRouteList) + staticRouteList.Items = []v1alpha1.StaticRoute{} + return nil + }) + crIDs, err := r.listStaticRouteCRIDs() + assert.NoError(t, err) + assert.Equal(t, 0, crIDs.Len()) + + // list returns items + k8sClient.EXPECT().List(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + staticRouteList := list.(*v1alpha1.StaticRouteList) + staticRouteList.Items = []v1alpha1.StaticRoute{ + {ObjectMeta: metav1.ObjectMeta{UID: "uid1"}}, + {ObjectMeta: metav1.ObjectMeta{UID: "uid2"}}, + } + return nil + }) + crIDs, err = r.listStaticRouteCRIDs() + assert.NoError(t, err) + assert.Equal(t, 2, crIDs.Len()) + assert.True(t, crIDs.Has("uid1")) + assert.True(t, crIDs.Has("uid2")) +} + +func TestStaticRouteReconciler_deleteStaticRouteByName(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + + k8sClient := mock_client.NewMockClient(mockCtl) + mockStaticRouteClient := mocks.NewMockStaticRoutesClient(mockCtl) + + service := &staticroute.StaticRouteService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + StaticRouteClient: mockStaticRouteClient, + }, + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + + r := &StaticRouteReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + } + + // listStaticRouteCRIDs returns an error + errList := errors.New("list error") + patch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(r), "listStaticRouteCRIDs", func(_ *StaticRouteReconciler) (sets.Set[string], error) { + return nil, errList + }) + defer patch.Reset() + + err := r.deleteStaticRouteByName("dummy-name", "dummy-ns") + assert.Equal(t, err, errList) + + // listStaticRouteCRIDs returns items, and deletion fails + patch.Reset() + patch.ApplyPrivateMethod(reflect.TypeOf(r), "listStaticRouteCRIDs", func(_ *StaticRouteReconciler) (sets.Set[string], error) { + return sets.New[string]("uid1"), nil + }) + patch.ApplyMethod(reflect.TypeOf(service), "ListStaticRouteByName", func(_ *staticroute.StaticRouteService, _ string, _ string) []*model.StaticRoutes { + return []*model.StaticRoutes{ + { + Id: pointy.String("route-id-1"), + Path: pointy.String("/orgs/org123/projects/pro123/vpcs/vpc123/static-routes/route-id-1"), + Tags: []model.Tag{{Scope: pointy.String(common.TagScopeStaticRouteCRUID), Tag: pointy.String("uid1")}}, + }, + { + Id: pointy.String("route-id-2"), + Path: pointy.String("/orgs/org123/projects/pro123/vpcs/vpc123/static-routes/route-id-2"), + Tags: []model.Tag{{Scope: pointy.String(common.TagScopeStaticRouteCRUID), Tag: pointy.String("uid2")}}, + }, + } + }) + + patch.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRouteByPath", func(_ *staticroute.StaticRouteService, orgId string, projectId string, vpcId string, uid string) error { + if uid == "route-id-2" { + return errors.New("delete failed") + } + return nil + }) + + err = r.deleteStaticRouteByName("dummy-name", "dummy-ns") + assert.Error(t, err) + patch.Reset() +} diff --git a/pkg/controllers/subnet/namespace_handler.go b/pkg/controllers/subnet/namespace_handler.go index ee2f935d3..57d667248 100644 --- a/pkg/controllers/subnet/namespace_handler.go +++ b/pkg/controllers/subnet/namespace_handler.go @@ -18,7 +18,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ) -// Subnet controller should watch event of namespace, when there are some updates of namespace labels, +// Subnet controller should watch event of Namespace, when there are some updates of Namespace labels, // controller should build tags and update VpcSubnet according to new labels. type EnqueueRequestForNamespace struct { @@ -26,22 +26,22 @@ type EnqueueRequestForNamespace struct { } func (e *EnqueueRequestForNamespace) Create(_ context.Context, _ event.CreateEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace create event, do nothing") + log.V(1).Info("Namespace create event, do nothing") } func (e *EnqueueRequestForNamespace) Delete(_ context.Context, _ event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace delete event, do nothing") + log.V(1).Info("Namespace delete event, do nothing") } func (e *EnqueueRequestForNamespace) Generic(_ context.Context, _ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace generic event, do nothing") + log.V(1).Info("Namespace generic event, do nothing") } func (e *EnqueueRequestForNamespace) Update(_ context.Context, updateEvent event.UpdateEvent, l workqueue.RateLimitingInterface) { obj := updateEvent.ObjectNew.(*v1.Namespace) - err := reconcileSubnet(e.Client, obj.Name, l) + err := requeueSubnet(e.Client, obj.Name, l) if err != nil { - log.Error(err, "failed to reconcile subnet") + log.Error(err, "Failed to requeue Subnet") } } @@ -52,9 +52,9 @@ var PredicateFuncsNs = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldObj := e.ObjectOld.(*v1.Namespace) newObj := e.ObjectNew.(*v1.Namespace) - log.V(1).Info("receive namespace update event", "name", oldObj.Name) + log.V(1).Info("Receive Namespace update event", "Name", oldObj.Name) if reflect.DeepEqual(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) { - log.Info("labels of namespace are not changed", "name", oldObj.Name) + log.Info("Labels of Namespace are not changed", "Name", oldObj.Name) return false } return true @@ -64,17 +64,17 @@ var PredicateFuncsNs = predicate.Funcs{ }, } -func reconcileSubnet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { +func requeueSubnet(c client.Client, ns string, q workqueue.RateLimitingInterface) error { subnetList := &v1alpha1.SubnetList{} - err := c.List(context.Background(), subnetList, client.InNamespace(namespace)) + err := c.List(context.Background(), subnetList, client.InNamespace(ns)) if err != nil { - log.Error(err, "failed to list all the subnets") + log.Error(err, "Failed to list all the Subnets") return err } for _, subnet_item := range subnetList.Items { - log.Info("reconcile subnet because namespace update", - "namespace", subnet_item.Namespace, "name", subnet_item.Name) + log.Info("Requeue Subnet because Namespace update", + "Namespace", subnet_item.Namespace, "Name", subnet_item.Name) q.Add(reconcile.Request{ NamespacedName: types.NamespacedName{ Name: subnet_item.Name, diff --git a/pkg/controllers/subnet/namespace_handler_test.go b/pkg/controllers/subnet/namespace_handler_test.go new file mode 100644 index 000000000..88c1b8615 --- /dev/null +++ b/pkg/controllers/subnet/namespace_handler_test.go @@ -0,0 +1,128 @@ +package subnet + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" +) + +type MockRateLimitingInterface struct { + mock.Mock +} + +func (m *MockRateLimitingInterface) Add(item interface{}) { +} + +func (m *MockRateLimitingInterface) Len() int { + return 0 +} + +func (m *MockRateLimitingInterface) Get() (item interface{}, shutdown bool) { + return +} + +func (m *MockRateLimitingInterface) Done(item interface{}) { + return +} + +func (m *MockRateLimitingInterface) ShutDown() { +} + +func (m *MockRateLimitingInterface) ShutDownWithDrain() { +} + +func (m *MockRateLimitingInterface) ShuttingDown() bool { + return true +} + +func (m *MockRateLimitingInterface) AddAfter(item interface{}, duration time.Duration) { + return +} + +func (m *MockRateLimitingInterface) AddRateLimited(item interface{}) { + m.Called(item) +} + +func (m *MockRateLimitingInterface) Forget(item interface{}) { + m.Called(item) +} + +func (m *MockRateLimitingInterface) NumRequeues(item interface{}) int { + args := m.Called(item) + return args.Int(0) +} + +func TestEnqueueRequestForNamespace(t *testing.T) { + fakeClient := fake.NewClientBuilder().Build() + queue := new(MockRateLimitingInterface) + handler := &EnqueueRequestForNamespace{Client: fakeClient} + + t.Run("Update event with changed labels", func(t *testing.T) { + oldNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "old"}, + }, + } + newNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "new"}, + }, + } + updateEvent := event.UpdateEvent{ + ObjectOld: oldNamespace, + ObjectNew: newNamespace, + } + queue.On("Add", mock.Anything).Return() + + handler.Update(context.Background(), updateEvent, queue) + }) + + t.Run("Update event with unchanged labels", func(t *testing.T) { + oldNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "same"}, + }, + } + newNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "same"}, + }, + } + updateEvent := event.UpdateEvent{ + ObjectOld: oldNamespace, + ObjectNew: newNamespace, + } + + queue.On("Add", mock.Anything).Return() + + handler.Update(context.Background(), updateEvent, queue) + + queue.AssertNotCalled(t, "Add", mock.Anything) + }) + + t.Run("Requeue subnet function", func(t *testing.T) { + ns := "test-ns" + + schem := fake.NewClientBuilder().Build().Scheme() + v1alpha1.AddToScheme(schem) + queue.On("Add", mock.Anything).Return() + + err := requeueSubnet(fakeClient, ns, queue) + + assert.NoError(t, err) + queue.AssertNumberOfCalls(t, "Add", 0) + }) +} diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 4a5daf27c..3e7fa27d3 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "reflect" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -16,18 +18,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" - "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -50,102 +48,148 @@ type SubnetReconciler struct { } func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.Subnet{} - log.Info("reconciling subnet CR", "subnet", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling Subnet", "Subnet", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnet) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch Subnet CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) - } - - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) - if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { - controllerutil.AddFinalizer(obj, servicecommon.SubnetFinalizerName) - - if obj.Spec.AccessMode == "" { - log.Info("obj.Spec.AccessMode set", "subnet", req.NamespacedName) - obj.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) - } - - if obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) - if vpcNetworkConfig == nil { - err := fmt.Errorf("operate failed: cannot get configuration for Subnet CR") - log.Error(nil, "failed to find VPCNetworkConfig for Subnet CR", "subnet", req.NamespacedName, "namespace %s", obj.Namespace) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize - } - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + subnetCR := &v1alpha1.Subnet{} + if err := r.Client.Get(ctx, req.NamespacedName, subnetCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetByName(req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete NSX Subnet", "Subnet", req.NamespacedName) return ResultRequeue, err } - log.V(1).Info("added finalizer on subnet CR", "subnet", req.NamespacedName) + return ResultNormal, nil } - tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) - if tags == nil { - return ResultRequeue, errors.New("failed to generate subnet tags") + log.Error(err, "Unable to fetch Subnet CR", "req", req.NamespacedName) + return ResultRequeue, err + } + if !subnetCR.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) + if err := r.deleteSubnetByID(string(subnetCR.GetUID())); err != nil { + log.Error(err, "Failed to delete NSX Subnet, retrying", "Subnet", req.NamespacedName) + deleteFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } - vpcInfoList := r.VPCService.ListVPCInfo(req.Namespace) - if len(vpcInfoList) == 0 { - return ResultRequeueAfter10sec, nil + if err := r.Client.Delete(ctx, subnetCR); err != nil { + log.Error(err, "Failed to delete Subnet CR, retrying", "Subnet", req.NamespacedName) + deleteFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } - if _, err := r.SubnetService.CreateOrUpdateSubnet(obj, vpcInfoList[0], tags); err != nil { - if errors.As(err, &util.ExceedTagsError{}) { - log.Error(err, "exceed tags limit, would not retry", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultNormal, nil - } - log.Error(err, "operate failed, would retry exponentially", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + log.Info("Successfully deleted Subnet", "Subnet", req.NamespacedName) + deleteSuccess(r, ctx, subnetCR) + return ResultNormal, nil + } + + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) + + // Spec mutation check and update if necessary + specChanged := false + if subnetCR.Spec.AccessMode == "" { + subnetCR.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) + specChanged = true + } + + if subnetCR.Spec.IPv4SubnetSize == 0 { + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(subnetCR.Namespace) + if vpcNetworkConfig == nil { + err := fmt.Errorf("VPCNetworkConfig not found for Subnet CR") + log.Error(nil, "Failed to find VPCNetworkConfig", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) return ResultRequeue, err } - if err := r.updateSubnetStatus(obj); err != nil { - log.Error(err, "update subnet status failed, would retry exponentially", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + subnetCR.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize + specChanged = true + } + if specChanged { + if err := r.Client.Update(ctx, subnetCR); err != nil { + log.Error(err, "Failed to update Subnet", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) return ResultRequeue, err } - updateSuccess(r, ctx, obj) - } else { - if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) - if err := r.DeleteSubnet(*obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnet", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, servicecommon.SubnetFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnet", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "subnet", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - log.Info("finalizers cannot be recognized", "subnet", req.NamespacedName) + log.Info("Updated Subnet CR", "Subnet", req.NamespacedName) + } + + tags := r.SubnetService.GenerateSubnetNSTags(subnetCR) + if tags == nil { + log.Error(nil, "Failed to generate Subnet tags", "Subnet", req.NamespacedName) + return ResultRequeue, errors.New("failed to generate Subnet tags") + } + // List VPC Info + vpcInfoList := r.VPCService.ListVPCInfo(req.Namespace) + if len(vpcInfoList) == 0 { + log.Info("No VPC info found, requeueing", "Namespace", req.Namespace) + return ResultRequeueAfter10sec, nil + } + // Create or update the subnet in NSX + if _, err := r.SubnetService.CreateOrUpdateSubnet(subnetCR, vpcInfoList[0], tags); err != nil { + if errors.As(err, &nsxutil.ExceedTagsError{}) { + log.Error(err, "Tags limit exceeded, not retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultNormal, nil } + log.Error(err, "Failed to create/update Subnet, retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err + } + // Update status + if err := r.updateSubnetStatus(subnetCR); err != nil { + log.Error(err, "Failed to update Subnet status, retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } + updateSuccess(r, ctx, subnetCR) return ctrl.Result{}, nil } -func (r *SubnetReconciler) DeleteSubnet(obj v1alpha1.Subnet) error { - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, string(obj.GetUID())) - if len(nsxSubnets) == 0 { - log.Info("no subnet found for subnet CR", "uid", string(obj.GetUID())) - return nil +func (r *SubnetReconciler) deleteSubnetByID(subnetID string) error { + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, subnetID) + return r.deleteSubnets(nsxSubnets) +} + +func (r *SubnetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) error { + for _, nsxSubnet := range nsxSubnets { + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) + if portNums > 0 { + err := fmt.Errorf("cannot delete Subnet %s, still attached by %d port(s)", *nsxSubnet.Id, portNums) + log.Error(err, "Delete Subnet from NSX failed") + return err + } + if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { + log.Error(err, "Failed to delete Subnet", "ID", *nsxSubnet.Id) + return err + } + log.Info("Successfully deleted Subnet", "ID", *nsxSubnet.Id) } - portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnets[0].Id)) - if portNums > 0 { - err := errors.New("subnet still attached by port") - log.Error(err, "", "ID", *nsxSubnets[0].Id) + log.Info("Successfully cleaned Subnets") + return nil +} + +func (r *SubnetReconciler) deleteStaleSubnets(nsxSubnets []*model.VpcSubnet) error { + crdSubnetIDs, err := r.listSubnetIDsFromCRs(context.Background()) + if err != nil { + log.Error(err, "Failed to list Subnet CRs") return err } - return r.SubnetService.DeleteSubnet(*nsxSubnets[0]) + crdSubnetIDsSet := sets.NewString(crdSubnetIDs...) + nsxSubnetsToDelete := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetCRUID) + if crdSubnetIDsSet.Has(uid) { + log.Info("Skipping deletion, Subnet CR still exists in K8s", "ID", *nsxSubnet.Id) + continue + } + nsxSubnetsToDelete = append(nsxSubnetsToDelete, nsxSubnet) + } + log.Info("Cleaning stale Subnets", "Count", len(nsxSubnetsToDelete)) + return r.deleteSubnets(nsxSubnetsToDelete) +} + +func (r *SubnetReconciler) deleteSubnetByName(name, ns string) error { + nsxSubnets := r.SubnetService.ListSubnetByName(ns, name) + return r.deleteStaleSubnets(nsxSubnets) } func (r *SubnetReconciler) updateSubnetStatus(obj *v1alpha1.Subnet) error { @@ -163,7 +207,7 @@ func (r *SubnetReconciler) updateSubnetStatus(obj *v1alpha1.Subnet) error { for _, status := range statusList { obj.Status.NetworkAddresses = append(obj.Status.NetworkAddresses, *status.NetworkAddress) obj.Status.GatewayAddresses = append(obj.Status.GatewayAddresses, *status.GatewayAddress) - // DHCPServerAddress is only for the subnet with DHCP enabled + // DHCPServerAddress is only for the Subnet with DHCP enabled if status.DhcpServerAddress != nil { obj.Status.DHCPServerAddresses = append(obj.Status.DHCPServerAddresses, *status.DhcpServerAddress) } @@ -177,7 +221,7 @@ func (r *SubnetReconciler) setSubnetReadyStatusTrue(ctx context.Context, subnet Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Subnet has been successfully created/updated", - Reason: "SubnetCreated", + Reason: "SubnetReady", LastTransitionTime: transitionTime, }, } @@ -209,9 +253,9 @@ func (r *SubnetReconciler) updateSubnetStatusConditions(ctx context.Context, sub } if conditionsUpdated { if err := r.Client.Status().Update(ctx, subnet); err != nil { - log.Error(err, "failed to update subnet status", "Name", subnet.Name, "Namespace", subnet.Namespace) + log.Error(err, "Failed to update Subnet status", "Name", subnet.Name, "Namespace", subnet.Namespace) } else { - log.Info("updated Subnet", "Name", subnet.Name, "Namespace", subnet.Namespace, "New Conditions", newConditions) + log.Info("Updated Subnet", "Name", subnet.Name, "Namespace", subnet.Namespace, "New Conditions", newConditions) } } } @@ -220,7 +264,7 @@ func (r *SubnetReconciler) mergeSubnetStatusCondition(ctx context.Context, subne matchedCondition := getExistingConditionOfType(newCondition.Type, subnet.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { - log.V(2).Info("conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) + log.V(2).Info("Conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) return false } @@ -266,28 +310,8 @@ func deleteSuccess(r *SubnetReconciler, _ context.Context, o *v1alpha1.Subnet) { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnet) } -func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.Subnet{}). - WithEventFilter(predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }). - Watches( - &v1.Namespace{}, - &EnqueueRequestForNamespace{Client: mgr.GetClient()}, - builder.WithPredicates(PredicateFuncsNs), - ). - WithOptions( - controller.Options{ - MaxConcurrentReconciles: common.NumReconcile(), - }). - Complete(r) -} - func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider) error { + // Create the Subnet Reconciler with the necessary services and configuration subnetReconciler := &SubnetReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -296,57 +320,79 @@ func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService VPCService: vpcService, Recorder: mgr.GetEventRecorderFor("subnet-controller"), } - if err := subnetReconciler.Start(mgr); err != nil { - log.Error(err, "failed to create controller", "controller", "Subnet") + // Start the controller + if err := subnetReconciler.start(mgr); err != nil { + log.Error(err, "Failed to create controller", "controller", "Subnet") return err } - go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetReconciler.CollectGarbage) + // Start garbage collector in a separate goroutine + go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetReconciler.collectGarbage) return nil } -// Start setup manager -func (r *SubnetReconciler) Start(mgr ctrl.Manager) error { - err := r.setupWithManager(mgr) - if err != nil { - return err - } - return nil +// start sets up the manager for the Subnet Reconciler +func (r *SubnetReconciler) start(mgr ctrl.Manager) error { + return r.setupWithManager(mgr) } -// CollectGarbage implements the interface GarbageCollector method. -func (r *SubnetReconciler) CollectGarbage(ctx context.Context) { - log.Info("subnet garbage collector started") +// setupWithManager configures the controller to watch Subnet resources +func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.Subnet{}). + // Watches for changes in Namespaces and triggers reconciliation + Watches( + &v1.Namespace{}, + &EnqueueRequestForNamespace{Client: mgr.GetClient()}, + builder.WithPredicates(PredicateFuncsNs), + ). + // Set controller options, including max concurrent reconciles + WithOptions( + controller.Options{ + MaxConcurrentReconciles: common.NumReconcile(), + }). + Complete(r) +} + +func (r *SubnetReconciler) listSubnetIDsFromCRs(ctx context.Context) ([]string, error) { crdSubnetList := &v1alpha1.SubnetList{} err := r.Client.List(ctx, crdSubnetList) if err != nil { - log.Error(err, "failed to list subnet CR") - return - } - var nsxSubnetList []*model.VpcSubnet - for _, subnet := range crdSubnetList.Items { - nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnet(string(subnet.UID))...) - } - if len(nsxSubnetList) == 0 { - return + return nil, err } - crdSubnetIDs := sets.NewString() + crdSubnetIDs := make([]string, 0, len(crdSubnetList.Items)) for _, sr := range crdSubnetList.Items { - crdSubnetIDs.Insert(string(sr.UID)) + crdSubnetIDs = append(crdSubnetIDs, string(sr.UID)) } + return crdSubnetIDs, nil +} - for _, elem := range nsxSubnetList { - uid := util.FindTag(elem.Tags, servicecommon.TagScopeSubnetCRUID) - if crdSubnetIDs.Has(uid) { - continue - } +// collectGarbage implements the interface GarbageCollector method. +func (r *SubnetReconciler) collectGarbage(ctx context.Context) { + startTime := time.Now() + defer func() { + log.Info("Subnet garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() - log.Info("GC collected Subnet CR", "UID", elem) + crdSubnetIDs, err := r.listSubnetIDsFromCRs(ctx) + if err != nil { + log.Error(err, "Failed to list Subnet CRs") + return + } + crdSubnetIDsSet := sets.New[string](crdSubnetIDs...) + + subnetUIDs := r.SubnetService.ListSubnetIDsFromNSXSubnets() + subnetIDsToDelete := subnetUIDs.Difference(crdSubnetIDsSet) + for subnetID := range subnetIDsToDelete { + nsxSubnets := r.SubnetService.ListSubnetCreatedBySubnet(subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeSubnet) - err = r.SubnetService.DeleteSubnet(*elem) - if err != nil { + + log.Info("Subnet garbage collection, cleaning stale Subnets", "Count", len(nsxSubnets)) + if err := r.deleteSubnets(nsxSubnets); err != nil { + log.Error(err, "Subnet garbage collection, failed to delete NSX subnet", "SubnetUID", subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeSubnet) } else { + log.Info("Subnet garbage collection, successfully deleted NSX subnet", "SubnetUID", subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeSubnet) } } diff --git a/pkg/controllers/subnet/subnet_controller_test.go b/pkg/controllers/subnet/subnet_controller_test.go index 04f680226..eccd24915 100644 --- a/pkg/controllers/subnet/subnet_controller_test.go +++ b/pkg/controllers/subnet/subnet_controller_test.go @@ -2,25 +2,37 @@ package subnet import ( "context" + "errors" "reflect" "testing" + "time" "github.com/agiledragon/gomonkey/v2" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) func TestSubnetReconciler_GarbageCollector(t *testing.T) { + subnetStore := &subnet.SubnetStore{} service := &subnet.SubnetService{ + SubnetStore: subnetStore, Service: common.Service{ NSXConfig: &config.NSXOperatorConfig{ NsxConfig: &config.NsxConfig{ @@ -29,49 +41,64 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { }, }, } + serviceSubnetPort := &subnetport.SubnetPortService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + r := &SubnetReconciler{ + Client: k8sClient, + Scheme: nil, + SubnetService: service, + SubnetPortService: serviceSubnetPort, + } + // Subnet doesn't have TagScopeSubnetSetCRId (not belong to SubnetSet) - // gc collect item "2345", local store has more item than k8s cache - patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - tags1 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("2345")}} - tags2 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("1234")}} + // gc collect item "fake-id1", local store has more item than k8s cache + patch := gomonkey.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + res := sets.New[string]("fake-id1", "fake-id2") + return res + }) + patch.ApplyMethod(reflect.TypeOf(r.SubnetService.SubnetStore), "GetByIndex", func(_ *subnet.SubnetStore, _ string) []*model.VpcSubnet { + tags1 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-id1")}} + tags2 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-id2")}} var a []*model.VpcSubnet - id1 := "2345" + id1 := "fake-id1" a = append(a, &model.VpcSubnet{Id: &id1, Tags: tags1}) - id2 := "1234" + id2 := "fake-id2" a = append(a, &model.VpcSubnet{Id: &id2, Tags: tags2}) return a }) - patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { + patch.ApplyMethod(reflect.TypeOf(r.SubnetPortService), "GetPortsOfSubnet", func(_ *subnetport.SubnetPortService, _ string) (ports []*model.VpcSubnetPort) { + return nil + }) + patch.ApplyMethod(reflect.TypeOf(r.SubnetService), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { return nil }) - mockCtl := gomock.NewController(t) - k8sClient := mock_client.NewMockClient(mockCtl) - r := &SubnetReconciler{ - Client: k8sClient, - Scheme: nil, - SubnetService: service, - } ctx := context.Background() srList := &v1alpha1.SubnetList{} k8sClient.EXPECT().List(ctx, srList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { a := list.(*v1alpha1.SubnetList) a.Items = append(a.Items, v1alpha1.Subnet{}) a.Items[0].ObjectMeta = metav1.ObjectMeta{} - a.Items[0].UID = "1234" + a.Items[0].UID = "fake-id2" return nil }) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) // local store has same item as k8s cache patch.Reset() - patch.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("1234")}} - var a []*model.VpcSubnet - id := "1234" - a = append(a, &model.VpcSubnet{Id: &id, Tags: tags}) - return a + patch = gomonkey.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + res := sets.New[string]("fake-id2") + return res }) patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { assert.FailNow(t, "should not be called") @@ -81,21 +108,233 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { a := list.(*v1alpha1.SubnetList) a.Items = append(a.Items, v1alpha1.Subnet{}) a.Items[0].ObjectMeta = metav1.ObjectMeta{} - a.Items[0].UID = "1234" + a.Items[0].UID = "fake-id2" return nil }) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) // local store has no item patch.Reset() - patch.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - return []*model.VpcSubnet{} + patch = patch.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + return nil }) patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { assert.FailNow(t, "should not be called") return nil }) k8sClient.EXPECT().List(ctx, srList).Return(nil).Times(1) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) patch.Reset() } + +type fakeRecorder struct{} + +func (recorder fakeRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} + +func (recorder fakeRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func (recorder fakeRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func createFakeSubnetReconciler() *SubnetReconciler { + service := &vpc.VPCService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + }, + } + subnetService := &subnet.SubnetService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + UseAVILoadBalancer: false, + }, + }, + }, + SubnetStore: &subnet.SubnetStore{}, + } + + subnetPortService := &subnetport.SubnetPortService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + }, + SubnetPortStore: nil, + } + + return &SubnetReconciler{ + Client: fake.NewClientBuilder().Build(), + Scheme: fake.NewClientBuilder().Build().Scheme(), + VPCService: service, + SubnetService: subnetService, + SubnetPortService: subnetPortService, + Recorder: &fakeRecorder{}, + } +} + +func TestSubnetReconciler_Reconcile(t *testing.T) { + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "test-subnet", + }, + } + + createNewSubnet := func() *v1alpha1.Subnet { + return &v1alpha1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet", + Namespace: "default", + UID: "fake-subnet-uid", + }, + Spec: v1alpha1.SubnetSpec{ + IPv4SubnetSize: 0, + AccessMode: "", + }, + } + } + + reconciler := createFakeSubnetReconciler() + ctx := context.Background() + + // When the Subnet CR does not exist + t.Run("Subnet CR not found", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByName", func(_ *SubnetReconciler, name, ns string) error { + return nil + }) + defer patches.Reset() + + result, err := reconciler.Reconcile(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, ctrl.Result{}, result) + patches.Reset() + }) + + t.Run("Get Subnet CR return other error should retry", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + patches := gomonkey.ApplyMethod(reflect.TypeOf(reconciler.Client), "Get", func(_ client.Client, _ context.Context, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return errors.New("get Subnet CR error") + }) + defer patches.Reset() + patches.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByName", func(_ *SubnetReconciler, name, ns string) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.ErrorContains(t, err, "get Subnet CR error") + assert.Equal(t, ResultRequeue, result) + }) + + // When the Subnet CR is being deleted should delete the Subnet and return no error + t.Run("Subnet CR being deleted", func(t *testing.T) { + t.Log("When the Subnet CR is being deleted should delete the Subnet and return no error") + v1alpha1.AddToScheme(reconciler.Scheme) + subnetCR := createNewSubnet() + now := metav1.NewTime(time.Now()) + subnetCR.DeletionTimestamp = &now + + createErr := reconciler.Client.Create(ctx, subnetCR) + defer reconciler.Client.Delete(ctx, subnetCR) + assert.NoError(t, createErr) + + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByID", func(_ *SubnetReconciler, _ string) error { + return nil + }) + defer patches.Reset() + patches.ApplyMethod(reflect.TypeOf(reconciler.Client), "Delete", func(_ client.Client, _ context.Context, _ client.Object, _ ...client.DeleteOption) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.ErrorContains(t, err, "VPCNetworkConfig not found") + assert.Equal(t, ResultRequeue, result) + }) + + // When an error occurs during reconciliation, should return an error and requeue" + t.Run("Create or Update Subnet Failure", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + + subnetCR := createNewSubnet() + createErr := reconciler.Client.Create(ctx, subnetCR) + assert.NoError(t, createErr) + defer reconciler.Client.Delete(ctx, subnetCR) + + vpcConfig := &common.VPCNetworkConfigInfo{DefaultSubnetSize: 16} + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler.VPCService), "GetVPCNetworkConfigByNamespace", func(_ *vpc.VPCService, ns string) *common.VPCNetworkConfigInfo { + return vpcConfig + }) + defer patches.Reset() + + tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-tag")}} + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "GenerateSubnetNSTags", func(_ *subnet.SubnetService, obj client.Object) []model.Tag { + return tags + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, ns string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {OrgID: "org-id", ProjectID: "project-id", VPCID: "vpc-id", ID: "fake-id"}, + } + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "CreateOrUpdateSubnet", func(_ *subnet.SubnetService, obj client.Object, vpcInfo common.VPCResourceInfo, tags []model.Tag) (string, error) { + return "", errors.New("create or update failed") + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.Error(t, err) + assert.Equal(t, ctrl.Result{Requeue: true}, result) + }) + + // When updating the Subnet spec, should update the Subnet CR spec if not set + t.Run("Update Subnet CR spec", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + + subnetCR := createNewSubnet() + createErr := reconciler.Client.Create(ctx, subnetCR) + assert.NoError(t, createErr) + defer reconciler.Client.Delete(ctx, subnetCR) + + vpcConfig := &common.VPCNetworkConfigInfo{DefaultSubnetSize: 16} + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler.VPCService), "GetVPCNetworkConfigByNamespace", func(_ *vpc.VPCService, ns string) *common.VPCNetworkConfigInfo { + return vpcConfig + }) + defer patches.Reset() + + tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-tag")}} + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "GenerateSubnetNSTags", func(_ *subnet.SubnetService, obj client.Object) []model.Tag { + return tags + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, ns string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {OrgID: "org-id", ProjectID: "project-id", VPCID: "vpc-id", ID: "fake-id"}, + } + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "CreateOrUpdateSubnet", func(_ *subnet.SubnetService, obj client.Object, vpcInfo common.VPCResourceInfo, tags []model.Tag) (string, error) { + return "", nil + }) + + patches.ApplyPrivateMethod(reflect.TypeOf(reconciler), "updateSubnetStatus", func(_ *SubnetReconciler, obj *v1alpha1.Subnet) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.NoError(t, err) + assert.NoError(t, reconciler.Client.Get(ctx, req.NamespacedName, subnetCR)) + assert.Equal(t, 16, subnetCR.Spec.IPv4SubnetSize) + assert.Equal(t, ctrl.Result{}, result) + }) +} diff --git a/pkg/controllers/subnetport/addressbinding_webhook.go b/pkg/controllers/subnetport/addressbinding_webhook.go new file mode 100644 index 000000000..c00db77bd --- /dev/null +++ b/pkg/controllers/subnetport/addressbinding_webhook.go @@ -0,0 +1,64 @@ +package subnetport + +import ( + "context" + "fmt" + "net/http" + "reflect" + + admissionv1 "k8s.io/api/admission/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +// Create validator instead of using the existing one in controller-runtime because the existing one can't +// inspect admission.Request in Handle function. + +//+kubebuilder:webhook:path=/validate-crd-nsx-vmware-com-v1alpha1-addressbinding,mutating=false,failurePolicy=fail,sideEffects=None,groups=crd.nsx.vmware.com,resources=addressbindings,verbs=create;update,versions=v1alpha1,name=addressbinding.validating.crd.nsx.vmware.com,admissionReviewVersions=v1 + +type AddressBindingValidator struct { + Client client.Client + decoder admission.Decoder +} + +// Handle handles admission requests. +func (v *AddressBindingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + ab := &v1alpha1.AddressBinding{} + if req.Operation == admissionv1.Delete { + return admission.Allowed("") + } else { + err := v.decoder.Decode(req, ab) + if err != nil { + log.Error(err, "error while decoding AddressBinding", "AddressBinding", req.Namespace+"/"+req.Name) + return admission.Errored(http.StatusBadRequest, err) + } + } + switch req.Operation { + case admissionv1.Create: + existingAddressBindingList := &v1alpha1.AddressBindingList{} + abIndexValue := fmt.Sprintf("%s/%s", ab.Namespace, ab.Spec.VMName) + err := v.Client.List(context.TODO(), existingAddressBindingList, client.MatchingFields{util.AddressBindingNamespaceVMIndexKey: abIndexValue}) + if err != nil { + log.Error(err, "failed to list AddressBinding from cache", "indexValue", abIndexValue) + return admission.Errored(http.StatusInternalServerError, err) + } + for _, existingAddressBinding := range existingAddressBindingList.Items { + if ab.Name != existingAddressBinding.Name && ab.Spec.InterfaceName == existingAddressBinding.Spec.InterfaceName { + return admission.Denied("interface already has AddressBinding") + } + } + case admissionv1.Update: + oldAddressBinding := &v1alpha1.AddressBinding{} + if err := v.decoder.DecodeRaw(req.OldObject, oldAddressBinding); err != nil { + log.Error(err, "error while decoding AddressBinding", "AddressBinding", req.Namespace+"/"+req.Name) + return admission.Errored(http.StatusBadRequest, err) + } + if !reflect.DeepEqual(ab.Spec, oldAddressBinding.Spec) { + return admission.Denied("update AddressBinding is not allowed") + } + } + return admission.Allowed("") +} diff --git a/pkg/controllers/subnetport/addressbinding_webhook_test.go b/pkg/controllers/subnetport/addressbinding_webhook_test.go new file mode 100644 index 000000000..9fa8f7791 --- /dev/null +++ b/pkg/controllers/subnetport/addressbinding_webhook_test.go @@ -0,0 +1,134 @@ +package subnetport + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +func TestAddressBindingValidator_Handle(t *testing.T) { + req1, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab1", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf1", + }, + }) + req1New, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab1", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf1new", + }, + }) + req2, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab2", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf2", + }, + }) + type args struct { + req admission.Request + } + tests := []struct { + name string + args args + prepareFunc func(*testing.T, client.Client, context.Context) *gomonkey.Patches + want admission.Response + }{ + { + name: "delete", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Delete}}}, + want: admission.Allowed(""), + }, + { + name: "create decode error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create}}}, + want: admission.Errored(http.StatusBadRequest, fmt.Errorf("there is no content to decode")), + }, + { + name: "create", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req1}}}}, + want: admission.Allowed(""), + }, + { + name: "create list error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req1}}}}, + prepareFunc: func(t *testing.T, client client.Client, ctx context.Context) *gomonkey.Patches { + return gomonkey.ApplyMethodSeq(client, "List", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("mock error")}, + Times: 1, + }}) + }, + want: admission.Errored(http.StatusInternalServerError, fmt.Errorf("mock error")), + }, + { + name: "create dup", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req2}}}}, + want: admission.Denied("interface already has AddressBinding"), + }, + { + name: "update decode error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Update, Object: runtime.RawExtension{Raw: req1}}}}, + want: admission.Errored(http.StatusBadRequest, fmt.Errorf("there is no content to decode")), + }, + { + name: "update changed", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Update, Object: runtime.RawExtension{Raw: req1New}, OldObject: runtime.RawExtension{Raw: req1}}}}, + want: admission.Denied("update AddressBinding is not allowed"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := clientgoscheme.Scheme + v1alpha1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&v1alpha1.AddressBinding{}).WithIndex(&v1alpha1.AddressBinding{}, util.AddressBindingNamespaceVMIndexKey, addressBindingNamespaceVMIndexFunc).Build() + decoder := admission.NewDecoder(scheme) + ctx := context.TODO() + client.Create(ctx, &v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab2a", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf2", + }, + }) + if tt.prepareFunc != nil { + patches := tt.prepareFunc(t, client, ctx) + defer patches.Reset() + } + v := &AddressBindingValidator{ + Client: client, + decoder: decoder, + } + assert.Equalf(t, tt.want, v.Handle(ctx, tt.args.req), "Handle()") + }) + } +} diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index 1cf28af32..fa82c15d8 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -10,24 +10,25 @@ import ( "os" "reflect" "strings" + "time" vmv1alpha1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" @@ -57,18 +58,27 @@ type SubnetPortReconciler struct { // +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports/finalizers,verbs=update func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - subnetPort := &v1alpha1.SubnetPort{} log.Info("reconciling subnetport CR", "subnetport", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("finished reconciling SubnetPort", "SubnetPort", req.NamespacedName, "duration", time.Since(startTime)) + }() metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetPort) + subnetPort := &v1alpha1.SubnetPort{} if err := r.Client.Get(ctx, req.NamespacedName, subnetPort); err != nil { - log.Error(err, "unable to fetch subnetport CR", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetPortByName(ctx, req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NSX SubnetPort", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err + } + return common.ResultNormal, nil + } + log.Error(err, "unable to fetch SubnetPort CR", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err } - if len(subnetPort.Spec.SubnetSet) > 0 && len(subnetPort.Spec.Subnet) > 0 { err := errors.New("subnet and subnetset should not be configured at the same time") log.Error(err, "failed to get subnet/subnetset of the subnetport", "subnetport", req.NamespacedName) @@ -78,15 +88,6 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) if subnetPort.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetPort) - if !controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { - controllerutil.AddFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) - if err := r.Client.Update(ctx, subnetPort); err != nil { - log.Error(err, "add finalizer", "subnetport", req.NamespacedName) - updateFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - log.Info("added finalizer on subnetport CR", "subnetport", req.NamespacedName) - } old_status := subnetPort.Status.DeepCopy() isParentResourceTerminating, nsxSubnetPath, err := r.CheckAndGetSubnetPathForSubnetPort(ctx, subnetPort) @@ -145,27 +146,16 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) } updateSuccess(r, ctx, subnetPort) } else { - if controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { - metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) - if err := r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetport", req.NamespacedName) - deleteFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - controllerutil.RemoveFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) - if err := r.Client.Update(ctx, subnetPort); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetport", req.NamespacedName) - deleteFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - log.Info("removed finalizer", "subnetport", req.NamespacedName) - deleteSuccess(r, ctx, subnetPort) - } else { - log.Info("finalizers cannot be recognized", "subnetport", req.NamespacedName) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) + subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) + if err := r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "SubnetPort", req.NamespacedName) + deleteFail(r, ctx, subnetPort, &err) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, subnetPort) } - return ctrl.Result{}, nil + return common.ResultNormal, nil } func subnetPortNamespaceVMIndexFunc(obj client.Object) []string { @@ -191,6 +181,29 @@ func addressBindingNamespaceVMIndexFunc(obj client.Object) []string { } } +func (r *SubnetPortReconciler) deleteSubnetPortByName(ctx context.Context, ns string, name string) error { + // When deleting SubnetPort by Name and Namespace, skip the SubnetPort belonging to the existed SubnetPort CR + nsxSubnetPorts := r.SubnetPortService.ListSubnetPortByName(ns, name) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) + if err != nil { + log.Error(err, "failed to list SubnetPort CRs") + return err + } + + for _, nsxSubnetPort := range nsxSubnetPorts { + if crSubnetPortIDsSet.Has(*nsxSubnetPort.Id) { + log.Info("skipping deletion, SubnetPort CR still exists in K8s", "ID", *nsxSubnetPort.Id) + continue + } + if err := r.SubnetPortService.DeleteSubnetPort(nsxSubnetPort); err != nil { + return err + } + } + log.Info("successfully deleted nsxSubnetPort", "namespace", ns, "name", name) + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &v1alpha1.SubnetPort{}, util.SubnetPortNamespaceVMIndexKey, subnetPortNamespaceVMIndexFunc); err != nil { @@ -201,18 +214,6 @@ func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.SubnetPort{}). - WithEventFilter( - predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - switch e.Object.(type) { - case *v1alpha1.AddressBinding: - return true - } - return false - }, - }, - ). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -257,7 +258,7 @@ func (r *SubnetPortReconciler) vmMapFunc(_ context.Context, vm client.Object) [] return requests } -func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService *subnet.SubnetService, vpcService *vpc.VPCService) { +func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService *subnet.SubnetService, vpcService *vpc.VPCService, hookServer webhook.Server) { subnetPortReconciler := SubnetPortReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -270,6 +271,15 @@ func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.S log.Error(err, "failed to create controller", "controller", "SubnetPort") os.Exit(1) } + if hookServer != nil { + hookServer.Register("/validate-crd-nsx-vmware-com-v1alpha1-addressbinding", + &webhook.Admission{ + Handler: &AddressBindingValidator{ + Client: mgr.GetClient(), + decoder: admission.NewDecoder(mgr.GetScheme()), + }, + }) + } go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetPortReconciler.CollectGarbage) } @@ -290,24 +300,17 @@ func (r *SubnetPortReconciler) CollectGarbage(ctx context.Context) { if len(nsxSubnetPortSet) == 0 { return } - subnetPortList := &v1alpha1.SubnetPortList{} - err := r.Client.List(ctx, subnetPortList) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) if err != nil { - log.Error(err, "failed to list SubnetPort CR") return } - CRSubnetPortSet := sets.New[string]() - for _, subnetPort := range subnetPortList.Items { - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) - CRSubnetPortSet.Insert(subnetPortID) - } - - diffSet := nsxSubnetPortSet.Difference(CRSubnetPortSet) + diffSet := nsxSubnetPortSet.Difference(crSubnetPortIDsSet) for elem := range diffSet { log.V(1).Info("GC collected SubnetPort CR", "UID", elem) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - err = r.SubnetPortService.DeleteSubnetPort(elem) + err = r.SubnetPortService.DeleteSubnetPortById(elem) if err != nil { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) } else { @@ -322,7 +325,7 @@ func (r *SubnetPortReconciler) setSubnetPortReadyStatusTrue(ctx context.Context, Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX subnet port has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "SubnetPortReady", LastTransitionTime: transitionTime, }, } @@ -332,13 +335,13 @@ func (r *SubnetPortReconciler) setSubnetPortReadyStatusTrue(ctx context.Context, func (r *SubnetPortReconciler) setSubnetPortReadyStatusFalse(ctx context.Context, subnetPort *v1alpha1.SubnetPort, transitionTime metav1.Time, err *error) { newConditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX subnet port could not be created/updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the SubnetPort CR. Error: %v", *err, ), + Reason: "SubnetPortNotReady", LastTransitionTime: transitionTime, }, } @@ -359,7 +362,7 @@ func (r *SubnetPortReconciler) UpdateSubnetPortStatusConditions(ctx context.Cont } } -func (r *SubnetPortReconciler) mergeSubnetPortStatusCondition(ctx context.Context, subnetPort *v1alpha1.SubnetPort, newCondition *v1alpha1.Condition) bool { +func (r *SubnetPortReconciler) mergeSubnetPortStatusCondition(_ context.Context, subnetPort *v1alpha1.SubnetPort, newCondition *v1alpha1.Condition) bool { matchedCondition := getExistingConditionOfType(newCondition.Type, subnetPort.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { @@ -420,7 +423,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co _, err = r.SubnetService.GetSubnetByPath(subnetPath) if err != nil { log.Info("previous NSX subnet is deleted, deleting the stale subnet port", "subnetPort.UID", subnetPort.UID, "subnetPath", subnetPath) - if err = r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { + if err = r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { log.Error(err, "failed to delete the stale subnetport", "subnetport.UID", subnetPort.UID) return } @@ -439,7 +442,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co log.Error(err, "subnet CR not found", "subnet CR", namespacedName) return } - if subnet != nil && !subnet.DeletionTimestamp.IsZero() { + if !subnet.DeletionTimestamp.IsZero() { isStale = true err := fmt.Errorf("subnet %s is being deleted, cannot operate subnetport %s", namespacedName, subnetPort.Name) return true, "", err @@ -464,7 +467,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co log.Error(err, "subnetSet CR not found", "subnet CR", namespacedName) return } - if subnetSet != nil && !subnetSet.DeletionTimestamp.IsZero() { + if !subnetSet.DeletionTimestamp.IsZero() { isStale = true err = fmt.Errorf("subnetset %s is being deleted, cannot operate subnetport %s", namespacedName, subnetPort.Name) return diff --git a/pkg/controllers/subnetport/subnetport_controller_test.go b/pkg/controllers/subnetport/subnetport_controller_test.go index fca295125..f5db99e82 100644 --- a/pkg/controllers/subnetport/subnetport_controller_test.go +++ b/pkg/controllers/subnetport/subnetport_controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -76,41 +77,37 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { }) defer patchesGetSubnetByPath.Reset() - // not found - errNotFound := errors.New("not found") + // fail to get + errFailToGet := errors.New("failed to get CR") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errFailToGet) + _, ret := r.Reconcile(ctx, req) + assert.Equal(t, errFailToGet, ret) + + // not found and deletion success + errNotFound := apierrors.NewNotFound(v1alpha1.Resource("subnetport"), "") k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) - _, err := r.Reconcile(ctx, req) - assert.Equal(t, err, errNotFound) - // update fails - sp := &v1alpha1.SubnetPort{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" + patchesDeleteSubnetPortByName := gomonkey.ApplyFunc((*SubnetPortReconciler).deleteSubnetPortByName, + func(r *SubnetPortReconciler, ctx context.Context, ns string, name string) error { return nil }) - subnet := &v1alpha1.Subnet{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), subnet).Return(nil).AnyTimes().Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - s := obj.(*v1alpha1.Subnet) - s.Name = sp.Spec.Subnet - return nil - }) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) - patchesSuccess := gomonkey.ApplyFunc(updateSuccess, - func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort) { - }) - defer patchesSuccess.Reset() - patchesUpdateFail := gomonkey.ApplyFunc(updateFail, - func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort, e *error) { + defer patchesDeleteSubnetPortByName.Reset() + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, nil, ret) + + // not found and deletion failed + err := errors.New("Deletion failed") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + patchesDeleteSubnetPortByName = gomonkey.ApplyFunc((*SubnetPortReconciler).deleteSubnetPortByName, + func(r *SubnetPortReconciler, ctx context.Context, ns string, name string) error { + return err }) - defer patchesUpdateFail.Reset() - _, ret := r.Reconcile(ctx, req) + defer patchesDeleteSubnetPortByName.Reset() + _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) // both subnet and subnetset are configured + sp := &v1alpha1.SubnetPort{} k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -118,6 +115,10 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.Spec.SubnetSet = "subnetset2" return nil }) + patchesUpdateFail := gomonkey.ApplyFunc(updateFail, + func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort, e *error) { + }) + defer patchesUpdateFail.Reset() err = errors.New("subnet and subnetset should not be configured at the same time") _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) @@ -134,7 +135,6 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return requests }) defer patchesVmMapFunc.Reset() - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -154,8 +154,8 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { defer patchesCreateOrUpdateSubnetPort.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) + // happy path - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -183,6 +183,10 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return portState, nil }) defer patchesCreateOrUpdateSubnetPort.Reset() + patchesSuccess := gomonkey.ApplyFunc(updateSuccess, + func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort) { + }) + defer patchesSuccess.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, nil, ret) @@ -193,15 +197,14 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.Spec.Subnet = "subnet1" time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} return nil }) err = errors.New("DeleteSubnetPort failed") - patchesDeleteSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { return err }) - defer patchesDeleteSubnetPort.Reset() + defer patchesDeleteSubnetPortById.Reset() patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort, func(s *subnetport.SubnetPortService, obj interface{}, nsxSubnet *model.VpcSubnet, contextID string, tags *map[string]string) (*model.SegmentPortState, error) { assert.FailNow(t, "should not be called") @@ -215,46 +218,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) - // handle deletion event - update subnetport failed in deletion event - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} - return nil - }) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, - func(s *subnetport.SubnetPortService, uid string) error { - return nil - }) - defer patchesDeleteSubnetPort.Reset() - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, err, ret) - // handle deletion event - successfully deleted - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} - return nil - }) - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, - func(s *subnetport.SubnetPortService, uid string) error { - return nil - }) - defer patchesDeleteSubnetPort.Reset() - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, nil, ret) - - // handle deletion event - unknown finalizers k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -263,20 +227,22 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.ObjectMeta.DeletionTimestamp = &time return nil }) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { - assert.FailNow(t, "should not be called") return nil }) - defer patchesDeleteSubnetPort.Reset() + defer patchesDeleteSubnetPortById.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, nil, ret) } func TestSubnetPortReconciler_GarbageCollector(t *testing.T) { // gc collect item "2345", local store has more item than k8s cache + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) service := &subnetport.SubnetPortService{ Service: common.Service{ + Client: k8sClient, NSXConfig: &config.NSXOperatorConfig{ NsxConfig: &config.NsxConfig{ EnforcementPoint: "vmc-enforcementpoint", @@ -292,13 +258,12 @@ func TestSubnetPortReconciler_GarbageCollector(t *testing.T) { return a }) defer patchesListNSXSubnetPortIDForCR.Reset() - patchesDeleteSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { return nil }) - defer patchesDeleteSubnetPort.Reset() - mockCtl := gomock.NewController(t) - k8sClient := mock_client.NewMockClient(mockCtl) + defer patchesDeleteSubnetPortById.Reset() + r := &SubnetPortReconciler{ Client: k8sClient, Scheme: nil, diff --git a/pkg/controllers/subnetset/namespace_handler.go b/pkg/controllers/subnetset/namespace_handler.go index 1098fd596..d7c4e038c 100644 --- a/pkg/controllers/subnetset/namespace_handler.go +++ b/pkg/controllers/subnetset/namespace_handler.go @@ -18,7 +18,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ) -// SubnetSet controller should watch event of namespace, when there are some updates of namespace labels, +// SubnetSet controller should watch event of Namespace, when there are some updates of Namespace labels, // controller should build tags and update VpcSubnetSetSet according to new labels. type EnqueueRequestForNamespace struct { @@ -26,20 +26,20 @@ type EnqueueRequestForNamespace struct { } func (e *EnqueueRequestForNamespace) Create(_ context.Context, _ event.CreateEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace create event, do nothing") + log.V(1).Info("Namespace create event, do nothing") } func (e *EnqueueRequestForNamespace) Delete(_ context.Context, _ event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace delete event, do nothing") + log.V(1).Info("Namespace delete event, do nothing") } func (e *EnqueueRequestForNamespace) Generic(_ context.Context, _ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace generic event, do nothing") + log.V(1).Info("Namespace generic event, do nothing") } func (e *EnqueueRequestForNamespace) Update(_ context.Context, updateEvent event.UpdateEvent, l workqueue.RateLimitingInterface) { obj := updateEvent.ObjectNew.(*v1.Namespace) - err := reconcileSubnetSet(e.Client, obj.Name, l) + err := requeueSubnetSet(e.Client, obj.Name, l) if err != nil { log.Error(err, "failed to reconcile subnet") } @@ -52,9 +52,9 @@ var PredicateFuncsNs = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldObj := e.ObjectOld.(*v1.Namespace) newObj := e.ObjectNew.(*v1.Namespace) - log.V(1).Info("receive namespace update event", "name", oldObj.Name) + log.V(1).Info("Receive Namespace update event", "name", oldObj.Name) if reflect.DeepEqual(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) { - log.Info("label of namespace is not changed, ignore it", "name", oldObj.Name) + log.Info("label of Namespace is not changed, ignore it", "name", oldObj.Name) return false } return true @@ -64,21 +64,20 @@ var PredicateFuncsNs = predicate.Funcs{ }, } -func reconcileSubnetSet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { +func requeueSubnetSet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { subnetSetList := &v1alpha1.SubnetSetList{} err := c.List(context.Background(), subnetSetList, client.InNamespace(namespace)) if err != nil { - log.Error(err, "failed to list all the subnets") + log.Error(err, "Failed to list all the Subnets") return err } - for _, subnet_set_item := range subnetSetList.Items { - log.Info("reconcile subnet set because namespace update", - "namespace", subnet_set_item.Namespace, "name", subnet_set_item.Name) + for _, subnetSet := range subnetSetList.Items { + log.Info("Requeue SubnetSet because Namespace updated", "Namespace", subnetSet.Namespace, "Name", subnetSet.Name) q.Add(reconcile.Request{ NamespacedName: types.NamespacedName{ - Name: subnet_set_item.Name, - Namespace: subnet_set_item.Namespace, + Name: subnetSet.Name, + Namespace: subnetSet.Namespace, }, }) } diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 9bc4ebbc5..c9502cc78 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "reflect" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -16,17 +18,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" - "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/pkg/util" ) @@ -49,94 +50,97 @@ type SubnetSetReconciler struct { } func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.SubnetSet{} - log.Info("reconciling subnetset CR", "subnetset", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling SubnetSet", "SubnetSet", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + subnetsetCR := &v1alpha1.SubnetSet{} metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetSet) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch subnetset CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + if err := r.Client.Get(ctx, req.NamespacedName, subnetsetCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetBySubnetSetName(ctx, req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete NSX Subnet", "SubnetSet", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + log.Error(err, "Unable to fetch SubnetSet CR", "SubnetSet", req.NamespacedName) + return ResultRequeue, err + } + if !subnetsetCR.ObjectMeta.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) + err := r.deleteSubnetForSubnetSet(*subnetsetCR, false, false) + if err != nil { + log.Error(err, "Failed to delete NSX Subnet, retrying", "SubnetSet", req.NamespacedName) + deleteFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + if err := r.Client.Delete(ctx, subnetsetCR); err != nil { + log.Error(err, "Failed to delete SubnetSet CR, retrying", "SubnetSet", req.NamespacedName) + deleteFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + deleteSuccess(r, ctx, subnetsetCR) + return ResultNormal, nil } - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) - if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { - controllerutil.AddFinalizer(obj, servicecommon.SubnetSetFinalizerName) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) - if obj.Spec.AccessMode == "" { - obj.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) - } - if obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) - if vpcNetworkConfig == nil { - err := fmt.Errorf("failed to find VPCNetworkConfig for namespace %s", obj.Namespace) - log.Error(err, "operate failed, would retry exponentially", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize - } - if !util.IsPowerOfTwo(obj.Spec.IPv4SubnetSize) { - errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", obj.Spec.IPv4SubnetSize) - log.Error(nil, errorMsg, "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, errorMsg) - return ResultNormal, nil - } + specChanged := false + if subnetsetCR.Spec.AccessMode == "" { + subnetsetCR.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) + specChanged = true + } + if subnetsetCR.Spec.IPv4SubnetSize == 0 { + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(subnetsetCR.Namespace) + if vpcNetworkConfig == nil { + err := fmt.Errorf("failed to find VPCNetworkConfig for Namespace %s", subnetsetCR.Namespace) + log.Error(err, "Operate failed, would retry exponentially", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + subnetsetCR.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize + specChanged = true + } + if !util.IsPowerOfTwo(subnetsetCR.Spec.IPv4SubnetSize) { + errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", subnetsetCR.Spec.IPv4SubnetSize) + log.Error(nil, errorMsg, "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, errorMsg) + return ResultNormal, nil + } - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on subnetset CR", "subnetset", req.NamespacedName) + if specChanged { + err := r.Client.Update(ctx, subnetsetCR) + if err != nil { + log.Error(err, "Update SubnetSet failed", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err } + } - // update subnetset tags if labels of namespace changed - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.UID)) - if len(nsxSubnets) > 0 { - tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) - if tags == nil { - return ResultRequeue, errors.New("failed to generate subnet tags") - } - // tags cannot exceed maximum size 26 - if len(tags) > servicecommon.TagsCountMax { - errorMsg := fmt.Sprintf("tags cannot exceed maximum size 26, tags length: %d", len(tags)) - log.Error(nil, "exceed tags limit, would not retry", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, errorMsg) - return ResultNormal, nil - } - if err := r.SubnetService.UpdateSubnetSetTags(obj.Namespace, nsxSubnets, tags); err != nil { - log.Error(err, "failed to update subnetset tags") - } + // update SubnetSet tags if labels of namespace changed + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(subnetsetCR.UID)) + if len(nsxSubnets) > 0 { + tags := r.SubnetService.GenerateSubnetNSTags(subnetsetCR) + if tags == nil { + log.Error(nil, "Failed to generate SubnetSet tags", "SubnetSet", req.NamespacedName) + return ResultRequeue, errors.New("failed to generate SubnetSet tags") } - updateSuccess(r, ctx, obj) - } else { - if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) - hasStaleSubnetPorts, err := r.DeleteSubnetForSubnetSet(*obj, false) - if err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetset", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - if hasStaleSubnetPorts { - err := fmt.Errorf("stale subnet ports found while deleting subnetset %v", req.NamespacedName) - log.Error(err, "deletion failed, delete all the subnetports first", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, servicecommon.SubnetSetFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetset", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "subnetset", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - log.Info("finalizers cannot be recognized", "subnetset", req.NamespacedName) + // tags cannot exceed maximum size 26 + if len(tags) > servicecommon.TagsCountMax { + errorMsg := fmt.Sprintf("tags cannot exceed maximum size 26, tags length: %d", len(tags)) + log.Error(nil, "Exceed tags limit, would not retry", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, errorMsg) + return ResultNormal, nil + } + if err := r.SubnetService.UpdateSubnetSetTags(subnetsetCR.Namespace, nsxSubnets, tags); err != nil { + log.Error(err, "Failed to update SubnetSet tags", "SubnetSet", req.NamespacedName) } } + updateSuccess(r, ctx, subnetsetCR) + return ctrl.Result{}, nil } @@ -163,7 +167,7 @@ func deleteSuccess(r *SubnetSetReconciler, _ context.Context, o *v1alpha1.Subnet metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } -func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, subnetset *v1alpha1.SubnetSet, transitionTime metav1.Time) { +func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, subnetSet *v1alpha1.SubnetSet, transitionTime metav1.Time) { newConditions := []v1alpha1.Condition{ { Type: v1alpha1.Ready, @@ -173,10 +177,10 @@ func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, s LastTransitionTime: transitionTime, }, } - r.updateSubnetSetStatusConditions(ctx, subnetset, newConditions) + r.updateSubnetSetStatusConditions(ctx, subnetSet, newConditions) } -func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, subnetset *v1alpha1.SubnetSet, transitionTime metav1.Time, m string) { +func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, subnetSet *v1alpha1.SubnetSet, transitionTime metav1.Time, m string) { newConditions := []v1alpha1.Condition{ { Type: v1alpha1.Ready, @@ -189,30 +193,30 @@ func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, if m != "" { newConditions[0].Message = m } - r.updateSubnetSetStatusConditions(ctx, subnetset, newConditions) + r.updateSubnetSetStatusConditions(ctx, subnetSet, newConditions) } -func (r *SubnetSetReconciler) updateSubnetSetStatusConditions(ctx context.Context, subnetset *v1alpha1.SubnetSet, newConditions []v1alpha1.Condition) { +func (r *SubnetSetReconciler) updateSubnetSetStatusConditions(ctx context.Context, subnetSet *v1alpha1.SubnetSet, newConditions []v1alpha1.Condition) { conditionsUpdated := false for i := range newConditions { - if r.mergeSubnetSetStatusCondition(ctx, subnetset, &newConditions[i]) { + if r.mergeSubnetSetStatusCondition(ctx, subnetSet, &newConditions[i]) { conditionsUpdated = true } } if conditionsUpdated { - if err := r.Client.Status().Update(ctx, subnetset); err != nil { - log.Error(err, "failed to update status", "Name", subnetset.Name, "Namespace", subnetset.Namespace) + if err := r.Client.Status().Update(ctx, subnetSet); err != nil { + log.Error(err, "Failed to update status", "Name", subnetSet.Name, "Namespace", subnetSet.Namespace) } else { - log.Info("updated SubnetSet", "Name", subnetset.Name, "Namespace", subnetset.Namespace, "New Conditions", newConditions) + log.Info("Updated SubnetSet", "Name", subnetSet.Name, "Namespace", subnetSet.Namespace, "New Conditions", newConditions) } } } -func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, subnetset *v1alpha1.SubnetSet, newCondition *v1alpha1.Condition) bool { - matchedCondition := getExistingConditionOfType(newCondition.Type, subnetset.Status.Conditions) +func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, subnetSet *v1alpha1.SubnetSet, newCondition *v1alpha1.Condition) bool { + matchedCondition := getExistingConditionOfType(newCondition.Type, subnetSet.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { - log.V(2).Info("conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) + log.V(2).Info("Conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) return false } @@ -221,7 +225,7 @@ func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, matchedCondition.Message = newCondition.Message matchedCondition.Status = newCondition.Status } else { - subnetset.Status.Conditions = append(subnetset.Status.Conditions, *newCondition) + subnetSet.Status.Conditions = append(subnetSet.Status.Conditions, *newCondition) } return true } @@ -252,74 +256,122 @@ func (r *SubnetSetReconciler) setupWithManager(mgr ctrl.Manager) error { // CollectGarbage collect Subnet which there is no port attached on it. // it implements the interface GarbageCollector method. func (r *SubnetSetReconciler) CollectGarbage(ctx context.Context) { - log.Info("subnetset garbage collector started") - subnetSetList := &v1alpha1.SubnetSetList{} - err := r.Client.List(ctx, subnetSetList) + startTime := time.Now() + defer func() { + log.Info("SubnetSet garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + crdSubnetSetList := &v1alpha1.SubnetSetList{} + err := r.Client.List(ctx, crdSubnetSetList) if err != nil { - log.Error(err, "failed to list SubnetSet CR") - return - } - var nsxSubnetList []*model.VpcSubnet - for _, subnetSet := range subnetSetList.Items { - nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnetSet(string(subnetSet.UID))...) - } - if len(nsxSubnetList) == 0 { + log.Error(err, "Failed to list SubnetSet CRs") return } - subnetSetIDs := sets.New[string]() - for _, subnetSet := range subnetSetList.Items { - if _, err := r.DeleteSubnetForSubnetSet(subnetSet, true); err != nil { + crdSubnetSetIDsSet := sets.New[string]() + for _, subnetSet := range crdSubnetSetList.Items { + crdSubnetSetIDsSet.Insert(string(subnetSet.UID)) + if err := r.deleteSubnetForSubnetSet(subnetSet, true, true); err != nil { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } - subnetSetIDs.Insert(string(subnetSet.UID)) } - for _, subnet := range nsxSubnetList { - if !r.SubnetService.IsOrphanSubnet(*subnet, subnetSetIDs) { - continue - } - if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { + + subnetSetIDs := r.SubnetService.ListSubnetSetIDsFromNSXSubnets() + subnetSetIDsToDelete := subnetSetIDs.Difference(crdSubnetSetIDsSet) + for subnetSetID := range subnetSetIDsToDelete { + nsxSubnets := r.SubnetService.ListSubnetCreatedBySubnetSet(subnetSetID) + log.Info("SubnetSet garbage collection, cleaning stale Subnets for SubnetSet", "Count", len(nsxSubnets)) + if _, err := r.deleteSubnets(nsxSubnets); err != nil { + log.Error(err, "SubnetSet garbage collection, failed to delete NSX subnet", "SubnetSetUID", subnetSetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { + log.Info("SubnetSet garbage collection, successfully deleted NSX subnet", "SubnetSetUID", subnetSetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } } } -func (r *SubnetSetReconciler) DeleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, updataStatus bool) (bool, error) { - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.GetUID())) - hitError := false - hasStaleSubnetPorts := false - for _, subnet := range nsxSubnets { - r.SubnetService.LockSubnet(subnet.Path) - portNums := len(r.SubnetPortService.GetPortsOfSubnet(*subnet.Id)) +func (r *SubnetSetReconciler) deleteSubnetBySubnetSetName(ctx context.Context, subnetSetName, ns string) error { + nsxSubnets := r.SubnetService.ListSubnetBySubnetSetName(ns, subnetSetName) + return r.deleteStaleSubnets(ctx, nsxSubnets) +} + +func (r *SubnetSetReconciler) deleteSubnetForSubnetSet(subnetSet v1alpha1.SubnetSet, updateStatus, ignoreStaleSubnetPort bool) error { + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(subnetSet.GetUID())) + hasStaleSubnetPort, deleteErr := r.deleteSubnets(nsxSubnets) + if updateStatus { + if err := r.SubnetService.UpdateSubnetSetStatus(&subnetSet); err != nil { + return err + } + } + if deleteErr != nil { + return deleteErr + } + if hasStaleSubnetPort && !ignoreStaleSubnetPort { + return fmt.Errorf("stale Subnet ports found while deleting Subnet for SubnetSet %s/%s", subnetSet.Name, subnetSet.Namespace) + } + return nil +} + +// deleteSubnets deletes all the specified NSX Subnets. +// If any of the Subnets have stale SubnetPorts, they are skipped. The final result returns true. +// If there is an error while deleting any NSX Subnet, it is skipped, and the final result returns an error. +func (r *SubnetSetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) (hasStalePort bool, err error) { + var deleteErrs []error + for _, nsxSubnet := range nsxSubnets { + r.SubnetService.LockSubnet(nsxSubnet.Path) + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) if portNums > 0 { - hasStaleSubnetPorts = true - r.SubnetService.UnlockSubnet(subnet.Path) + r.SubnetService.UnlockSubnet(nsxSubnet.Path) + hasStalePort = true + log.Info("Skipped deleting NSX Subnet due to stale ports", "nsxSubnet", *nsxSubnet.Id) continue } - if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { - log.Error(err, "fail to delete subnet from subnetset cr", "ID", *subnet.Id) - hitError = true + if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { + r.SubnetService.UnlockSubnet(nsxSubnet.Path) + deleteErr := fmt.Errorf("failed to delete NSX Subnet/%s: %+v", *nsxSubnet.Id, err) + deleteErrs = append(deleteErrs, deleteErr) + log.Error(deleteErr, "Skipping to next Subnet") + continue } - r.SubnetService.UnlockSubnet(subnet.Path) + r.SubnetService.UnlockSubnet(nsxSubnet.Path) + } + if len(deleteErrs) > 0 { + err = fmt.Errorf("multiple errors occurred while deleting Subnets: %v", deleteErrs) + return + } + log.Info("Successfully deleted all specified NSX Subnets", "subnetCount", len(nsxSubnets)) + return +} + +func (r *SubnetSetReconciler) deleteStaleSubnets(ctx context.Context, nsxSubnets []*model.VpcSubnet) error { + crdSubnetSetIDsSet, err := r.SubnetService.ListSubnetSetID(ctx) + if err != nil { + log.Error(err, "Failed to list SubnetSet CRs") + return err } - if updataStatus { - if err := r.SubnetService.UpdateSubnetSetStatus(&obj); err != nil { - return hasStaleSubnetPorts, err + nsxSubnetsToDelete := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetSetCRUID) + if crdSubnetSetIDsSet.Has(uid) { + log.Info("Skipping deletion, SubnetSet CR still exists in K8s", "ID", *nsxSubnet.Id) + continue } + nsxSubnetsToDelete = append(nsxSubnetsToDelete, nsxSubnet) } - if hitError { - return hasStaleSubnetPorts, errors.New("error occurs when deleting subnet") + log.Info("Cleaning stale Subnets for SubnetSet", "Count", len(nsxSubnetsToDelete)) + hasStaleSubnetPort, err := r.deleteSubnets(nsxSubnetsToDelete) + if err != nil || hasStaleSubnetPort { + return fmt.Errorf("failed to delete stale Subnets, error: %v, hasStaleSubnetPort: %t", err, hasStaleSubnetPort) } - return hasStaleSubnetPorts, nil + return nil } func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider, - enableWebhook bool, + hookServer webhook.Server, ) error { subnetsetReconciler := &SubnetSetReconciler{ Client: mgr.GetClient(), @@ -329,8 +381,8 @@ func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetServ VPCService: vpcService, Recorder: mgr.GetEventRecorderFor("subnetset-controller"), } - if err := subnetsetReconciler.Start(mgr, enableWebhook); err != nil { - log.Error(err, "failed to create controller", "controller", "Subnet") + if err := subnetsetReconciler.Start(mgr, hookServer); err != nil { + log.Error(err, "Failed to create controller", "controller", "SubnetSet") return err } go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetsetReconciler.CollectGarbage) @@ -338,19 +390,12 @@ func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetServ } // Start setup manager -func (r *SubnetSetReconciler) Start(mgr ctrl.Manager, enableWebhook bool) error { +func (r *SubnetSetReconciler) Start(mgr ctrl.Manager, hookServer webhook.Server) error { err := r.setupWithManager(mgr) if err != nil { return err } - if enableWebhook { - hookServer := webhook.NewServer(webhook.Options{ - Port: config.WebhookServerPort, - CertDir: config.WebhookCertDir, - }) - if err := mgr.Add(hookServer); err != nil { - return err - } + if hookServer != nil { hookServer.Register("/validate-crd-nsx-vmware-com-v1alpha1-subnetset", &webhook.Admission{ Handler: &SubnetSetValidator{ diff --git a/pkg/controllers/subnetset/subnetset_webhook.go b/pkg/controllers/subnetset/subnetset_webhook.go index 9ee99e475..fe0b0c691 100644 --- a/pkg/controllers/subnetset/subnetset_webhook.go +++ b/pkg/controllers/subnetset/subnetset_webhook.go @@ -22,7 +22,7 @@ var NSXOperatorSA = "system:serviceaccount:vmware-system-nsx:ncp-svc-account" // inspect admission.Request in Handle function. // +kubebuilder:webhook:path=/validate-crd-nsx-vmware-com-v1alpha1-subnetset,mutating=false,failurePolicy=fail,sideEffects=None, -//groups=nsx.vmware.com.nsx.vmware.com,resources=subnetsets,verbs=create;update,versions=v1alpha1, +//groups=crd.nsx.vmware.com,resources=subnetsets,verbs=create;update,versions=v1alpha1, //name=default.subnetset.validating.crd.nsx.vmware.com,admissionReviewVersions=v1 type SubnetSetValidator struct { diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index c43f6b45c..d0922bc6d 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -92,6 +92,7 @@ type Client struct { VPCLBSClient vpcs.VpcLbsClient VpcLbVirtualServersClient vpcs.VpcLbVirtualServersClient VpcLbPoolsClient vpcs.VpcLbPoolsClient + VpcAttachmentClient vpcs.AttachmentsClient ProjectClient orgs.ProjectsClient TransitGatewayClient projects.TransitGatewaysClient TransitGatewayAttachmentClient transit_gateways.AttachmentsClient @@ -186,6 +187,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(restConnector(cluster)) vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(restConnector(cluster)) + vpcAttachmentClient := vpcs.NewAttachmentsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -234,6 +236,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { VPCLBSClient: vpcLBSClient, VpcLbVirtualServersClient: vpcLbVirtualServersClient, VpcLbPoolsClient: vpcLbPoolsClient, + VpcAttachmentClient: vpcAttachmentClient, ProjectClient: projectClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/services.go b/pkg/nsx/services/common/services.go index b42eb3094..c23a08f29 100644 --- a/pkg/nsx/services/common/services.go +++ b/pkg/nsx/services/common/services.go @@ -29,7 +29,7 @@ type SubnetServiceProvider interface { GetSubnetByPath(path string) (*model.VpcSubnet, error) GetSubnetsByIndex(key, value string) []*model.VpcSubnet CreateOrUpdateSubnet(obj client.Object, vpcInfo VPCResourceInfo, tags []model.Tag) (string, error) - GenerateSubnetNSTags(obj client.Object, nsUID string) []model.Tag + GenerateSubnetNSTags(obj client.Object) []model.Tag LockSubnet(path *string) UnlockSubnet(path *string) } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index ddc7d2bbc..47984e2ca 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -82,7 +82,6 @@ const ( TagValueShareCreatedForInfra string = "infra" TagValueShareCreatedForProject string = "project" TagValueShareNotCreated string = "notShared" - TagValueGroupAvi string = "avi" TagValueSLB string = "SLB" AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config" AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace" @@ -103,17 +102,8 @@ const ( IPPoolTypePublic = "Public" IPPoolTypePrivate = "Private" - NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" - T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" - - SecurityPolicyFinalizerName = "securitypolicy.crd.nsx.vmware.com/finalizer" - NetworkPolicyFinalizerName = "networkpolicy.crd.nsx.vmware.com/finalizer" - StaticRouteFinalizerName = "staticroute.crd.nsx.vmware.com/finalizer" - SubnetFinalizerName = "subnet.crd.nsx.vmware.com/finalizer" - SubnetSetFinalizerName = "subnetset.crd.nsx.vmware.com/finalizer" - SubnetPortFinalizerName = "subnetport.crd.nsx.vmware.com/finalizer" - NetworkInfoFinalizerName = "networkinfo.crd.nsx.vmware.com/finalizer" - PodFinalizerName = "pod.crd.nsx.vmware.com/finalizer" + NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" + T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" IPPoolFinalizerName = "ippool.crd.nsx.vmware.com/finalizer" IPAddressAllocationFinalizerName = "ipaddressallocation.crd.nsx.vmware.com/finalizer" @@ -122,20 +112,21 @@ const ( IndexKeyNodeName = "IndexKeyNodeName" GCValidationInterval uint16 = 720 - RuleIngress = "ingress" - RuleEgress = "egress" - RuleActionAllow = "allow" - RuleActionDrop = "isolation" - RuleActionReject = "reject" - RuleAnyPorts = "all" - DefaultProject = "default" - SecurityPolicyPrefix = "sp" - NetworkPolicyPrefix = "np" - TargetGroupSuffix = "scope" - SrcGroupSuffix = "src" - DstGroupSuffix = "dst" - IpSetGroupSuffix = "ipset" - ShareSuffix = "share" + RuleIngress = "ingress" + RuleEgress = "egress" + RuleActionAllow = "allow" + RuleActionDrop = "isolation" + RuleActionReject = "reject" + RuleAnyPorts = "all" + DefaultProject = "default" + DefaultVpcAttachmentId = "default" + SecurityPolicyPrefix = "sp" + NetworkPolicyPrefix = "np" + TargetGroupSuffix = "scope" + SrcGroupSuffix = "src" + DstGroupSuffix = "dst" + IpSetGroupSuffix = "ipset" + ShareSuffix = "share" ) var ( @@ -161,6 +152,7 @@ var ( ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" ResourceTypeLBService = "LBService" + ResourceTypeVpcAttachment = "VpcAttachment" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" @@ -168,6 +160,7 @@ var ( ResourceTypeChildRule = "ChildRule" ResourceTypeChildGroup = "ChildGroup" ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" + ResourceTypeChildVpcAttachment = "ChildVpcAttachment" ResourceTypeChildResourceReference = "ChildResourceReference" ResourceTypeTlsCertificate = "TlsCertificate" ResourceTypeLBHttpProfile = "LBHttpProfile" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go index a2c96d3a0..ddc0dfac5 100644 --- a/pkg/nsx/services/common/wrap.go +++ b/pkg/nsx/services/common/wrap.go @@ -82,3 +82,18 @@ func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, erro } return []*data.StructValue{dataValue.(*data.StructValue)}, nil } + +func (service *Service) WrapAttachment(attachment *model.VpcAttachment) ([]*data.StructValue, error) { + attachment.ResourceType = pointy.String(ResourceTypeVpcAttachment) + childVpcAttachment := model.ChildVpcAttachment{ + Id: attachment.Id, + MarkedForDelete: attachment.MarkedForDelete, + ResourceType: ResourceTypeChildVpcAttachment, + VpcAttachment: attachment, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpcAttachment, childVpcAttachment.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} diff --git a/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go b/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go index a84aa88d0..231ebadab 100644 --- a/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go +++ b/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go @@ -208,36 +208,35 @@ func (s *IPBlocksInfoService) getIPBlockCIDRsByVPCConfig(vpcConfigList []v1alpha return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("default project not found, try later") } - // for all VPC path, get VPCConnectivityProfile from VPC - vpcStore := &VPCStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.VpcBindingType(), - }} - queryParam := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeVpc) - count, err := s.SearchResource(common.ResourceTypeVpc, queryParam, vpcStore, nil) + // for all VPC path, get VPCConnectivityProfile from VPC attachment + vpcAttachmentStore := NewVpcAttachmentStore() + queryParam := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeVpcAttachment) + count, err := s.SearchResource(common.ResourceTypeVpcAttachment, queryParam, vpcAttachmentStore, nil) if err != nil { + log.Error(err, "failed to query VPC attachment") return externalIPCIDRs, privateTGWIPCIDRs, err } - log.V(2).Info("successfully fetch all Vpc from NSX", "count", count) + log.V(2).Info("successfully fetch all VPC Attachment from NSX", "count", count) for vpcPath := range vpcs { vpcResInfo, err := common.ParseVPCResourcePath(vpcPath) if err != nil { return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("invalid VPC path %s", vpcPath) } - - obj := vpcStore.GetByKey(vpcPath) - if obj == nil { - return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("failed to get VPC %s from NSX", vpcPath) - } - vpc := obj.(*model.Vpc) - log.V(2).Info("successfully fetch VPC", "path", vpcPath) - // for pre-created vpc, mark as default for those under default project + // for pre-created VPC, mark as default for those under default project vpcProjectPath := fmt.Sprintf("/orgs/%s/projects/%s", vpcResInfo.OrgID, vpcResInfo.ProjectID) + vpcAttachments := vpcAttachmentStore.GetByVpcPath(vpcPath) + if len(vpcAttachments) == 0 { + err = fmt.Errorf("no VPC attachment found") + log.Error(err, "get VPC attachment", "VPC Path", vpcPath) + return externalIPCIDRs, privateTGWIPCIDRs, err + } + log.V(2).Info("successfully fetch VPC attachment", "path", vpcPath, "VPC Attachment", vpcAttachments[0]) + vpcConnectivityProfile := vpcAttachments[0].VpcConnectivityProfile if vpcProjectPath == s.defaultProject { - vpcConnectivityProfileProjectMap[*vpc.VpcConnectivityProfile] = true + vpcConnectivityProfileProjectMap[*vpcConnectivityProfile] = true } else { - vpcConnectivityProfileProjectMap[*vpc.VpcConnectivityProfile] = false + vpcConnectivityProfileProjectMap[*vpcConnectivityProfile] = false } } diff --git a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go index c13d5d8bc..774529044 100644 --- a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go +++ b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "testing" + "time" "github.com/agiledragon/gomonkey" "github.com/golang/mock/gomock" @@ -12,6 +13,8 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -36,6 +39,7 @@ var ( vpcConnectivityProfilePath1 = "/orgs/default/projects/default/vpc-connectivity-profiles/vpc-connectivity-profile-1" vpcConnectivityProfilePath2 = "/orgs/default/projects/default/vpc-connectivity-profiles/vpc-connectivity-profile-2" vpcPath = "/orgs/default/projects/default/vpcs/vpc-1" + vpcAttachmentPath = vpcPath + "/attachments/default" projectPath = "/orgs/default/projects/default" ) @@ -55,12 +59,13 @@ func createService(t *testing.T) (*IPBlocksInfoService, *gomock.Controller, *moc func fakeSearchResource(_ *common.Service, resourceTypeValue string, _ string, store common.Store, _ common.Filter) (uint64, error) { var count uint64 switch resourceTypeValue { - case common.ResourceTypeVpc: - vpc := &model.Vpc{ - Path: &vpcPath, + case common.ResourceTypeVpcAttachment: + vpcAttachment := &model.VpcAttachment{ + ParentPath: &vpcPath, + Path: &vpcAttachmentPath, VpcConnectivityProfile: &vpcConnectivityProfilePath2, } - store.Apply(vpc) + store.Apply(vpcAttachment) count = 1 case common.ResourceTypeVpcConnectivityProfile: vpcConnectivityProfile1 := &model.VpcConnectivityProfile{ @@ -167,3 +172,107 @@ func TestIPBlocksInfoService_SyncIPBlocksInfo(t *testing.T) { err := service.SyncIPBlocksInfo(context.TODO()) assert.Equal(t, nil, err) } + +func TestIPBlocksInfoService_StartPeriodicSync(t *testing.T) { + ipBlocksInfoService := &IPBlocksInfoService{ + Service: common.Service{}, + SyncTask: NewIPBlocksInfoSyncTask(time.Millisecond*100, time.Millisecond*50), + } + done := make(chan bool) + go func() { + syncIPBlocksInfoPatch := gomonkey.ApplyMethod(reflect.TypeOf(ipBlocksInfoService), "SyncIPBlocksInfo", func(_ *IPBlocksInfoService, cxt context.Context) error { + return fmt.Errorf("mock error") + }) + defer syncIPBlocksInfoPatch.Reset() + ipBlocksInfoService.StartPeriodicSync() + done <- true + }() + + time.Sleep(time.Millisecond * 20) + ipBlocksInfoService.SyncTask.resetChan <- struct{}{} + + select { + case <-done: + t.Error("StartPeriodicSync stop unexpectedly") + case <-time.After(time.Millisecond * 500): + // Stop StartPeriodicSync after some time + } +} + +func TestIPBlocksInfoService_getIPBlockCIDRsFromStore(t *testing.T) { + ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.IpAddressBlockBindingType(), + }} + ipblock1 := model.IpAddressBlock{ + Path: &ipBlocksPath1, + } + ipBlockStore.Apply(&ipblock1) + service := &IPBlocksInfoService{} + + // Fetch non-existed IPBlocks + pathSet := sets.New[string]() + pathSet.Insert(ipBlocksPath2) + _, err := service.getIPBlockCIDRsFromStore(pathSet, ipBlockStore) + assert.ErrorContains(t, err, "failed to get IPBlock") + + // No CIDR in IPBlocks + pathSet = sets.New[string]() + pathSet.Insert(ipBlocksPath1) + _, err = service.getIPBlockCIDRsFromStore(pathSet, ipBlockStore) + assert.ErrorContains(t, err, "failed to get CIDR from ipblock") +} + +func TestIPBlocksInfoService_createOrUpdateIPBlocksInfo(t *testing.T) { + service, mockController, mockK8sClient := createService(t) + defer mockController.Finish() + + ipBlocksInfo := v1alpha1.IPBlocksInfo{} + mockErr := fmt.Errorf("mock error") + + // Fail to get IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(mockErr) + + err := service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) + + // Fail to create IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(apierrors.NewNotFound(v1alpha1.Resource("IPBlocksInfo"), "ipBlocksInfoName")) + mockK8sClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(mockErr) + err = service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) + + // // Fail to udpate IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + ipBlocksInfoCR := obj.(*v1alpha1.IPBlocksInfo) + ipBlocksInfoCR.ExternalIPCIDRs = []string{ipBlocksMap[ipBlocksPath4]} + return nil + }) + mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(mockErr) + err = service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) +} + +func TestIsDefaultNetworkConfigCR(t *testing.T) { + testCRD1 := v1alpha1.VPCNetworkConfiguration{} + testCRD1.Name = "test-1" + testCRD2 := v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationDefaultNetworkConfig: "invalid", + }, + }, + } + testCRD2.Name = "test-2" + testCRD3 := v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationDefaultNetworkConfig: "true", + }, + }, + } + testCRD3.Name = "test-3" + assert.Equal(t, isDefaultNetworkConfigCR(testCRD1), false) + assert.Equal(t, isDefaultNetworkConfigCR(testCRD2), false) + assert.Equal(t, isDefaultNetworkConfigCR(testCRD3), true) +} diff --git a/pkg/nsx/services/ipblocksinfo/store.go b/pkg/nsx/services/ipblocksinfo/store.go index d09740c63..4fd050620 100644 --- a/pkg/nsx/services/ipblocksinfo/store.go +++ b/pkg/nsx/services/ipblocksinfo/store.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" ) @@ -17,6 +18,8 @@ func keyFunc(obj interface{}) (string, error) { return *v.Path, nil case *model.IpAddressBlock: return *v.Path, nil + case *model.VpcAttachment: + return *v.Path, nil default: return "", fmt.Errorf("keyFunc doesn't support unknown type %s", v) } @@ -72,27 +75,55 @@ func (is *IPBlockStore) Apply(i interface{}) error { return nil } -type VPCStore struct { +type VpcAttachmentStore struct { common.ResourceStore } -func (vs *VPCStore) Apply(i interface{}) error { +func NewVpcAttachmentStore() *VpcAttachmentStore { + return &VpcAttachmentStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.VpcAttachmentBindingType(), + }} +} + +func (vas *VpcAttachmentStore) Apply(i interface{}) error { if i == nil { return nil } - vpc := i.(*model.Vpc) - if vpc.MarkedForDelete != nil && *vpc.MarkedForDelete { - err := vs.Delete(vpc) - log.V(1).Info("delete VPC from store", "VPC", vpc) + attachment := i.(*model.VpcAttachment) + if attachment.MarkedForDelete != nil && *attachment.MarkedForDelete { + err := vas.Delete(attachment) + log.V(1).Info("delete VPC attachment from store", "VpcAttachment", attachment) if err != nil { return err } } else { - err := vs.Add(vpc) - log.V(1).Info("add VPC to store", "VPC", vpc) + err := vas.Add(attachment) + log.V(1).Info("add VPC attachment to store", "VpcAttachment", attachment) if err != nil { return err } } return nil } + +func (vas *VpcAttachmentStore) GetByKey(path string) *model.VpcAttachment { + obj := vas.ResourceStore.GetByKey(path) + if obj != nil { + attachment := obj.(*model.VpcAttachment) + return attachment + } + return nil +} + +func (vas *VpcAttachmentStore) GetByVpcPath(vpcPath string) []*model.VpcAttachment { + result := []*model.VpcAttachment{} + attachments := vas.ResourceStore.List() + for _, item := range attachments { + attachment := item.(*model.VpcAttachment) + if attachment != nil && *attachment.ParentPath == vpcPath { + result = append(result, attachment) + } + } + return result +} diff --git a/pkg/nsx/services/ipblocksinfo/store_test.go b/pkg/nsx/services/ipblocksinfo/store_test.go index 1159a03a0..ec675bc48 100644 --- a/pkg/nsx/services/ipblocksinfo/store_test.go +++ b/pkg/nsx/services/ipblocksinfo/store_test.go @@ -5,15 +5,24 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +var ( + fakeVpcPath = "vpc-path" + fakeVpcProfilePath = "vpc-connectivity-profile-path" + fakeIpBlockPath = "ip-block-path" + fakeVpcAttachmentPath = fakeVpcPath + "/attachments/defaults" + fakeDeleted = true ) func Test_KeyFunc(t *testing.T) { - vpcPath := "vpc-path" - vpc := model.Vpc{Path: &vpcPath} - vpcProfilePath := "vpc-connectivity-profile-path" - vpcProfile := model.VpcConnectivityProfile{Path: &vpcProfilePath} - ipBlockPath := "ip-block-path" - ipBlock := model.IpAddressBlock{Path: &ipBlockPath} + vpc := model.Vpc{Path: &fakeVpcPath} + vpcProfile := model.VpcConnectivityProfile{Path: &fakeVpcProfilePath} + ipBlock := model.IpAddressBlock{Path: &fakeIpBlockPath} + notSupported := struct{}{} type args struct { obj interface{} @@ -23,32 +32,180 @@ func Test_KeyFunc(t *testing.T) { name string expectedKey string item args + expectedErr bool }{ { name: "Vpc", item: args{obj: &vpc}, - expectedKey: vpcPath, + expectedKey: fakeVpcPath, }, { name: "VpcConnectivityProfile", item: args{obj: &vpcProfile}, - expectedKey: vpcProfilePath, + expectedKey: fakeVpcProfilePath, }, { name: "IpBlock", item: args{obj: &ipBlock}, - expectedKey: ipBlockPath, + expectedKey: fakeIpBlockPath, + }, + { + name: "NotSupported", + item: args{obj: ¬Supported}, + expectedErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := keyFunc(tt.item.obj) - assert.Nil(t, err) - if got != tt.expectedKey { - t.Errorf("keyFunc() = %v, want %v", got, tt.expectedKey) + if !tt.expectedErr { + assert.Nil(t, err) + if got != tt.expectedKey { + t.Errorf("keyFunc() = %v, want %v", got, tt.expectedKey) + } + } else { + assert.NotNil(t, err) } + + }) + } + +} + +func TestVPCConnectivityProfileStore_Apply(t *testing.T) { + vpcConnectivityProfileStore := &VPCConnectivityProfileStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.VpcConnectivityProfileBindingType(), + }} + + profile1 := model.VpcConnectivityProfile{ + Path: &fakeVpcProfilePath, + } + profile2 := model.VpcConnectivityProfile{ + Path: &fakeVpcProfilePath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &profile1}}, + {"Delete", args{i: &profile2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := vpcConnectivityProfileStore.Apply(tt.args.i) + assert.Nil(t, err) + }) + } +} + +func TestIPBlockStore_Apply(t *testing.T) { + ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.IpAddressBlockBindingType(), + }} + + ipblock1 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, + } + ipblock2 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &ipblock1}}, + {"Delete", args{i: &ipblock2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ipBlockStore.Apply(tt.args.i) + assert.Nil(t, err) }) } +} +func TestVpcAttachmentStore_Apply(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() + + attachment1 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + } + attachment2 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &attachment1}}, + {"Delete", args{i: &attachment2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := vpcAttachmentStore.Apply(tt.args.i) + assert.Nil(t, err) + }) + } +} +func TestVpcAttachmentStore_GetByKey(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() + + attachment := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + } + + err := vpcAttachmentStore.Apply(&attachment) + assert.Nil(t, err) + + t.Run("GetByKey", func(t *testing.T) { + got := vpcAttachmentStore.GetByKey(fakeVpcAttachmentPath) + assert.NotNil(t, got) + assert.Equal(t, &attachment, got) + }) +} + +func TestVpcAttachmentStore_GetByVpcPath(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() + + attachment1 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + } + attachment2 := model.VpcAttachment{ + Path: common.String(fakeVpcAttachmentPath + "2"), + ParentPath: &fakeVpcPath, + } + + err := vpcAttachmentStore.Apply(&attachment1) + assert.Nil(t, err) + err = vpcAttachmentStore.Apply(&attachment2) + assert.Nil(t, err) + t.Run("GetByVpcPath", func(t *testing.T) { + got := vpcAttachmentStore.GetByVpcPath(fakeVpcPath) + assert.NotNil(t, got) + assert.Equal(t, 2, len(got)) + assert.Contains(t, got, &attachment1) + assert.Contains(t, got, &attachment2) + }) } diff --git a/pkg/nsx/services/securitypolicy/builder.go b/pkg/nsx/services/securitypolicy/builder.go index b1a31e45e..6a7bfb56a 100644 --- a/pkg/nsx/services/securitypolicy/builder.go +++ b/pkg/nsx/services/securitypolicy/builder.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strconv" "strings" @@ -14,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" @@ -39,24 +39,15 @@ var ( Int64 = common.Int64 ) -func (service *SecurityPolicyService) buildSecurityPolicyName(obj *v1alpha1.SecurityPolicy, createdFor string) string { - if IsVPCEnabled(service) { - // For VPC scenario, we use obj.Name as the NSX resource display name for both SecurityPolicy and NetworkPolicy. - return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", "", "", "") - } - prefix := common.SecurityPolicyPrefix - if createdFor != common.ResourceTypeSecurityPolicy { - prefix = common.NetworkPolicyPrefix - } - // For T1 scenario, we use ns-name as the key resource name for SecurityPolicy, it is to be consistent with the - // previous solutions. - return util.GenerateTruncName(common.MaxNameLength, strings.Join([]string{obj.Namespace, obj.Name}, common.ConnectorUnderline), prefix, "", "", "") +func (service *SecurityPolicyService) buildSecurityPolicyName(obj *v1alpha1.SecurityPolicy) string { + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", "", "", "") } func (service *SecurityPolicyService) buildSecurityPolicyID(obj *v1alpha1.SecurityPolicy, createdFor string) string { if IsVPCEnabled(service) { return util.GenerateIDByObject(obj) } + prefix := common.SecurityPolicyPrefix if createdFor != common.ResourceTypeSecurityPolicy { prefix = common.NetworkPolicyPrefix @@ -76,7 +67,7 @@ func (service *SecurityPolicyService) buildSecurityPolicy(obj *v1alpha1.Security nsxSecurityPolicy := &model.SecurityPolicy{} nsxSecurityPolicy.Id = String(service.buildSecurityPolicyID(obj, createdFor)) - nsxSecurityPolicy.DisplayName = String(service.buildSecurityPolicyName(obj, createdFor)) + nsxSecurityPolicy.DisplayName = String(service.buildSecurityPolicyName(obj)) // TODO: confirm the sequence number: offset nsxSecurityPolicy.SequenceNumber = Int64(int64(obj.Spec.Priority)) @@ -93,7 +84,7 @@ func (service *SecurityPolicyService) buildSecurityPolicy(obj *v1alpha1.Security currentSet := sets.Set[string]{} for ruleIdx, r := range obj.Spec.Rules { rule := r - // A rule containing named port may expand to multiple rules if the name maps to multiple port numbers. + // A rule containing named port may be expanded to multiple rules if the named ports map to multiple port numbers. expandRules, buildGroups, buildGroupShares, err := service.buildRuleAndGroups(obj, &rule, ruleIdx, createdFor) if err != nil { log.Error(err, "failed to build rule and groups", "rule", rule, "ruleIndex", ruleIdx) @@ -204,11 +195,6 @@ func (service *SecurityPolicyService) buildTargetTags(obj *v1alpha1.SecurityPoli rule *v1alpha1.SecurityPolicyRule, ruleIdx int, createdFor string, ) []model.Tag { basicTags := service.buildBasicTags(obj, createdFor) - sort.Slice(*targets, func(i, j int) bool { - k1, _ := json.Marshal((*targets)[i]) - k2, _ := json.Marshal((*targets)[j]) - return string(k1) < string(k2) - }) serializedBytes, _ := json.Marshal(*targets) targetTags := []model.Tag{ { @@ -242,7 +228,7 @@ func (service *SecurityPolicyService) buildTargetTags(obj *v1alpha1.SecurityPoli targetTags = append(targetTags, model.Tag{ Scope: String(common.TagScopeRuleID), - Tag: String(service.buildRuleID(obj, rule, ruleIdx, createdFor)), + Tag: String(service.buildRuleID(obj, ruleIdx, createdFor)), }, ) } @@ -379,7 +365,8 @@ func (service *SecurityPolicyService) buildAppliedGroupID(obj *v1alpha1.Security if IsVPCEnabled(service) { suffix := common.TargetGroupSuffix if ruleIdx != -1 { - suffix = strings.Join([]string{strconv.Itoa(ruleIdx), suffix}, common.ConnectorUnderline) + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) } return util.GenerateIDByObjectWithSuffix(obj, suffix) } @@ -415,17 +402,13 @@ func (service *SecurityPolicyService) buildAppliedGroupPath(obj *v1alpha1.Securi // build appliedTo group display name for both policy and rule levels. func (service *SecurityPolicyService) buildAppliedGroupName(obj *v1alpha1.SecurityPolicy, ruleIdx int) string { - var rule *v1alpha1.SecurityPolicyRule if ruleIdx != -1 { - rule = &(obj.Spec.Rules[ruleIdx]) - ruleName := strings.Join([]string{obj.Name, strconv.Itoa(ruleIdx)}, common.ConnectorUnderline) - if len(rule.Name) > 0 { - ruleName = rule.Name - } - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", common.TargetGroupSuffix, "", "") + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix := strings.Join([]string{ruleHash, common.TargetGroupSuffix}, common.ConnectorUnderline) + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", suffix, "", "") } - ruleName := strings.Join([]string{obj.Namespace, obj.Name}, common.ConnectorUnderline) - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", common.TargetGroupSuffix, "", "") + + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", common.TargetGroupSuffix, "", "") } func (service *SecurityPolicyService) buildRuleAndGroups(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, @@ -598,22 +581,44 @@ func (service *SecurityPolicyService) buildRuleOutGroup(obj *v1alpha1.SecurityPo return nsxRuleDstGroup, nsxRuleSrcGroupPath, nsxRuleDstGroupPath, nsxGroupShare, nil } -func (service *SecurityPolicyService) buildRuleID(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, ruleIdx int, createdFor string) string { - serializedBytes, _ := json.Marshal(rule) - ruleHash := fmt.Sprintf("%s", util.Sha1(string(serializedBytes))) - ruleIdxStr := fmt.Sprintf("%d", ruleIdx) +func (service *SecurityPolicyService) buildRuleID(obj *v1alpha1.SecurityPolicy, ruleIdx int, createdFor string) string { + ruleIndexHash := service.buildRuleHashString(&(obj.Spec.Rules[ruleIdx])) + if IsVPCEnabled(service) { - suffix := strings.Join([]string{ruleIdxStr, ruleHash}, common.ConnectorUnderline) - return util.GenerateIDByObjectWithSuffix(obj, suffix) + ruleIndexHash = service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + return util.GenerateIDByObjectWithSuffix(obj, ruleIndexHash) } + prefix := common.SecurityPolicyPrefix if createdFor == common.ResourceTypeNetworkPolicy { prefix = common.NetworkPolicyPrefix } - return util.GenerateID(fmt.Sprintf("%s", obj.UID), prefix, ruleHash, ruleIdxStr) + ruleIdxStr := fmt.Sprintf("%d", ruleIdx) + return strings.Join([]string{prefix, string(obj.UID), ruleIndexHash, ruleIdxStr}, common.ConnectorUnderline) } -func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.SecurityPolicyRule, portIdx, portNumber int, hasNamedport bool, createdFor string) (string, error) { +// A rule containing named port may be expanded to multiple NSX rules if the name ports map to multiple port numbers. +// So, in VPC network, the rule port numbers, which either are defined in rule Port or resolved from named port, will be appended as CR rule baseID to distinguish them. +// For T1, the portIdx and portAddressIdx are appended as suffix. +func (service *SecurityPolicyService) buildExpandedRuleID(obj *v1alpha1.SecurityPolicy, ruleIdx int, portIdx, portAddressIdx int, + hasNamedport bool, portNumber int, createdFor string, +) string { + ruleBaseID := service.buildRuleID(obj, ruleIdx, createdFor) + + if IsVPCEnabled(service) { + portNumberSuffix := "" + if !hasNamedport { + portNumberSuffix = service.buildRulePortsNumberString(obj.Spec.Rules[ruleIdx].Ports) + } else { + portNumberSuffix = service.buildRulePortNumberString(obj.Spec.Rules[ruleIdx].Ports[portIdx], portNumber) + } + return strings.Join([]string{ruleBaseID, portNumberSuffix}, common.ConnectorUnderline) + } + + return strings.Join([]string{ruleBaseID, strconv.Itoa(portIdx), strconv.Itoa(portAddressIdx)}, common.ConnectorUnderline) +} + +func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.SecurityPolicyRule, portIdx int, hasNamedport bool, portNumber int, createdFor string) (string, error) { var ruleName string var ruleAct string @@ -651,7 +656,7 @@ func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.Securi ruleName = strings.Join([]string{ruleName, suffix}, common.ConnectorUnderline) } } else { - ruleName = service.buildRulePortsString(&rule.Ports, suffix) + ruleName = service.buildRulePortsString(rule.Ports, suffix) } if !hasNamedport { @@ -659,11 +664,11 @@ func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.Securi } else { // For the security policy rule with namedPort, it will be expanded to the multiple security policy rules based on resolution of named port. // e.g. input: security policy's rule name: TCP.http_UDP.1234_ingress_allow, - // expand to NSX security policy rules with name TCP.http_UDP.1234_TCP.80_ingress_allow and TCP.http_UDP.1234_UDP.1234_ingress_allow. + // expand to NSX security policy rules with name TCP.http_UDP.1234.TCP.80_ingress_allow and TCP.http_UDP.1234.UDP.1234_ingress_allow. // in case that user defined input security policy's rule name: sp_namedport_rule, // expand to NSX security policy rules with name sp_namedport_rule.TCP.80_ingress_allow and sp_namedport_rule.UDP.1234_ingress_allow. index := strings.Index(ruleName, common.ConnectorUnderline+suffix) - return util.GenerateTruncName(common.MaxNameLength, ruleName[:index]+"."+service.buildRulePortString(&rule.Ports[portIdx], true, portNumber), "", suffix, "", ""), nil + return util.GenerateTruncName(common.MaxNameLength, ruleName[:index]+"."+service.buildRulePortString(rule.Ports[portIdx], portNumber), "", suffix, "", ""), nil } } @@ -747,24 +752,25 @@ func (service *SecurityPolicyService) buildRulePeerGroupID(obj *v1alpha1.Securit if isSource == true { suffix = common.SrcGroupSuffix } + if IsVPCEnabled(service) { - suffix = strings.Join([]string{strconv.Itoa(ruleIdx), suffix}, common.ConnectorUnderline) + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) return util.GenerateIDByObjectWithSuffix(obj, suffix) } + return util.GenerateID(string(obj.UID), common.SecurityPolicyPrefix, suffix, strconv.Itoa(ruleIdx)) } func (service *SecurityPolicyService) buildRulePeerGroupName(obj *v1alpha1.SecurityPolicy, ruleIdx int, isSource bool) string { - rule := &(obj.Spec.Rules[ruleIdx]) suffix := common.DstGroupSuffix if isSource == true { suffix = common.SrcGroupSuffix } - ruleName := strings.Join([]string{obj.Name, strconv.Itoa(ruleIdx)}, common.ConnectorUnderline) - if len(rule.Name) > 0 { - ruleName = rule.Name - } - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", suffix, "", "") + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) + + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", suffix, "", "") } func (service *SecurityPolicyService) buildRulePeerGroupPath(obj *v1alpha1.SecurityPolicy, ruleIdx int, isSource, infraGroupShared, projectGroupShared bool, vpcInfo *common.VPCResourceInfo) (string, error) { @@ -913,15 +919,11 @@ func (service *SecurityPolicyService) buildRulePeerGroup(obj *v1alpha1.SecurityP return &rulePeerGroup, rulePeerGroupPath, nil, err } -func (service *SecurityPolicyService) buildExpandedRuleId(ruleBaseId string, portIdx int, portAddressIdx int) string { - return strings.Join([]string{ruleBaseId, strconv.Itoa(portIdx), strconv.Itoa(portAddressIdx)}, common.ConnectorUnderline) -} - // Build rule basic info, ruleIdx is the index of the rules of security policy, // portIdx is the index of rule's ports, portAddressIdx is the index // of multiple port number if one named port maps to multiple port numbers. func (service *SecurityPolicyService) buildRuleBasicInfo(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, ruleIdx int, portIdx int, portAddressIdx int, - portNumber int, hasNamedport bool, createdFor string, + hasNamedport bool, portNumber int, createdFor string, ) (*model.Rule, error) { ruleAction, err := getRuleAction(rule) if err != nil { @@ -931,13 +933,13 @@ func (service *SecurityPolicyService) buildRuleBasicInfo(obj *v1alpha1.SecurityP if err != nil { return nil, err } - displayName, err := service.buildRuleDisplayName(rule, portIdx, portNumber, hasNamedport, createdFor) + displayName, err := service.buildRuleDisplayName(rule, portIdx, hasNamedport, portNumber, createdFor) if err != nil { log.Error(err, "failed to build rule's display name", "securityPolicyUID", obj.UID, "rule", rule, "createdFor", createdFor) } nsxRule := model.Rule{ - Id: String(service.buildExpandedRuleId(service.buildRuleID(obj, rule, ruleIdx, createdFor), portIdx, portAddressIdx)), + Id: String(service.buildExpandedRuleID(obj, ruleIdx, portIdx, portAddressIdx, hasNamedport, portNumber, createdFor)), DisplayName: &displayName, Direction: &ruleDirection, SequenceNumber: Int64(int64(ruleIdx)), @@ -957,13 +959,6 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy groupTypeTag = String(common.TagValueGroupSource) peers = &rule.Sources } - - // TODO: abstract sort func for both peers and targets - sort.Slice(*peers, func(i, j int) bool { - k1, _ := json.Marshal((*peers)[i]) - k2, _ := json.Marshal((*peers)[j]) - return string(k1) < string(k2) - }) serializedBytes, _ := json.Marshal(*peers) peerTags := []model.Tag{ @@ -973,7 +968,7 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy }, { Scope: String(common.TagScopeRuleID), - Tag: String(service.buildRuleID(obj, rule, ruleIdx, createdFor)), + Tag: String(service.buildRuleID(obj, ruleIdx, createdFor)), }, { Scope: String(common.TagScopeSelectorHash), @@ -1009,6 +1004,7 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy ) } } + return peerTags } @@ -1847,7 +1843,7 @@ func (service *SecurityPolicyService) getNamespaceUID(ns string) (nsUid types.UI return namespace_uid } -func (service *SecurityPolicyService) buildRulePortString(port *v1alpha1.SecurityPolicyPort, hasNamedport bool, portNumber int) string { +func (service *SecurityPolicyService) buildRulePortString(port v1alpha1.SecurityPolicyPort, portNumber int) string { protocol := string(port.Protocol) // Build the rule port string name for non named port. // This is a common case where the string is built from port definition. For instance, @@ -1858,36 +1854,77 @@ func (service *SecurityPolicyService) buildRulePortString(port *v1alpha1.Securit // - protocol: UDP // port: 3308 // The built port string is: UDP.3308 - if !hasNamedport { - if port.EndPort != 0 { - return fmt.Sprintf("%s.%s.%d", protocol, (port.Port).String(), port.EndPort) - } - return fmt.Sprintf("%s.%s", protocol, (port.Port).String()) - } else { + if port.Port.Type == intstr.String && portNumber > 0 { // Build the rule port string name for named port. // The port string is built from specific port number resolved from named port. return fmt.Sprintf("%s.%d", protocol, portNumber) } + if port.EndPort != 0 { + return fmt.Sprintf("%s.%s.%d", protocol, (port.Port).String(), port.EndPort) + } + return fmt.Sprintf("%s.%s", protocol, (port.Port).String()) } -func (service *SecurityPolicyService) buildRulePortsString(ports *[]v1alpha1.SecurityPolicyPort, suffix string) string { +func (service *SecurityPolicyService) buildRulePortsString(ports []v1alpha1.SecurityPolicyPort, suffix string) string { portsString := "" - if ports == nil || len(*ports) == 0 { + if ports == nil || len(ports) == 0 { portsString = common.RuleAnyPorts } else { - for idx, p := range *ports { + portStrings := make([]string, len(ports)) + for idx, p := range ports { port := p - portString := service.buildRulePortString(&port, false, -1) - if idx == 0 { - portsString = portString - } else { - portsString = strings.Join([]string{portsString, portString}, common.ConnectorUnderline) - } + portStrings[idx] = service.buildRulePortString(port, -1) } + portsString = strings.Join(portStrings, common.ConnectorUnderline) } + return util.GenerateTruncName(common.MaxNameLength, portsString, "", suffix, "", "") } +func (service *SecurityPolicyService) buildRulePortNumberString(port v1alpha1.SecurityPolicyPort, portNumber int) string { + // Build the rule port number string name for non named port. + // This is a common case where the string is built from port definition. For instance, + // - protocol: TCP + // port: 8282 + // endPort: 8286 + // The built port number string is: 8282.8286 + // - protocol: UDP + // port: 3308 + // The built port number string is: 3308 + if port.Port.Type == intstr.String && portNumber > 0 { + // Build the rule port number string name for named port. + // The port number string is built from specific port number resolved from named port. + return fmt.Sprintf("%d", portNumber) + } + if port.EndPort != 0 { + return fmt.Sprintf("%s.%d", (port.Port).String(), port.EndPort) + } + return (port.Port).String() +} + +func (service *SecurityPolicyService) buildRulePortsNumberString(ports []v1alpha1.SecurityPolicyPort) string { + if ports == nil || len(ports) == 0 { + return common.RuleAnyPorts + } + + portNumStrings := make([]string, len(ports)) + for idx, p := range ports { + port := p + portNumStrings[idx] = service.buildRulePortNumberString(port, -1) + } + return strings.Join(portNumStrings, common.ConnectorUnderline) +} + +func (service *SecurityPolicyService) buildLimitedRuleHashString(rule *v1alpha1.SecurityPolicyRule) string { + serializedBytes, _ := json.Marshal(rule) + return util.Sha1(string(serializedBytes))[:common.HashLength] +} + +func (service *SecurityPolicyService) buildRuleHashString(rule *v1alpha1.SecurityPolicyRule) string { + serializedBytes, _ := json.Marshal(rule) + return util.Sha1(string(serializedBytes)) +} + func (service *SecurityPolicyService) BuildNetworkPolicyAllowPolicyID(uid string) string { return strings.Join([]string{uid, common.RuleActionAllow}, common.ConnectorUnderline) } diff --git a/pkg/nsx/services/securitypolicy/builder_test.go b/pkg/nsx/services/securitypolicy/builder_test.go index 5fbca2def..aef63e991 100644 --- a/pkg/nsx/services/securitypolicy/builder_test.go +++ b/pkg/nsx/services/securitypolicy/builder_test.go @@ -35,19 +35,19 @@ func TestBuildSecurityPolicy(t *testing.T) { ) podSelectorRule0Name00 := "rule-with-pod-ns-selector_ingress_allow" - podSelectorRule0IDPort000 := "sp_uidA_0_2c822e90b1377b346014adfa583f08a99dee52a8_0_0" + podSelectorRule0IDPort000 := "sp_uidA_2c822e90b1377b346014adfa583f08a99dee52a8_0_0_0" podSelectorRule1Name00 := "rule-with-ns-selector_ingress_allow" - podSelectorRule1IDPort000 := "sp_uidA_1_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_0_0" + podSelectorRule1IDPort000 := "sp_uidA_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_1_0_0" vmSelectorRule0Name00 := "rule-with-VM-selector_egress_isolation" - vmSelectorRule0IDPort000 := "sp_uidB_0_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0" + vmSelectorRule0IDPort000 := "sp_uidB_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0_0" vmSelectorRule1Name00 := "rule-with-ns-selector_egress_isolation" - vmSelectorRule1IDPort000 := "sp_uidB_1_7d721f087be35f0bf318f4847b5acdc3d2b91446_0_0" + vmSelectorRule1IDPort000 := "sp_uidB_7d721f087be35f0bf318f4847b5acdc3d2b91446_1_0_0" vmSelectorRule2Name00 := "all_egress_isolation" - vmSelectorRule2IDPort000 := "sp_uidB_2_a40c813916cc397fcd2260e48cc773d4c9b08565_0_0" + vmSelectorRule2IDPort000 := "sp_uidB_a40c813916cc397fcd2260e48cc773d4c9b08565_2_0_0" tests := []struct { name string @@ -96,7 +96,7 @@ func TestBuildSecurityPolicy(t *testing.T) { name: "security-policy-with-VM-selector For T1", inputPolicy: &spWithVMSelector, expectedPolicy: &model.SecurityPolicy{ - DisplayName: common.String("sp_ns1_spB"), + DisplayName: common.String("spB"), Id: common.String("sp_uidB"), Scope: []string{"/infra/domains/k8scl-one/groups/sp_uidB_scope"}, SequenceNumber: &seq0, @@ -201,19 +201,19 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { defer patches.Reset() podSelectorRule0Name00 := "rule-with-pod-ns-selector_ingress_allow" - podSelectorRule0IDPort000 := "spA_uidA_0_2c822e90b1377b346014adfa583f08a99dee52a8_0_0" + podSelectorRule0IDPort000 := "spA_uidA_2c822e90_all" podSelectorRule1Name00 := "rule-with-ns-selector_ingress_allow" - podSelectorRule1IDPort000 := "spA_uidA_1_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_0_0" + podSelectorRule1IDPort000 := "spA_uidA_2a4595d0_53" vmSelectorRule0Name00 := "rule-with-VM-selector_egress_isolation" - vmSelectorRule0IDPort000 := "spB_uidB_0_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0" + vmSelectorRule0IDPort000 := "spB_uidB_67410606_all" vmSelectorRule1Name00 := "rule-with-ns-selector_egress_isolation" - vmSelectorRule1IDPort000 := "spB_uidB_1_7d721f087be35f0bf318f4847b5acdc3d2b91446_0_0" + vmSelectorRule1IDPort000 := "spB_uidB_7d721f08_all" vmSelectorRule2Name00 := "all_egress_isolation" - vmSelectorRule2IDPort000 := "spB_uidB_2_a40c813916cc397fcd2260e48cc773d4c9b08565_0_0" + vmSelectorRule2IDPort000 := "spB_uidB_a40c8139_all" tests := []struct { name string @@ -234,10 +234,10 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { Id: &podSelectorRule0IDPort000, DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_0_scope"}, + Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_2c822e90_scope"}, SequenceNumber: &seq0, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_0_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_2c822e90_src"}, Action: &nsxRuleActionAllow, Tags: vpcBasicTags, }, @@ -249,7 +249,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { Scope: []string{"ANY"}, SequenceNumber: &seq1, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_1_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_2a4595d0_src"}, Action: &nsxRuleActionAllow, ServiceEntries: []*data.StructValue{serviceEntry}, Tags: vpcBasicTags, @@ -270,9 +270,9 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule0Name00, Id: &vmSelectorRule0IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_0_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_67410606_dst"}, Direction: &nsxRuleDirectionOut, - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_0_scope"}, + Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_67410606_scope"}, SequenceNumber: &seq0, Services: []string{"ANY"}, SourceGroups: []string{"ANY"}, @@ -282,7 +282,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule1Name00, Id: &vmSelectorRule1IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spB_uidB_1_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spB_uidB_7d721f08_dst"}, Direction: &nsxRuleDirectionOut, Scope: []string{"ANY"}, SequenceNumber: &seq1, @@ -295,7 +295,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule2Name00, Id: &vmSelectorRule2IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_2_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_a40c8139_dst"}, Direction: &nsxRuleDirectionOut, Scope: []string{"ANY"}, SequenceNumber: &seq2, @@ -354,7 +354,7 @@ func TestBuildTargetTags(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyCRName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyCRUID - ruleTagID0 := service.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy) + ruleTagID0 := service.buildRuleID(&spWithPodSelector, 0, common.ResourceTypeSecurityPolicy) tests := []struct { name string inputPolicy *v1alpha1.SecurityPolicy @@ -437,7 +437,7 @@ func TestBuildTargetTags(t *testing.T) { } func TestBuildPeerTags(t *testing.T) { - ruleTagID0 := service.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy) + ruleTagID0 := service.buildRuleID(&spWithPodSelector, 0, common.ResourceTypeSecurityPolicy) tests := []struct { name string inputPolicy *v1alpha1.SecurityPolicy @@ -825,10 +825,7 @@ func TestUpdateMixedExpressionsMatchExpression(t *testing.T) { } var securityPolicyWithMultipleNormalPorts = v1alpha1.SecurityPolicy{ - ObjectMeta: v1.ObjectMeta{ - Namespace: "null", - Name: "null", - }, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spMulPorts", UID: "spMulPortsuidA"}, Spec: v1alpha1.SecurityPolicySpec{ Rules: []v1alpha1.SecurityPolicyRule{ { @@ -862,25 +859,37 @@ var securityPolicyWithMultipleNormalPorts = v1alpha1.SecurityPolicy{ }, }, }, + { + Action: &allowAction, + Direction: &directionOut, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 80}, + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 1234}, + EndPort: 1234, + }, + }, + }, }, }, } var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ - ObjectMeta: v1.ObjectMeta{ - Namespace: "null", - Name: "null", - }, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spNamedPorts", UID: "spNamedPortsuidA"}, Spec: v1alpha1.SecurityPolicySpec{ Rules: []v1alpha1.SecurityPolicyRule{ { - Name: "TCP.http_UDP.1234.1235_ingress_allow", + Name: "user-defined-rule-namedport", Action: &allowAction, Direction: &directionIn, Ports: []v1alpha1.SecurityPolicyPort{ { Protocol: "TCP", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "http"}, + Port: intstr.IntOrString{Type: intstr.String, StrVal: "http"}, // http port is 80 }, { Protocol: "UDP", @@ -889,6 +898,45 @@ var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ }, }, }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "https"}, // http port is 443 + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 1236}, + EndPort: 1237, + }, + }, + }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "web"}, + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 533}, + }, + }, + }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "db"}, + }, + }, + }, }, }, } @@ -896,22 +944,52 @@ var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ func TestBuildRulePortsString(t *testing.T) { tests := []struct { name string - inputPorts *[]v1alpha1.SecurityPolicyPort + inputPorts []v1alpha1.SecurityPolicyPort suffix string expectedRulePortsString string }{ { name: "build-string-for-multiple-ports-without-named-port", - inputPorts: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, suffix: "ingress_allow", expectedRulePortsString: "TCP.80_UDP.1234.1235_ingress_allow", }, { - name: "build-string-for-multiple-ports-without-one-named-port", - inputPorts: &securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, + name: "build-string-for-multiple-ports-userdefinedrule-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[1].Ports, + suffix: "egress_drop", + expectedRulePortsString: "TCP.88_UDP.1236.1237_egress_drop", + }, + { + name: "build-string-for-multiple-ports-start-end-port-same-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[2].Ports, + suffix: "egress_allow", + expectedRulePortsString: "TCP.80_UDP.1234.1234_egress_allow", + }, + { + name: "build-string-for-multiple-ports-with-http-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, suffix: "ingress_allow", expectedRulePortsString: "TCP.http_UDP.1234.1235_ingress_allow", }, + { + name: "build-string-for-multiple-ports-with-https-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[1].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.https_UDP.1236.1237_ingress_allow", + }, + { + name: "build-string-for-multiple-ports-with-web-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[2].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.web_UDP.533_ingress_allow", + }, + { + name: "build-string-for-multiple-ports-with-db-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[3].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.db_ingress_allow", + }, { name: "build-string-for-nil-ports", inputPorts: nil, @@ -927,6 +1005,61 @@ func TestBuildRulePortsString(t *testing.T) { } } +func TestBuildRulePortsNumberString(t *testing.T) { + tests := []struct { + name string + inputPorts []v1alpha1.SecurityPolicyPort + expectedRulePortsString string + }{ + { + name: "build-string-for-multiple-ports-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, + expectedRulePortsString: "80_1234.1235", + }, + { + name: "build-string-for-multiple-ports-userdefinedrule-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[1].Ports, + expectedRulePortsString: "88_1236.1237", + }, + { + name: "build-string-for-multiple-ports-start-end-port-same-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[2].Ports, + expectedRulePortsString: "80_1234.1234", + }, + { + name: "build-string-for-multiple-ports-with-http-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, + expectedRulePortsString: "http_1234.1235", + }, + { + name: "build-string-for-multiple-ports-with-https-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[1].Ports, + expectedRulePortsString: "https_1236.1237", + }, + { + name: "build-string-for-multiple-ports-with-web-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[2].Ports, + expectedRulePortsString: "web_533", + }, + { + name: "build-string-for-multiple-ports-with-db-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[3].Ports, + expectedRulePortsString: "db", + }, + { + name: "build-string-for-nil-ports", + inputPorts: nil, + expectedRulePortsString: "all", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + observedString := service.buildRulePortsNumberString(tt.inputPorts) + assert.Equal(t, tt.expectedRulePortsString, observedString) + }) + } +} + func TestBuildRuleDisplayName(t *testing.T) { tests := []struct { name string @@ -934,6 +1067,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule *v1alpha1.SecurityPolicyRule ruleIdx int portIdx int + hasNamedPort bool + portNumber int createdFor string expectedRuleDisplayName string }{ @@ -943,6 +1078,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], ruleIdx: 0, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeNetworkPolicy, expectedRuleDisplayName: "TCP.80_UDP.1234.1235_ingress_allow", }, @@ -952,6 +1089,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], ruleIdx: 1, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeNetworkPolicy, expectedRuleDisplayName: "MultipleNormalPorts-rule1", }, @@ -961,19 +1100,143 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], ruleIdx: 1, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeSecurityPolicy, expectedRuleDisplayName: "MultipleNormalPorts-rule1_egress_isolation", }, + { + name: "build-display-name-for-user-defined-rulename-with-one-named-http-port", + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleDisplayName: "user-defined-rule-namedport.TCP.80_ingress_allow", + }, + { + name: "build-display-name-for-multiple-ports-with-one-named-https-port", + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[1], + ruleIdx: 1, + portIdx: 0, + hasNamedPort: true, + portNumber: 443, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleDisplayName: "TCP.https_UDP.1236.1237.TCP.443_ingress_allow", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - observedDisplayName, observedError := service.buildRuleDisplayName(tt.inputRule, tt.portIdx, -1, false, tt.createdFor) + observedDisplayName, observedError := service.buildRuleDisplayName(tt.inputRule, tt.portIdx, tt.hasNamedPort, tt.portNumber, tt.createdFor) assert.Equal(t, tt.expectedRuleDisplayName, observedDisplayName) assert.Equal(t, nil, observedError) }) } } +func TestBuildExpandedRuleID(t *testing.T) { + svc := &SecurityPolicyService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "cluster1", + }, + }, + }, + } + + tests := []struct { + name string + vpcEnabled bool + inputSecurityPolicy *v1alpha1.SecurityPolicy + inputRule *v1alpha1.SecurityPolicyRule + ruleIdx int + portIdx int + portAddressIdx int + hasNamedPort bool + portNumber int + createdFor string + expectedRuleID string + }{ + { + name: "build-ruleID-for-multiple-ports-0-for-vpc", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "spMulPorts_spMulPortsuidA_d0b8e36c_80_1234.1235", + }, + { + name: "build-ruleID-for-multiple-ports-0-for-T1", + vpcEnabled: false, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "sp_spMulPortsuidA_d0b8e36cf858e76624b9706c3c8e77b6006c0e10_0_0_0", + }, + { + name: "build-ruleID-for-multiple-ports-1-for-vpc-NP", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], + ruleIdx: 1, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeNetworkPolicy, + expectedRuleID: "spMulPorts_spMulPortsuidA_555356be_88_1236.1237", + }, + { + name: "build-ruleID-for-multiple-ports-with-one-named-port-for-VPC", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "spNamedPorts_spNamedPortsuidA_3f7c7d8c_80", + }, + { + name: "build-ruleID-for-multiple-ports-with-one-named-port-for-T1", + vpcEnabled: false, + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "sp_spNamedPortsuidA_3f7c7d8c8449687178002f23599add04bf0c3250_0_0_0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc.NSXConfig.EnableVPCNetwork = tt.vpcEnabled + observedRuleID := svc.buildExpandedRuleID(tt.inputSecurityPolicy, tt.ruleIdx, tt.portIdx, tt.portAddressIdx, tt.hasNamedPort, tt.portNumber, tt.createdFor) + assert.Equal(t, tt.expectedRuleID, observedRuleID) + }) + } +} + func TestBuildSecurityPolicyName(t *testing.T) { svc := &SecurityPolicyService{ Service: common.Service{ @@ -1004,7 +1267,7 @@ func TestBuildSecurityPolicyName(t *testing.T) { }, }, createdFor: common.ResourceTypeSecurityPolicy, - expName: "sp_ns1_securitypolicy1", + expName: "securitypolicy1", expId: "sp_uid1", }, { @@ -1052,7 +1315,7 @@ func TestBuildSecurityPolicyName(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { svc.NSXConfig.EnableVPCNetwork = tc.vpcEnabled - name := svc.buildSecurityPolicyName(tc.obj, tc.createdFor) + name := svc.buildSecurityPolicyName(tc.obj) assert.Equal(t, tc.expName, name) assert.True(t, len(name) <= common.MaxNameLength) id := svc.buildSecurityPolicyID(tc.obj, tc.createdFor) @@ -1093,51 +1356,51 @@ func TestBuildGroupName(t *testing.T) { expId string }{ { - name: "src rule without name", + name: "src peer group for rule without user-defined name", ruleIdx: 0, isSource: true, enableVPC: true, - expName: "sp1_0_src", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_src", + expName: "sp1_d0b8e36c_src", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_src", }, { - name: "dst rule without name", + name: "dst peer group for rule without user-defined name", ruleIdx: 0, isSource: false, enableVPC: true, - expName: "sp1_0_dst", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_dst", + expName: "sp1_d0b8e36c_dst", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_dst", }, { - name: "dst rule without name with T1", + name: "dst peer group for rule without user-defined name for T1", ruleIdx: 0, isSource: false, enableVPC: false, - expName: "sp1_0_dst", + expName: "sp1_d0b8e36c_dst", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_dst", }, { - name: "src rule with name", + name: "src peer group for rule with user-defined name", ruleIdx: 1, isSource: true, enableVPC: true, - expName: "MultipleNormalPorts-rule1_src", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_src", + expName: "sp1_555356be_src", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_src", }, { - name: "dst rule with name", + name: "dst peer group for rule with user-defined name", ruleIdx: 1, isSource: false, enableVPC: true, - expName: "MultipleNormalPorts-rule1_dst", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_dst", + expName: "sp1_555356be_dst", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_dst", }, { - name: "dst rule with name with T1", + name: "dst peer group for rule with user-defined name for T1", ruleIdx: 1, isSource: false, enableVPC: false, - expName: "MultipleNormalPorts-rule1_dst", + expName: "sp1_555356be_dst", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_dst", }, } { @@ -1161,31 +1424,45 @@ func TestBuildGroupName(t *testing.T) { expId string }{ { - name: "rule without name", + name: "applied group for rule without user-defined name", ruleIdx: 0, enableVPC: true, - expName: "sp1_0_scope", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_scope", + expName: "sp1_d0b8e36c_scope", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_scope", }, { - name: "rule with name", + name: "applied group for rule with user-defined name", ruleIdx: 1, enableVPC: true, - expName: "MultipleNormalPorts-rule1_scope", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_scope", + expName: "sp1_555356be_scope", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_scope", + }, + { + name: "applied group for rule without user-defined name", + ruleIdx: 0, + enableVPC: false, + expName: "sp1_d0b8e36c_scope", + expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_scope", + }, + { + name: "applied group fpr rule with user-defined name for T1", + ruleIdx: 1, + enableVPC: false, + expName: "sp1_555356be_scope", + expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_scope", }, { name: "policy applied group", ruleIdx: -1, enableVPC: true, - expName: "ns1_sp1_scope", + expName: "sp1_scope", expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_scope", }, { - name: "policy applied group with T1", + name: "policy applied group for T1", ruleIdx: -1, enableVPC: false, - expName: "ns1_sp1_scope", + expName: "sp1_scope", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_scope", }, } { diff --git a/pkg/nsx/services/securitypolicy/expand.go b/pkg/nsx/services/securitypolicy/expand.go index 91ee24139..5a78496ef 100644 --- a/pkg/nsx/services/securitypolicy/expand.go +++ b/pkg/nsx/services/securitypolicy/expand.go @@ -28,7 +28,7 @@ func (service *SecurityPolicyService) expandRule(obj *v1alpha1.SecurityPolicy, r var nsxGroups []*model.Group if len(rule.Ports) == 0 { - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, -1, false, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, false, -1, createdFor) if err != nil { return nil, nil, err } @@ -39,7 +39,7 @@ func (service *SecurityPolicyService) expandRule(obj *v1alpha1.SecurityPolicy, r // Check if there is a namedport in the rule hasNamedPort := service.hasNamedPort(rule) if !hasNamedPort { - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, -1, false, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, false, -1, createdFor) if err != nil { return nil, nil, err } @@ -88,7 +88,7 @@ func (service *SecurityPolicyService) expandRuleByPort(obj *v1alpha1.SecurityPol if err != nil { // In case there is no more valid ip set selected, so clear the stale ip set group in NSX if stale ips exist if errors.As(err, &nsxutil.NoEffectiveOption{}) { - groups := service.groupStore.GetByIndex(common.TagScopeRuleID, service.buildRuleID(obj, rule, ruleIdx, createdFor)) + groups := service.groupStore.GetByIndex(common.TagScopeRuleID, service.buildRuleID(obj, ruleIdx, createdFor)) var ipSetGroup *model.Group for _, group := range groups { ipSetGroup = group @@ -121,7 +121,7 @@ func (service *SecurityPolicyService) expandRuleByService(obj *v1alpha1.Security ) ([]*model.Group, *model.Rule, error) { var nsxGroups []*model.Group - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, portIdx, portAddressIdx, portAddress.Port, true, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, portIdx, portAddressIdx, true, portAddress.Port, createdFor) if err != nil { return nil, nil, err } diff --git a/pkg/nsx/services/securitypolicy/expand_test.go b/pkg/nsx/services/securitypolicy/expand_test.go index 7f30e60ac..56c30485e 100644 --- a/pkg/nsx/services/securitypolicy/expand_test.go +++ b/pkg/nsx/services/securitypolicy/expand_test.go @@ -24,9 +24,23 @@ import ( func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { sp := &v1alpha1.SecurityPolicy{ ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spA", UID: "uidA"}, + Spec: v1alpha1.SecurityPolicySpec{ + Rules: []v1alpha1.SecurityPolicyRule{ + { + Action: &allowAction, + Direction: &directionIn, + Sources: []v1alpha1.SecurityPolicyPeer{ + { + PodSelector: &v1.LabelSelector{ + MatchLabels: map[string]string{"pod_selector_1": "pod_value_1"}, + }, + }, + }, + }, + }, + }, } - rule := v1alpha1.SecurityPolicyRule{} nsxRule := model.Rule{ DisplayName: &ruleNameWithPodSelector00, Id: &ruleIDPort000, @@ -66,7 +80,7 @@ func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { DisplayName: &policyGroupName, Expression: []*data.StructValue{blockExpression}, // build ipset group tags from input securitypolicy and securitypolicy rule - Tags: service.buildPeerTags(sp, &rule, 0, false, false, false, common.ResourceTypeSecurityPolicy), + Tags: service.buildPeerTags(sp, &sp.Spec.Rules[0], 0, false, false, false, common.ResourceTypeSecurityPolicy), } type args struct { @@ -82,7 +96,7 @@ func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, service.buildRuleIPSetGroup(sp, &rule, tt.args.obj, tt.args.ips, 0, common.ResourceTypeSecurityPolicy), "buildRuleIPSetGroup(%v, %v)", + assert.Equalf(t, tt.want, service.buildRuleIPSetGroup(sp, &sp.Spec.Rules[0], tt.args.obj, tt.args.ips, 0, common.ResourceTypeSecurityPolicy), "buildRuleIPSetGroup(%v, %v)", tt.args.obj, tt.args.ips) }) } diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 3d06ad578..ad2b63233 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -144,6 +144,7 @@ func (s *SecurityPolicyService) setUpStore(indexScope string) { keyFunc, cache.Indexers{ indexScope: indexBySecurityPolicyUID, common.TagScopeNetworkPolicyUID: indexByNetworkPolicyUID, + common.TagScopeNamespace: indexBySecurityPolicyNamespace, }), BindingType: model.SecurityPolicyBindingType(), }} @@ -1104,6 +1105,30 @@ func (service *SecurityPolicyService) ListNetworkPolicyID() sets.Set[string] { return service.getGCSecurityPolicyIDSet(indexScope) } +func (service *SecurityPolicyService) ListSecurityPolicyByName(ns, name string) []*model.SecurityPolicy { + var result []*model.SecurityPolicy + securityPolicies := service.securityPolicyStore.GetByIndex(common.TagScopeNamespace, ns) + for _, securityPolicy := range securityPolicies { + securityPolicyCRName := nsxutil.FindTag(securityPolicy.Tags, common.TagValueScopeSecurityPolicyName) + if securityPolicyCRName == name { + result = append(result, securityPolicy) + } + } + return result +} + +func (service *SecurityPolicyService) ListNetworkPolicyByName(ns, name string) []*model.SecurityPolicy { + var result []*model.SecurityPolicy + securityPolicies := service.securityPolicyStore.GetByIndex(common.TagScopeNamespace, ns) + for _, securityPolicy := range securityPolicies { + securityPolicyCRName := nsxutil.FindTag(securityPolicy.Tags, common.TagScopeNetworkPolicyName) + if securityPolicyCRName == name { + result = append(result, securityPolicy) + } + } + return result +} + func (service *SecurityPolicyService) Cleanup(ctx context.Context) error { // Delete all the security policies in store uids := service.ListSecurityPolicyID() diff --git a/pkg/nsx/services/securitypolicy/firewall_test.go b/pkg/nsx/services/securitypolicy/firewall_test.go index 85969bc3f..38d268ee1 100644 --- a/pkg/nsx/services/securitypolicy/firewall_test.go +++ b/pkg/nsx/services/securitypolicy/firewall_test.go @@ -42,8 +42,8 @@ var ( tagScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID tagScopeRuleID = common.TagScopeRuleID tagScopeSelectorHash = common.TagScopeSelectorHash - spName = "sp_ns1_spA" - spGroupName = "ns1_spA_scope" + spName = "spA" + spGroupName = "spA_scope" spID = "sp_uidA" spID2 = "sp_uidB" spGroupID = "sp_uidA_scope" @@ -1073,11 +1073,11 @@ func TestCreateOrUpdateSecurityPolicy(t *testing.T) { mockVPCService := common.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - podSelectorRule0IDPort000 := fakeService.buildExpandedRuleId(fakeService.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy), 0, 0) - podSelectorRule1IDPort000 := fakeService.buildExpandedRuleId(fakeService.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[1], 1, common.ResourceTypeSecurityPolicy), 0, 0) + podSelectorRule0IDPort000 := fakeService.buildExpandedRuleID(&spWithPodSelector, 0, 0, 0, false, -1, common.ResourceTypeSecurityPolicy) + podSelectorRule1IDPort000 := fakeService.buildExpandedRuleID(&spWithPodSelector, 1, 0, 0, false, -1, common.ResourceTypeSecurityPolicy) - podSelectorRule0Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[0], 0, -1, false, common.ResourceTypeSecurityPolicy) - podSelectorRule1Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[1], 0, -1, false, common.ResourceTypeSecurityPolicy) + podSelectorRule0Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[0], 0, false, -1, common.ResourceTypeSecurityPolicy) + podSelectorRule1Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[1], 0, false, -1, common.ResourceTypeSecurityPolicy) type args struct { spObj *v1alpha1.SecurityPolicy @@ -1494,13 +1494,13 @@ func TestGetFinalSecurityPolicyResouceFromNetworkPolicy(t *testing.T) { Rules: []model.Rule{ { DisplayName: common.String("TCP.6001_ingress_allow"), - Id: common.String("np-app-access_uidNP_allow_0_6c2a026ca143812daa72699fb924ee36b33b5cdc_0_0"), + Id: common.String("np-app-access_uidNP_allow_6c2a026c_6001"), DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, Scope: []string{"ANY"}, SequenceNumber: &seq0, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/np-app-access_uidNP_allow_0_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/np-app-access_uidNP_allow_6c2a026c_src"}, Action: &nsxRuleActionAllow, ServiceEntries: []*data.StructValue{serviceEntry}, Tags: npAllowBasicTags, @@ -1516,7 +1516,7 @@ func TestGetFinalSecurityPolicyResouceFromNetworkPolicy(t *testing.T) { Rules: []model.Rule{ { DisplayName: common.String("ingress_isolation"), - Id: common.String("np-app-access_uidNP_isolation_0_114fed106ef3b5eae2a583f312435e84c02ca97f_0_0"), + Id: common.String("np-app-access_uidNP_isolation_114fed10_all"), DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/np-app-access_uidNP_isolation_scope"}, diff --git a/pkg/nsx/services/securitypolicy/store.go b/pkg/nsx/services/securitypolicy/store.go index b7c60026d..d69dd6b4f 100644 --- a/pkg/nsx/services/securitypolicy/store.go +++ b/pkg/nsx/services/securitypolicy/store.go @@ -76,6 +76,15 @@ func indexGroupFunc(obj interface{}) ([]string, error) { } } +func indexBySecurityPolicyNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.SecurityPolicy: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("indexBySecurityPolicyNamespace doesn't support unknown type") + } +} + var filterRuleTag = func(v []model.Tag) []string { res := make([]string, 0, 5) for _, tag := range v { diff --git a/pkg/nsx/services/staticroute/staticroute.go b/pkg/nsx/services/staticroute/staticroute.go index d09b37c4a..4a007b538 100644 --- a/pkg/nsx/services/staticroute/staticroute.go +++ b/pkg/nsx/services/staticroute/staticroute.go @@ -40,6 +40,7 @@ func InitializeStaticRoute(commonService common.Service, vpcService common.VPCSe staticRouteStore := &StaticRouteStore{} staticRouteStore.Indexer = cache.NewIndexer(keyFunc, cache.Indexers{ common.TagScopeStaticRouteCRUID: indexFunc, + common.TagScopeNamespace: indexStaticRouteNamespace, }) staticRouteStore.BindingType = model.StaticRoutesBindingType() staticRouteService.StaticRouteStore = staticRouteStore @@ -148,6 +149,19 @@ func (service *StaticRouteService) DeleteStaticRoute(obj *v1alpha1.StaticRoute) return service.DeleteStaticRouteByPath(vpcResourceInfo.OrgID, vpcResourceInfo.ProjectID, vpcResourceInfo.VPCID, id) } +func (service *StaticRouteService) ListStaticRouteByName(ns, name string) []*model.StaticRoutes { + var result []*model.StaticRoutes + staticroutes := service.StaticRouteStore.GetByIndex(common.TagScopeNamespace, ns) + for _, staticroute := range staticroutes { + sr := staticroute.(*model.StaticRoutes) + tagname := nsxutil.FindTag(sr.Tags, common.TagScopeStaticRouteCRName) + if tagname == name { + result = append(result, staticroute.(*model.StaticRoutes)) + } + } + return result +} + func (service *StaticRouteService) ListStaticRoute() []*model.StaticRoutes { staticRoutes := service.StaticRouteStore.List() staticRouteSet := []*model.StaticRoutes{} diff --git a/pkg/nsx/services/staticroute/store.go b/pkg/nsx/services/staticroute/store.go index b4d13153d..1d64ce2a2 100644 --- a/pkg/nsx/services/staticroute/store.go +++ b/pkg/nsx/services/staticroute/store.go @@ -29,17 +29,26 @@ func indexFunc(obj interface{}) ([]string, error) { res := make([]string, 0, 5) switch v := obj.(type) { case *model.StaticRoutes: - return filterTag(v.Tags), nil + return filterTag(v.Tags, common.TagScopeStaticRouteCRUID), nil default: break } return res, nil } -var filterTag = func(v []model.Tag) []string { +func indexStaticRouteNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.StaticRoutes: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("indexByStaticRouteNamespace doesn't support unknown type") + } +} + +var filterTag = func(v []model.Tag, tagScope string) []string { res := make([]string, 0, 5) for _, tag := range v { - if *tag.Scope == common.TagScopeStaticRouteCRUID { + if *tag.Scope == tagScope { res = append(res, *tag.Tag) } } diff --git a/pkg/nsx/services/subnet/store.go b/pkg/nsx/services/subnet/store.go index bcd12a4cb..6073e3cdc 100644 --- a/pkg/nsx/services/subnet/store.go +++ b/pkg/nsx/services/subnet/store.go @@ -39,6 +39,24 @@ func subnetIndexFunc(obj interface{}) ([]string, error) { } } +func subnetIndexVMNamespaceFunc(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnet: + return filterTag(o.Tags, common.TagScopeVMNamespace), nil + default: + return nil, errors.New("subnetIndexVMNamespaceFunc doesn't support unknown type") + } +} + +func subnetIndexNamespaceFunc(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnet: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("subnetIndexNamespaceFunc doesn't support unknown type") + } +} + // subnetIndexFunc is used to filter out NSX Subnets which are tagged with CR UID. func subnetSetIndexFunc(obj interface{}) ([]string, error) { switch o := obj.(type) { diff --git a/pkg/nsx/services/subnet/store_test.go b/pkg/nsx/services/subnet/store_test.go index 4db3c39c1..3307f073e 100644 --- a/pkg/nsx/services/subnet/store_test.go +++ b/pkg/nsx/services/subnet/store_test.go @@ -49,7 +49,7 @@ func Test_IndexFunc(t *testing.T) { func Test_KeyFunc(t *testing.T) { id := "test_id" subnet := model.VpcSubnet{Id: &id} - t.Run("1", func(t *testing.T) { + t.Run("subnetKeyFunc", func(t *testing.T) { got, _ := keyFunc(&subnet) if got != "test_id" { t.Errorf("keyFunc() = %v, want %v", got, "test_id") @@ -62,7 +62,9 @@ func Test_InitializeSubnetStore(t *testing.T) { cluster, _ := nsx.NewCluster(config2) rc, _ := cluster.NewRestConnector() - subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeSubnetCRUID: subnetIndexFunc}) + subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeSubnetCRUID: subnetIndexFunc, + }) service := SubnetService{ Service: common.Service{ NSXClient: &nsx.Client{ @@ -107,25 +109,27 @@ func Test_InitializeSubnetStore(t *testing.T) { } func TestSubnetStore_Apply(t *testing.T) { - subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeSubnetCRUID: subnetIndexFunc}) + subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeSubnetCRUID: subnetIndexFunc, + }) resourceStore := common.ResourceStore{ Indexer: subnetCacheIndexer, BindingType: model.SecurityPolicyBindingType(), } subnetStore := &SubnetStore{ResourceStore: resourceStore} type args struct { - i interface{} + subnetVPC interface{} } tests := []struct { name string args args wantErr assert.ErrorAssertionFunc }{ - {"1", args{i: &model.VpcSubnet{Id: common.String("1")}}, assert.NoError}, + {"subnet with id", args{subnetVPC: &model.VpcSubnet{Id: common.String("fake-subnet-id")}}, assert.NoError}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.wantErr(t, subnetStore.Apply(tt.args.i), fmt.Sprintf("Apply(%v)", tt.args.i)) + tt.wantErr(t, subnetStore.Apply(tt.args.subnetVPC), fmt.Sprintf("Apply(%v)", tt.args.subnetVPC)) }) } } diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index 29806d0af..9f6bf9bee 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -58,6 +58,8 @@ func InitializeSubnetService(service common.Service) (*SubnetService, error) { Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ common.TagScopeSubnetCRUID: subnetIndexFunc, common.TagScopeSubnetSetCRUID: subnetSetIndexFunc, + common.TagScopeVMNamespace: subnetIndexVMNamespaceFunc, + common.TagScopeNamespace: subnetIndexNamespaceFunc, }), BindingType: model.VpcSubnetBindingType(), }, @@ -84,10 +86,10 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, vpcInfo co uid := string(obj.GetUID()) nsxSubnet, err := service.buildSubnet(obj, tags) if err != nil { - log.Error(err, "failed to build Subnet") + log.Error(err, "Failed to build Subnet") return "", err } - // Only check whether needs update when obj is v1alpha1.Subnet + // Only check whether it needs update when obj is v1alpha1.Subnet if subnet, ok := obj.(*v1alpha1.Subnet); ok { existingSubnet := service.SubnetStore.GetByKey(service.BuildSubnetID(subnet)) changed := false @@ -95,9 +97,15 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, vpcInfo co changed = true } else { changed = common.CompareResource(SubnetToComparable(existingSubnet), SubnetToComparable(nsxSubnet)) + if changed { + // Only tags are expected to be updated + // inherit other fields from the existing Subnet + existingSubnet.Tags = nsxSubnet.Tags + nsxSubnet = existingSubnet + } } if !changed { - log.Info("subnet not changed, skip updating", "subnet.Id", uid) + log.Info("Subnet not changed, skip updating", "SubnetId", uid) return uid, nil } } @@ -126,8 +134,17 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet Jitter: 0, Steps: 6, } + // Failure of CheckRealizeState may result in the creation of an existing Subnet. + // For Subnets, it's important to reuse the already created NSXSubnet. + // For SubnetSets, since the ID includes a random value, the created NSX Subnet needs to be deleted and recreated. if err = realizeService.CheckRealizeState(backoff, *nsxSubnet.Path, "RealizedLogicalSwitch"); err != nil { log.Error(err, "failed to check subnet realization state", "ID", *nsxSubnet.Id) + // Delete the subnet if realization check fails, avoiding creating duplicate subnets continuously. + deleteErr := service.DeleteSubnet(*nsxSubnet) + if deleteErr != nil { + log.Error(deleteErr, "failed to delete subnet after realization check failure", "ID", *nsxSubnet.Id) + return "", fmt.Errorf("realization check failed: %v; deletion failed: %v", err, deleteErr) + } return "", err } if err = service.SubnetStore.Apply(nsxSubnet); err != nil { @@ -174,28 +191,44 @@ func (service *SubnetService) ListSubnetCreatedBySubnetSet(id string) []*model.V return service.SubnetStore.GetByIndex(common.TagScopeSubnetSetCRUID, id) } -func (service *SubnetService) ListSubnetSetID(ctx context.Context) sets.Set[string] { +func (service *SubnetService) ListSubnetSetID(ctx context.Context) (sets.Set[string], error) { crdSubnetSetList := &v1alpha1.SubnetSetList{} - subnetsetIDs := sets.New[string]() err := service.Client.List(ctx, crdSubnetSetList) if err != nil { - log.Error(err, "failed to list subnetset CR") - return subnetsetIDs + return nil, err } + + crdSubnetSetIDs := sets.New[string]() for _, subnetset := range crdSubnetSetList.Items { - subnetsetIDs.Insert(string(subnetset.UID)) + crdSubnetSetIDs.Insert(string(subnetset.UID)) } - return subnetsetIDs + return crdSubnetSetIDs, nil } -// check if subnet belongs to a subnetset, if yes, check if that subnetset still exists -func (service *SubnetService) IsOrphanSubnet(subnet model.VpcSubnet, subnetsetIDs sets.Set[string]) bool { - for _, tag := range subnet.Tags { - if *tag.Scope == common.TagScopeSubnetSetCRUID && subnetsetIDs.Has(*tag.Tag) { - return false +func (service *SubnetService) ListSubnetByName(ns, name string) []*model.VpcSubnet { + nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeVMNamespace, ns) + res := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + tagName := nsxutil.FindTag(nsxSubnet.Tags, common.TagScopeSubnetCRName) + if tagName == name { + res = append(res, nsxSubnet) } } - return true + return res +} + +func (service *SubnetService) ListSubnetBySubnetSetName(ns, subnetSetName string) []*model.VpcSubnet { + nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeVMNamespace, ns) + nsxSubnetsOfDefaultPodSubnetSet := service.SubnetStore.GetByIndex(common.TagScopeNamespace, ns) + nsxSubnets = append(nsxSubnets, nsxSubnetsOfDefaultPodSubnetSet...) + res := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + tagName := nsxutil.FindTag(nsxSubnet.Tags, common.TagScopeSubnetSetCRName) + if tagName == subnetSetName { + res = append(res, nsxSubnet) + } + } + return res } func (service *SubnetService) DeleteIPAllocation(orgID, projectID, vpcID, subnetID string) error { @@ -309,26 +342,47 @@ func (service *SubnetService) GetSubnetByPath(path string) (*model.VpcSubnet, er return nsxSubnet, err } -func (service *SubnetService) ListSubnetID() sets.Set[string] { +func (service *SubnetService) ListSubnetSetIDsFromNSXSubnets() sets.Set[string] { + subnetSetIDs := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetSetCRUID) + return subnetSetIDs +} + +func (service *SubnetService) ListSubnetIDsFromNSXSubnets() sets.Set[string] { + subnetIDs := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetCRUID) + return subnetIDs +} + +// ListIndexFuncValues returns all the indexed values of the given index +// Index maps the indexed value to a set of keys in the store that match on that value: type Index map[string]sets.String +// see the getIndexValues function in k8s.io/client-go/tools/cache/thread_safe_store.go +func (service *SubnetService) ListAllSubnet() []*model.VpcSubnet { + var allNSXSubnets []*model.VpcSubnet + // ListSubnetCreatedBySubnet subnets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetCRUID) + for subnetID := range subnets { + nsxSubnets := service.ListSubnetCreatedBySubnet(subnetID) + allNSXSubnets = append(allNSXSubnets, nsxSubnets...) + } + // ListSubnetCreatedBySubnetSet subnetSets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetSetCRUID) - return subnets.Union(subnetSets) + for subnetSetID := range subnetSets { + nsxSubnets := service.ListSubnetCreatedBySubnetSet(subnetSetID) + allNSXSubnets = append(allNSXSubnets, nsxSubnets...) + } + return allNSXSubnets } func (service *SubnetService) Cleanup(ctx context.Context) error { - uids := service.ListSubnetID() - log.Info("cleaning up subnet", "count", len(uids)) - for uid := range uids { - nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(uid)) - for _, nsxSubnet := range nsxSubnets { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSubnet(*nsxSubnet) - if err != nil { - return err - } + allNSXSubnets := service.ListAllSubnet() + log.Info("cleaning up Subnet", "Count", len(allNSXSubnets)) + for _, nsxSubnet := range allNSXSubnets { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + err := service.DeleteSubnet(*nsxSubnet) + if err != nil { + return err } } } @@ -339,12 +393,15 @@ func (service *SubnetService) GetSubnetsByIndex(key, value string) []*model.VpcS return service.SubnetStore.GetByIndex(key, value) } -func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) []model.Tag { +func (service *SubnetService) GenerateSubnetNSTags(obj client.Object) []model.Tag { + ns := obj.GetNamespace() namespace := &v1.Namespace{} namespacedName := types.NamespacedName{ Name: ns, } + // Get the namespace object from the Kubernetes API if err := service.Client.Get(context.Background(), namespacedName, namespace); err != nil { + log.Error(err, "Failed to get Namespace", "Namespace", ns) return nil } nsUID := string(namespace.UID) @@ -355,14 +412,8 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) model.Tag{Scope: String(common.TagScopeVMNamespaceUID), Tag: String(nsUID)}, model.Tag{Scope: String(common.TagScopeVMNamespace), Tag: String(obj.GetNamespace())}) case *v1alpha1.SubnetSet: - findLabelDefaultPodSubnetSet := false - for k, v := range o.Labels { - if k == common.LabelDefaultSubnetSet && v == common.LabelDefaultPodSubnetSet { - findLabelDefaultPodSubnetSet = true - break - } - } - if findLabelDefaultPodSubnetSet { + isDefaultPodSubnetSet := o.Labels[common.LabelDefaultSubnetSet] == common.LabelDefaultPodSubnetSet + if isDefaultPodSubnetSet { tags = append(tags, model.Tag{Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String(nsUID)}, model.Tag{Scope: common.String(common.TagScopeNamespace), Tag: common.String(obj.GetNamespace())}) @@ -372,6 +423,7 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) model.Tag{Scope: common.String(common.TagScopeVMNamespace), Tag: common.String(obj.GetNamespace())}) } } + // Append Namespace labels as tags for k, v := range namespace.Labels { tags = append(tags, model.Tag{Scope: common.String(k), Tag: common.String(v)}) } @@ -379,7 +431,7 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) } func (service *SubnetService) UpdateSubnetSetTags(ns string, vpcSubnets []*model.VpcSubnet, tags []model.Tag) error { - for i := range vpcSubnets { + for i, vpcSubnet := range vpcSubnets { subnetSet := &v1alpha1.SubnetSet{} var name string @@ -397,27 +449,32 @@ func (service *SubnetService) UpdateSubnetSetTags(ns string, vpcSubnets []*model } } - if matchNamespace { - if err := service.Client.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, subnetSet); err != nil { - return err - } - newTags := append(service.buildBasicTags(subnetSet), tags...) - changed := common.CompareResource(SubnetToComparable(vpcSubnets[i]), SubnetToComparable(&model.VpcSubnet{Tags: newTags})) - if !changed { - log.Info("NSX subnet tags unchanged, skip updating") - continue - } - vpcSubnets[i].Tags = newTags - vpcInfo, err := common.ParseVPCResourcePath(*vpcSubnets[i].Path) - if err != nil { - err := fmt.Errorf("failed to parse NSX VPC path for Subnet %s: %s", *vpcSubnets[i].Path, err) - return err - } - if _, err := service.createOrUpdateSubnet(subnetSet, vpcSubnets[i], &vpcInfo); err != nil { - return err - } - log.Info("successfully updated subnet set tags", "subnetSet", subnetSet) + // Skip this subnet if the Namespace doesn't match + if !matchNamespace { + log.Info("Namespace mismatch, skipping subnet", "Subnet", *vpcSubnet.Id, "Namespace", ns) + continue + } + + if err := service.Client.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, subnetSet); err != nil { + return fmt.Errorf("failed to get SubnetSet %s in Namespace %s: %w", name, ns, err) + } + newTags := append(service.buildBasicTags(subnetSet), tags...) + changed := common.CompareResource(SubnetToComparable(vpcSubnets[i]), SubnetToComparable(&model.VpcSubnet{Tags: newTags})) + if !changed { + log.Info("NSX Subnet tags unchanged, skipping update", "subnet", *vpcSubnet.Id) + continue + } + vpcSubnets[i].Tags = newTags + + vpcInfo, err := common.ParseVPCResourcePath(*vpcSubnets[i].Path) + if err != nil { + err := fmt.Errorf("failed to parse NSX VPC path for Subnet %s: %s", *vpcSubnets[i].Path, err) + return err + } + if _, err := service.createOrUpdateSubnet(subnetSet, vpcSubnets[i], &vpcInfo); err != nil { + return fmt.Errorf("failed to update Subnet %s in SubnetSet %s: %w", *vpcSubnet.Id, subnetSet.Name, err) } + log.Info("Successfully updated SubnetSet tags", "subnetSet", subnetSet, "Subnet", *vpcSubnet.Id) } return nil } diff --git a/pkg/nsx/services/subnet/subnet_test.go b/pkg/nsx/services/subnet/subnet_test.go new file mode 100644 index 000000000..cf65d0477 --- /dev/null +++ b/pkg/nsx/services/subnet/subnet_test.go @@ -0,0 +1,90 @@ +package subnet + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func TestGenerateSubnetNSTags(t *testing.T) { + scheme := clientgoscheme.Scheme + v1alpha1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + + service := &SubnetService{ + Service: common.Service{ + Client: fakeClient, + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + UseAVILoadBalancer: false, + }, + }, + }, + } + + // Create a test namespace + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + UID: "namespace-uid", + Labels: map[string]string{ + "env": "test", + }, + }, + } + + assert.NoError(t, fakeClient.Create(context.TODO(), namespace)) + + // Define the Subnet object + subnet := &v1alpha1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet", + Namespace: "test-ns", + }, + } + + // Generate tags for the Subnet + tags := service.GenerateSubnetNSTags(subnet) + + // Validate the tags + assert.NotNil(t, tags) + assert.Equal(t, 3, len(tags)) // 3 tags should be generated + + // Check specific tags + assert.Equal(t, "namespace-uid", *tags[0].Tag) + assert.Equal(t, "test-ns", *tags[1].Tag) + assert.Equal(t, "test", *tags[2].Tag) + + // Define the SubnetSet object + subnetSet := &v1alpha1.SubnetSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet-set", + Namespace: "test-ns", + Labels: map[string]string{ + common.LabelDefaultSubnetSet: common.LabelDefaultPodSubnetSet, + }, + }, + } + + // Generate tags for the SubnetSet + tagsSet := service.GenerateSubnetNSTags(subnetSet) + + // Validate the tags for SubnetSet + assert.NotNil(t, tagsSet) + assert.Equal(t, 3, len(tagsSet)) // 3 tags should be generated + assert.Equal(t, "namespace-uid", *tagsSet[0].Tag) + assert.Equal(t, "test-ns", *tagsSet[1].Tag) +} diff --git a/pkg/nsx/services/subnetport/builder.go b/pkg/nsx/services/subnetport/builder.go index 0c8795f94..7ff5e29ce 100644 --- a/pkg/nsx/services/subnetport/builder.go +++ b/pkg/nsx/services/subnetport/builder.go @@ -51,9 +51,6 @@ func (service *SubnetPortService) buildSubnetPort(obj interface{}, nsxSubnet *mo return nil, err } nsxSubnetPortPath := fmt.Sprintf("%s/ports/%s", *nsxSubnet.Path, nsxSubnetPortID) - if err != nil { - return nil, err - } namespace := &corev1.Namespace{} namespacedName := types.NamespacedName{ Name: objNamespace, diff --git a/pkg/nsx/services/subnetport/store.go b/pkg/nsx/services/subnetport/store.go index 6a01e4d68..c5496a20e 100644 --- a/pkg/nsx/services/subnetport/store.go +++ b/pkg/nsx/services/subnetport/store.go @@ -67,6 +67,24 @@ func subnetPortIndexBySubnetID(obj interface{}) ([]string, error) { } } +func subnetPortIndexNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnetPort: + return filterTag(o.Tags, common.TagScopeVMNamespace), nil + default: + return nil, errors.New("subnetPortIndexNamespace doesn't support unknown type") + } +} + +func subnetPortIndexPodNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnetPort: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("subnetPortIndexPodNamespace doesn't support unknown type") + } +} + // SubnetPortStore is a store for SubnetPorts type SubnetPortStore struct { common.ResourceStore diff --git a/pkg/nsx/services/subnetport/subnetport.go b/pkg/nsx/services/subnetport/subnetport.go index 36c2d947a..8f0cd7031 100644 --- a/pkg/nsx/services/subnetport/subnetport.go +++ b/pkg/nsx/services/subnetport/subnetport.go @@ -52,6 +52,8 @@ func InitializeSubnetPort(service servicecommon.Service) (*SubnetPortService, er cache.Indexers{ servicecommon.TagScopeSubnetPortCRUID: subnetPortIndexByCRUID, servicecommon.TagScopePodUID: subnetPortIndexByPodUID, + servicecommon.TagScopeVMNamespace: subnetPortIndexNamespace, + servicecommon.TagScopeNamespace: subnetPortIndexPodNamespace, servicecommon.IndexKeySubnetID: subnetPortIndexBySubnetID, }), BindingType: model.VpcSubnetPortBindingType(), @@ -176,7 +178,7 @@ func (service *SubnetPortService) CheckSubnetPortState(obj interface{}, nsxSubne if realizestate.IsRealizeStateError(err) { log.Error(err, "the created subnet port is in error realization state, cleaning the resource", "subnetport", portID) // only recreate subnet port on RealizationErrorStateError. - if err := service.DeleteSubnetPort(portID); err != nil { + if err := service.DeleteSubnetPortById(portID); err != nil { log.Error(err, "cleanup error subnetport failed", "subnetport", portID) return nil, err } @@ -213,26 +215,30 @@ func (service *SubnetPortService) GetSubnetPortState(obj interface{}, nsxSubnetP return &nsxSubnetPortState, nil } -func (service *SubnetPortService) DeleteSubnetPort(portID string) error { - nsxSubnetPort := service.SubnetPortStore.GetByKey(portID) - if nsxSubnetPort == nil || nsxSubnetPort.Id == nil { - log.Info("NSX subnet port is not found in store, skip deleting it", "id", portID) - return nil - } +func (service *SubnetPortService) DeleteSubnetPort(nsxSubnetPort *model.VpcSubnetPort) error { nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID := nsxutil.ParseVPCPath(*nsxSubnetPort.Path) - err := service.NSXClient.PortClient.Delete(nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID, portID) + err := service.NSXClient.PortClient.Delete(nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID, *nsxSubnetPort.Id) err = nsxutil.TransNSXApiError(err) if err != nil { log.Error(err, "failed to delete subnetport", "nsxSubnetPort.Path", *nsxSubnetPort.Path) return err } - if err = service.SubnetPortStore.Delete(portID); err != nil { + if err = service.SubnetPortStore.Delete(*nsxSubnetPort.Id); err != nil { return err } - log.Info("successfully deleted nsxSubnetPort", "nsxSubnetPortID", portID) + log.Info("successfully deleted nsxSubnetPort", "nsxSubnetPortID", *nsxSubnetPort.Id) return nil } +func (service *SubnetPortService) DeleteSubnetPortById(portID string) error { + nsxSubnetPort := service.SubnetPortStore.GetByKey(portID) + if nsxSubnetPort == nil || nsxSubnetPort.Id == nil { + log.Info("NSX subnet port is not found in store, skip deleting it", "id", portID) + return nil + } + return service.DeleteSubnetPort(nsxSubnetPort) +} + func (service *SubnetPortService) ListNSXSubnetPortIDForCR() sets.Set[string] { log.V(2).Info("listing subnet port CR UIDs") subnetPortSet := sets.New[string]() @@ -305,6 +311,46 @@ func (service *SubnetPortService) GetPortsOfSubnet(nsxSubnetID string) (ports [] return subnetPortList } +func (service *SubnetPortService) ListSubnetPortIDsFromCRs(ctx context.Context) (sets.Set[string], error) { + subnetPortList := &v1alpha1.SubnetPortList{} + err := service.Client.List(ctx, subnetPortList) + if err != nil { + log.Error(err, "failed to list SubnetPort CR") + return nil, err + } + + crSubnetPortIDsSet := sets.New[string]() + for _, subnetPort := range subnetPortList.Items { + subnetPortID := service.BuildSubnetPortId(&subnetPort.ObjectMeta) + crSubnetPortIDsSet.Insert(subnetPortID) + } + return crSubnetPortIDsSet, nil +} + +func (service *SubnetPortService) ListSubnetPortByName(ns string, name string) []*model.VpcSubnetPort { + var result []*model.VpcSubnetPort + subnetports := service.SubnetPortStore.GetByIndex(servicecommon.TagScopeVMNamespace, ns) + for _, subnetport := range subnetports { + tagname := nsxutil.FindTag(subnetport.Tags, servicecommon.TagScopeSubnetPortCRName) + if tagname == name { + result = append(result, subnetport) + } + } + return result +} + +func (service *SubnetPortService) ListSubnetPortByPodName(ns string, name string) []*model.VpcSubnetPort { + var result []*model.VpcSubnetPort + subnetports := service.SubnetPortStore.GetByIndex(servicecommon.TagScopeNamespace, ns) + for _, subnetport := range subnetports { + tagname := nsxutil.FindTag(subnetport.Tags, servicecommon.TagScopePodName) + if tagname == name { + result = append(result, subnetport) + } + } + return result +} + func (service *SubnetPortService) Cleanup(ctx context.Context) error { subnetPorts := service.SubnetPortStore.List() log.Info("cleanup subnetports", "count", len(subnetPorts)) @@ -314,7 +360,7 @@ func (service *SubnetPortService) Cleanup(ctx context.Context) error { case <-ctx.Done(): return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) default: - err := service.DeleteSubnetPort(subnetPortID) + err := service.DeleteSubnetPortById(subnetPortID) if err != nil { log.Error(err, "cleanup subnetport failed", "subnetPortID", subnetPortID) return err diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index c05038a47..2ac9aaac6 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -82,10 +82,6 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe Scope: common.String(common.TagScopeVPCManagedBy), Tag: common.String(common.AutoCreatedVPCTagValue)}) } - if nc.VPCConnectivityProfile != "" { - vpc.VpcConnectivityProfile = &nc.VPCConnectivityProfile - } - vpc.PrivateIps = nc.PrivateIPs return vpc, nil } @@ -106,3 +102,12 @@ func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSiz lbs.RelaxScaleValidation = relaxScaleValidation return lbs, nil } + +func buildVpcAttachment(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster string, vpcconnectiveprofile string) (*model.VpcAttachment, error) { + attachment := &model.VpcAttachment{} + attachment.VpcConnectivityProfile = &vpcconnectiveprofile + attachment.DisplayName = common.String(common.DefaultVpcAttachmentId) + attachment.Id = common.String(common.DefaultVpcAttachmentId) + attachment.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + return attachment, nil +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 2ae232f1a..ce8fe5872 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -441,6 +441,7 @@ func (s *VPCService) DeleteLBMonitorProfile(id string) error { log.Info("successfully deleted NCP created lbMonitorProfile", "lbMonitorProfile", id) return nil } + func (s *VPCService) ListLBVirtualServer() []model.LBVirtualServer { store := &ResourceStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), @@ -699,13 +700,13 @@ func (s *VPCService) IsLBProviderChanged(existingVPC *model.Vpc, lbProvider LBPr return false } func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPCNetworkConfigInfo, lbProvider LBProvider) (*model.Vpc, error) { - // check from VPC store if vpc already exist + // check from VPC store if VPC already exist ns := obj.Namespace updateVpc := false nsObj := &v1.Namespace{} - // get name obj + // get Namespace if err := s.Client.Get(ctx, types.NamespacedName{Name: obj.Namespace}, nsObj); err != nil { - log.Error(err, "unable to fetch namespace", "name", obj.Namespace) + log.Error(err, "unable to fetch Namespace", "Name", obj.Namespace) return nil, err } @@ -774,7 +775,8 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) } // build HAPI request - orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NSXProject, createdVpc, createdLBS) + createdAttachment, _ := buildVpcAttachment(obj, nsObj, s.NSXConfig.Cluster, nc.VPCConnectivityProfile) + orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NSXProject, createdVpc, createdLBS, createdAttachment) if err != nil { log.Error(err, "failed to build HAPI request") return nil, err @@ -850,6 +852,28 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC } } + // Check VpcAttachment realization + if createdAttachment != nil { + newAttachment, err := s.NSXClient.VpcAttachmentClient.Get(nc.Org, nc.NSXProject, *createdVpc.Id, *createdAttachment.Id) + if err != nil || newAttachment.VpcConnectivityProfile == nil { + log.Error(err, "failed to read VPC attachment object after creating or updating", "VpcAttachment", createdAttachment.Id) + return nil, err + } + log.V(2).Info("check VPC attachment realization state", "VpcAttachment", *createdAttachment.Id) + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(util.NSXTLBVSDefaultRetry, *newAttachment.Path, ""); err != nil { + log.Error(err, "failed to check VPC attachment realization state", "VpcAttachment", *createdAttachment.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created VPC attachment is in error realization state, cleaning the resource", "VpcAttachment", *createdAttachment.Id) + // delete the nsx vpc object and re-create it in the next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, err + } + } + return nil, err + } + } return &newVpc, nil } diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go index 8b16e771b..0322173d4 100644 --- a/pkg/nsx/services/vpc/wrap.go +++ b/pkg/nsx/services/vpc/wrap.go @@ -5,9 +5,9 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" ) -func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) { +func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService, attachment *model.VpcAttachment) (*model.OrgRoot, error) { + var vpcChildren []*data.StructValue if lbs != nil { - var vpcChildren []*data.StructValue childrenLBS, err := s.WrapLBS(lbs) if err != nil { return nil, err @@ -15,6 +15,14 @@ func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, l vpcChildren = append(vpcChildren, childrenLBS...) vpc.Children = vpcChildren } + if attachment != nil { + childrenAttachment, err := s.WrapAttachment(attachment) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, childrenAttachment...) + vpc.Children = vpcChildren + } var projectChildren []*data.StructValue childrenVPC, err := s.WrapVPC(vpc) if err != nil { diff --git a/pkg/nsx/services/vpc/wrap_test.go b/pkg/nsx/services/vpc/wrap_test.go index b51338705..9720f2188 100644 --- a/pkg/nsx/services/vpc/wrap_test.go +++ b/pkg/nsx/services/vpc/wrap_test.go @@ -15,6 +15,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { nsxtProject string vpc *model.Vpc lbs *model.LBService + attachment *model.VpcAttachment } tests := []struct { name string @@ -30,6 +31,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { nsxtProject: "testproject", vpc: &model.Vpc{}, lbs: &model.LBService{}, + attachment: &model.VpcAttachment{}, }, want: &model.OrgRoot{ResourceType: pointy.String("OrgRoot")}, wantChildren: 1, @@ -39,7 +41,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &VPCService{} - got, err := s.WrapHierarchyVPC(tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs) + got, err := s.WrapHierarchyVPC(tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs, tt.args.attachment) if !tt.wantErr(t, err, fmt.Sprintf("WrapHierarchyVPC(%v, %v, %v, %v)", tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs)) { return } diff --git a/pkg/util/utils.go b/pkg/util/utils.go index e3e663c72..8d1e06b4e 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -21,7 +21,6 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" t1v1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" @@ -31,33 +30,13 @@ import ( ) const ( - wcpSystemResource = "vmware-system-shared-t1" - HashLength int = 8 - SubnetTypeSubnet = "subnet" - SubnetTypeSubnetSet = "subnetset" + wcpSystemResource = "vmware-system-shared-t1" ) var ( - String = common.String - basicTags = []string{ - common.TagScopeCluster, common.TagScopeVersion, - common.TagScopeStaticRouteCRName, common.TagScopeStaticRouteCRUID, - common.TagValueScopeSecurityPolicyName, common.TagValueScopeSecurityPolicyUID, - common.TagScopeNetworkPolicyName, common.TagScopeNetworkPolicyUID, - common.TagScopeSubnetCRName, common.TagScopeSubnetCRUID, - common.TagScopeSubnetPortCRName, common.TagScopeSubnetPortCRUID, - common.TagScopeIPPoolCRName, common.TagScopeIPPoolCRUID, - common.TagScopeSubnetSetCRName, common.TagScopeSubnetSetCRUID, - } - tagsScopeSet = sets.New[string]() + String = common.String ) -func init() { - for _, tag := range basicTags { - tagsScopeSet.Insert(tag) - } -} - var log = &logger.Log func NormalizeLabels(matchLabels *map[string]string) *map[string]string { @@ -106,11 +85,11 @@ func NormalizeId(name string) string { return newName } hashString := Sha1(name) - nameLength := common.MaxIdLength - HashLength - 1 + nameLength := common.MaxIdLength - common.HashLength - 1 for strings.ContainsAny(string(newName[nameLength-1]), "-._") { nameLength-- } - newName = fmt.Sprintf("%s-%s", newName[:nameLength], hashString[:HashLength]) + newName = fmt.Sprintf("%s-%s", newName[:nameLength], hashString[:common.HashLength]) return newName } @@ -331,24 +310,6 @@ func If(condition bool, trueVal, falseVal interface{}) interface{} { } } -func GetMapValues(in interface{}) []string { - if in == nil { - return make([]string, 0) - } - switch in.(type) { - case map[string]string: - ssMap := in.(map[string]string) - values := make([]string, 0, len(ssMap)) - for _, v := range ssMap { - values = append(values, v) - } - return values - default: - log.Info("Unsupported map format") - return nil - } -} - // the changes map contains key/value map that you want to change. // if giving empty value for a key in changes map like: "mykey":"", that means removing this annotation from k8s resource func UpdateK8sResourceAnnotation(client client.Client, ctx context.Context, k8sObj client.Object, changes map[string]string) error { @@ -499,19 +460,6 @@ func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []mo return tags } -func AppendTags(basicTags, extraTags []model.Tag) []model.Tag { - if basicTags == nil { - log.Info("AppendTags", "basicTags", basicTags, "extra tags", extraTags) - return nil - } - for _, tag := range extraTags { - if !tagsScopeSet.Has(*tag.Scope) { - basicTags = append(basicTags, tag) - } - } - return basicTags -} - func Capitalize(s string) string { if s == "" { return "" @@ -521,7 +469,7 @@ func Capitalize(s string) string { func GetRandomIndexString() string { uuidStr := uuid.NewString() - return Sha1(uuidStr)[:HashLength] + return Sha1(uuidStr)[:common.HashLength] } // IsPowerOfTwo checks if a given number is a power of 2 diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 87b121e5d..a46ca0a9f 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -30,7 +30,7 @@ func TestNormalizeName(t *testing.T) { shortName := strings.Repeat("a", 256) assert.Equal(t, NormalizeName(shortName), shortName) longName := strings.Repeat("a", 257) - assert.Equal(t, NormalizeName(longName), fmt.Sprintf("%s_%s", strings.Repeat("a", 256-HashLength-1), "0c103888")) + assert.Equal(t, NormalizeName(longName), fmt.Sprintf("%s_%s", strings.Repeat("a", 256-common.HashLength-1), "0c103888")) } func TestNormalizeLabelKey(t *testing.T) { diff --git a/test/e2e/nsx_subnet_test.go b/test/e2e/nsx_subnet_test.go index 1421a14db..cf15c438f 100644 --- a/test/e2e/nsx_subnet_test.go +++ b/test/e2e/nsx_subnet_test.go @@ -88,7 +88,7 @@ func transSearchResponsetoSubnet(response model.SearchResponse) []model.VpcSubne return resources } -func fetchSubnet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { +func fetchSubnetBySubnetSet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { tags := []string{common.TagScopeSubnetSetCRUID, string(subnetSet.UID)} results, err := testData.queryResource(common.ResourceTypeSubnet, tags) assertNil(t, err) @@ -96,6 +96,7 @@ func fetchSubnet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { assertTrue(t, len(subnets) > 0, "No NSX subnet found") return subnets[0] } + func defaultSubnetSet(t *testing.T) { // 1. Check whether default-vm-subnetset and default-pod-subnetset are created. err := testData.waitForCRReadyOrDeleted(defaultTimeout, SubnetSetCRType, E2ENamespace, common.DefaultVMSubnetSet, Ready) @@ -127,11 +128,11 @@ func defaultSubnetSet(t *testing.T) { assertNil(t, err) labelKey, labelValue := "subnet-e2e", "add" ns.Labels[labelKey] = labelValue - ns, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet := fetchSubnet(t, subnetSet) + vpcSubnet := fetchSubnetBySubnetSet(t, subnetSet) found := false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey && *tag.Tag == labelValue { @@ -142,12 +143,14 @@ func defaultSubnetSet(t *testing.T) { assertTrue(t, found, "Failed to add tags for NSX subnet %s", *(vpcSubnet.Id)) // 6. Check updating NSX subnet tags. + ns, err = testData.clientset.CoreV1().Namespaces().Get(context.TODO(), E2ENamespace, v1.GetOptions{}) + assertNil(t, err) labelValue = "update" ns.Labels[labelKey] = labelValue - ns, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet = fetchSubnet(t, subnetSet) + vpcSubnet = fetchSubnetBySubnetSet(t, subnetSet) found = false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey && *tag.Tag == labelValue { @@ -158,11 +161,14 @@ func defaultSubnetSet(t *testing.T) { assertTrue(t, found, "Failed to update tags for NSX subnet %s", *(vpcSubnet.Id)) // 7. Check deleting NSX subnet tags. + ns, err = testData.clientset.CoreV1().Namespaces().Get(context.TODO(), E2ENamespace, v1.GetOptions{}) + assertNil(t, err) delete(ns.Labels, labelKey) - _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + newNs, err := testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet = fetchSubnet(t, subnetSet) + t.Logf("new Namespace: %+v", newNs) + vpcSubnet = fetchSubnetBySubnetSet(t, subnetSet) found = false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey { @@ -288,6 +294,9 @@ func SubnetCIDR(t *testing.T) { assertNil(t, err) allocatedSubnet, err := testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) assertNil(t, err) + nsxSubnets := testData.fetchSubnetByNamespace(t, E2ENamespace, false) + assert.Equal(t, 1, len(nsxSubnets)) + targetCIDR := allocatedSubnet.Status.NetworkAddresses[0] err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Delete(context.TODO(), subnet.Name, v1.DeleteOptions{}) assertNil(t, err) @@ -300,10 +309,13 @@ func SubnetCIDR(t *testing.T) { return false, err }) assertNil(t, err) + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, true) + assert.Equal(t, true, len(nsxSubnets) <= 1) subnet.Spec.IPAddresses = []string{targetCIDR} _, err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Create(context.TODO(), subnet, v1.CreateOptions{}) if err != nil && errors.IsAlreadyExists(err) { + t.Logf("Create Subnet error: %+v", err) err = nil } assertNil(t, err) @@ -312,4 +324,35 @@ func SubnetCIDR(t *testing.T) { allocatedSubnet, err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) assertNil(t, err) assert.Equal(t, targetCIDR, allocatedSubnet.Status.NetworkAddresses[0]) + + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, false) + assert.Equal(t, 1, len(nsxSubnets)) + + err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Delete(context.TODO(), subnet.Name, v1.DeleteOptions{}) + assertNil(t, err) + + err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 100*time.Second, false, func(ctx context.Context) (bool, error) { + _, err := testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return true, nil + } + return false, err + }) + assertNil(t, err) + + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, true) + assert.Equal(t, true, len(nsxSubnets) <= 1) +} + +func (data *TestData) fetchSubnetByNamespace(t *testing.T, ns string, isMarkForDelete bool) (res []model.VpcSubnet) { + tags := []string{common.TagScopeNamespace, ns} + results, err := testData.queryResource(common.ResourceTypeSubnet, tags) + assertNil(t, err) + subnets := transSearchResponsetoSubnet(results) + for _, subnet := range subnets { + if *subnet.MarkedForDelete == isMarkForDelete { + res = append(res, subnet) + } + } + return }