From 21831450ee806767b0d3a1164e7b93b21164373c Mon Sep 17 00:00:00 2001 From: Yanjun Zhou Date: Tue, 1 Oct 2024 16:41:32 +0800 Subject: [PATCH 01/18] Support immutable fields in Subnet/Subnetset (#789) NSX does not support change either from static IP Allocation to DHCP or from DHCP to static IP Allocation. This PR updates the Subnet/SubnetSet CRD to make DHCPConfig immutable; and also - Prevent all the immutable fields in Subnet/SubnetSet from being updated from a value to empty - Fixes a bug in subnet update to inherit other fields from the existing Subnet to avoid overriding some immutable fields like IP Addresses and IP Blocks by empty during the update. Signed-off-by: Yanjun Zhou --- build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml | 12 ++++++++++++ .../yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml | 10 ++++++++++ pkg/apis/vpc/v1alpha1/subnet_types.go | 5 +++++ pkg/apis/vpc/v1alpha1/subnetset_types.go | 4 ++++ pkg/nsx/services/subnet/subnet.go | 6 ++++++ 5 files changed, 37 insertions(+) diff --git a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml index ab63cacb5..afe3cdcf0 100644 --- a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml +++ b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnets.yaml @@ -59,6 +59,9 @@ spec: default: false type: boolean type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf accessMode: description: Access mode of Subnet, accessible only from within VPC or from outside VPC. @@ -89,6 +92,15 @@ spec: - message: Value is immutable rule: self == oldSelf type: object + x-kubernetes-validations: + - message: DHCPConfig is required once set + rule: '!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)' + - message: ipv4SubnetSize is required once set + rule: '!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)' + - message: accessMode is required once set + rule: '!has(oldSelf.accessMode) || has(self.accessMode)' + - message: ipAddresses is required once set + rule: '!has(oldSelf.ipAddresses) || has(self.ipAddresses)' status: description: SubnetStatus defines the observed state of Subnet. properties: diff --git a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml index a2803e167..662aa2e7f 100644 --- a/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml +++ b/build/yaml/crd/vpc/crd.nsx.vmware.com_subnetsets.yaml @@ -59,6 +59,9 @@ spec: default: false type: boolean type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf accessMode: description: Access mode of Subnet, accessible only from within VPC or from outside VPC. @@ -79,6 +82,13 @@ spec: - message: Value is immutable rule: self == oldSelf type: object + x-kubernetes-validations: + - message: DHCPConfig is required once set + rule: '!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)' + - message: accessMode is required once set + rule: '!has(oldSelf.accessMode) || has(self.accessMode)' + - message: ipv4SubnetSize is required once set + rule: '!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)' status: description: SubnetSetStatus defines the observed state of SubnetSet. properties: diff --git a/pkg/apis/vpc/v1alpha1/subnet_types.go b/pkg/apis/vpc/v1alpha1/subnet_types.go index b7d00c6e8..eeae36548 100644 --- a/pkg/apis/vpc/v1alpha1/subnet_types.go +++ b/pkg/apis/vpc/v1alpha1/subnet_types.go @@ -16,6 +16,10 @@ const ( ) // SubnetSpec defines the desired state of Subnet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)", message="DHCPConfig is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipAddresses) || has(self.ipAddresses)", message="ipAddresses is required once set" type SubnetSpec struct { // Size of Subnet based upon estimated workload count. // +kubebuilder:validation:Maximum:=65536 @@ -32,6 +36,7 @@ type SubnetSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" IPAddresses []string `json:"ipAddresses,omitempty"` // DHCPConfig DHCP configuration. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" DHCPConfig DHCPConfig `json:"DHCPConfig,omitempty"` } diff --git a/pkg/apis/vpc/v1alpha1/subnetset_types.go b/pkg/apis/vpc/v1alpha1/subnetset_types.go index c486dc603..1154411e4 100644 --- a/pkg/apis/vpc/v1alpha1/subnetset_types.go +++ b/pkg/apis/vpc/v1alpha1/subnetset_types.go @@ -8,6 +8,9 @@ import ( ) // SubnetSetSpec defines the desired state of SubnetSet. +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.DHCPConfig) || has(self.DHCPConfig)", message="DHCPConfig is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.accessMode) || has(self.accessMode)", message="accessMode is required once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv4SubnetSize) || has(self.ipv4SubnetSize)", message="ipv4SubnetSize is required once set" type SubnetSetSpec struct { // Size of Subnet based upon estimated workload count. // +kubebuilder:validation:Maximum:=65536 @@ -19,6 +22,7 @@ type SubnetSetSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" AccessMode AccessMode `json:"accessMode,omitempty"` // DHCPConfig DHCP configuration. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" DHCPConfig DHCPConfig `json:"DHCPConfig,omitempty"` } diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index 29806d0af..c4f28d637 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -95,6 +95,12 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, vpcInfo co changed = true } else { changed = common.CompareResource(SubnetToComparable(existingSubnet), SubnetToComparable(nsxSubnet)) + if changed { + // Only tags are expected to be updated + // inherit other fields from the existing Subnet + existingSubnet.Tags = nsxSubnet.Tags + nsxSubnet = existingSubnet + } } if !changed { log.Info("subnet not changed, skip updating", "subnet.Id", uid) From b91000647023e3107757ce1989d4a69d4db587cf Mon Sep 17 00:00:00 2001 From: Yanjun Zhou Date: Thu, 3 Oct 2024 10:28:22 +0800 Subject: [PATCH 02/18] Enhance the IPBlocksInfo unit tests (#790) Signed-off-by: Yanjun Zhou --- .../ipblocksinfo/ipblocksinfo_test.go | 107 ++++++++++++++ pkg/nsx/services/ipblocksinfo/store_test.go | 139 ++++++++++++++++-- 2 files changed, 234 insertions(+), 12 deletions(-) diff --git a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go index c13d5d8bc..0abdb424e 100644 --- a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go +++ b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "testing" + "time" "github.com/agiledragon/gomonkey" "github.com/golang/mock/gomock" @@ -12,6 +13,8 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -167,3 +170,107 @@ func TestIPBlocksInfoService_SyncIPBlocksInfo(t *testing.T) { err := service.SyncIPBlocksInfo(context.TODO()) assert.Equal(t, nil, err) } + +func TestIPBlocksInfoService_StartPeriodicSync(t *testing.T) { + ipBlocksInfoService := &IPBlocksInfoService{ + Service: common.Service{}, + SyncTask: NewIPBlocksInfoSyncTask(time.Millisecond*100, time.Millisecond*50), + } + done := make(chan bool) + go func() { + syncIPBlocksInfoPatch := gomonkey.ApplyMethod(reflect.TypeOf(ipBlocksInfoService), "SyncIPBlocksInfo", func(_ *IPBlocksInfoService, cxt context.Context) error { + return fmt.Errorf("mock error") + }) + defer syncIPBlocksInfoPatch.Reset() + ipBlocksInfoService.StartPeriodicSync() + done <- true + }() + + time.Sleep(time.Millisecond * 20) + ipBlocksInfoService.SyncTask.resetChan <- struct{}{} + + select { + case <-done: + t.Error("StartPeriodicSync stop unexpectedly") + case <-time.After(time.Millisecond * 500): + // Stop StartPeriodicSync after some time + } +} + +func TestIPBlocksInfoService_getIPBlockCIDRsFromStore(t *testing.T) { + ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.IpAddressBlockBindingType(), + }} + ipblock1 := model.IpAddressBlock{ + Path: &ipBlocksPath1, + } + ipBlockStore.Apply(&ipblock1) + service := &IPBlocksInfoService{} + + // Fetch non-existed IPBlocks + pathSet := sets.New[string]() + pathSet.Insert(ipBlocksPath2) + _, err := service.getIPBlockCIDRsFromStore(pathSet, ipBlockStore) + assert.ErrorContains(t, err, "failed to get IPBlock") + + // No CIDR in IPBlocks + pathSet = sets.New[string]() + pathSet.Insert(ipBlocksPath1) + _, err = service.getIPBlockCIDRsFromStore(pathSet, ipBlockStore) + assert.ErrorContains(t, err, "failed to get CIDR from ipblock") +} + +func TestIPBlocksInfoService_createOrUpdateIPBlocksInfo(t *testing.T) { + service, mockController, mockK8sClient := createService(t) + defer mockController.Finish() + + ipBlocksInfo := v1alpha1.IPBlocksInfo{} + mockErr := fmt.Errorf("mock error") + + // Fail to get IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(mockErr) + + err := service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) + + // Fail to create IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(apierrors.NewNotFound(v1alpha1.Resource("IPBlocksInfo"), "ipBlocksInfoName")) + mockK8sClient.EXPECT().Create(gomock.Any(), gomock.Any()).Return(mockErr) + err = service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) + + // // Fail to udpate IPBlocksInfo CR + mockK8sClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + ipBlocksInfoCR := obj.(*v1alpha1.IPBlocksInfo) + ipBlocksInfoCR.ExternalIPCIDRs = []string{ipBlocksMap[ipBlocksPath4]} + return nil + }) + mockK8sClient.EXPECT().Update(gomock.Any(), gomock.Any()).Return(mockErr) + err = service.createOrUpdateIPBlocksInfo(context.TODO(), &ipBlocksInfo, false) + assert.ErrorIs(t, err, mockErr) +} + +func TestIsDefaultNetworkConfigCR(t *testing.T) { + testCRD1 := v1alpha1.VPCNetworkConfiguration{} + testCRD1.Name = "test-1" + testCRD2 := v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationDefaultNetworkConfig: "invalid", + }, + }, + } + testCRD2.Name = "test-2" + testCRD3 := v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationDefaultNetworkConfig: "true", + }, + }, + } + testCRD3.Name = "test-3" + assert.Equal(t, isDefaultNetworkConfigCR(testCRD1), false) + assert.Equal(t, isDefaultNetworkConfigCR(testCRD2), false) + assert.Equal(t, isDefaultNetworkConfigCR(testCRD3), true) +} diff --git a/pkg/nsx/services/ipblocksinfo/store_test.go b/pkg/nsx/services/ipblocksinfo/store_test.go index 1159a03a0..23a3ebee6 100644 --- a/pkg/nsx/services/ipblocksinfo/store_test.go +++ b/pkg/nsx/services/ipblocksinfo/store_test.go @@ -5,15 +5,23 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +var ( + fakeVpcPath = "vpc-path" + fakeVpcProfilePath = "vpc-connectivity-profile-path" + fakeIpBlockPath = "ip-block-path" + fakeDeleted = true ) func Test_KeyFunc(t *testing.T) { - vpcPath := "vpc-path" - vpc := model.Vpc{Path: &vpcPath} - vpcProfilePath := "vpc-connectivity-profile-path" - vpcProfile := model.VpcConnectivityProfile{Path: &vpcProfilePath} - ipBlockPath := "ip-block-path" - ipBlock := model.IpAddressBlock{Path: &ipBlockPath} + vpc := model.Vpc{Path: &fakeVpcPath} + vpcProfile := model.VpcConnectivityProfile{Path: &fakeVpcProfilePath} + ipBlock := model.IpAddressBlock{Path: &fakeIpBlockPath} + notSupported := struct{}{} type args struct { obj interface{} @@ -23,32 +31,139 @@ func Test_KeyFunc(t *testing.T) { name string expectedKey string item args + expectedErr bool }{ { name: "Vpc", item: args{obj: &vpc}, - expectedKey: vpcPath, + expectedKey: fakeVpcPath, }, { name: "VpcConnectivityProfile", item: args{obj: &vpcProfile}, - expectedKey: vpcProfilePath, + expectedKey: fakeVpcProfilePath, }, { name: "IpBlock", item: args{obj: &ipBlock}, - expectedKey: ipBlockPath, + expectedKey: fakeIpBlockPath, + }, + { + name: "NotSupported", + item: args{obj: ¬Supported}, + expectedErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := keyFunc(tt.item.obj) - assert.Nil(t, err) - if got != tt.expectedKey { - t.Errorf("keyFunc() = %v, want %v", got, tt.expectedKey) + if !tt.expectedErr { + assert.Nil(t, err) + if got != tt.expectedKey { + t.Errorf("keyFunc() = %v, want %v", got, tt.expectedKey) + } + } else { + assert.NotNil(t, err) } + + }) + } + +} + +func TestVPCConnectivityProfileStore_Apply(t *testing.T) { + vpcConnectivityProfileStore := &VPCConnectivityProfileStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.VpcConnectivityProfileBindingType(), + }} + + profile1 := model.VpcConnectivityProfile{ + Path: &fakeVpcProfilePath, + } + profile2 := model.VpcConnectivityProfile{ + Path: &fakeVpcProfilePath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &profile1}}, + {"Delete", args{i: &profile2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := vpcConnectivityProfileStore.Apply(tt.args.i) + assert.Nil(t, err) }) } +} +func TestVPCStore_Apply(t *testing.T) { + vpcStore := &VPCStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.VpcBindingType(), + }} + + vpc1 := model.Vpc{ + Path: &fakeVpcPath, + } + vpc2 := model.Vpc{ + Path: &fakeVpcPath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &vpc1}}, + {"Delete", args{i: &vpc2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := vpcStore.Apply(tt.args.i) + assert.Nil(t, err) + }) + } +} + +func TestIPBlockStore_Apply(t *testing.T) { + ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.IpAddressBlockBindingType(), + }} + + ipblock1 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, + } + ipblock2 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, + MarkedForDelete: &fakeDeleted, + } + + type args struct { + i interface{} + } + tests := []struct { + name string + args args + }{ + {"Add", args{i: &ipblock1}}, + {"Delete", args{i: &ipblock2}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ipBlockStore.Apply(tt.args.i) + assert.Nil(t, err) + }) + } } From f4d9664c8952f3feb405718de830424c874164fe Mon Sep 17 00:00:00 2001 From: Xiaopei Liu Date: Wed, 2 Oct 2024 17:58:35 +0800 Subject: [PATCH 03/18] Change VPC CR Conditions For IPAddressAllocation, SecurityPolicy, StaticRoute, SubnetPort CR, change to use "reason" for simplified message and "message" for details messages. according to https://github.com/kubernetes/apimachinery/blob/d4f471b82f0a17cda946aeba446770563f92114d/pkg/apis/meta/v1/types.go#L1407 Reason : reason contains a programmatic identifier indicating the reason for the condition Message : message is a human readable message indicating details about the transition --- .../ipaddressallocation_controller.go | 10 +++++----- .../ipaddressallocation_controller_test.go | 2 +- .../securitypolicy/securitypolicy_controller.go | 10 +++++----- pkg/controllers/staticroute/staticroute_controller.go | 6 +++--- pkg/controllers/subnet/subnet_controller.go | 2 +- pkg/controllers/subnetport/subnetport_controller.go | 10 +++++----- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go index a9c0a5a73..a52eb7fbb 100644 --- a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller.go @@ -70,13 +70,13 @@ func updateFail(r *IPAddressAllocationReconciler, c context.Context, o *v1alpha1 func (r *IPAddressAllocationReconciler) setReadyStatusFalse(ctx context.Context, ipaddressallocation *v1alpha1.IPAddressAllocation, transitionTime metav1.Time, err *error) { conditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX IPAddressAllocation could not be created or updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the IPAddressAllocation CR. Error: %v", *err, ), + Reason: "IPAddressAllocationNotReady", LastTransitionTime: transitionTime, }, } @@ -93,7 +93,7 @@ func (r *IPAddressAllocationReconciler) setReadyStatusTrue(ctx context.Context, Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX IPAddressAllocation has been successfully created/updated", - Reason: "", + Reason: "IPAddressAllocationReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go index 2eee4de45..2699943cd 100644 --- a/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go +++ b/pkg/controllers/ipaddressallocation/ipaddressallocation_controller_test.go @@ -51,7 +51,7 @@ func TestIPAddressAllocationController_setReadyStatusTrue(t *testing.T) { Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX IPAddressAllocation has been successfully created/updated", - Reason: "", + Reason: "IPAddressAllocationReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller.go b/pkg/controllers/securitypolicy/securitypolicy_controller.go index 830dec59d..778d052f9 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller.go @@ -243,7 +243,7 @@ func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusTrue(ctx context. Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Security Policy has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "SecurityPolicyReady", LastTransitionTime: transitionTime, }, } @@ -253,13 +253,13 @@ func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusTrue(ctx context. func (r *SecurityPolicyReconciler) setSecurityPolicyReadyStatusFalse(ctx context.Context, secPolicy *v1alpha1.SecurityPolicy, transitionTime metav1.Time, err *error) { newConditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX Security Policy could not be created/updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the SecurityPolicy CR. Error: %v", *err, ), + Reason: "SecurityPolicyNotReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/staticroute/staticroute_controller.go b/pkg/controllers/staticroute/staticroute_controller.go index 9e8e1769b..265ff29ad 100644 --- a/pkg/controllers/staticroute/staticroute_controller.go +++ b/pkg/controllers/staticroute/staticroute_controller.go @@ -135,7 +135,7 @@ func (r *StaticRouteReconciler) setStaticRouteReadyStatusTrue(ctx context.Contex Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Static Route has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "StaticRouteReady", LastTransitionTime: transitionTime, }, } @@ -147,8 +147,8 @@ func (r *StaticRouteReconciler) setStaticRouteReadyStatusFalse(ctx context.Conte { Type: v1alpha1.Ready, Status: v1.ConditionFalse, - Message: "NSX Static Route could not be created/updated/deleted", - Reason: fmt.Sprintf("Error occurred while processing the Static Route CR. Please check the config and try again. Error: %v", *err), + Message: fmt.Sprintf("Error occurred while processing the Static Route CR. Please check the config and try again. Error: %v", *err), + Reason: "StaticRouteNotReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 4a5daf27c..13da216f6 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -177,7 +177,7 @@ func (r *SubnetReconciler) setSubnetReadyStatusTrue(ctx context.Context, subnet Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX Subnet has been successfully created/updated", - Reason: "SubnetCreated", + Reason: "SubnetReady", LastTransitionTime: transitionTime, }, } diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index 1cf28af32..ee6a15cfc 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -322,7 +322,7 @@ func (r *SubnetPortReconciler) setSubnetPortReadyStatusTrue(ctx context.Context, Type: v1alpha1.Ready, Status: v1.ConditionTrue, Message: "NSX subnet port has been successfully created/updated", - Reason: "NSX API returned 200 response code for PATCH", + Reason: "SubnetPortReady", LastTransitionTime: transitionTime, }, } @@ -332,13 +332,13 @@ func (r *SubnetPortReconciler) setSubnetPortReadyStatusTrue(ctx context.Context, func (r *SubnetPortReconciler) setSubnetPortReadyStatusFalse(ctx context.Context, subnetPort *v1alpha1.SubnetPort, transitionTime metav1.Time, err *error) { newConditions := []v1alpha1.Condition{ { - Type: v1alpha1.Ready, - Status: v1.ConditionFalse, - Message: "NSX subnet port could not be created/updated", - Reason: fmt.Sprintf( + Type: v1alpha1.Ready, + Status: v1.ConditionFalse, + Message: fmt.Sprintf( "error occurred while processing the SubnetPort CR. Error: %v", *err, ), + Reason: "SubnetPortNotReady", LastTransitionTime: transitionTime, }, } From 7d40f201b8f032a3a47ec46fd06beb5d91ec37ea Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Fri, 27 Sep 2024 11:29:11 +0800 Subject: [PATCH 04/18] Delete the vs/lb pool created for pre-created VPC The pre-created VPC is shared by many ns. Once the ns deleted, the pre-created VPC will not be deleted. But the vs/lb pool created for service under pre-created VPC should be released. Delete vpc/lb pool created for SLB by NCP in cleanup --- pkg/clean/clean.go | 4 +- pkg/nsx/client.go | 6 ++ pkg/nsx/services/common/types.go | 4 ++ pkg/nsx/services/vpc/builder.go | 14 ++++ pkg/nsx/services/vpc/store.go | 4 ++ pkg/nsx/services/vpc/vpc.go | 111 +++++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 2 deletions(-) diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index f7a26f3c3..c315bfa13 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -174,8 +174,8 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Clien AddCleanupService(wrapInitializeSubnetService(commonService)). AddCleanupService(wrapInitializeSecurityPolicy(commonService)). AddCleanupService(wrapInitializeStaticRoute(commonService)). - AddCleanupService(wrapInitializeIPAddressAllocation(commonService)). - AddCleanupService(wrapInitializeVPC(commonService)) + AddCleanupService(wrapInitializeVPC(commonService)). + AddCleanupService(wrapInitializeIPAddressAllocation(commonService)) return cleanupService, nil } diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 3af7aa23f..c43f6b45c 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -90,6 +90,8 @@ type Client struct { RealizedStateClient realized_state.RealizedEntitiesClient IPAddressAllocationClient vpcs.IpAddressAllocationsClient VPCLBSClient vpcs.VpcLbsClient + VpcLbVirtualServersClient vpcs.VpcLbVirtualServersClient + VpcLbPoolsClient vpcs.VpcLbPoolsClient ProjectClient orgs.ProjectsClient TransitGatewayClient projects.TransitGatewaysClient TransitGatewayAttachmentClient transit_gateways.AttachmentsClient @@ -182,6 +184,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster)) vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) + vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(restConnector(cluster)) + vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -228,6 +232,8 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, VPCLBSClient: vpcLBSClient, + VpcLbVirtualServersClient: vpcLbVirtualServersClient, + VpcLbPoolsClient: vpcLbPoolsClient, ProjectClient: projectClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 31ec87ff3..ddc7d2bbc 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -21,10 +21,12 @@ const ( MaxIdLength int = 255 MaxNameLength int = 255 MaxSubnetNameLength int = 80 + VPCLbResourcePathMinSegments int = 8 PriorityNetworkPolicyAllowRule int = 2010 PriorityNetworkPolicyIsolationRule int = 2090 TagScopeNCPCluster string = "ncp/cluster" TagScopeNCPProjectUID string = "ncp/project_uid" + TagScopeNCPCreateFor string = "ncp/created_for" TagScopeNCPVIFProjectUID string = "ncp/vif_project_uid" TagScopeNCPPod string = "ncp/pod" TagScopeNCPVNETInterface string = "ncp/vnet_interface" @@ -175,6 +177,8 @@ var ( ResourceTypeLBSourceIpPersistenceProfile = "LBSourceIpPersistenceProfile" ResourceTypeLBHttpMonitorProfile = "LBHttpMonitorProfile" ResourceTypeLBTcpMonitorProfile = "LBTcpMonitorProfile" + ResourceTypeLBVirtualServer = "LBVirtualServer" + ResourceTypeLBPool = "LBPool" // ResourceTypeClusterControlPlane is used by NSXServiceAccountController ResourceTypeClusterControlPlane = "clustercontrolplane" diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 2bd6a9201..c05038a47 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -32,6 +32,20 @@ func generateLBSKey(lbs model.LBService) (string, error) { return combineVPCIDAndLBSID(vpcID, *lbs.Id), nil } +func generateVirtualServerKey(vs model.LBVirtualServer) (string, error) { + if vs.Path == nil || *vs.Path == "" { + return "", fmt.Errorf("LBVirtualServer path is nil or empty") + } + return *vs.Path, nil +} + +func generatePoolKey(pool model.LBPool) (string, error) { + if pool.Path == nil || *pool.Path == "" { + return "", fmt.Errorf("LBPool path is nil or empty") + } + return *pool.Path, nil +} + func combineVPCIDAndLBSID(vpcID, lbsID string) string { return fmt.Sprintf("%s_%s", vpcID, lbsID) } diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 067b369a7..74c7b765a 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -15,6 +15,10 @@ func keyFunc(obj interface{}) (string, error) { return *v.Id, nil case *model.LBService: return generateLBSKey(*v) + case *model.LBVirtualServer: + return generateVirtualServerKey(*v) + case *model.LBPool: + return generatePoolKey(*v) default: return "", errors.New("keyFunc doesn't support unknown type") } diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 049cd2a23..3e9e69481 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -239,6 +239,13 @@ func (s *VPCService) addClusterTag(query string) string { return query + " AND " + tagParam } +func (s *VPCService) addNCPCreatedForTag(query string) string { + tagScopeClusterKey := strings.Replace(common.TagScopeNCPCreateFor, "/", "\\/", -1) + tagScopeClusterValue := strings.Replace(common.TagValueSLB, ":", "\\:", -1) + tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) + return query + " AND " + tagParam +} + func (s *VPCService) ListCert() []model.TlsCertificate { store := &ResourceStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), @@ -429,7 +436,84 @@ func (s *VPCService) DeleteLBMonitorProfile(id string) error { log.Info("successfully deleted NCP created lbMonitorProfile", "lbMonitorProfile", id) return nil } +func (s *VPCService) ListLBVirtualServer() []model.LBVirtualServer { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBVirtualServerBindingType(), + }} + query := fmt.Sprintf("(%s:%s)", + common.ResourceType, common.ResourceTypeLBVirtualServer) + query = s.addClusterTag(query) + query = s.addNCPCreatedForTag(query) + count, searcherr := s.SearchResource("", query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query LBVirtualServer", "query", query) + } else { + log.V(1).Info("query LBVirtualServer", "count", count) + } + lbVirtualServers := store.List() + lbVirtualServersSet := []model.LBVirtualServer{} + for _, lbVirtualServer := range lbVirtualServers { + lbVirtualServersSet = append(lbVirtualServersSet, *lbVirtualServer.(*model.LBVirtualServer)) + } + return lbVirtualServersSet +} + +func (s *VPCService) DeleteLBVirtualServer(path string) error { + lbVirtualServersClient := s.NSXClient.VpcLbVirtualServersClient + boolValue := false + paths := strings.Split(path, "/") + + if len(paths) < common.VPCLbResourcePathMinSegments { + // skip virtual server under infra + log.Info("failed to parse virtual server path", "path", path) + return nil + } + if err := lbVirtualServersClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil { + return err + } + log.Info("successfully deleted NCP created lbVirtualServer", "lbVirtualServer", path) + return nil +} +func (s *VPCService) ListLBPool() []model.LBPool { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBPoolBindingType(), + }} + query := fmt.Sprintf("(%s:%s)", + common.ResourceType, common.ResourceTypeLBPool) + query = s.addClusterTag(query) + query = s.addNCPCreatedForTag(query) + count, searcherr := s.SearchResource("", query, store, nil) + if searcherr != nil { + log.Error(searcherr, "failed to query LBPool", "query", query) + } else { + log.V(1).Info("query LBPool", "count", count) + } + lbPools := store.List() + lbPoolsSet := []model.LBPool{} + for _, lbPool := range lbPools { + lbPoolsSet = append(lbPoolsSet, *lbPool.(*model.LBPool)) + } + return lbPoolsSet +} + +func (s *VPCService) DeleteLBPool(path string) error { + lbPoolsClient := s.NSXClient.VpcLbPoolsClient + boolValue := false + paths := strings.Split(path, "/") + if len(paths) < 8 { + // skip lb pool under infra + log.Info("failed to parse lb pool path", "path", path) + return nil + } + if err := lbPoolsClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil { + return err + } + log.Info("successfully deleted NCP created lbPool", "lbPool", path) + return nil +} func (s *VPCService) IsSharedVPCNamespaceByNS(ns string) (bool, error) { shared_ns, err := s.getSharedVPCNamespaceFromNS(ns) if err != nil { @@ -917,6 +1001,33 @@ func (s *VPCService) Cleanup(ctx context.Context) error { } } } + + // Clean vs/lb pool created for pre-created vpc + lbVirtualServers := s.ListLBVirtualServer() + log.Info("cleaning up lbVirtualServers", "Count", len(lbVirtualServers)) + for _, lbVirtualServer := range lbVirtualServers { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteLBVirtualServer(*lbVirtualServer.Path); err != nil { + return err + } + } + } + + lbPools := s.ListLBPool() + log.Info("cleaning up lbPools", "Count", len(lbPools)) + for _, lbPool := range lbPools { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := s.DeleteLBPool(*lbPool.Path); err != nil { + return err + } + } + } // We don't clean client_ssl_profile as client_ssl_profile is not created by ncp or nsx-operator return nil } From 0445ed3501ad373e907d215a4dc7b617124345a1 Mon Sep 17 00:00:00 2001 From: Deng Yun Date: Wed, 9 Oct 2024 20:17:12 +0800 Subject: [PATCH 05/18] Remove rule index from rule/group ID/Name and unify name for VPC and T1 (#785) This patch is to: 1. Remove SecurityPolicy rule index from rule ID and for VPC mode, and keep T1 mode rule ID unchanged with rule index. 2. Remove SecurityPolicy rule index from group ID for VPC mode, and keep T1 mode group ID unchanged with rule index. 3. Remove rule index from NSX group/rule name, and unify the NSX resource name for VPC and T1 network, including SecurityPolicy, rule, and group. 4. Reduce length of rule hash string to 8 chars for VPC mode. --- pkg/nsx/services/securitypolicy/builder.go | 197 +++++---- .../services/securitypolicy/builder_test.go | 401 +++++++++++++++--- pkg/nsx/services/securitypolicy/expand.go | 8 +- .../services/securitypolicy/expand_test.go | 20 +- .../services/securitypolicy/firewall_test.go | 18 +- pkg/util/utils.go | 13 +- pkg/util/utils_test.go | 2 +- 7 files changed, 493 insertions(+), 166 deletions(-) diff --git a/pkg/nsx/services/securitypolicy/builder.go b/pkg/nsx/services/securitypolicy/builder.go index b1a31e45e..6a7bfb56a 100644 --- a/pkg/nsx/services/securitypolicy/builder.go +++ b/pkg/nsx/services/securitypolicy/builder.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strconv" "strings" @@ -14,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" @@ -39,24 +39,15 @@ var ( Int64 = common.Int64 ) -func (service *SecurityPolicyService) buildSecurityPolicyName(obj *v1alpha1.SecurityPolicy, createdFor string) string { - if IsVPCEnabled(service) { - // For VPC scenario, we use obj.Name as the NSX resource display name for both SecurityPolicy and NetworkPolicy. - return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", "", "", "") - } - prefix := common.SecurityPolicyPrefix - if createdFor != common.ResourceTypeSecurityPolicy { - prefix = common.NetworkPolicyPrefix - } - // For T1 scenario, we use ns-name as the key resource name for SecurityPolicy, it is to be consistent with the - // previous solutions. - return util.GenerateTruncName(common.MaxNameLength, strings.Join([]string{obj.Namespace, obj.Name}, common.ConnectorUnderline), prefix, "", "", "") +func (service *SecurityPolicyService) buildSecurityPolicyName(obj *v1alpha1.SecurityPolicy) string { + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", "", "", "") } func (service *SecurityPolicyService) buildSecurityPolicyID(obj *v1alpha1.SecurityPolicy, createdFor string) string { if IsVPCEnabled(service) { return util.GenerateIDByObject(obj) } + prefix := common.SecurityPolicyPrefix if createdFor != common.ResourceTypeSecurityPolicy { prefix = common.NetworkPolicyPrefix @@ -76,7 +67,7 @@ func (service *SecurityPolicyService) buildSecurityPolicy(obj *v1alpha1.Security nsxSecurityPolicy := &model.SecurityPolicy{} nsxSecurityPolicy.Id = String(service.buildSecurityPolicyID(obj, createdFor)) - nsxSecurityPolicy.DisplayName = String(service.buildSecurityPolicyName(obj, createdFor)) + nsxSecurityPolicy.DisplayName = String(service.buildSecurityPolicyName(obj)) // TODO: confirm the sequence number: offset nsxSecurityPolicy.SequenceNumber = Int64(int64(obj.Spec.Priority)) @@ -93,7 +84,7 @@ func (service *SecurityPolicyService) buildSecurityPolicy(obj *v1alpha1.Security currentSet := sets.Set[string]{} for ruleIdx, r := range obj.Spec.Rules { rule := r - // A rule containing named port may expand to multiple rules if the name maps to multiple port numbers. + // A rule containing named port may be expanded to multiple rules if the named ports map to multiple port numbers. expandRules, buildGroups, buildGroupShares, err := service.buildRuleAndGroups(obj, &rule, ruleIdx, createdFor) if err != nil { log.Error(err, "failed to build rule and groups", "rule", rule, "ruleIndex", ruleIdx) @@ -204,11 +195,6 @@ func (service *SecurityPolicyService) buildTargetTags(obj *v1alpha1.SecurityPoli rule *v1alpha1.SecurityPolicyRule, ruleIdx int, createdFor string, ) []model.Tag { basicTags := service.buildBasicTags(obj, createdFor) - sort.Slice(*targets, func(i, j int) bool { - k1, _ := json.Marshal((*targets)[i]) - k2, _ := json.Marshal((*targets)[j]) - return string(k1) < string(k2) - }) serializedBytes, _ := json.Marshal(*targets) targetTags := []model.Tag{ { @@ -242,7 +228,7 @@ func (service *SecurityPolicyService) buildTargetTags(obj *v1alpha1.SecurityPoli targetTags = append(targetTags, model.Tag{ Scope: String(common.TagScopeRuleID), - Tag: String(service.buildRuleID(obj, rule, ruleIdx, createdFor)), + Tag: String(service.buildRuleID(obj, ruleIdx, createdFor)), }, ) } @@ -379,7 +365,8 @@ func (service *SecurityPolicyService) buildAppliedGroupID(obj *v1alpha1.Security if IsVPCEnabled(service) { suffix := common.TargetGroupSuffix if ruleIdx != -1 { - suffix = strings.Join([]string{strconv.Itoa(ruleIdx), suffix}, common.ConnectorUnderline) + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) } return util.GenerateIDByObjectWithSuffix(obj, suffix) } @@ -415,17 +402,13 @@ func (service *SecurityPolicyService) buildAppliedGroupPath(obj *v1alpha1.Securi // build appliedTo group display name for both policy and rule levels. func (service *SecurityPolicyService) buildAppliedGroupName(obj *v1alpha1.SecurityPolicy, ruleIdx int) string { - var rule *v1alpha1.SecurityPolicyRule if ruleIdx != -1 { - rule = &(obj.Spec.Rules[ruleIdx]) - ruleName := strings.Join([]string{obj.Name, strconv.Itoa(ruleIdx)}, common.ConnectorUnderline) - if len(rule.Name) > 0 { - ruleName = rule.Name - } - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", common.TargetGroupSuffix, "", "") + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix := strings.Join([]string{ruleHash, common.TargetGroupSuffix}, common.ConnectorUnderline) + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", suffix, "", "") } - ruleName := strings.Join([]string{obj.Namespace, obj.Name}, common.ConnectorUnderline) - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", common.TargetGroupSuffix, "", "") + + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", common.TargetGroupSuffix, "", "") } func (service *SecurityPolicyService) buildRuleAndGroups(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, @@ -598,22 +581,44 @@ func (service *SecurityPolicyService) buildRuleOutGroup(obj *v1alpha1.SecurityPo return nsxRuleDstGroup, nsxRuleSrcGroupPath, nsxRuleDstGroupPath, nsxGroupShare, nil } -func (service *SecurityPolicyService) buildRuleID(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, ruleIdx int, createdFor string) string { - serializedBytes, _ := json.Marshal(rule) - ruleHash := fmt.Sprintf("%s", util.Sha1(string(serializedBytes))) - ruleIdxStr := fmt.Sprintf("%d", ruleIdx) +func (service *SecurityPolicyService) buildRuleID(obj *v1alpha1.SecurityPolicy, ruleIdx int, createdFor string) string { + ruleIndexHash := service.buildRuleHashString(&(obj.Spec.Rules[ruleIdx])) + if IsVPCEnabled(service) { - suffix := strings.Join([]string{ruleIdxStr, ruleHash}, common.ConnectorUnderline) - return util.GenerateIDByObjectWithSuffix(obj, suffix) + ruleIndexHash = service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + return util.GenerateIDByObjectWithSuffix(obj, ruleIndexHash) } + prefix := common.SecurityPolicyPrefix if createdFor == common.ResourceTypeNetworkPolicy { prefix = common.NetworkPolicyPrefix } - return util.GenerateID(fmt.Sprintf("%s", obj.UID), prefix, ruleHash, ruleIdxStr) + ruleIdxStr := fmt.Sprintf("%d", ruleIdx) + return strings.Join([]string{prefix, string(obj.UID), ruleIndexHash, ruleIdxStr}, common.ConnectorUnderline) } -func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.SecurityPolicyRule, portIdx, portNumber int, hasNamedport bool, createdFor string) (string, error) { +// A rule containing named port may be expanded to multiple NSX rules if the name ports map to multiple port numbers. +// So, in VPC network, the rule port numbers, which either are defined in rule Port or resolved from named port, will be appended as CR rule baseID to distinguish them. +// For T1, the portIdx and portAddressIdx are appended as suffix. +func (service *SecurityPolicyService) buildExpandedRuleID(obj *v1alpha1.SecurityPolicy, ruleIdx int, portIdx, portAddressIdx int, + hasNamedport bool, portNumber int, createdFor string, +) string { + ruleBaseID := service.buildRuleID(obj, ruleIdx, createdFor) + + if IsVPCEnabled(service) { + portNumberSuffix := "" + if !hasNamedport { + portNumberSuffix = service.buildRulePortsNumberString(obj.Spec.Rules[ruleIdx].Ports) + } else { + portNumberSuffix = service.buildRulePortNumberString(obj.Spec.Rules[ruleIdx].Ports[portIdx], portNumber) + } + return strings.Join([]string{ruleBaseID, portNumberSuffix}, common.ConnectorUnderline) + } + + return strings.Join([]string{ruleBaseID, strconv.Itoa(portIdx), strconv.Itoa(portAddressIdx)}, common.ConnectorUnderline) +} + +func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.SecurityPolicyRule, portIdx int, hasNamedport bool, portNumber int, createdFor string) (string, error) { var ruleName string var ruleAct string @@ -651,7 +656,7 @@ func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.Securi ruleName = strings.Join([]string{ruleName, suffix}, common.ConnectorUnderline) } } else { - ruleName = service.buildRulePortsString(&rule.Ports, suffix) + ruleName = service.buildRulePortsString(rule.Ports, suffix) } if !hasNamedport { @@ -659,11 +664,11 @@ func (service *SecurityPolicyService) buildRuleDisplayName(rule *v1alpha1.Securi } else { // For the security policy rule with namedPort, it will be expanded to the multiple security policy rules based on resolution of named port. // e.g. input: security policy's rule name: TCP.http_UDP.1234_ingress_allow, - // expand to NSX security policy rules with name TCP.http_UDP.1234_TCP.80_ingress_allow and TCP.http_UDP.1234_UDP.1234_ingress_allow. + // expand to NSX security policy rules with name TCP.http_UDP.1234.TCP.80_ingress_allow and TCP.http_UDP.1234.UDP.1234_ingress_allow. // in case that user defined input security policy's rule name: sp_namedport_rule, // expand to NSX security policy rules with name sp_namedport_rule.TCP.80_ingress_allow and sp_namedport_rule.UDP.1234_ingress_allow. index := strings.Index(ruleName, common.ConnectorUnderline+suffix) - return util.GenerateTruncName(common.MaxNameLength, ruleName[:index]+"."+service.buildRulePortString(&rule.Ports[portIdx], true, portNumber), "", suffix, "", ""), nil + return util.GenerateTruncName(common.MaxNameLength, ruleName[:index]+"."+service.buildRulePortString(rule.Ports[portIdx], portNumber), "", suffix, "", ""), nil } } @@ -747,24 +752,25 @@ func (service *SecurityPolicyService) buildRulePeerGroupID(obj *v1alpha1.Securit if isSource == true { suffix = common.SrcGroupSuffix } + if IsVPCEnabled(service) { - suffix = strings.Join([]string{strconv.Itoa(ruleIdx), suffix}, common.ConnectorUnderline) + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) return util.GenerateIDByObjectWithSuffix(obj, suffix) } + return util.GenerateID(string(obj.UID), common.SecurityPolicyPrefix, suffix, strconv.Itoa(ruleIdx)) } func (service *SecurityPolicyService) buildRulePeerGroupName(obj *v1alpha1.SecurityPolicy, ruleIdx int, isSource bool) string { - rule := &(obj.Spec.Rules[ruleIdx]) suffix := common.DstGroupSuffix if isSource == true { suffix = common.SrcGroupSuffix } - ruleName := strings.Join([]string{obj.Name, strconv.Itoa(ruleIdx)}, common.ConnectorUnderline) - if len(rule.Name) > 0 { - ruleName = rule.Name - } - return util.GenerateTruncName(common.MaxNameLength, ruleName, "", suffix, "", "") + ruleHash := service.buildLimitedRuleHashString(&(obj.Spec.Rules[ruleIdx])) + suffix = strings.Join([]string{ruleHash, suffix}, common.ConnectorUnderline) + + return util.GenerateTruncName(common.MaxNameLength, obj.Name, "", suffix, "", "") } func (service *SecurityPolicyService) buildRulePeerGroupPath(obj *v1alpha1.SecurityPolicy, ruleIdx int, isSource, infraGroupShared, projectGroupShared bool, vpcInfo *common.VPCResourceInfo) (string, error) { @@ -913,15 +919,11 @@ func (service *SecurityPolicyService) buildRulePeerGroup(obj *v1alpha1.SecurityP return &rulePeerGroup, rulePeerGroupPath, nil, err } -func (service *SecurityPolicyService) buildExpandedRuleId(ruleBaseId string, portIdx int, portAddressIdx int) string { - return strings.Join([]string{ruleBaseId, strconv.Itoa(portIdx), strconv.Itoa(portAddressIdx)}, common.ConnectorUnderline) -} - // Build rule basic info, ruleIdx is the index of the rules of security policy, // portIdx is the index of rule's ports, portAddressIdx is the index // of multiple port number if one named port maps to multiple port numbers. func (service *SecurityPolicyService) buildRuleBasicInfo(obj *v1alpha1.SecurityPolicy, rule *v1alpha1.SecurityPolicyRule, ruleIdx int, portIdx int, portAddressIdx int, - portNumber int, hasNamedport bool, createdFor string, + hasNamedport bool, portNumber int, createdFor string, ) (*model.Rule, error) { ruleAction, err := getRuleAction(rule) if err != nil { @@ -931,13 +933,13 @@ func (service *SecurityPolicyService) buildRuleBasicInfo(obj *v1alpha1.SecurityP if err != nil { return nil, err } - displayName, err := service.buildRuleDisplayName(rule, portIdx, portNumber, hasNamedport, createdFor) + displayName, err := service.buildRuleDisplayName(rule, portIdx, hasNamedport, portNumber, createdFor) if err != nil { log.Error(err, "failed to build rule's display name", "securityPolicyUID", obj.UID, "rule", rule, "createdFor", createdFor) } nsxRule := model.Rule{ - Id: String(service.buildExpandedRuleId(service.buildRuleID(obj, rule, ruleIdx, createdFor), portIdx, portAddressIdx)), + Id: String(service.buildExpandedRuleID(obj, ruleIdx, portIdx, portAddressIdx, hasNamedport, portNumber, createdFor)), DisplayName: &displayName, Direction: &ruleDirection, SequenceNumber: Int64(int64(ruleIdx)), @@ -957,13 +959,6 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy groupTypeTag = String(common.TagValueGroupSource) peers = &rule.Sources } - - // TODO: abstract sort func for both peers and targets - sort.Slice(*peers, func(i, j int) bool { - k1, _ := json.Marshal((*peers)[i]) - k2, _ := json.Marshal((*peers)[j]) - return string(k1) < string(k2) - }) serializedBytes, _ := json.Marshal(*peers) peerTags := []model.Tag{ @@ -973,7 +968,7 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy }, { Scope: String(common.TagScopeRuleID), - Tag: String(service.buildRuleID(obj, rule, ruleIdx, createdFor)), + Tag: String(service.buildRuleID(obj, ruleIdx, createdFor)), }, { Scope: String(common.TagScopeSelectorHash), @@ -1009,6 +1004,7 @@ func (service *SecurityPolicyService) buildPeerTags(obj *v1alpha1.SecurityPolicy ) } } + return peerTags } @@ -1847,7 +1843,7 @@ func (service *SecurityPolicyService) getNamespaceUID(ns string) (nsUid types.UI return namespace_uid } -func (service *SecurityPolicyService) buildRulePortString(port *v1alpha1.SecurityPolicyPort, hasNamedport bool, portNumber int) string { +func (service *SecurityPolicyService) buildRulePortString(port v1alpha1.SecurityPolicyPort, portNumber int) string { protocol := string(port.Protocol) // Build the rule port string name for non named port. // This is a common case where the string is built from port definition. For instance, @@ -1858,36 +1854,77 @@ func (service *SecurityPolicyService) buildRulePortString(port *v1alpha1.Securit // - protocol: UDP // port: 3308 // The built port string is: UDP.3308 - if !hasNamedport { - if port.EndPort != 0 { - return fmt.Sprintf("%s.%s.%d", protocol, (port.Port).String(), port.EndPort) - } - return fmt.Sprintf("%s.%s", protocol, (port.Port).String()) - } else { + if port.Port.Type == intstr.String && portNumber > 0 { // Build the rule port string name for named port. // The port string is built from specific port number resolved from named port. return fmt.Sprintf("%s.%d", protocol, portNumber) } + if port.EndPort != 0 { + return fmt.Sprintf("%s.%s.%d", protocol, (port.Port).String(), port.EndPort) + } + return fmt.Sprintf("%s.%s", protocol, (port.Port).String()) } -func (service *SecurityPolicyService) buildRulePortsString(ports *[]v1alpha1.SecurityPolicyPort, suffix string) string { +func (service *SecurityPolicyService) buildRulePortsString(ports []v1alpha1.SecurityPolicyPort, suffix string) string { portsString := "" - if ports == nil || len(*ports) == 0 { + if ports == nil || len(ports) == 0 { portsString = common.RuleAnyPorts } else { - for idx, p := range *ports { + portStrings := make([]string, len(ports)) + for idx, p := range ports { port := p - portString := service.buildRulePortString(&port, false, -1) - if idx == 0 { - portsString = portString - } else { - portsString = strings.Join([]string{portsString, portString}, common.ConnectorUnderline) - } + portStrings[idx] = service.buildRulePortString(port, -1) } + portsString = strings.Join(portStrings, common.ConnectorUnderline) } + return util.GenerateTruncName(common.MaxNameLength, portsString, "", suffix, "", "") } +func (service *SecurityPolicyService) buildRulePortNumberString(port v1alpha1.SecurityPolicyPort, portNumber int) string { + // Build the rule port number string name for non named port. + // This is a common case where the string is built from port definition. For instance, + // - protocol: TCP + // port: 8282 + // endPort: 8286 + // The built port number string is: 8282.8286 + // - protocol: UDP + // port: 3308 + // The built port number string is: 3308 + if port.Port.Type == intstr.String && portNumber > 0 { + // Build the rule port number string name for named port. + // The port number string is built from specific port number resolved from named port. + return fmt.Sprintf("%d", portNumber) + } + if port.EndPort != 0 { + return fmt.Sprintf("%s.%d", (port.Port).String(), port.EndPort) + } + return (port.Port).String() +} + +func (service *SecurityPolicyService) buildRulePortsNumberString(ports []v1alpha1.SecurityPolicyPort) string { + if ports == nil || len(ports) == 0 { + return common.RuleAnyPorts + } + + portNumStrings := make([]string, len(ports)) + for idx, p := range ports { + port := p + portNumStrings[idx] = service.buildRulePortNumberString(port, -1) + } + return strings.Join(portNumStrings, common.ConnectorUnderline) +} + +func (service *SecurityPolicyService) buildLimitedRuleHashString(rule *v1alpha1.SecurityPolicyRule) string { + serializedBytes, _ := json.Marshal(rule) + return util.Sha1(string(serializedBytes))[:common.HashLength] +} + +func (service *SecurityPolicyService) buildRuleHashString(rule *v1alpha1.SecurityPolicyRule) string { + serializedBytes, _ := json.Marshal(rule) + return util.Sha1(string(serializedBytes)) +} + func (service *SecurityPolicyService) BuildNetworkPolicyAllowPolicyID(uid string) string { return strings.Join([]string{uid, common.RuleActionAllow}, common.ConnectorUnderline) } diff --git a/pkg/nsx/services/securitypolicy/builder_test.go b/pkg/nsx/services/securitypolicy/builder_test.go index 5fbca2def..aef63e991 100644 --- a/pkg/nsx/services/securitypolicy/builder_test.go +++ b/pkg/nsx/services/securitypolicy/builder_test.go @@ -35,19 +35,19 @@ func TestBuildSecurityPolicy(t *testing.T) { ) podSelectorRule0Name00 := "rule-with-pod-ns-selector_ingress_allow" - podSelectorRule0IDPort000 := "sp_uidA_0_2c822e90b1377b346014adfa583f08a99dee52a8_0_0" + podSelectorRule0IDPort000 := "sp_uidA_2c822e90b1377b346014adfa583f08a99dee52a8_0_0_0" podSelectorRule1Name00 := "rule-with-ns-selector_ingress_allow" - podSelectorRule1IDPort000 := "sp_uidA_1_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_0_0" + podSelectorRule1IDPort000 := "sp_uidA_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_1_0_0" vmSelectorRule0Name00 := "rule-with-VM-selector_egress_isolation" - vmSelectorRule0IDPort000 := "sp_uidB_0_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0" + vmSelectorRule0IDPort000 := "sp_uidB_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0_0" vmSelectorRule1Name00 := "rule-with-ns-selector_egress_isolation" - vmSelectorRule1IDPort000 := "sp_uidB_1_7d721f087be35f0bf318f4847b5acdc3d2b91446_0_0" + vmSelectorRule1IDPort000 := "sp_uidB_7d721f087be35f0bf318f4847b5acdc3d2b91446_1_0_0" vmSelectorRule2Name00 := "all_egress_isolation" - vmSelectorRule2IDPort000 := "sp_uidB_2_a40c813916cc397fcd2260e48cc773d4c9b08565_0_0" + vmSelectorRule2IDPort000 := "sp_uidB_a40c813916cc397fcd2260e48cc773d4c9b08565_2_0_0" tests := []struct { name string @@ -96,7 +96,7 @@ func TestBuildSecurityPolicy(t *testing.T) { name: "security-policy-with-VM-selector For T1", inputPolicy: &spWithVMSelector, expectedPolicy: &model.SecurityPolicy{ - DisplayName: common.String("sp_ns1_spB"), + DisplayName: common.String("spB"), Id: common.String("sp_uidB"), Scope: []string{"/infra/domains/k8scl-one/groups/sp_uidB_scope"}, SequenceNumber: &seq0, @@ -201,19 +201,19 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { defer patches.Reset() podSelectorRule0Name00 := "rule-with-pod-ns-selector_ingress_allow" - podSelectorRule0IDPort000 := "spA_uidA_0_2c822e90b1377b346014adfa583f08a99dee52a8_0_0" + podSelectorRule0IDPort000 := "spA_uidA_2c822e90_all" podSelectorRule1Name00 := "rule-with-ns-selector_ingress_allow" - podSelectorRule1IDPort000 := "spA_uidA_1_2a4595d0dd582c2ae5613245ad7b39de5ade2e20_0_0" + podSelectorRule1IDPort000 := "spA_uidA_2a4595d0_53" vmSelectorRule0Name00 := "rule-with-VM-selector_egress_isolation" - vmSelectorRule0IDPort000 := "spB_uidB_0_67410606c486d2ba38002ed076a2a4211c9d49b5_0_0" + vmSelectorRule0IDPort000 := "spB_uidB_67410606_all" vmSelectorRule1Name00 := "rule-with-ns-selector_egress_isolation" - vmSelectorRule1IDPort000 := "spB_uidB_1_7d721f087be35f0bf318f4847b5acdc3d2b91446_0_0" + vmSelectorRule1IDPort000 := "spB_uidB_7d721f08_all" vmSelectorRule2Name00 := "all_egress_isolation" - vmSelectorRule2IDPort000 := "spB_uidB_2_a40c813916cc397fcd2260e48cc773d4c9b08565_0_0" + vmSelectorRule2IDPort000 := "spB_uidB_a40c8139_all" tests := []struct { name string @@ -234,10 +234,10 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { Id: &podSelectorRule0IDPort000, DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_0_scope"}, + Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_2c822e90_scope"}, SequenceNumber: &seq0, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_0_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_2c822e90_src"}, Action: &nsxRuleActionAllow, Tags: vpcBasicTags, }, @@ -249,7 +249,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { Scope: []string{"ANY"}, SequenceNumber: &seq1, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_1_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spA_uidA_2a4595d0_src"}, Action: &nsxRuleActionAllow, ServiceEntries: []*data.StructValue{serviceEntry}, Tags: vpcBasicTags, @@ -270,9 +270,9 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule0Name00, Id: &vmSelectorRule0IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_0_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_67410606_dst"}, Direction: &nsxRuleDirectionOut, - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_0_scope"}, + Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_67410606_scope"}, SequenceNumber: &seq0, Services: []string{"ANY"}, SourceGroups: []string{"ANY"}, @@ -282,7 +282,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule1Name00, Id: &vmSelectorRule1IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spB_uidB_1_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/spB_uidB_7d721f08_dst"}, Direction: &nsxRuleDirectionOut, Scope: []string{"ANY"}, SequenceNumber: &seq1, @@ -295,7 +295,7 @@ func TestBuildSecurityPolicyForVPC(t *testing.T) { { DisplayName: &vmSelectorRule2Name00, Id: &vmSelectorRule2IDPort000, - DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_2_dst"}, + DestinationGroups: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spB_uidB_a40c8139_dst"}, Direction: &nsxRuleDirectionOut, Scope: []string{"ANY"}, SequenceNumber: &seq2, @@ -354,7 +354,7 @@ func TestBuildTargetTags(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyCRName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyCRUID - ruleTagID0 := service.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy) + ruleTagID0 := service.buildRuleID(&spWithPodSelector, 0, common.ResourceTypeSecurityPolicy) tests := []struct { name string inputPolicy *v1alpha1.SecurityPolicy @@ -437,7 +437,7 @@ func TestBuildTargetTags(t *testing.T) { } func TestBuildPeerTags(t *testing.T) { - ruleTagID0 := service.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy) + ruleTagID0 := service.buildRuleID(&spWithPodSelector, 0, common.ResourceTypeSecurityPolicy) tests := []struct { name string inputPolicy *v1alpha1.SecurityPolicy @@ -825,10 +825,7 @@ func TestUpdateMixedExpressionsMatchExpression(t *testing.T) { } var securityPolicyWithMultipleNormalPorts = v1alpha1.SecurityPolicy{ - ObjectMeta: v1.ObjectMeta{ - Namespace: "null", - Name: "null", - }, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spMulPorts", UID: "spMulPortsuidA"}, Spec: v1alpha1.SecurityPolicySpec{ Rules: []v1alpha1.SecurityPolicyRule{ { @@ -862,25 +859,37 @@ var securityPolicyWithMultipleNormalPorts = v1alpha1.SecurityPolicy{ }, }, }, + { + Action: &allowAction, + Direction: &directionOut, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 80}, + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 1234}, + EndPort: 1234, + }, + }, + }, }, }, } var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ - ObjectMeta: v1.ObjectMeta{ - Namespace: "null", - Name: "null", - }, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spNamedPorts", UID: "spNamedPortsuidA"}, Spec: v1alpha1.SecurityPolicySpec{ Rules: []v1alpha1.SecurityPolicyRule{ { - Name: "TCP.http_UDP.1234.1235_ingress_allow", + Name: "user-defined-rule-namedport", Action: &allowAction, Direction: &directionIn, Ports: []v1alpha1.SecurityPolicyPort{ { Protocol: "TCP", - Port: intstr.IntOrString{Type: intstr.String, StrVal: "http"}, + Port: intstr.IntOrString{Type: intstr.String, StrVal: "http"}, // http port is 80 }, { Protocol: "UDP", @@ -889,6 +898,45 @@ var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ }, }, }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "https"}, // http port is 443 + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 1236}, + EndPort: 1237, + }, + }, + }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "web"}, + }, + { + Protocol: "UDP", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: 533}, + }, + }, + }, + { + Action: &allowAction, + Direction: &directionIn, + Ports: []v1alpha1.SecurityPolicyPort{ + { + Protocol: "TCP", + Port: intstr.IntOrString{Type: intstr.String, StrVal: "db"}, + }, + }, + }, }, }, } @@ -896,22 +944,52 @@ var securityPolicyWithOneNamedPort = v1alpha1.SecurityPolicy{ func TestBuildRulePortsString(t *testing.T) { tests := []struct { name string - inputPorts *[]v1alpha1.SecurityPolicyPort + inputPorts []v1alpha1.SecurityPolicyPort suffix string expectedRulePortsString string }{ { name: "build-string-for-multiple-ports-without-named-port", - inputPorts: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, suffix: "ingress_allow", expectedRulePortsString: "TCP.80_UDP.1234.1235_ingress_allow", }, { - name: "build-string-for-multiple-ports-without-one-named-port", - inputPorts: &securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, + name: "build-string-for-multiple-ports-userdefinedrule-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[1].Ports, + suffix: "egress_drop", + expectedRulePortsString: "TCP.88_UDP.1236.1237_egress_drop", + }, + { + name: "build-string-for-multiple-ports-start-end-port-same-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[2].Ports, + suffix: "egress_allow", + expectedRulePortsString: "TCP.80_UDP.1234.1234_egress_allow", + }, + { + name: "build-string-for-multiple-ports-with-http-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, suffix: "ingress_allow", expectedRulePortsString: "TCP.http_UDP.1234.1235_ingress_allow", }, + { + name: "build-string-for-multiple-ports-with-https-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[1].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.https_UDP.1236.1237_ingress_allow", + }, + { + name: "build-string-for-multiple-ports-with-web-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[2].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.web_UDP.533_ingress_allow", + }, + { + name: "build-string-for-multiple-ports-with-db-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[3].Ports, + suffix: "ingress_allow", + expectedRulePortsString: "TCP.db_ingress_allow", + }, { name: "build-string-for-nil-ports", inputPorts: nil, @@ -927,6 +1005,61 @@ func TestBuildRulePortsString(t *testing.T) { } } +func TestBuildRulePortsNumberString(t *testing.T) { + tests := []struct { + name string + inputPorts []v1alpha1.SecurityPolicyPort + expectedRulePortsString string + }{ + { + name: "build-string-for-multiple-ports-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[0].Ports, + expectedRulePortsString: "80_1234.1235", + }, + { + name: "build-string-for-multiple-ports-userdefinedrule-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[1].Ports, + expectedRulePortsString: "88_1236.1237", + }, + { + name: "build-string-for-multiple-ports-start-end-port-same-without-named-port", + inputPorts: securityPolicyWithMultipleNormalPorts.Spec.Rules[2].Ports, + expectedRulePortsString: "80_1234.1234", + }, + { + name: "build-string-for-multiple-ports-with-http-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[0].Ports, + expectedRulePortsString: "http_1234.1235", + }, + { + name: "build-string-for-multiple-ports-with-https-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[1].Ports, + expectedRulePortsString: "https_1236.1237", + }, + { + name: "build-string-for-multiple-ports-with-web-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[2].Ports, + expectedRulePortsString: "web_533", + }, + { + name: "build-string-for-multiple-ports-with-db-named-port", + inputPorts: securityPolicyWithOneNamedPort.Spec.Rules[3].Ports, + expectedRulePortsString: "db", + }, + { + name: "build-string-for-nil-ports", + inputPorts: nil, + expectedRulePortsString: "all", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + observedString := service.buildRulePortsNumberString(tt.inputPorts) + assert.Equal(t, tt.expectedRulePortsString, observedString) + }) + } +} + func TestBuildRuleDisplayName(t *testing.T) { tests := []struct { name string @@ -934,6 +1067,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule *v1alpha1.SecurityPolicyRule ruleIdx int portIdx int + hasNamedPort bool + portNumber int createdFor string expectedRuleDisplayName string }{ @@ -943,6 +1078,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], ruleIdx: 0, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeNetworkPolicy, expectedRuleDisplayName: "TCP.80_UDP.1234.1235_ingress_allow", }, @@ -952,6 +1089,8 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], ruleIdx: 1, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeNetworkPolicy, expectedRuleDisplayName: "MultipleNormalPorts-rule1", }, @@ -961,19 +1100,143 @@ func TestBuildRuleDisplayName(t *testing.T) { inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], ruleIdx: 1, portIdx: 0, + hasNamedPort: false, + portNumber: -1, createdFor: common.ResourceTypeSecurityPolicy, expectedRuleDisplayName: "MultipleNormalPorts-rule1_egress_isolation", }, + { + name: "build-display-name-for-user-defined-rulename-with-one-named-http-port", + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleDisplayName: "user-defined-rule-namedport.TCP.80_ingress_allow", + }, + { + name: "build-display-name-for-multiple-ports-with-one-named-https-port", + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[1], + ruleIdx: 1, + portIdx: 0, + hasNamedPort: true, + portNumber: 443, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleDisplayName: "TCP.https_UDP.1236.1237.TCP.443_ingress_allow", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - observedDisplayName, observedError := service.buildRuleDisplayName(tt.inputRule, tt.portIdx, -1, false, tt.createdFor) + observedDisplayName, observedError := service.buildRuleDisplayName(tt.inputRule, tt.portIdx, tt.hasNamedPort, tt.portNumber, tt.createdFor) assert.Equal(t, tt.expectedRuleDisplayName, observedDisplayName) assert.Equal(t, nil, observedError) }) } } +func TestBuildExpandedRuleID(t *testing.T) { + svc := &SecurityPolicyService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "cluster1", + }, + }, + }, + } + + tests := []struct { + name string + vpcEnabled bool + inputSecurityPolicy *v1alpha1.SecurityPolicy + inputRule *v1alpha1.SecurityPolicyRule + ruleIdx int + portIdx int + portAddressIdx int + hasNamedPort bool + portNumber int + createdFor string + expectedRuleID string + }{ + { + name: "build-ruleID-for-multiple-ports-0-for-vpc", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "spMulPorts_spMulPortsuidA_d0b8e36c_80_1234.1235", + }, + { + name: "build-ruleID-for-multiple-ports-0-for-T1", + vpcEnabled: false, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "sp_spMulPortsuidA_d0b8e36cf858e76624b9706c3c8e77b6006c0e10_0_0_0", + }, + { + name: "build-ruleID-for-multiple-ports-1-for-vpc-NP", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithMultipleNormalPorts, + inputRule: &securityPolicyWithMultipleNormalPorts.Spec.Rules[1], + ruleIdx: 1, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: false, + portNumber: -1, + createdFor: common.ResourceTypeNetworkPolicy, + expectedRuleID: "spMulPorts_spMulPortsuidA_555356be_88_1236.1237", + }, + { + name: "build-ruleID-for-multiple-ports-with-one-named-port-for-VPC", + vpcEnabled: true, + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "spNamedPorts_spNamedPortsuidA_3f7c7d8c_80", + }, + { + name: "build-ruleID-for-multiple-ports-with-one-named-port-for-T1", + vpcEnabled: false, + inputSecurityPolicy: &securityPolicyWithOneNamedPort, + inputRule: &securityPolicyWithOneNamedPort.Spec.Rules[0], + ruleIdx: 0, + portIdx: 0, + portAddressIdx: 0, + hasNamedPort: true, + portNumber: 80, + createdFor: common.ResourceTypeSecurityPolicy, + expectedRuleID: "sp_spNamedPortsuidA_3f7c7d8c8449687178002f23599add04bf0c3250_0_0_0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc.NSXConfig.EnableVPCNetwork = tt.vpcEnabled + observedRuleID := svc.buildExpandedRuleID(tt.inputSecurityPolicy, tt.ruleIdx, tt.portIdx, tt.portAddressIdx, tt.hasNamedPort, tt.portNumber, tt.createdFor) + assert.Equal(t, tt.expectedRuleID, observedRuleID) + }) + } +} + func TestBuildSecurityPolicyName(t *testing.T) { svc := &SecurityPolicyService{ Service: common.Service{ @@ -1004,7 +1267,7 @@ func TestBuildSecurityPolicyName(t *testing.T) { }, }, createdFor: common.ResourceTypeSecurityPolicy, - expName: "sp_ns1_securitypolicy1", + expName: "securitypolicy1", expId: "sp_uid1", }, { @@ -1052,7 +1315,7 @@ func TestBuildSecurityPolicyName(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { svc.NSXConfig.EnableVPCNetwork = tc.vpcEnabled - name := svc.buildSecurityPolicyName(tc.obj, tc.createdFor) + name := svc.buildSecurityPolicyName(tc.obj) assert.Equal(t, tc.expName, name) assert.True(t, len(name) <= common.MaxNameLength) id := svc.buildSecurityPolicyID(tc.obj, tc.createdFor) @@ -1093,51 +1356,51 @@ func TestBuildGroupName(t *testing.T) { expId string }{ { - name: "src rule without name", + name: "src peer group for rule without user-defined name", ruleIdx: 0, isSource: true, enableVPC: true, - expName: "sp1_0_src", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_src", + expName: "sp1_d0b8e36c_src", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_src", }, { - name: "dst rule without name", + name: "dst peer group for rule without user-defined name", ruleIdx: 0, isSource: false, enableVPC: true, - expName: "sp1_0_dst", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_dst", + expName: "sp1_d0b8e36c_dst", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_dst", }, { - name: "dst rule without name with T1", + name: "dst peer group for rule without user-defined name for T1", ruleIdx: 0, isSource: false, enableVPC: false, - expName: "sp1_0_dst", + expName: "sp1_d0b8e36c_dst", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_dst", }, { - name: "src rule with name", + name: "src peer group for rule with user-defined name", ruleIdx: 1, isSource: true, enableVPC: true, - expName: "MultipleNormalPorts-rule1_src", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_src", + expName: "sp1_555356be_src", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_src", }, { - name: "dst rule with name", + name: "dst peer group for rule with user-defined name", ruleIdx: 1, isSource: false, enableVPC: true, - expName: "MultipleNormalPorts-rule1_dst", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_dst", + expName: "sp1_555356be_dst", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_dst", }, { - name: "dst rule with name with T1", + name: "dst peer group for rule with user-defined name for T1", ruleIdx: 1, isSource: false, enableVPC: false, - expName: "MultipleNormalPorts-rule1_dst", + expName: "sp1_555356be_dst", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_dst", }, } { @@ -1161,31 +1424,45 @@ func TestBuildGroupName(t *testing.T) { expId string }{ { - name: "rule without name", + name: "applied group for rule without user-defined name", ruleIdx: 0, enableVPC: true, - expName: "sp1_0_scope", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_scope", + expName: "sp1_d0b8e36c_scope", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_d0b8e36c_scope", }, { - name: "rule with name", + name: "applied group for rule with user-defined name", ruleIdx: 1, enableVPC: true, - expName: "MultipleNormalPorts-rule1_scope", - expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_scope", + expName: "sp1_555356be_scope", + expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_555356be_scope", + }, + { + name: "applied group for rule without user-defined name", + ruleIdx: 0, + enableVPC: false, + expName: "sp1_d0b8e36c_scope", + expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_0_scope", + }, + { + name: "applied group fpr rule with user-defined name for T1", + ruleIdx: 1, + enableVPC: false, + expName: "sp1_555356be_scope", + expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_1_scope", }, { name: "policy applied group", ruleIdx: -1, enableVPC: true, - expName: "ns1_sp1_scope", + expName: "sp1_scope", expId: "sp1_c5db1800-ce4c-11de-bedc-84a0de00c35b_scope", }, { - name: "policy applied group with T1", + name: "policy applied group for T1", ruleIdx: -1, enableVPC: false, - expName: "ns1_sp1_scope", + expName: "sp1_scope", expId: "sp_c5db1800-ce4c-11de-bedc-84a0de00c35b_scope", }, } { diff --git a/pkg/nsx/services/securitypolicy/expand.go b/pkg/nsx/services/securitypolicy/expand.go index 91ee24139..5a78496ef 100644 --- a/pkg/nsx/services/securitypolicy/expand.go +++ b/pkg/nsx/services/securitypolicy/expand.go @@ -28,7 +28,7 @@ func (service *SecurityPolicyService) expandRule(obj *v1alpha1.SecurityPolicy, r var nsxGroups []*model.Group if len(rule.Ports) == 0 { - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, -1, false, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, false, -1, createdFor) if err != nil { return nil, nil, err } @@ -39,7 +39,7 @@ func (service *SecurityPolicyService) expandRule(obj *v1alpha1.SecurityPolicy, r // Check if there is a namedport in the rule hasNamedPort := service.hasNamedPort(rule) if !hasNamedPort { - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, -1, false, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, 0, 0, false, -1, createdFor) if err != nil { return nil, nil, err } @@ -88,7 +88,7 @@ func (service *SecurityPolicyService) expandRuleByPort(obj *v1alpha1.SecurityPol if err != nil { // In case there is no more valid ip set selected, so clear the stale ip set group in NSX if stale ips exist if errors.As(err, &nsxutil.NoEffectiveOption{}) { - groups := service.groupStore.GetByIndex(common.TagScopeRuleID, service.buildRuleID(obj, rule, ruleIdx, createdFor)) + groups := service.groupStore.GetByIndex(common.TagScopeRuleID, service.buildRuleID(obj, ruleIdx, createdFor)) var ipSetGroup *model.Group for _, group := range groups { ipSetGroup = group @@ -121,7 +121,7 @@ func (service *SecurityPolicyService) expandRuleByService(obj *v1alpha1.Security ) ([]*model.Group, *model.Rule, error) { var nsxGroups []*model.Group - nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, portIdx, portAddressIdx, portAddress.Port, true, createdFor) + nsxRule, err := service.buildRuleBasicInfo(obj, rule, ruleIdx, portIdx, portAddressIdx, true, portAddress.Port, createdFor) if err != nil { return nil, nil, err } diff --git a/pkg/nsx/services/securitypolicy/expand_test.go b/pkg/nsx/services/securitypolicy/expand_test.go index 7f30e60ac..56c30485e 100644 --- a/pkg/nsx/services/securitypolicy/expand_test.go +++ b/pkg/nsx/services/securitypolicy/expand_test.go @@ -24,9 +24,23 @@ import ( func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { sp := &v1alpha1.SecurityPolicy{ ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "spA", UID: "uidA"}, + Spec: v1alpha1.SecurityPolicySpec{ + Rules: []v1alpha1.SecurityPolicyRule{ + { + Action: &allowAction, + Direction: &directionIn, + Sources: []v1alpha1.SecurityPolicyPeer{ + { + PodSelector: &v1.LabelSelector{ + MatchLabels: map[string]string{"pod_selector_1": "pod_value_1"}, + }, + }, + }, + }, + }, + }, } - rule := v1alpha1.SecurityPolicyRule{} nsxRule := model.Rule{ DisplayName: &ruleNameWithPodSelector00, Id: &ruleIDPort000, @@ -66,7 +80,7 @@ func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { DisplayName: &policyGroupName, Expression: []*data.StructValue{blockExpression}, // build ipset group tags from input securitypolicy and securitypolicy rule - Tags: service.buildPeerTags(sp, &rule, 0, false, false, false, common.ResourceTypeSecurityPolicy), + Tags: service.buildPeerTags(sp, &sp.Spec.Rules[0], 0, false, false, false, common.ResourceTypeSecurityPolicy), } type args struct { @@ -82,7 +96,7 @@ func TestSecurityPolicyService_buildRuleIPGroup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, service.buildRuleIPSetGroup(sp, &rule, tt.args.obj, tt.args.ips, 0, common.ResourceTypeSecurityPolicy), "buildRuleIPSetGroup(%v, %v)", + assert.Equalf(t, tt.want, service.buildRuleIPSetGroup(sp, &sp.Spec.Rules[0], tt.args.obj, tt.args.ips, 0, common.ResourceTypeSecurityPolicy), "buildRuleIPSetGroup(%v, %v)", tt.args.obj, tt.args.ips) }) } diff --git a/pkg/nsx/services/securitypolicy/firewall_test.go b/pkg/nsx/services/securitypolicy/firewall_test.go index 85969bc3f..38d268ee1 100644 --- a/pkg/nsx/services/securitypolicy/firewall_test.go +++ b/pkg/nsx/services/securitypolicy/firewall_test.go @@ -42,8 +42,8 @@ var ( tagScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID tagScopeRuleID = common.TagScopeRuleID tagScopeSelectorHash = common.TagScopeSelectorHash - spName = "sp_ns1_spA" - spGroupName = "ns1_spA_scope" + spName = "spA" + spGroupName = "spA_scope" spID = "sp_uidA" spID2 = "sp_uidB" spGroupID = "sp_uidA_scope" @@ -1073,11 +1073,11 @@ func TestCreateOrUpdateSecurityPolicy(t *testing.T) { mockVPCService := common.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - podSelectorRule0IDPort000 := fakeService.buildExpandedRuleId(fakeService.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[0], 0, common.ResourceTypeSecurityPolicy), 0, 0) - podSelectorRule1IDPort000 := fakeService.buildExpandedRuleId(fakeService.buildRuleID(&spWithPodSelector, &spWithPodSelector.Spec.Rules[1], 1, common.ResourceTypeSecurityPolicy), 0, 0) + podSelectorRule0IDPort000 := fakeService.buildExpandedRuleID(&spWithPodSelector, 0, 0, 0, false, -1, common.ResourceTypeSecurityPolicy) + podSelectorRule1IDPort000 := fakeService.buildExpandedRuleID(&spWithPodSelector, 1, 0, 0, false, -1, common.ResourceTypeSecurityPolicy) - podSelectorRule0Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[0], 0, -1, false, common.ResourceTypeSecurityPolicy) - podSelectorRule1Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[1], 0, -1, false, common.ResourceTypeSecurityPolicy) + podSelectorRule0Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[0], 0, false, -1, common.ResourceTypeSecurityPolicy) + podSelectorRule1Name00, _ := fakeService.buildRuleDisplayName(&spWithPodSelector.Spec.Rules[1], 0, false, -1, common.ResourceTypeSecurityPolicy) type args struct { spObj *v1alpha1.SecurityPolicy @@ -1494,13 +1494,13 @@ func TestGetFinalSecurityPolicyResouceFromNetworkPolicy(t *testing.T) { Rules: []model.Rule{ { DisplayName: common.String("TCP.6001_ingress_allow"), - Id: common.String("np-app-access_uidNP_allow_0_6c2a026ca143812daa72699fb924ee36b33b5cdc_0_0"), + Id: common.String("np-app-access_uidNP_allow_6c2a026c_6001"), DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, Scope: []string{"ANY"}, SequenceNumber: &seq0, Services: []string{"ANY"}, - SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/np-app-access_uidNP_allow_0_src"}, + SourceGroups: []string{"/orgs/default/projects/projectQuality/infra/domains/default/groups/np-app-access_uidNP_allow_6c2a026c_src"}, Action: &nsxRuleActionAllow, ServiceEntries: []*data.StructValue{serviceEntry}, Tags: npAllowBasicTags, @@ -1516,7 +1516,7 @@ func TestGetFinalSecurityPolicyResouceFromNetworkPolicy(t *testing.T) { Rules: []model.Rule{ { DisplayName: common.String("ingress_isolation"), - Id: common.String("np-app-access_uidNP_isolation_0_114fed106ef3b5eae2a583f312435e84c02ca97f_0_0"), + Id: common.String("np-app-access_uidNP_isolation_114fed10_all"), DestinationGroups: []string{"ANY"}, Direction: &nsxRuleDirectionIn, Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/np-app-access_uidNP_isolation_scope"}, diff --git a/pkg/util/utils.go b/pkg/util/utils.go index e3e663c72..efe098e55 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -31,10 +31,9 @@ import ( ) const ( - wcpSystemResource = "vmware-system-shared-t1" - HashLength int = 8 - SubnetTypeSubnet = "subnet" - SubnetTypeSubnetSet = "subnetset" + wcpSystemResource = "vmware-system-shared-t1" + SubnetTypeSubnet = "subnet" + SubnetTypeSubnetSet = "subnetset" ) var ( @@ -106,11 +105,11 @@ func NormalizeId(name string) string { return newName } hashString := Sha1(name) - nameLength := common.MaxIdLength - HashLength - 1 + nameLength := common.MaxIdLength - common.HashLength - 1 for strings.ContainsAny(string(newName[nameLength-1]), "-._") { nameLength-- } - newName = fmt.Sprintf("%s-%s", newName[:nameLength], hashString[:HashLength]) + newName = fmt.Sprintf("%s-%s", newName[:nameLength], hashString[:common.HashLength]) return newName } @@ -521,7 +520,7 @@ func Capitalize(s string) string { func GetRandomIndexString() string { uuidStr := uuid.NewString() - return Sha1(uuidStr)[:HashLength] + return Sha1(uuidStr)[:common.HashLength] } // IsPowerOfTwo checks if a given number is a power of 2 diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 87b121e5d..a46ca0a9f 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -30,7 +30,7 @@ func TestNormalizeName(t *testing.T) { shortName := strings.Repeat("a", 256) assert.Equal(t, NormalizeName(shortName), shortName) longName := strings.Repeat("a", 257) - assert.Equal(t, NormalizeName(longName), fmt.Sprintf("%s_%s", strings.Repeat("a", 256-HashLength-1), "0c103888")) + assert.Equal(t, NormalizeName(longName), fmt.Sprintf("%s_%s", strings.Repeat("a", 256-common.HashLength-1), "0c103888")) } func TestNormalizeLabelKey(t *testing.T) { From edb6207b88aa358dd5bafc6c842d2dbbc022011a Mon Sep 17 00:00:00 2001 From: Deng Yun Date: Fri, 11 Oct 2024 13:57:12 +0800 Subject: [PATCH 06/18] Remove NetworkPolicy/SecurityPolicy Finalizer (#793) This patch is to 1. Remove Finalizer for NetworkPolicy CR. 2. Remove Finalizer for SecurityPolicy in VPC network. 3. For T1 network work, the upgrade SecurityPolicy from V4.1.2 still has existing finalizer, however, the new created SecurityPolicy has no finalizer any longer after upgrade. After Finalizer is removed, the deletion process of NetworkPolicy/SecurityPolicy is modified 1. Add a new indexer function for tag nsx-op/namespace for SecurityPolicy store. 2. Once the K8s CR(NetworkPolicy/SecurityPolicy) is deleted, the CR will be deleted at once usually if without finalizer. So, there no deletion timestamp could be found in the CR. It's needed to handle K8s deletion even when k8s client finds that the CR not found. 3.Using nsx-op/namespace tag to fileter the corresponding NSX resources in the same namespace with delete request, and then comparing CR name tag value, either SecurityPolicy CR or NetworkPolciy CR, with delete request CR name to filter the NSX SecurityPolicy resources to be deleted. 4.If there are multiple NSX resources with the same CR anme tag value in the NSX store. It's must check the the K8s CR exist to decide which CR was deleted already, or the new created in the same namespace with the same name. --- .../networkpolicy/networkpolicy_controller.go | 99 ++++++++++------- .../securitypolicy_controller.go | 105 ++++++++++++------ .../securitypolicy_controller_test.go | 60 ++++++++-- pkg/nsx/services/common/types.go | 7 +- pkg/nsx/services/securitypolicy/firewall.go | 25 +++++ pkg/nsx/services/securitypolicy/store.go | 9 ++ 6 files changed, 213 insertions(+), 92 deletions(-) diff --git a/pkg/controllers/networkpolicy/networkpolicy_controller.go b/pkg/controllers/networkpolicy/networkpolicy_controller.go index 66097ee98..ac332d138 100644 --- a/pkg/controllers/networkpolicy/networkpolicy_controller.go +++ b/pkg/controllers/networkpolicy/networkpolicy_controller.go @@ -18,7 +18,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -100,21 +99,21 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) if err := r.Client.Get(ctx, req.NamespacedName, networkPolicy); err != nil { - log.Error(err, "unable to fetch network policy", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + // IgnoreNotFound returns nil on NotFound errors. + if client.IgnoreNotFound(err) == nil { + if err := r.deleteNetworkPolicyByName(req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NetworkPolicy", "networkpolicy", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + // In case that client is unable to check CR + log.Error(err, "client is unable to fetch NetworkPolicy CR", "req", req.NamespacedName) + return ResultRequeue, err } if networkPolicy.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) - if !controllerutil.ContainsFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) { - controllerutil.AddFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) - if err := r.Client.Update(ctx, networkPolicy); err != nil { - log.Error(err, "add finalizer", "networkpolicy", req.NamespacedName) - updateFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on networkpolicy", "networkpolicy", req.NamespacedName) - } if err := r.Service.CreateOrUpdateSecurityPolicy(networkPolicy); err != nil { if errors.As(err, &nsxutil.RestrictionError{}) { @@ -135,25 +134,14 @@ func (r *NetworkPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques updateSuccess(r, ctx, networkPolicy) cleanNetworkPolicyErrorAnnotation(ctx, networkPolicy, r.Client) } else { - if controllerutil.ContainsFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - if err := r.Service.DeleteSecurityPolicy(networkPolicy, false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) - deleteFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(networkPolicy, servicecommon.NetworkPolicyFinalizerName) - if err := r.Client.Update(ctx, networkPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) - deleteFail(r, ctx, networkPolicy, &err) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "networkpolicy", req.NamespacedName) - deleteSuccess(r, ctx, networkPolicy) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "networkpolicy", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + + if err := r.Service.DeleteSecurityPolicy(networkPolicy, false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "networkpolicy", req.NamespacedName) + deleteFail(r, ctx, networkPolicy, &err) + return ResultRequeue, err } + deleteSuccess(r, ctx, networkPolicy) } return ResultNormal, nil @@ -186,19 +174,12 @@ func (r *NetworkPolicyReconciler) CollectGarbage(ctx context.Context) { if len(nsxPolicySet) == 0 { return } - policyList := &networkingv1.NetworkPolicyList{} - err := r.Client.List(ctx, policyList) + + CRPolicySet, err := r.listNetworkPolciyCRIDs() if err != nil { - log.Error(err, "failed to list NetworkPolicy") return } - CRPolicySet := sets.New[string]() - for _, policy := range policyList.Items { - CRPolicySet.Insert(r.Service.BuildNetworkPolicyAllowPolicyID(string(policy.UID))) - CRPolicySet.Insert(r.Service.BuildNetworkPolicyIsolationPolicyID(string(policy.UID))) - } - diffSet := nsxPolicySet.Difference(CRPolicySet) for elem := range diffSet { log.V(1).Info("GC collected NetworkPolicy", "ID", elem) @@ -212,6 +193,46 @@ func (r *NetworkPolicyReconciler) CollectGarbage(ctx context.Context) { } } +func (r *NetworkPolicyReconciler) deleteNetworkPolicyByName(ns, name string) error { + nsxSecurityPolicies := r.Service.ListNetworkPolicyByName(ns, name) + + CRPolicySet, err := r.listNetworkPolciyCRIDs() + if err != nil { + return err + } + for _, item := range nsxSecurityPolicies { + uid := nsxutil.FindTag(item.Tags, servicecommon.TagScopeNetworkPolicyUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, NetworkPolicy CR still exists in K8s", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + continue + } + + log.Info("deleting NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + if err := r.Service.DeleteSecurityPolicy(types.UID(uid), false, false, servicecommon.ResourceTypeNetworkPolicy); err != nil { + log.Error(err, "failed to delete NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + return err + } + log.Info("successfully deleted NetworkPolicy", "networkPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + } + return nil +} + +func (r *NetworkPolicyReconciler) listNetworkPolciyCRIDs() (sets.Set[string], error) { + networkPolicyList := &networkingv1.NetworkPolicyList{} + err := r.Client.List(context.Background(), networkPolicyList) + if err != nil { + log.Error(err, "failed to list NetworkPolicy CRs") + return nil, err + } + + CRPolicySet := sets.New[string]() + for _, policy := range networkPolicyList.Items { + CRPolicySet.Insert(r.Service.BuildNetworkPolicyAllowPolicyID(string(policy.UID))) + CRPolicySet.Insert(r.Service.BuildNetworkPolicyIsolationPolicyID(string(policy.UID))) + } + return CRPolicySet, nil +} + func StartNetworkPolicyController(mgr ctrl.Manager, commonService servicecommon.Service, vpcService servicecommon.VPCServiceProvider) { networkPolicyReconcile := NetworkPolicyReconciler{ Client: mgr.GetClient(), diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller.go b/pkg/controllers/securitypolicy/securitypolicy_controller.go index 778d052f9..1220eac0a 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller.go @@ -137,12 +137,21 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque obj = &v1alpha1.SecurityPolicy{} } - log.Info("reconciling securitypolicy CR", "securitypolicy", req.NamespacedName) + log.Info("reconciling SecurityPolicy CR", "securitypolicy", req.NamespacedName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResType) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch security policy CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + // IgnoreNotFound returns nil on NotFound errors. + if client.IgnoreNotFound(err) == nil { + if err := r.deleteSecurityPolicyByName(req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete SecurityPolicy", "securitypolicy", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + // In case that client is unable to check CR + log.Error(err, "client is unable to fetch SecurityPolicy CR", "req", req.NamespacedName) + return ResultRequeue, err } isZero := false @@ -152,7 +161,6 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque case *crdv1alpha1.SecurityPolicy: o := obj.(*crdv1alpha1.SecurityPolicy) isZero = o.ObjectMeta.DeletionTimestamp.IsZero() - finalizerName = servicecommon.SecurityPolicyFinalizerName realObj = securitypolicy.VPCToT1(o) case *v1alpha1.SecurityPolicy: realObj = obj.(*v1alpha1.SecurityPolicy) @@ -170,15 +178,6 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque if isZero { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResType) - if !controllerutil.ContainsFinalizer(obj, finalizerName) { - controllerutil.AddFinalizer(obj, finalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "securitypolicy", req.NamespacedName) - updateFail(r, ctx, realObj, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on securitypolicy CR", "securitypolicy", req.NamespacedName) - } if isCRInSysNs, err := util.IsSystemNamespace(r.Client, req.Namespace, nil); err != nil { err = errors.New("fetch namespace associated with security policy CR failed") @@ -213,25 +212,24 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque cleanSecurityPolicyErrorAnnotation(ctx, realObj, securitypolicy.IsVPCEnabled(r.Service), r.Client) } else { log.Info("reconciling CR to delete securitypolicy", "securitypolicy", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + + // For T1 upgrade, the upgraded CRs still has finalizer if controllerutil.ContainsFinalizer(obj, finalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - if err := r.Service.DeleteSecurityPolicy(realObj.UID, false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) - deleteFail(r, ctx, realObj, &err) - return ResultRequeue, err - } controllerutil.RemoveFinalizer(obj, finalizerName) if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) + log.Error(err, "finalizer remove failed, would retry exponentially", "securitypolicy", req.NamespacedName) deleteFail(r, ctx, realObj, &err) return ResultRequeue, err } log.V(1).Info("removed finalizer", "securitypolicy", req.NamespacedName) - deleteSuccess(r, ctx, realObj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "securitypolicy", req.NamespacedName) } + if err := r.Service.DeleteSecurityPolicy(realObj.UID, false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "securitypolicy", req.NamespacedName) + deleteFail(r, ctx, realObj, &err) + return ResultRequeue, err + } + deleteSuccess(r, ctx, realObj) } return ResultNormal, nil @@ -361,16 +359,59 @@ func (r *SecurityPolicyReconciler) CollectGarbage(ctx context.Context) { return } + CRPolicySet, err := r.listSecurityPolciyCRIDs() + if err != nil { + return + } + + diffSet := nsxPolicySet.Difference(CRPolicySet) + for elem := range diffSet { + log.V(1).Info("GC collected SecurityPolicy CR", "securityPolicyUID", elem) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) + err = r.Service.DeleteSecurityPolicy(types.UID(elem), true, false, servicecommon.ResourceTypeSecurityPolicy) + if err != nil { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) + } else { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) + } + } +} + +func (r *SecurityPolicyReconciler) deleteSecurityPolicyByName(ns, name string) error { + nsxSecurityPolicies := r.Service.ListSecurityPolicyByName(ns, name) + + CRPolicySet, err := r.listSecurityPolciyCRIDs() + if err != nil { + return err + } + for _, item := range nsxSecurityPolicies { + uid := nsxutil.FindTag(item.Tags, servicecommon.TagValueScopeSecurityPolicyUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, SecurityPolicy CR still exists in K8s", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + continue + } + + log.Info("deleting SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + if err := r.Service.DeleteSecurityPolicy(types.UID(uid), false, false, servicecommon.ResourceTypeSecurityPolicy); err != nil { + log.Error(err, "failed to delete SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + return err + } + log.Info("successfully deleted SecurityPolicy", "securityPolicyUID", uid, "nsxSecurityPolicyId", *item.Id) + } + return nil +} + +func (r *SecurityPolicyReconciler) listSecurityPolciyCRIDs() (sets.Set[string], error) { var objectList client.ObjectList if securitypolicy.IsVPCEnabled(r.Service) { objectList = &crdv1alpha1.SecurityPolicyList{} } else { objectList = &v1alpha1.SecurityPolicyList{} } - err := r.Client.List(ctx, objectList) + err := r.Client.List(context.Background(), objectList) if err != nil { log.Error(err, "failed to list SecurityPolicy CR") - return + return nil, err } CRPolicySet := sets.New[string]() @@ -387,17 +428,7 @@ func (r *SecurityPolicyReconciler) CollectGarbage(ctx context.Context) { } } - diffSet := nsxPolicySet.Difference(CRPolicySet) - for elem := range diffSet { - log.V(1).Info("GC collected SecurityPolicy CR", "securityPolicyUID", elem) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResType) - err = r.Service.DeleteSecurityPolicy(types.UID(elem), true, false, servicecommon.ResourceTypeSecurityPolicy) - if err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResType) - } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResType) - } - } + return CRPolicySet, nil } // It is triggered by associated controller like pod, namespace, etc. diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller_test.go b/pkg/controllers/securitypolicy/securitypolicy_controller_test.go index 49f70a42c..c245d01bc 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller_test.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller_test.go @@ -36,6 +36,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/securitypolicy" + "github.com/vmware-tanzu/nsx-operator/pkg/util" ) func fakeService() *securitypolicy.SecurityPolicyService { @@ -193,8 +194,13 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { // not found errNotFound := errors.New("not found") k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) - _, err := r.Reconcile(ctx, req) + deleteSecurityPolicyByNamePatch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(r), "deleteSecurityPolicyByName", func(_ *SecurityPolicyReconciler, name, ns string) error { + return nil + }) + defer deleteSecurityPolicyByNamePatch.Reset() + result, err := r.Reconcile(ctx, req) assert.Equal(t, err, errNotFound) + assert.Equal(t, ResultRequeue, result) // NSX version check failed case sp := &v1alpha1.SecurityPolicy{} @@ -202,10 +208,10 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { return false }) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - patches := gomonkey.ApplyFunc(updateFail, + updateFailPatch := gomonkey.ApplyFunc(updateFail, func(r *SecurityPolicyReconciler, c context.Context, o *v1alpha1.SecurityPolicy, e *error) { }) - defer patches.Reset() + defer updateFailPatch.Reset() result, ret := r.Reconcile(ctx, req) resultRequeueAfter5mins := controllerruntime.Result{Requeue: true, RequeueAfter: 5 * time.Minute} assert.Equal(t, nil, ret) @@ -217,29 +223,60 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { }) defer checkNsxVersionPatch.Reset() - // DeletionTimestamp.IsZero = ture, client update failed + // DeletionTimestamp.IsZero = ture, create security policy in SystemNamespace k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) - _, ret = r.Reconcile(ctx, req) + err = errors.New("fetch namespace associated with security policy CR failed") + IsSystemNamespacePatch := gomonkey.ApplyFunc(util.IsSystemNamespace, func(client client.Client, ns string, obj *v1.Namespace, + ) (bool, error) { + return true, errors.New("fetch namespace associated with security policy CR failed") + }) + + result, ret = r.Reconcile(ctx, req) + IsSystemNamespacePatch.Reset() assert.Equal(t, err, ret) + assert.Equal(t, ResultRequeue, result) - // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.SecurityPolicyFinalizerName + // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName and update fails k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SecurityPolicy) time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time + v1sp.Finalizers = []string{common.T1SecurityPolicyFinalizerName} return nil }) + patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteSecurityPolicy", func(_ *securitypolicy.SecurityPolicyService, UID interface{}, isGc bool, isVPCCleanup bool) error { assert.FailNow(t, "should not be called") return nil }) - _, ret = r.Reconcile(ctx, req) + + err = errors.New("finalizer remove failed, would retry exponentially") + k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) + deleteFailPatch := gomonkey.ApplyFunc(deleteFail, + func(r *SecurityPolicyReconciler, c context.Context, o *v1alpha1.SecurityPolicy, e *error) { + }) + defer deleteFailPatch.Reset() + result, ret = r.Reconcile(ctx, req) + assert.Equal(t, ret, err) + assert.Equal(t, ResultRequeue, result) + patch.Reset() + + // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.SecurityPolicyFinalizerName + k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { + v1sp := obj.(*v1alpha1.SecurityPolicy) + time := metav1.Now() + v1sp.ObjectMeta.DeletionTimestamp = &time + return nil + }) + patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteSecurityPolicy", func(_ *securitypolicy.SecurityPolicyService, UID interface{}, isGc bool, isVPCCleanup bool) error { + return nil + }) + result, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) + assert.Equal(t, ResultNormal, result) patch.Reset() - // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName + // DeletionTimestamp.IsZero = false, Finalizers include util.SecurityPolicyFinalizerName k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SecurityPolicy) time := metav1.Now() @@ -251,8 +288,9 @@ func TestSecurityPolicyReconciler_Reconcile(t *testing.T) { return nil }) k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) - _, ret = r.Reconcile(ctx, req) + result, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) + assert.Equal(t, ResultNormal, result) patch.Reset() } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index ddc7d2bbc..454fe6849 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -103,11 +103,8 @@ const ( IPPoolTypePublic = "Public" IPPoolTypePrivate = "Private" - NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" - T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" - - SecurityPolicyFinalizerName = "securitypolicy.crd.nsx.vmware.com/finalizer" - NetworkPolicyFinalizerName = "networkpolicy.crd.nsx.vmware.com/finalizer" + NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" + T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" StaticRouteFinalizerName = "staticroute.crd.nsx.vmware.com/finalizer" SubnetFinalizerName = "subnet.crd.nsx.vmware.com/finalizer" SubnetSetFinalizerName = "subnetset.crd.nsx.vmware.com/finalizer" diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 3d06ad578..ad2b63233 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -144,6 +144,7 @@ func (s *SecurityPolicyService) setUpStore(indexScope string) { keyFunc, cache.Indexers{ indexScope: indexBySecurityPolicyUID, common.TagScopeNetworkPolicyUID: indexByNetworkPolicyUID, + common.TagScopeNamespace: indexBySecurityPolicyNamespace, }), BindingType: model.SecurityPolicyBindingType(), }} @@ -1104,6 +1105,30 @@ func (service *SecurityPolicyService) ListNetworkPolicyID() sets.Set[string] { return service.getGCSecurityPolicyIDSet(indexScope) } +func (service *SecurityPolicyService) ListSecurityPolicyByName(ns, name string) []*model.SecurityPolicy { + var result []*model.SecurityPolicy + securityPolicies := service.securityPolicyStore.GetByIndex(common.TagScopeNamespace, ns) + for _, securityPolicy := range securityPolicies { + securityPolicyCRName := nsxutil.FindTag(securityPolicy.Tags, common.TagValueScopeSecurityPolicyName) + if securityPolicyCRName == name { + result = append(result, securityPolicy) + } + } + return result +} + +func (service *SecurityPolicyService) ListNetworkPolicyByName(ns, name string) []*model.SecurityPolicy { + var result []*model.SecurityPolicy + securityPolicies := service.securityPolicyStore.GetByIndex(common.TagScopeNamespace, ns) + for _, securityPolicy := range securityPolicies { + securityPolicyCRName := nsxutil.FindTag(securityPolicy.Tags, common.TagScopeNetworkPolicyName) + if securityPolicyCRName == name { + result = append(result, securityPolicy) + } + } + return result +} + func (service *SecurityPolicyService) Cleanup(ctx context.Context) error { // Delete all the security policies in store uids := service.ListSecurityPolicyID() diff --git a/pkg/nsx/services/securitypolicy/store.go b/pkg/nsx/services/securitypolicy/store.go index b7c60026d..d69dd6b4f 100644 --- a/pkg/nsx/services/securitypolicy/store.go +++ b/pkg/nsx/services/securitypolicy/store.go @@ -76,6 +76,15 @@ func indexGroupFunc(obj interface{}) ([]string, error) { } } +func indexBySecurityPolicyNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.SecurityPolicy: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("indexBySecurityPolicyNamespace doesn't support unknown type") + } +} + var filterRuleTag = func(v []model.Tag) []string { res := make([]string, 0, 5) for _, tag := range v { From 762863a3c50ebdf596e916711bab29c2a35af65a Mon Sep 17 00:00:00 2001 From: wenqi Date: Fri, 11 Oct 2024 15:32:58 +0800 Subject: [PATCH 07/18] Remove Subnet Finalizer (#769) Remove SubnetSet Finalizer Add unit-test for Subnet controller Signed-off-by: Wenqi Qiu --- pkg/controllers/common/utils.go | 2 +- pkg/controllers/subnet/namespace_handler.go | 26 +- .../subnet/namespace_handler_test.go | 128 +++++++ pkg/controllers/subnet/subnet_controller.go | 321 +++++++++++------- .../subnet/subnet_controller_test.go | 227 ++++++++++++- .../subnetset/namespace_handler.go | 27 +- .../subnetset/subnetset_controller.go | 280 ++++++++------- pkg/nsx/services/common/services.go | 2 +- pkg/nsx/services/common/types.go | 3 - pkg/nsx/services/subnet/store.go | 18 + pkg/nsx/services/subnet/store_test.go | 16 +- pkg/nsx/services/subnet/subnet.go | 100 ++++-- pkg/nsx/services/subnet/subnet_test.go | 90 +++++ pkg/util/utils.go | 37 +- test/e2e/nsx_subnet_test.go | 57 +++- 15 files changed, 974 insertions(+), 360 deletions(-) create mode 100644 pkg/controllers/subnet/namespace_handler_test.go create mode 100644 pkg/nsx/services/subnet/subnet_test.go diff --git a/pkg/controllers/common/utils.go b/pkg/controllers/common/utils.go index b71f0b2a3..35c3e0f65 100644 --- a/pkg/controllers/common/utils.go +++ b/pkg/controllers/common/utils.go @@ -42,7 +42,7 @@ func AllocateSubnetFromSubnetSet(subnetSet *v1alpha1.SubnetSet, vpcService servi return *nsxSubnet.Path, nil } } - tags := subnetService.GenerateSubnetNSTags(subnetSet, subnetSet.Namespace) + tags := subnetService.GenerateSubnetNSTags(subnetSet) if tags == nil { return "", errors.New("failed to generate subnet tags") } diff --git a/pkg/controllers/subnet/namespace_handler.go b/pkg/controllers/subnet/namespace_handler.go index ee2f935d3..57d667248 100644 --- a/pkg/controllers/subnet/namespace_handler.go +++ b/pkg/controllers/subnet/namespace_handler.go @@ -18,7 +18,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ) -// Subnet controller should watch event of namespace, when there are some updates of namespace labels, +// Subnet controller should watch event of Namespace, when there are some updates of Namespace labels, // controller should build tags and update VpcSubnet according to new labels. type EnqueueRequestForNamespace struct { @@ -26,22 +26,22 @@ type EnqueueRequestForNamespace struct { } func (e *EnqueueRequestForNamespace) Create(_ context.Context, _ event.CreateEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace create event, do nothing") + log.V(1).Info("Namespace create event, do nothing") } func (e *EnqueueRequestForNamespace) Delete(_ context.Context, _ event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace delete event, do nothing") + log.V(1).Info("Namespace delete event, do nothing") } func (e *EnqueueRequestForNamespace) Generic(_ context.Context, _ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace generic event, do nothing") + log.V(1).Info("Namespace generic event, do nothing") } func (e *EnqueueRequestForNamespace) Update(_ context.Context, updateEvent event.UpdateEvent, l workqueue.RateLimitingInterface) { obj := updateEvent.ObjectNew.(*v1.Namespace) - err := reconcileSubnet(e.Client, obj.Name, l) + err := requeueSubnet(e.Client, obj.Name, l) if err != nil { - log.Error(err, "failed to reconcile subnet") + log.Error(err, "Failed to requeue Subnet") } } @@ -52,9 +52,9 @@ var PredicateFuncsNs = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldObj := e.ObjectOld.(*v1.Namespace) newObj := e.ObjectNew.(*v1.Namespace) - log.V(1).Info("receive namespace update event", "name", oldObj.Name) + log.V(1).Info("Receive Namespace update event", "Name", oldObj.Name) if reflect.DeepEqual(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) { - log.Info("labels of namespace are not changed", "name", oldObj.Name) + log.Info("Labels of Namespace are not changed", "Name", oldObj.Name) return false } return true @@ -64,17 +64,17 @@ var PredicateFuncsNs = predicate.Funcs{ }, } -func reconcileSubnet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { +func requeueSubnet(c client.Client, ns string, q workqueue.RateLimitingInterface) error { subnetList := &v1alpha1.SubnetList{} - err := c.List(context.Background(), subnetList, client.InNamespace(namespace)) + err := c.List(context.Background(), subnetList, client.InNamespace(ns)) if err != nil { - log.Error(err, "failed to list all the subnets") + log.Error(err, "Failed to list all the Subnets") return err } for _, subnet_item := range subnetList.Items { - log.Info("reconcile subnet because namespace update", - "namespace", subnet_item.Namespace, "name", subnet_item.Name) + log.Info("Requeue Subnet because Namespace update", + "Namespace", subnet_item.Namespace, "Name", subnet_item.Name) q.Add(reconcile.Request{ NamespacedName: types.NamespacedName{ Name: subnet_item.Name, diff --git a/pkg/controllers/subnet/namespace_handler_test.go b/pkg/controllers/subnet/namespace_handler_test.go new file mode 100644 index 000000000..88c1b8615 --- /dev/null +++ b/pkg/controllers/subnet/namespace_handler_test.go @@ -0,0 +1,128 @@ +package subnet + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" +) + +type MockRateLimitingInterface struct { + mock.Mock +} + +func (m *MockRateLimitingInterface) Add(item interface{}) { +} + +func (m *MockRateLimitingInterface) Len() int { + return 0 +} + +func (m *MockRateLimitingInterface) Get() (item interface{}, shutdown bool) { + return +} + +func (m *MockRateLimitingInterface) Done(item interface{}) { + return +} + +func (m *MockRateLimitingInterface) ShutDown() { +} + +func (m *MockRateLimitingInterface) ShutDownWithDrain() { +} + +func (m *MockRateLimitingInterface) ShuttingDown() bool { + return true +} + +func (m *MockRateLimitingInterface) AddAfter(item interface{}, duration time.Duration) { + return +} + +func (m *MockRateLimitingInterface) AddRateLimited(item interface{}) { + m.Called(item) +} + +func (m *MockRateLimitingInterface) Forget(item interface{}) { + m.Called(item) +} + +func (m *MockRateLimitingInterface) NumRequeues(item interface{}) int { + args := m.Called(item) + return args.Int(0) +} + +func TestEnqueueRequestForNamespace(t *testing.T) { + fakeClient := fake.NewClientBuilder().Build() + queue := new(MockRateLimitingInterface) + handler := &EnqueueRequestForNamespace{Client: fakeClient} + + t.Run("Update event with changed labels", func(t *testing.T) { + oldNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "old"}, + }, + } + newNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "new"}, + }, + } + updateEvent := event.UpdateEvent{ + ObjectOld: oldNamespace, + ObjectNew: newNamespace, + } + queue.On("Add", mock.Anything).Return() + + handler.Update(context.Background(), updateEvent, queue) + }) + + t.Run("Update event with unchanged labels", func(t *testing.T) { + oldNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "same"}, + }, + } + newNamespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + Labels: map[string]string{"key": "same"}, + }, + } + updateEvent := event.UpdateEvent{ + ObjectOld: oldNamespace, + ObjectNew: newNamespace, + } + + queue.On("Add", mock.Anything).Return() + + handler.Update(context.Background(), updateEvent, queue) + + queue.AssertNotCalled(t, "Add", mock.Anything) + }) + + t.Run("Requeue subnet function", func(t *testing.T) { + ns := "test-ns" + + schem := fake.NewClientBuilder().Build().Scheme() + v1alpha1.AddToScheme(schem) + queue.On("Add", mock.Anything).Return() + + err := requeueSubnet(fakeClient, ns, queue) + + assert.NoError(t, err) + queue.AssertNumberOfCalls(t, "Add", 0) + }) +} diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 13da216f6..80c44a010 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "reflect" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -16,18 +18,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" - "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -50,102 +48,148 @@ type SubnetReconciler struct { } func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.Subnet{} - log.Info("reconciling subnet CR", "subnet", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling Subnet", "Subnet", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnet) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch Subnet CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) - } - - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) - if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { - controllerutil.AddFinalizer(obj, servicecommon.SubnetFinalizerName) - - if obj.Spec.AccessMode == "" { - log.Info("obj.Spec.AccessMode set", "subnet", req.NamespacedName) - obj.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) - } - - if obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) - if vpcNetworkConfig == nil { - err := fmt.Errorf("operate failed: cannot get configuration for Subnet CR") - log.Error(nil, "failed to find VPCNetworkConfig for Subnet CR", "subnet", req.NamespacedName, "namespace %s", obj.Namespace) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize - } - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + subnetCR := &v1alpha1.Subnet{} + if err := r.Client.Get(ctx, req.NamespacedName, subnetCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetByName(req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete NSX Subnet", "Subnet", req.NamespacedName) return ResultRequeue, err } - log.V(1).Info("added finalizer on subnet CR", "subnet", req.NamespacedName) + return ResultNormal, nil } - tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) - if tags == nil { - return ResultRequeue, errors.New("failed to generate subnet tags") + log.Error(err, "Unable to fetch Subnet CR", "req", req.NamespacedName) + return ResultRequeue, err + } + if !subnetCR.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) + if err := r.deleteSubnetByID(string(subnetCR.GetUID())); err != nil { + log.Error(err, "Failed to delete NSX Subnet, retrying", "Subnet", req.NamespacedName) + deleteFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } - vpcInfoList := r.VPCService.ListVPCInfo(req.Namespace) - if len(vpcInfoList) == 0 { - return ResultRequeueAfter10sec, nil + if err := r.Client.Delete(ctx, subnetCR); err != nil { + log.Error(err, "Failed to delete Subnet CR, retrying", "Subnet", req.NamespacedName) + deleteFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } - if _, err := r.SubnetService.CreateOrUpdateSubnet(obj, vpcInfoList[0], tags); err != nil { - if errors.As(err, &util.ExceedTagsError{}) { - log.Error(err, "exceed tags limit, would not retry", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultNormal, nil - } - log.Error(err, "operate failed, would retry exponentially", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + log.Info("Successfully deleted Subnet", "Subnet", req.NamespacedName) + deleteSuccess(r, ctx, subnetCR) + return ResultNormal, nil + } + + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) + + // Spec mutation check and update if necessary + specChanged := false + if subnetCR.Spec.AccessMode == "" { + subnetCR.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) + specChanged = true + } + + if subnetCR.Spec.IPv4SubnetSize == 0 { + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(subnetCR.Namespace) + if vpcNetworkConfig == nil { + err := fmt.Errorf("VPCNetworkConfig not found for Subnet CR") + log.Error(nil, "Failed to find VPCNetworkConfig", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) return ResultRequeue, err } - if err := r.updateSubnetStatus(obj); err != nil { - log.Error(err, "update subnet status failed, would retry exponentially", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) + subnetCR.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize + specChanged = true + } + if specChanged { + if err := r.Client.Update(ctx, subnetCR); err != nil { + log.Error(err, "Failed to update Subnet", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) return ResultRequeue, err } - updateSuccess(r, ctx, obj) - } else { - if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) - if err := r.DeleteSubnet(*obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnet", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, servicecommon.SubnetFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnet", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "subnet", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - log.Info("finalizers cannot be recognized", "subnet", req.NamespacedName) + log.Info("Updated Subnet CR", "Subnet", req.NamespacedName) + } + + tags := r.SubnetService.GenerateSubnetNSTags(subnetCR) + if tags == nil { + log.Error(nil, "Failed to generate Subnet tags", "Subnet", req.NamespacedName) + return ResultRequeue, errors.New("failed to generate Subnet tags") + } + // List VPC Info + vpcInfoList := r.VPCService.ListVPCInfo(req.Namespace) + if len(vpcInfoList) == 0 { + log.Info("No VPC info found, requeueing", "Namespace", req.Namespace) + return ResultRequeueAfter10sec, nil + } + // Create or update the subnet in NSX + if _, err := r.SubnetService.CreateOrUpdateSubnet(subnetCR, vpcInfoList[0], tags); err != nil { + if errors.As(err, &nsxutil.ExceedTagsError{}) { + log.Error(err, "Tags limit exceeded, not retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultNormal, nil } + log.Error(err, "Failed to create/update Subnet, retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err } + // Update status + if err := r.updateSubnetStatus(subnetCR); err != nil { + log.Error(err, "Failed to update Subnet status, retrying", "Subnet", req.NamespacedName) + updateFail(r, ctx, subnetCR, err.Error()) + return ResultRequeue, err + } + updateSuccess(r, ctx, subnetCR) return ctrl.Result{}, nil } -func (r *SubnetReconciler) DeleteSubnet(obj v1alpha1.Subnet) error { - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, string(obj.GetUID())) - if len(nsxSubnets) == 0 { - log.Info("no subnet found for subnet CR", "uid", string(obj.GetUID())) - return nil +func (r *SubnetReconciler) deleteSubnetByID(subnetID string) error { + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, subnetID) + return r.deleteSubnets(nsxSubnets) +} + +func (r *SubnetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) error { + for _, nsxSubnet := range nsxSubnets { + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) + if portNums > 0 { + err := fmt.Errorf("cannot delete Subnet %s, still attached by %d port(s)", *nsxSubnet.Id, portNums) + log.Error(err, "Delete Subnet from NSX failed") + return err + } + if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { + log.Error(err, "Failed to delete Subnet", "ID", *nsxSubnet.Id) + return err + } + log.Info("Successfully deleted Subnet", "ID", *nsxSubnet.Id) } - portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnets[0].Id)) - if portNums > 0 { - err := errors.New("subnet still attached by port") - log.Error(err, "", "ID", *nsxSubnets[0].Id) + log.Info("Successfully cleaned Subnets") + return nil +} + +func (r *SubnetReconciler) deleteStaleSubnets(nsxSubnets []*model.VpcSubnet) error { + crdSubnetIDs, err := r.listSubnetIDsFromCRs(context.Background()) + if err != nil { + log.Error(err, "Failed to list Subnet CRs") return err } - return r.SubnetService.DeleteSubnet(*nsxSubnets[0]) + crdSubnetIDsSet := sets.NewString(crdSubnetIDs...) + nsxSubnetsToDelete := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetCRUID) + if crdSubnetIDsSet.Has(uid) { + log.Info("Skipping deletion, Subnet CR still exists in K8s", "ID", *nsxSubnet.Id) + continue + } + nsxSubnetsToDelete = append(nsxSubnetsToDelete, nsxSubnet) + } + log.Info("Cleaning stale Subnets", "Count", len(nsxSubnetsToDelete)) + return r.deleteSubnets(nsxSubnetsToDelete) +} + +func (r *SubnetReconciler) deleteSubnetByName(name, ns string) error { + nsxSubnets := r.SubnetService.ListSubnetByName(ns, name) + return r.deleteStaleSubnets(nsxSubnets) } func (r *SubnetReconciler) updateSubnetStatus(obj *v1alpha1.Subnet) error { @@ -163,7 +207,7 @@ func (r *SubnetReconciler) updateSubnetStatus(obj *v1alpha1.Subnet) error { for _, status := range statusList { obj.Status.NetworkAddresses = append(obj.Status.NetworkAddresses, *status.NetworkAddress) obj.Status.GatewayAddresses = append(obj.Status.GatewayAddresses, *status.GatewayAddress) - // DHCPServerAddress is only for the subnet with DHCP enabled + // DHCPServerAddress is only for the Subnet with DHCP enabled if status.DhcpServerAddress != nil { obj.Status.DHCPServerAddresses = append(obj.Status.DHCPServerAddresses, *status.DhcpServerAddress) } @@ -209,9 +253,9 @@ func (r *SubnetReconciler) updateSubnetStatusConditions(ctx context.Context, sub } if conditionsUpdated { if err := r.Client.Status().Update(ctx, subnet); err != nil { - log.Error(err, "failed to update subnet status", "Name", subnet.Name, "Namespace", subnet.Namespace) + log.Error(err, "Failed to update Subnet status", "Name", subnet.Name, "Namespace", subnet.Namespace) } else { - log.Info("updated Subnet", "Name", subnet.Name, "Namespace", subnet.Namespace, "New Conditions", newConditions) + log.Info("Updated Subnet", "Name", subnet.Name, "Namespace", subnet.Namespace, "New Conditions", newConditions) } } } @@ -220,7 +264,7 @@ func (r *SubnetReconciler) mergeSubnetStatusCondition(ctx context.Context, subne matchedCondition := getExistingConditionOfType(newCondition.Type, subnet.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { - log.V(2).Info("conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) + log.V(2).Info("Conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) return false } @@ -266,20 +310,42 @@ func deleteSuccess(r *SubnetReconciler, _ context.Context, o *v1alpha1.Subnet) { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnet) } +func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider) error { + // Create the Subnet Reconciler with the necessary services and configuration + subnetReconciler := &SubnetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SubnetService: subnetService, + SubnetPortService: subnetPortService, + VPCService: vpcService, + Recorder: mgr.GetEventRecorderFor("subnet-controller"), + } + // Start the controller + if err := subnetReconciler.start(mgr); err != nil { + log.Error(err, "Failed to create controller", "controller", "Subnet") + return err + } + // Start garbage collector in a separate goroutine + go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetReconciler.collectGarbage) + return nil +} + +// start sets up the manager for the Subnet Reconciler +func (r *SubnetReconciler) start(mgr ctrl.Manager) error { + return r.setupWithManager(mgr) +} + +// setupWithManager configures the controller to watch Subnet resources func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.Subnet{}). - WithEventFilter(predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }). + // Watches for changes in Namespaces and triggers reconciliation Watches( &v1.Namespace{}, &EnqueueRequestForNamespace{Client: mgr.GetClient()}, builder.WithPredicates(PredicateFuncsNs), ). + // Set controller options, including max concurrent reconciles WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -287,66 +353,57 @@ func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { Complete(r) } -func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider) error { - subnetReconciler := &SubnetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - SubnetService: subnetService, - SubnetPortService: subnetPortService, - VPCService: vpcService, - Recorder: mgr.GetEventRecorderFor("subnet-controller"), - } - if err := subnetReconciler.Start(mgr); err != nil { - log.Error(err, "failed to create controller", "controller", "Subnet") - return err +func (r *SubnetReconciler) listSubnetIDsFromCRs(ctx context.Context) ([]string, error) { + crdSubnetList := &v1alpha1.SubnetList{} + err := r.Client.List(ctx, crdSubnetList) + if err != nil { + return nil, err } - go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetReconciler.CollectGarbage) - return nil -} -// Start setup manager -func (r *SubnetReconciler) Start(mgr ctrl.Manager) error { - err := r.setupWithManager(mgr) - if err != nil { - return err + crdSubnetIDs := make([]string, 0, len(crdSubnetList.Items)) + for _, sr := range crdSubnetList.Items { + crdSubnetIDs = append(crdSubnetIDs, string(sr.UID)) } - return nil + return crdSubnetIDs, nil } -// CollectGarbage implements the interface GarbageCollector method. -func (r *SubnetReconciler) CollectGarbage(ctx context.Context) { - log.Info("subnet garbage collector started") - crdSubnetList := &v1alpha1.SubnetList{} - err := r.Client.List(ctx, crdSubnetList) +// collectGarbage implements the interface GarbageCollector method. +func (r *SubnetReconciler) collectGarbage(ctx context.Context) { + startTime := time.Now() + defer func() { + log.Info("Subnet garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + crdSubnetIDs, err := r.listSubnetIDsFromCRs(ctx) if err != nil { - log.Error(err, "failed to list subnet CR") + log.Error(err, "Failed to list Subnet CRs") return } + var nsxSubnetList []*model.VpcSubnet - for _, subnet := range crdSubnetList.Items { - nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnet(string(subnet.UID))...) + for _, crdSubnetID := range crdSubnetIDs { + nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnet(crdSubnetID)...) } if len(nsxSubnetList) == 0 { + log.Info("No Subnets found in NSX, garbage collection complete") return } - crdSubnetIDs := sets.NewString() - for _, sr := range crdSubnetList.Items { - crdSubnetIDs.Insert(string(sr.UID)) - } - - for _, elem := range nsxSubnetList { - uid := util.FindTag(elem.Tags, servicecommon.TagScopeSubnetCRUID) - if crdSubnetIDs.Has(uid) { + crdSubnetIDsSet := sets.NewString(crdSubnetIDs...) + for _, nsxSubnet := range nsxSubnetList { + uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetCRUID) + if crdSubnetIDsSet.Has(uid) { continue } - log.Info("GC collected Subnet CR", "UID", elem) + log.Info("GC collected Subnet CR", "UID", uid) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeSubnet) - err = r.SubnetService.DeleteSubnet(*elem) - if err != nil { + + if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { + log.Error(err, "Failed to delete NSX subnet", "NSX Subnet UID", uid) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeSubnet) } else { + log.Info("Successfully deleted NSX subnet", "NSX Subnet UID", uid) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeSubnet) } } diff --git a/pkg/controllers/subnet/subnet_controller_test.go b/pkg/controllers/subnet/subnet_controller_test.go index 04f680226..aadfbc0e5 100644 --- a/pkg/controllers/subnet/subnet_controller_test.go +++ b/pkg/controllers/subnet/subnet_controller_test.go @@ -2,21 +2,30 @@ package subnet import ( "context" + "errors" "reflect" "testing" + "time" "github.com/agiledragon/gomonkey/v2" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) func TestSubnetReconciler_GarbageCollector(t *testing.T) { @@ -62,7 +71,7 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { return nil }) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) // local store has same item as k8s cache patch.Reset() @@ -84,7 +93,7 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { a.Items[0].UID = "1234" return nil }) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) // local store has no item patch.Reset() @@ -96,6 +105,218 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { return nil }) k8sClient.EXPECT().List(ctx, srList).Return(nil).Times(1) - r.CollectGarbage(ctx) + r.collectGarbage(ctx) patch.Reset() } + +type fakeRecorder struct{} + +func (recorder fakeRecorder) Event(object runtime.Object, eventtype, reason, message string) { +} + +func (recorder fakeRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func (recorder fakeRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { +} + +func createFakeSubnetReconciler() *SubnetReconciler { + service := &vpc.VPCService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + }, + } + subnetService := &subnet.SubnetService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + UseAVILoadBalancer: false, + }, + }, + }, + SubnetStore: &subnet.SubnetStore{}, + } + + subnetPortService := &subnetport.SubnetPortService{ + Service: common.Service{ + Client: nil, + NSXClient: &nsx.Client{}, + }, + SubnetPortStore: nil, + } + + return &SubnetReconciler{ + Client: fake.NewClientBuilder().Build(), + Scheme: fake.NewClientBuilder().Build().Scheme(), + VPCService: service, + SubnetService: subnetService, + SubnetPortService: subnetPortService, + Recorder: &fakeRecorder{}, + } +} + +func TestSubnetReconciler_Reconcile(t *testing.T) { + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "test-subnet", + }, + } + + createNewSubnet := func() *v1alpha1.Subnet { + return &v1alpha1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet", + Namespace: "default", + UID: "fake-subnet-uid", + }, + Spec: v1alpha1.SubnetSpec{ + IPv4SubnetSize: 0, + AccessMode: "", + }, + } + } + + reconciler := createFakeSubnetReconciler() + ctx := context.Background() + + // When the Subnet CR does not exist + t.Run("Subnet CR not found", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByName", func(_ *SubnetReconciler, name, ns string) error { + return nil + }) + defer patches.Reset() + + result, err := reconciler.Reconcile(ctx, req) + + assert.NoError(t, err) + assert.Equal(t, ctrl.Result{}, result) + patches.Reset() + }) + + t.Run("Get Subnet CR return other error should retry", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + patches := gomonkey.ApplyMethod(reflect.TypeOf(reconciler.Client), "Get", func(_ client.Client, _ context.Context, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return errors.New("get Subnet CR error") + }) + defer patches.Reset() + patches.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByName", func(_ *SubnetReconciler, name, ns string) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.ErrorContains(t, err, "get Subnet CR error") + assert.Equal(t, ResultRequeue, result) + }) + + // When the Subnet CR is being deleted should delete the Subnet and return no error + t.Run("Subnet CR being deleted", func(t *testing.T) { + t.Log("When the Subnet CR is being deleted should delete the Subnet and return no error") + v1alpha1.AddToScheme(reconciler.Scheme) + subnetCR := createNewSubnet() + now := metav1.NewTime(time.Now()) + subnetCR.DeletionTimestamp = &now + + createErr := reconciler.Client.Create(ctx, subnetCR) + defer reconciler.Client.Delete(ctx, subnetCR) + assert.NoError(t, createErr) + + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler), "deleteSubnetByID", func(_ *SubnetReconciler, _ string) error { + return nil + }) + defer patches.Reset() + patches.ApplyMethod(reflect.TypeOf(reconciler.Client), "Delete", func(_ client.Client, _ context.Context, _ client.Object, _ ...client.DeleteOption) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.ErrorContains(t, err, "VPCNetworkConfig not found") + assert.Equal(t, ResultRequeue, result) + }) + + // When an error occurs during reconciliation, should return an error and requeue" + t.Run("Create or Update Subnet Failure", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + + subnetCR := createNewSubnet() + createErr := reconciler.Client.Create(ctx, subnetCR) + assert.NoError(t, createErr) + defer reconciler.Client.Delete(ctx, subnetCR) + + vpcConfig := &common.VPCNetworkConfigInfo{DefaultSubnetSize: 16} + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler.VPCService), "GetVPCNetworkConfigByNamespace", func(_ *vpc.VPCService, ns string) *common.VPCNetworkConfigInfo { + return vpcConfig + }) + defer patches.Reset() + + tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-tag")}} + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "GenerateSubnetNSTags", func(_ *subnet.SubnetService, obj client.Object) []model.Tag { + return tags + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, ns string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {OrgID: "org-id", ProjectID: "project-id", VPCID: "vpc-id", ID: "fake-id"}, + } + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "CreateOrUpdateSubnet", func(_ *subnet.SubnetService, obj client.Object, vpcInfo common.VPCResourceInfo, tags []model.Tag) (string, error) { + return "", errors.New("create or update failed") + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.Error(t, err) + assert.Equal(t, ctrl.Result{Requeue: true}, result) + }) + + // When updating the Subnet spec, should update the Subnet CR spec if not set + t.Run("Update Subnet CR spec", func(t *testing.T) { + v1alpha1.AddToScheme(reconciler.Scheme) + + subnetCR := createNewSubnet() + createErr := reconciler.Client.Create(ctx, subnetCR) + assert.NoError(t, createErr) + defer reconciler.Client.Delete(ctx, subnetCR) + + vpcConfig := &common.VPCNetworkConfigInfo{DefaultSubnetSize: 16} + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(reconciler.VPCService), "GetVPCNetworkConfigByNamespace", func(_ *vpc.VPCService, ns string) *common.VPCNetworkConfigInfo { + return vpcConfig + }) + defer patches.Reset() + + tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-tag")}} + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "GenerateSubnetNSTags", func(_ *subnet.SubnetService, obj client.Object) []model.Tag { + return tags + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.VPCService), "ListVPCInfo", func(_ *vpc.VPCService, ns string) []common.VPCResourceInfo { + return []common.VPCResourceInfo{ + {OrgID: "org-id", ProjectID: "project-id", VPCID: "vpc-id", ID: "fake-id"}, + } + }) + + patches.ApplyMethod(reflect.TypeOf(reconciler.SubnetService), "CreateOrUpdateSubnet", func(_ *subnet.SubnetService, obj client.Object, vpcInfo common.VPCResourceInfo, tags []model.Tag) (string, error) { + return "", nil + }) + + patches.ApplyPrivateMethod(reflect.TypeOf(reconciler), "updateSubnetStatus", func(_ *SubnetReconciler, obj *v1alpha1.Subnet) error { + return nil + }) + + result, err := reconciler.Reconcile(ctx, req) + + assert.NoError(t, err) + assert.NoError(t, reconciler.Client.Get(ctx, req.NamespacedName, subnetCR)) + assert.Equal(t, 16, subnetCR.Spec.IPv4SubnetSize) + assert.Equal(t, ctrl.Result{}, result) + }) +} diff --git a/pkg/controllers/subnetset/namespace_handler.go b/pkg/controllers/subnetset/namespace_handler.go index 1098fd596..d7c4e038c 100644 --- a/pkg/controllers/subnetset/namespace_handler.go +++ b/pkg/controllers/subnetset/namespace_handler.go @@ -18,7 +18,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" ) -// SubnetSet controller should watch event of namespace, when there are some updates of namespace labels, +// SubnetSet controller should watch event of Namespace, when there are some updates of Namespace labels, // controller should build tags and update VpcSubnetSetSet according to new labels. type EnqueueRequestForNamespace struct { @@ -26,20 +26,20 @@ type EnqueueRequestForNamespace struct { } func (e *EnqueueRequestForNamespace) Create(_ context.Context, _ event.CreateEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace create event, do nothing") + log.V(1).Info("Namespace create event, do nothing") } func (e *EnqueueRequestForNamespace) Delete(_ context.Context, _ event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace delete event, do nothing") + log.V(1).Info("Namespace delete event, do nothing") } func (e *EnqueueRequestForNamespace) Generic(_ context.Context, _ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(1).Info("namespace generic event, do nothing") + log.V(1).Info("Namespace generic event, do nothing") } func (e *EnqueueRequestForNamespace) Update(_ context.Context, updateEvent event.UpdateEvent, l workqueue.RateLimitingInterface) { obj := updateEvent.ObjectNew.(*v1.Namespace) - err := reconcileSubnetSet(e.Client, obj.Name, l) + err := requeueSubnetSet(e.Client, obj.Name, l) if err != nil { log.Error(err, "failed to reconcile subnet") } @@ -52,9 +52,9 @@ var PredicateFuncsNs = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldObj := e.ObjectOld.(*v1.Namespace) newObj := e.ObjectNew.(*v1.Namespace) - log.V(1).Info("receive namespace update event", "name", oldObj.Name) + log.V(1).Info("Receive Namespace update event", "name", oldObj.Name) if reflect.DeepEqual(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) { - log.Info("label of namespace is not changed, ignore it", "name", oldObj.Name) + log.Info("label of Namespace is not changed, ignore it", "name", oldObj.Name) return false } return true @@ -64,21 +64,20 @@ var PredicateFuncsNs = predicate.Funcs{ }, } -func reconcileSubnetSet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { +func requeueSubnetSet(c client.Client, namespace string, q workqueue.RateLimitingInterface) error { subnetSetList := &v1alpha1.SubnetSetList{} err := c.List(context.Background(), subnetSetList, client.InNamespace(namespace)) if err != nil { - log.Error(err, "failed to list all the subnets") + log.Error(err, "Failed to list all the Subnets") return err } - for _, subnet_set_item := range subnetSetList.Items { - log.Info("reconcile subnet set because namespace update", - "namespace", subnet_set_item.Namespace, "name", subnet_set_item.Name) + for _, subnetSet := range subnetSetList.Items { + log.Info("Requeue SubnetSet because Namespace updated", "Namespace", subnetSet.Namespace, "Name", subnetSet.Name) q.Add(reconcile.Request{ NamespacedName: types.NamespacedName{ - Name: subnet_set_item.Name, - Namespace: subnet_set_item.Namespace, + Name: subnetSet.Name, + Namespace: subnetSet.Namespace, }, }) } diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 84d688526..7c182c2d7 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "reflect" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -16,7 +18,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -27,6 +28,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/metrics" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" "github.com/vmware-tanzu/nsx-operator/pkg/util" ) @@ -49,94 +51,97 @@ type SubnetSetReconciler struct { } func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.SubnetSet{} - log.Info("reconciling subnetset CR", "subnetset", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling SubnetSet", "SubnetSet", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + subnetsetCR := &v1alpha1.SubnetSet{} metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetSet) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch subnetset CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + if err := r.Client.Get(ctx, req.NamespacedName, subnetsetCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetBySubnetSetName(ctx, req.Name, req.Namespace); err != nil { + log.Error(err, "Failed to delete NSX Subnet", "SubnetSet", req.NamespacedName) + return ResultRequeue, err + } + return ResultNormal, nil + } + log.Error(err, "Unable to fetch SubnetSet CR", "req", req.NamespacedName) + return ResultRequeue, err + } + if !subnetsetCR.ObjectMeta.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) + err := r.deleteSubnetForSubnetSet(*subnetsetCR, false) + if err != nil { + log.Error(err, "Failed to delete NSX Subnet, retrying", "SubnetSet", req.NamespacedName) + deleteFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + if err := r.Client.Delete(ctx, subnetsetCR); err != nil { + log.Error(err, "Failed to delete SubnetSet CR, retrying", "SubnetSet", req.NamespacedName) + deleteFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + deleteSuccess(r, ctx, subnetsetCR) + return ResultNormal, nil } - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) - if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { - controllerutil.AddFinalizer(obj, servicecommon.SubnetSetFinalizerName) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) - if obj.Spec.AccessMode == "" { - obj.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) - } - if obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) - if vpcNetworkConfig == nil { - err := fmt.Errorf("failed to find VPCNetworkConfig for namespace %s", obj.Namespace) - log.Error(err, "operate failed, would retry exponentially", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize - } - if !util.IsPowerOfTwo(obj.Spec.IPv4SubnetSize) { - errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", obj.Spec.IPv4SubnetSize) - log.Error(nil, errorMsg, "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, errorMsg) - return ResultNormal, nil - } + specChanged := false + if subnetsetCR.Spec.AccessMode == "" { + subnetsetCR.Spec.AccessMode = v1alpha1.AccessMode(v1alpha1.AccessModePrivate) + specChanged = true + } + if subnetsetCR.Spec.IPv4SubnetSize == 0 { + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(subnetsetCR.Namespace) + if vpcNetworkConfig == nil { + err := fmt.Errorf("failed to find VPCNetworkConfig for namespace %s", subnetsetCR.Namespace) + log.Error(err, "Operate failed, would retry exponentially", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err + } + subnetsetCR.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultSubnetSize + specChanged = true + } + if !util.IsPowerOfTwo(subnetsetCR.Spec.IPv4SubnetSize) { + errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", subnetsetCR.Spec.IPv4SubnetSize) + log.Error(nil, errorMsg, "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, errorMsg) + return ResultNormal, nil + } - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on subnetset CR", "subnetset", req.NamespacedName) + if specChanged { + err := r.Client.Update(ctx, subnetsetCR) + if err != nil { + log.Error(err, "Update SubnetSet failed", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, err.Error()) + return ResultRequeue, err } + } - // update subnetset tags if labels of namespace changed - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.UID)) - if len(nsxSubnets) > 0 { - tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) - if tags == nil { - return ResultRequeue, errors.New("failed to generate subnet tags") - } - // tags cannot exceed maximum size 26 - if len(tags) > servicecommon.TagsCountMax { - errorMsg := fmt.Sprintf("tags cannot exceed maximum size 26, tags length: %d", len(tags)) - log.Error(nil, "exceed tags limit, would not retry", "subnet", req.NamespacedName) - updateFail(r, ctx, obj, errorMsg) - return ResultNormal, nil - } - if err := r.SubnetService.UpdateSubnetSetTags(obj.Namespace, nsxSubnets, tags); err != nil { - log.Error(err, "failed to update subnetset tags") - } + // update SubnetSet tags if labels of namespace changed + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(subnetsetCR.UID)) + if len(nsxSubnets) > 0 { + tags := r.SubnetService.GenerateSubnetNSTags(subnetsetCR) + if tags == nil { + log.Error(nil, "Failed to generate SubnetSet tags", "SubnetSet", req.NamespacedName) + return ResultRequeue, errors.New("failed to generate SubnetSet tags") } - updateSuccess(r, ctx, obj) - } else { - if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { - metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) - hasStaleSubnetPorts, err := r.DeleteSubnetForSubnetSet(*obj, false) - if err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetset", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - if hasStaleSubnetPorts { - err := fmt.Errorf("stale subnet ports found while deleting subnetset %v", req.NamespacedName) - log.Error(err, "deletion failed, delete all the subnetports first", "subnetset", req.NamespacedName) - updateFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, servicecommon.SubnetSetFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetset", req.NamespacedName) - deleteFail(r, ctx, obj, err.Error()) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "subnetset", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - log.Info("finalizers cannot be recognized", "subnetset", req.NamespacedName) + // tags cannot exceed maximum size 26 + if len(tags) > servicecommon.TagsCountMax { + errorMsg := fmt.Sprintf("tags cannot exceed maximum size 26, tags length: %d", len(tags)) + log.Error(nil, "Exceed tags limit, would not retry", "SubnetSet", req.NamespacedName) + updateFail(r, ctx, subnetsetCR, errorMsg) + return ResultNormal, nil + } + if err := r.SubnetService.UpdateSubnetSetTags(subnetsetCR.Namespace, nsxSubnets, tags); err != nil { + log.Error(err, "Failed to update SubnetSet tags", "SubnetSet", req.NamespacedName) } } + updateSuccess(r, ctx, subnetsetCR) + return ctrl.Result{}, nil } @@ -163,7 +168,7 @@ func deleteSuccess(r *SubnetSetReconciler, _ context.Context, o *v1alpha1.Subnet metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } -func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, subnetset *v1alpha1.SubnetSet, transitionTime metav1.Time) { +func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, subnetSet *v1alpha1.SubnetSet, transitionTime metav1.Time) { newConditions := []v1alpha1.Condition{ { Type: v1alpha1.Ready, @@ -173,10 +178,10 @@ func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx context.Context, s LastTransitionTime: transitionTime, }, } - r.updateSubnetSetStatusConditions(ctx, subnetset, newConditions) + r.updateSubnetSetStatusConditions(ctx, subnetSet, newConditions) } -func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, subnetset *v1alpha1.SubnetSet, transitionTime metav1.Time, m string) { +func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, subnetSet *v1alpha1.SubnetSet, transitionTime metav1.Time, m string) { newConditions := []v1alpha1.Condition{ { Type: v1alpha1.Ready, @@ -189,30 +194,30 @@ func (r *SubnetSetReconciler) setSubnetSetReadyStatusFalse(ctx context.Context, if m != "" { newConditions[0].Message = m } - r.updateSubnetSetStatusConditions(ctx, subnetset, newConditions) + r.updateSubnetSetStatusConditions(ctx, subnetSet, newConditions) } -func (r *SubnetSetReconciler) updateSubnetSetStatusConditions(ctx context.Context, subnetset *v1alpha1.SubnetSet, newConditions []v1alpha1.Condition) { +func (r *SubnetSetReconciler) updateSubnetSetStatusConditions(ctx context.Context, subnetSet *v1alpha1.SubnetSet, newConditions []v1alpha1.Condition) { conditionsUpdated := false for i := range newConditions { - if r.mergeSubnetSetStatusCondition(ctx, subnetset, &newConditions[i]) { + if r.mergeSubnetSetStatusCondition(ctx, subnetSet, &newConditions[i]) { conditionsUpdated = true } } if conditionsUpdated { - if err := r.Client.Status().Update(ctx, subnetset); err != nil { - log.Error(err, "failed to update status", "Name", subnetset.Name, "Namespace", subnetset.Namespace) + if err := r.Client.Status().Update(ctx, subnetSet); err != nil { + log.Error(err, "Failed to update status", "Name", subnetSet.Name, "Namespace", subnetSet.Namespace) } else { - log.Info("updated SubnetSet", "Name", subnetset.Name, "Namespace", subnetset.Namespace, "New Conditions", newConditions) + log.Info("Updated SubnetSet", "Name", subnetSet.Name, "Namespace", subnetSet.Namespace, "New Conditions", newConditions) } } } -func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, subnetset *v1alpha1.SubnetSet, newCondition *v1alpha1.Condition) bool { - matchedCondition := getExistingConditionOfType(newCondition.Type, subnetset.Status.Conditions) +func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, subnetSet *v1alpha1.SubnetSet, newCondition *v1alpha1.Condition) bool { + matchedCondition := getExistingConditionOfType(newCondition.Type, subnetSet.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { - log.V(2).Info("conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) + log.V(2).Info("Conditions already match", "New Condition", newCondition, "Existing Condition", matchedCondition) return false } @@ -221,7 +226,7 @@ func (r *SubnetSetReconciler) mergeSubnetSetStatusCondition(ctx context.Context, matchedCondition.Message = newCondition.Message matchedCondition.Status = newCondition.Status } else { - subnetset.Status.Conditions = append(subnetset.Status.Conditions, *newCondition) + subnetSet.Status.Conditions = append(subnetSet.Status.Conditions, *newCondition) } return true } @@ -252,11 +257,14 @@ func (r *SubnetSetReconciler) setupWithManager(mgr ctrl.Manager) error { // CollectGarbage collect Subnet which there is no port attached on it. // it implements the interface GarbageCollector method. func (r *SubnetSetReconciler) CollectGarbage(ctx context.Context) { - log.Info("subnetset garbage collector started") + startTime := time.Now() + defer func() { + log.Info("SubnetSet garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() subnetSetList := &v1alpha1.SubnetSetList{} err := r.Client.List(ctx, subnetSetList) if err != nil { - log.Error(err, "failed to list SubnetSet CR") + log.Error(err, "Failed to list SubnetSet CR") return } var nsxSubnetList []*model.VpcSubnet @@ -269,7 +277,7 @@ func (r *SubnetSetReconciler) CollectGarbage(ctx context.Context) { subnetSetIDs := sets.New[string]() for _, subnetSet := range subnetSetList.Items { - if _, err := r.DeleteSubnetForSubnetSet(subnetSet, true); err != nil { + if err := r.deleteSubnetForSubnetSet(subnetSet, true); err != nil { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) @@ -288,33 +296,75 @@ func (r *SubnetSetReconciler) CollectGarbage(ctx context.Context) { } } -func (r *SubnetSetReconciler) DeleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, updataStatus bool) (bool, error) { +func (r *SubnetSetReconciler) deleteSubnetBySubnetSetName(ctx context.Context, subnetSetName, ns string) error { + nsxSubnets := r.SubnetService.ListSubnetBySubnetSetName(ns, subnetSetName) + return r.deleteStaleSubnets(ctx, nsxSubnets) +} + +func (r *SubnetSetReconciler) deleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, updateStatus bool) error { nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.GetUID())) - hitError := false - hasStaleSubnetPorts := false - for _, subnet := range nsxSubnets { - r.SubnetService.LockSubnet(subnet.Path) - portNums := len(r.SubnetPortService.GetPortsOfSubnet(*subnet.Id)) + if err := r.deleteSubnets(nsxSubnets); err != nil { + return err + } + if updateStatus { + err := r.SubnetService.UpdateSubnetSetStatus(&obj) + if err != nil { + return err + } + } + return nil +} + +func (r *SubnetSetReconciler) listSubnetSetIDsFromCRs(ctx context.Context) ([]string, error) { + crdSubnetSetList := &v1alpha1.SubnetSetList{} + err := r.Client.List(ctx, crdSubnetSetList) + if err != nil { + return nil, err + } + + crdSubnetSetIDs := make([]string, 0, len(crdSubnetSetList.Items)) + for _, sr := range crdSubnetSetList.Items { + crdSubnetSetIDs = append(crdSubnetSetIDs, string(sr.UID)) + } + return crdSubnetSetIDs, nil +} + +func (r *SubnetSetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) error { + for _, nsxSubnet := range nsxSubnets { + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) if portNums > 0 { - hasStaleSubnetPorts = true - r.SubnetService.UnlockSubnet(subnet.Path) - continue + return fmt.Errorf("fail to delete Subnet/%s from SubnetSet CR, there is stale ports", *nsxSubnet.Id) } - if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { - log.Error(err, "fail to delete subnet from subnetset cr", "ID", *subnet.Id) - hitError = true + r.SubnetService.LockSubnet(nsxSubnet.Path) + err := r.SubnetService.DeleteSubnet(*nsxSubnet) + if err != nil { + r.SubnetService.UnlockSubnet(nsxSubnet.Path) + return fmt.Errorf("fail to delete Subnet/%s from SubnetSet CR: %+v", *nsxSubnet.Id, err) } - r.SubnetService.UnlockSubnet(subnet.Path) + r.SubnetService.UnlockSubnet(nsxSubnet.Path) } - if updataStatus { - if err := r.SubnetService.UpdateSubnetSetStatus(&obj); err != nil { - return hasStaleSubnetPorts, err - } + log.Info("Successfully deleted all Subnets", "subnetCount", len(nsxSubnets)) + return nil +} + +func (r *SubnetSetReconciler) deleteStaleSubnets(ctx context.Context, nsxSubnets []*model.VpcSubnet) error { + crdSubnetSetIDs, err := r.listSubnetSetIDsFromCRs(ctx) + if err != nil { + log.Error(err, "Failed to list Subnet CRs") + return err } - if hitError { - return hasStaleSubnetPorts, errors.New("error occurs when deleting subnet") + crdSubnetSetIDsSet := sets.NewString(crdSubnetSetIDs...) + nsxSubnetsToDelete := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetSetCRUID) + if crdSubnetSetIDsSet.Has(uid) { + log.Info("Skipping deletion, SubnetSet CR still exists in K8s", "ID", *nsxSubnet.Id) + continue + } + nsxSubnetsToDelete = append(nsxSubnetsToDelete, nsxSubnet) } - return hasStaleSubnetPorts, nil + log.Info("Cleaning stale Subnets for SubnetSet", "Count", len(nsxSubnetsToDelete)) + return r.deleteSubnets(nsxSubnetsToDelete) } func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, @@ -330,7 +380,7 @@ func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetServ Recorder: mgr.GetEventRecorderFor("subnetset-controller"), } if err := subnetsetReconciler.Start(mgr, enableWebhook); err != nil { - log.Error(err, "failed to create controller", "controller", "Subnet") + log.Error(err, "Failed to create controller", "controller", "SubnetSet") return err } go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetsetReconciler.CollectGarbage) diff --git a/pkg/nsx/services/common/services.go b/pkg/nsx/services/common/services.go index b42eb3094..c23a08f29 100644 --- a/pkg/nsx/services/common/services.go +++ b/pkg/nsx/services/common/services.go @@ -29,7 +29,7 @@ type SubnetServiceProvider interface { GetSubnetByPath(path string) (*model.VpcSubnet, error) GetSubnetsByIndex(key, value string) []*model.VpcSubnet CreateOrUpdateSubnet(obj client.Object, vpcInfo VPCResourceInfo, tags []model.Tag) (string, error) - GenerateSubnetNSTags(obj client.Object, nsUID string) []model.Tag + GenerateSubnetNSTags(obj client.Object) []model.Tag LockSubnet(path *string) UnlockSubnet(path *string) } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 454fe6849..02a559e67 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -82,7 +82,6 @@ const ( TagValueShareCreatedForInfra string = "infra" TagValueShareCreatedForProject string = "project" TagValueShareNotCreated string = "notShared" - TagValueGroupAvi string = "avi" TagValueSLB string = "SLB" AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config" AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace" @@ -106,8 +105,6 @@ const ( NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" StaticRouteFinalizerName = "staticroute.crd.nsx.vmware.com/finalizer" - SubnetFinalizerName = "subnet.crd.nsx.vmware.com/finalizer" - SubnetSetFinalizerName = "subnetset.crd.nsx.vmware.com/finalizer" SubnetPortFinalizerName = "subnetport.crd.nsx.vmware.com/finalizer" NetworkInfoFinalizerName = "networkinfo.crd.nsx.vmware.com/finalizer" PodFinalizerName = "pod.crd.nsx.vmware.com/finalizer" diff --git a/pkg/nsx/services/subnet/store.go b/pkg/nsx/services/subnet/store.go index bcd12a4cb..6073e3cdc 100644 --- a/pkg/nsx/services/subnet/store.go +++ b/pkg/nsx/services/subnet/store.go @@ -39,6 +39,24 @@ func subnetIndexFunc(obj interface{}) ([]string, error) { } } +func subnetIndexVMNamespaceFunc(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnet: + return filterTag(o.Tags, common.TagScopeVMNamespace), nil + default: + return nil, errors.New("subnetIndexVMNamespaceFunc doesn't support unknown type") + } +} + +func subnetIndexNamespaceFunc(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnet: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("subnetIndexNamespaceFunc doesn't support unknown type") + } +} + // subnetIndexFunc is used to filter out NSX Subnets which are tagged with CR UID. func subnetSetIndexFunc(obj interface{}) ([]string, error) { switch o := obj.(type) { diff --git a/pkg/nsx/services/subnet/store_test.go b/pkg/nsx/services/subnet/store_test.go index 4db3c39c1..3307f073e 100644 --- a/pkg/nsx/services/subnet/store_test.go +++ b/pkg/nsx/services/subnet/store_test.go @@ -49,7 +49,7 @@ func Test_IndexFunc(t *testing.T) { func Test_KeyFunc(t *testing.T) { id := "test_id" subnet := model.VpcSubnet{Id: &id} - t.Run("1", func(t *testing.T) { + t.Run("subnetKeyFunc", func(t *testing.T) { got, _ := keyFunc(&subnet) if got != "test_id" { t.Errorf("keyFunc() = %v, want %v", got, "test_id") @@ -62,7 +62,9 @@ func Test_InitializeSubnetStore(t *testing.T) { cluster, _ := nsx.NewCluster(config2) rc, _ := cluster.NewRestConnector() - subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeSubnetCRUID: subnetIndexFunc}) + subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeSubnetCRUID: subnetIndexFunc, + }) service := SubnetService{ Service: common.Service{ NSXClient: &nsx.Client{ @@ -107,25 +109,27 @@ func Test_InitializeSubnetStore(t *testing.T) { } func TestSubnetStore_Apply(t *testing.T) { - subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeSubnetCRUID: subnetIndexFunc}) + subnetCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeSubnetCRUID: subnetIndexFunc, + }) resourceStore := common.ResourceStore{ Indexer: subnetCacheIndexer, BindingType: model.SecurityPolicyBindingType(), } subnetStore := &SubnetStore{ResourceStore: resourceStore} type args struct { - i interface{} + subnetVPC interface{} } tests := []struct { name string args args wantErr assert.ErrorAssertionFunc }{ - {"1", args{i: &model.VpcSubnet{Id: common.String("1")}}, assert.NoError}, + {"subnet with id", args{subnetVPC: &model.VpcSubnet{Id: common.String("fake-subnet-id")}}, assert.NoError}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tt.wantErr(t, subnetStore.Apply(tt.args.i), fmt.Sprintf("Apply(%v)", tt.args.i)) + tt.wantErr(t, subnetStore.Apply(tt.args.subnetVPC), fmt.Sprintf("Apply(%v)", tt.args.subnetVPC)) }) } } diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index c4f28d637..caad39431 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -58,6 +58,8 @@ func InitializeSubnetService(service common.Service) (*SubnetService, error) { Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ common.TagScopeSubnetCRUID: subnetIndexFunc, common.TagScopeSubnetSetCRUID: subnetSetIndexFunc, + common.TagScopeVMNamespace: subnetIndexVMNamespaceFunc, + common.TagScopeNamespace: subnetIndexNamespaceFunc, }), BindingType: model.VpcSubnetBindingType(), }, @@ -84,10 +86,10 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, vpcInfo co uid := string(obj.GetUID()) nsxSubnet, err := service.buildSubnet(obj, tags) if err != nil { - log.Error(err, "failed to build Subnet") + log.Error(err, "Failed to build Subnet") return "", err } - // Only check whether needs update when obj is v1alpha1.Subnet + // Only check whether it needs update when obj is v1alpha1.Subnet if subnet, ok := obj.(*v1alpha1.Subnet); ok { existingSubnet := service.SubnetStore.GetByKey(service.BuildSubnetID(subnet)) changed := false @@ -103,7 +105,7 @@ func (service *SubnetService) CreateOrUpdateSubnet(obj client.Object, vpcInfo co } } if !changed { - log.Info("subnet not changed, skip updating", "subnet.Id", uid) + log.Info("Subnet not changed, skip updating", "SubnetId", uid) return uid, nil } } @@ -132,6 +134,9 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet Jitter: 0, Steps: 6, } + // Failure of CheckRealizeState may result in the creation of an existing Subnet. + // For Subnets, it's important to reuse the already created NSXSubnet. + // For SubnetSets, since the ID includes a random value, the created NSX Subnet needs to be deleted and recreated. if err = realizeService.CheckRealizeState(backoff, *nsxSubnet.Path, "RealizedLogicalSwitch"); err != nil { log.Error(err, "failed to check subnet realization state", "ID", *nsxSubnet.Id) return "", err @@ -194,6 +199,32 @@ func (service *SubnetService) ListSubnetSetID(ctx context.Context) sets.Set[stri return subnetsetIDs } +func (service *SubnetService) ListSubnetByName(ns, name string) []*model.VpcSubnet { + nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeVMNamespace, ns) + res := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + tagName := nsxutil.FindTag(nsxSubnet.Tags, common.TagScopeSubnetCRName) + if tagName == name { + res = append(res, nsxSubnet) + } + } + return res +} + +func (service *SubnetService) ListSubnetBySubnetSetName(ns, subnetSetName string) []*model.VpcSubnet { + nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeVMNamespace, ns) + nsxSubnetsOfDefaultPodSubnetSet := service.SubnetStore.GetByIndex(common.TagScopeNamespace, ns) + nsxSubnets = append(nsxSubnets, nsxSubnetsOfDefaultPodSubnetSet...) + res := make([]*model.VpcSubnet, 0, len(nsxSubnets)) + for _, nsxSubnet := range nsxSubnets { + tagName := nsxutil.FindTag(nsxSubnet.Tags, common.TagScopeSubnetSetCRName) + if tagName == subnetSetName { + res = append(res, nsxSubnet) + } + } + return res +} + // check if subnet belongs to a subnetset, if yes, check if that subnetset still exists func (service *SubnetService) IsOrphanSubnet(subnet model.VpcSubnet, subnetsetIDs sets.Set[string]) bool { for _, tag := range subnet.Tags { @@ -345,12 +376,15 @@ func (service *SubnetService) GetSubnetsByIndex(key, value string) []*model.VpcS return service.SubnetStore.GetByIndex(key, value) } -func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) []model.Tag { +func (service *SubnetService) GenerateSubnetNSTags(obj client.Object) []model.Tag { + ns := obj.GetNamespace() namespace := &v1.Namespace{} namespacedName := types.NamespacedName{ Name: ns, } + // Get the namespace object from the Kubernetes API if err := service.Client.Get(context.Background(), namespacedName, namespace); err != nil { + log.Error(err, "Failed to get Namespace", "Namespace", ns) return nil } nsUID := string(namespace.UID) @@ -361,14 +395,8 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) model.Tag{Scope: String(common.TagScopeVMNamespaceUID), Tag: String(nsUID)}, model.Tag{Scope: String(common.TagScopeVMNamespace), Tag: String(obj.GetNamespace())}) case *v1alpha1.SubnetSet: - findLabelDefaultPodSubnetSet := false - for k, v := range o.Labels { - if k == common.LabelDefaultSubnetSet && v == common.LabelDefaultPodSubnetSet { - findLabelDefaultPodSubnetSet = true - break - } - } - if findLabelDefaultPodSubnetSet { + isDefaultPodSubnetSet := o.Labels[common.LabelDefaultSubnetSet] == common.LabelDefaultPodSubnetSet + if isDefaultPodSubnetSet { tags = append(tags, model.Tag{Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String(nsUID)}, model.Tag{Scope: common.String(common.TagScopeNamespace), Tag: common.String(obj.GetNamespace())}) @@ -378,6 +406,7 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) model.Tag{Scope: common.String(common.TagScopeVMNamespace), Tag: common.String(obj.GetNamespace())}) } } + // Append Namespace labels as tags for k, v := range namespace.Labels { tags = append(tags, model.Tag{Scope: common.String(k), Tag: common.String(v)}) } @@ -385,7 +414,7 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) } func (service *SubnetService) UpdateSubnetSetTags(ns string, vpcSubnets []*model.VpcSubnet, tags []model.Tag) error { - for i := range vpcSubnets { + for i, vpcSubnet := range vpcSubnets { subnetSet := &v1alpha1.SubnetSet{} var name string @@ -403,27 +432,32 @@ func (service *SubnetService) UpdateSubnetSetTags(ns string, vpcSubnets []*model } } - if matchNamespace { - if err := service.Client.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, subnetSet); err != nil { - return err - } - newTags := append(service.buildBasicTags(subnetSet), tags...) - changed := common.CompareResource(SubnetToComparable(vpcSubnets[i]), SubnetToComparable(&model.VpcSubnet{Tags: newTags})) - if !changed { - log.Info("NSX subnet tags unchanged, skip updating") - continue - } - vpcSubnets[i].Tags = newTags - vpcInfo, err := common.ParseVPCResourcePath(*vpcSubnets[i].Path) - if err != nil { - err := fmt.Errorf("failed to parse NSX VPC path for Subnet %s: %s", *vpcSubnets[i].Path, err) - return err - } - if _, err := service.createOrUpdateSubnet(subnetSet, vpcSubnets[i], &vpcInfo); err != nil { - return err - } - log.Info("successfully updated subnet set tags", "subnetSet", subnetSet) + // Skip this subnet if the Namespace doesn't match + if !matchNamespace { + log.Info("Namespace mismatch, skipping subnet", "Subnet", *vpcSubnet.Id, "Namespace", ns) + continue + } + + if err := service.Client.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, subnetSet); err != nil { + return fmt.Errorf("failed to get SubnetSet %s in Namespace %s: %w", name, ns, err) + } + newTags := append(service.buildBasicTags(subnetSet), tags...) + changed := common.CompareResource(SubnetToComparable(vpcSubnets[i]), SubnetToComparable(&model.VpcSubnet{Tags: newTags})) + if !changed { + log.Info("NSX Subnet tags unchanged, skipping update", "subnet", *vpcSubnet.Id) + continue + } + vpcSubnets[i].Tags = newTags + + vpcInfo, err := common.ParseVPCResourcePath(*vpcSubnets[i].Path) + if err != nil { + err := fmt.Errorf("failed to parse NSX VPC path for Subnet %s: %s", *vpcSubnets[i].Path, err) + return err + } + if _, err := service.createOrUpdateSubnet(subnetSet, vpcSubnets[i], &vpcInfo); err != nil { + return fmt.Errorf("failed to update Subnet %s in SubnetSet %s: %w", *vpcSubnet.Id, subnetSet.Name, err) } + log.Info("Successfully updated SubnetSet tags", "subnetSet", subnetSet, "Subnet", *vpcSubnet.Id) } return nil } diff --git a/pkg/nsx/services/subnet/subnet_test.go b/pkg/nsx/services/subnet/subnet_test.go new file mode 100644 index 000000000..cf65d0477 --- /dev/null +++ b/pkg/nsx/services/subnet/subnet_test.go @@ -0,0 +1,90 @@ +package subnet + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func TestGenerateSubnetNSTags(t *testing.T) { + scheme := clientgoscheme.Scheme + v1alpha1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + + service := &SubnetService{ + Service: common.Service{ + Client: fakeClient, + NSXClient: &nsx.Client{}, + + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + UseAVILoadBalancer: false, + }, + }, + }, + } + + // Create a test namespace + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + UID: "namespace-uid", + Labels: map[string]string{ + "env": "test", + }, + }, + } + + assert.NoError(t, fakeClient.Create(context.TODO(), namespace)) + + // Define the Subnet object + subnet := &v1alpha1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet", + Namespace: "test-ns", + }, + } + + // Generate tags for the Subnet + tags := service.GenerateSubnetNSTags(subnet) + + // Validate the tags + assert.NotNil(t, tags) + assert.Equal(t, 3, len(tags)) // 3 tags should be generated + + // Check specific tags + assert.Equal(t, "namespace-uid", *tags[0].Tag) + assert.Equal(t, "test-ns", *tags[1].Tag) + assert.Equal(t, "test", *tags[2].Tag) + + // Define the SubnetSet object + subnetSet := &v1alpha1.SubnetSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-subnet-set", + Namespace: "test-ns", + Labels: map[string]string{ + common.LabelDefaultSubnetSet: common.LabelDefaultPodSubnetSet, + }, + }, + } + + // Generate tags for the SubnetSet + tagsSet := service.GenerateSubnetNSTags(subnetSet) + + // Validate the tags for SubnetSet + assert.NotNil(t, tagsSet) + assert.Equal(t, 3, len(tagsSet)) // 3 tags should be generated + assert.Equal(t, "namespace-uid", *tagsSet[0].Tag) + assert.Equal(t, "test-ns", *tagsSet[1].Tag) +} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index efe098e55..d1eebde26 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -21,7 +21,6 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" t1v1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" @@ -37,26 +36,9 @@ const ( ) var ( - String = common.String - basicTags = []string{ - common.TagScopeCluster, common.TagScopeVersion, - common.TagScopeStaticRouteCRName, common.TagScopeStaticRouteCRUID, - common.TagValueScopeSecurityPolicyName, common.TagValueScopeSecurityPolicyUID, - common.TagScopeNetworkPolicyName, common.TagScopeNetworkPolicyUID, - common.TagScopeSubnetCRName, common.TagScopeSubnetCRUID, - common.TagScopeSubnetPortCRName, common.TagScopeSubnetPortCRUID, - common.TagScopeIPPoolCRName, common.TagScopeIPPoolCRUID, - common.TagScopeSubnetSetCRName, common.TagScopeSubnetSetCRUID, - } - tagsScopeSet = sets.New[string]() + String = common.String ) -func init() { - for _, tag := range basicTags { - tagsScopeSet.Insert(tag) - } -} - var log = &logger.Log func NormalizeLabels(matchLabels *map[string]string) *map[string]string { @@ -441,6 +423,10 @@ func GenerateTruncName(limit int, resName string, prefix, suffix, project, clust return generateDisplayName(common.ConnectorUnderline, resName, prefix, suffix, project, cluster) } +func CombineNamespaceName(name, namespace string) string { + return fmt.Sprintf("%s/%s", namespace, name) +} + func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []model.Tag { tags := []model.Tag{ { @@ -498,19 +484,6 @@ func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []mo return tags } -func AppendTags(basicTags, extraTags []model.Tag) []model.Tag { - if basicTags == nil { - log.Info("AppendTags", "basicTags", basicTags, "extra tags", extraTags) - return nil - } - for _, tag := range extraTags { - if !tagsScopeSet.Has(*tag.Scope) { - basicTags = append(basicTags, tag) - } - } - return basicTags -} - func Capitalize(s string) string { if s == "" { return "" diff --git a/test/e2e/nsx_subnet_test.go b/test/e2e/nsx_subnet_test.go index 1421a14db..cf15c438f 100644 --- a/test/e2e/nsx_subnet_test.go +++ b/test/e2e/nsx_subnet_test.go @@ -88,7 +88,7 @@ func transSearchResponsetoSubnet(response model.SearchResponse) []model.VpcSubne return resources } -func fetchSubnet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { +func fetchSubnetBySubnetSet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { tags := []string{common.TagScopeSubnetSetCRUID, string(subnetSet.UID)} results, err := testData.queryResource(common.ResourceTypeSubnet, tags) assertNil(t, err) @@ -96,6 +96,7 @@ func fetchSubnet(t *testing.T, subnetSet *v1alpha1.SubnetSet) model.VpcSubnet { assertTrue(t, len(subnets) > 0, "No NSX subnet found") return subnets[0] } + func defaultSubnetSet(t *testing.T) { // 1. Check whether default-vm-subnetset and default-pod-subnetset are created. err := testData.waitForCRReadyOrDeleted(defaultTimeout, SubnetSetCRType, E2ENamespace, common.DefaultVMSubnetSet, Ready) @@ -127,11 +128,11 @@ func defaultSubnetSet(t *testing.T) { assertNil(t, err) labelKey, labelValue := "subnet-e2e", "add" ns.Labels[labelKey] = labelValue - ns, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet := fetchSubnet(t, subnetSet) + vpcSubnet := fetchSubnetBySubnetSet(t, subnetSet) found := false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey && *tag.Tag == labelValue { @@ -142,12 +143,14 @@ func defaultSubnetSet(t *testing.T) { assertTrue(t, found, "Failed to add tags for NSX subnet %s", *(vpcSubnet.Id)) // 6. Check updating NSX subnet tags. + ns, err = testData.clientset.CoreV1().Namespaces().Get(context.TODO(), E2ENamespace, v1.GetOptions{}) + assertNil(t, err) labelValue = "update" ns.Labels[labelKey] = labelValue - ns, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet = fetchSubnet(t, subnetSet) + vpcSubnet = fetchSubnetBySubnetSet(t, subnetSet) found = false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey && *tag.Tag == labelValue { @@ -158,11 +161,14 @@ func defaultSubnetSet(t *testing.T) { assertTrue(t, found, "Failed to update tags for NSX subnet %s", *(vpcSubnet.Id)) // 7. Check deleting NSX subnet tags. + ns, err = testData.clientset.CoreV1().Namespaces().Get(context.TODO(), E2ENamespace, v1.GetOptions{}) + assertNil(t, err) delete(ns.Labels, labelKey) - _, err = testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) + newNs, err := testData.clientset.CoreV1().Namespaces().Update(context.TODO(), ns, v1.UpdateOptions{}) time.Sleep(5 * time.Second) assertNil(t, err) - vpcSubnet = fetchSubnet(t, subnetSet) + t.Logf("new Namespace: %+v", newNs) + vpcSubnet = fetchSubnetBySubnetSet(t, subnetSet) found = false for _, tag := range vpcSubnet.Tags { if *tag.Scope == labelKey { @@ -288,6 +294,9 @@ func SubnetCIDR(t *testing.T) { assertNil(t, err) allocatedSubnet, err := testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) assertNil(t, err) + nsxSubnets := testData.fetchSubnetByNamespace(t, E2ENamespace, false) + assert.Equal(t, 1, len(nsxSubnets)) + targetCIDR := allocatedSubnet.Status.NetworkAddresses[0] err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Delete(context.TODO(), subnet.Name, v1.DeleteOptions{}) assertNil(t, err) @@ -300,10 +309,13 @@ func SubnetCIDR(t *testing.T) { return false, err }) assertNil(t, err) + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, true) + assert.Equal(t, true, len(nsxSubnets) <= 1) subnet.Spec.IPAddresses = []string{targetCIDR} _, err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Create(context.TODO(), subnet, v1.CreateOptions{}) if err != nil && errors.IsAlreadyExists(err) { + t.Logf("Create Subnet error: %+v", err) err = nil } assertNil(t, err) @@ -312,4 +324,35 @@ func SubnetCIDR(t *testing.T) { allocatedSubnet, err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) assertNil(t, err) assert.Equal(t, targetCIDR, allocatedSubnet.Status.NetworkAddresses[0]) + + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, false) + assert.Equal(t, 1, len(nsxSubnets)) + + err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Delete(context.TODO(), subnet.Name, v1.DeleteOptions{}) + assertNil(t, err) + + err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, 100*time.Second, false, func(ctx context.Context) (bool, error) { + _, err := testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return true, nil + } + return false, err + }) + assertNil(t, err) + + nsxSubnets = testData.fetchSubnetByNamespace(t, E2ENamespace, true) + assert.Equal(t, true, len(nsxSubnets) <= 1) +} + +func (data *TestData) fetchSubnetByNamespace(t *testing.T, ns string, isMarkForDelete bool) (res []model.VpcSubnet) { + tags := []string{common.TagScopeNamespace, ns} + results, err := testData.queryResource(common.ResourceTypeSubnet, tags) + assertNil(t, err) + subnets := transSearchResponsetoSubnet(results) + for _, subnet := range subnets { + if *subnet.MarkedForDelete == isMarkForDelete { + res = append(res, subnet) + } + } + return } From fa4dbe6e89bc721785a0a3e1b3a8f46c8ec8e759 Mon Sep 17 00:00:00 2001 From: Yanjun Zhou Date: Sat, 12 Oct 2024 18:51:30 +0800 Subject: [PATCH 08/18] Remove SubnetPort/Pod finalizer (#792) This PR removes the Finalizer in SubnetPort and Pod controller to provide a smooth deletion experience for the user while ensure the stale NSX SubnetPort will be deleted as expected. Signed-off-by: Yanjun Zhou --- pkg/controllers/pod/pod_controller.go | 95 +++++++------- .../subnetport/subnetport_controller.go | 118 ++++++++--------- .../subnetport/subnetport_controller_test.go | 121 +++++++----------- pkg/nsx/services/common/types.go | 2 - pkg/nsx/services/subnetport/builder.go | 3 - pkg/nsx/services/subnetport/store.go | 18 +++ pkg/nsx/services/subnetport/subnetport.go | 68 ++++++++-- 7 files changed, 222 insertions(+), 203 deletions(-) diff --git a/pkg/controllers/pod/pod_controller.go b/pkg/controllers/pod/pod_controller.go index c13959a71..5c8797ef6 100644 --- a/pkg/controllers/pod/pod_controller.go +++ b/pkg/controllers/pod/pod_controller.go @@ -7,18 +7,17 @@ import ( "context" "fmt" "os" + "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -45,14 +44,25 @@ type PodReconciler struct { } func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - pod := &v1.Pod{} log.Info("reconciling pod", "pod", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("finished reconciling Pod", "Pod", req.NamespacedName, "duration", time.Since(startTime)) + }() metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypePod) + pod := &v1.Pod{} if err := r.Client.Get(ctx, req.NamespacedName, pod); err != nil { - log.Error(err, "unable to fetch pod", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetPortByPodName(ctx, req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NSX SubnetPort", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err + } + return common.ResultNormal, nil + } + log.Error(err, "unable to fetch Pod", "Pod", req.NamespacedName) + return common.ResultRequeue, err } if pod.Spec.HostNetwork { log.Info("skipping handling hostnetwork pod", "pod", req.NamespacedName) @@ -65,16 +75,6 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R if !podIsDeleted(pod) { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypePod) - if !controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { - controllerutil.AddFinalizer(pod, servicecommon.PodFinalizerName) - if err := r.Client.Update(ctx, pod); err != nil { - log.Error(err, "add finalizer", "pod", req.NamespacedName) - updateFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - log.Info("added finalizer on pod", "pod", req.NamespacedName) - } - nsxSubnetPath, err := r.GetSubnetPathForPod(ctx, pod) if err != nil { log.Error(err, "failed to get NSX resource path from subnet", "pod.Name", pod.Name, "pod.UID", pod.UID) @@ -105,25 +105,13 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } updateSuccess(r, ctx, pod) } else { - if controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { - metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&pod.ObjectMeta) - if err := r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) - deleteFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - controllerutil.RemoveFinalizer(pod, servicecommon.PodFinalizerName) - if err := r.Client.Update(ctx, pod); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) - deleteFail(r, ctx, pod, &err) - return common.ResultRequeue, err - } - log.Info("removed finalizer", "pod", req.NamespacedName) - deleteSuccess(r, ctx, pod) - } else { - log.Info("finalizers cannot be recognized", "pod", req.NamespacedName) + subnetPortID := r.SubnetPortService.BuildSubnetPortId(&pod.ObjectMeta) + if err := r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) + deleteFail(r, ctx, pod, &err) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, pod) } return ctrl.Result{}, nil } @@ -147,14 +135,6 @@ func (r *PodReconciler) GetNodeByName(nodeName string) (*model.HostTransportNode func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1.Pod{}). - WithEventFilter( - predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }, - ). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -212,7 +192,7 @@ func (r *PodReconciler) CollectGarbage(ctx context.Context) { for elem := range diffSet { log.V(1).Info("GC collected Pod", "NSXSubnetPortID", elem) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - err = r.SubnetPortService.DeleteSubnetPort(elem) + err = r.SubnetPortService.DeleteSubnetPortById(elem) if err != nil { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } else { @@ -221,17 +201,17 @@ func (r *PodReconciler) CollectGarbage(ctx context.Context) { } } -func updateFail(r *PodReconciler, c context.Context, o *v1.Pod, e *error) { +func updateFail(r *PodReconciler, _ context.Context, o *v1.Pod, e *error) { r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailUpdate, fmt.Sprintf("%v", *e)) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypePod) } -func deleteFail(r *PodReconciler, c context.Context, o *v1.Pod, e *error) { +func deleteFail(r *PodReconciler, _ context.Context, o *v1.Pod, e *error) { r.Recorder.Event(o, v1.EventTypeWarning, common.ReasonFailDelete, fmt.Sprintf("%v", *e)) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } -func updateSuccess(r *PodReconciler, c context.Context, o *v1.Pod) { +func updateSuccess(r *PodReconciler, _ context.Context, o *v1.Pod) { r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "Pod CR has been successfully updated") metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypePod) } @@ -264,3 +244,26 @@ func (r *PodReconciler) GetSubnetPathForPod(ctx context.Context, pod *v1.Pod) (s func podIsDeleted(pod *v1.Pod) bool { return !pod.ObjectMeta.DeletionTimestamp.IsZero() || pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed" } + +func (r *PodReconciler) deleteSubnetPortByPodName(ctx context.Context, ns string, name string) error { + // When deleting SubnetPort by Name and Namespace, skip the SubnetPort belonging to the existed SubnetPort CR + nsxSubnetPorts := r.SubnetPortService.ListSubnetPortByPodName(ns, name) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) + if err != nil { + log.Error(err, "failed to list SubnetPort CRs") + return err + } + + for _, nsxSubnetPort := range nsxSubnetPorts { + if crSubnetPortIDsSet.Has(*nsxSubnetPort.Id) { + log.Info("skipping deletion, Pod CR still exists in K8s", "ID", *nsxSubnetPort.Id) + continue + } + if err := r.SubnetPortService.DeleteSubnetPort(nsxSubnetPort); err != nil { + return err + } + } + log.Info("successfully deleted nsxSubnetPort for Pod", "namespace", ns, "name", name) + return nil +} diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index ee6a15cfc..468995878 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -10,21 +10,20 @@ import ( "os" "reflect" "strings" + "time" vmv1alpha1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -57,18 +56,27 @@ type SubnetPortReconciler struct { // +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports/finalizers,verbs=update func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - subnetPort := &v1alpha1.SubnetPort{} log.Info("reconciling subnetport CR", "subnetport", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("finished reconciling SubnetPort", "SubnetPort", req.NamespacedName, "duration", time.Since(startTime)) + }() metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetPort) + subnetPort := &v1alpha1.SubnetPort{} if err := r.Client.Get(ctx, req.NamespacedName, subnetPort); err != nil { - log.Error(err, "unable to fetch subnetport CR", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) + if apierrors.IsNotFound(err) { + if err := r.deleteSubnetPortByName(ctx, req.Namespace, req.Name); err != nil { + log.Error(err, "failed to delete NSX SubnetPort", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err + } + return common.ResultNormal, nil + } + log.Error(err, "unable to fetch SubnetPort CR", "SubnetPort", req.NamespacedName) + return common.ResultRequeue, err } - if len(subnetPort.Spec.SubnetSet) > 0 && len(subnetPort.Spec.Subnet) > 0 { err := errors.New("subnet and subnetset should not be configured at the same time") log.Error(err, "failed to get subnet/subnetset of the subnetport", "subnetport", req.NamespacedName) @@ -78,15 +86,6 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) if subnetPort.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetPort) - if !controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { - controllerutil.AddFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) - if err := r.Client.Update(ctx, subnetPort); err != nil { - log.Error(err, "add finalizer", "subnetport", req.NamespacedName) - updateFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - log.Info("added finalizer on subnetport CR", "subnetport", req.NamespacedName) - } old_status := subnetPort.Status.DeepCopy() isParentResourceTerminating, nsxSubnetPath, err := r.CheckAndGetSubnetPathForSubnetPort(ctx, subnetPort) @@ -145,27 +144,16 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) } updateSuccess(r, ctx, subnetPort) } else { - if controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { - metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) - if err := r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetport", req.NamespacedName) - deleteFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - controllerutil.RemoveFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) - if err := r.Client.Update(ctx, subnetPort); err != nil { - log.Error(err, "deletion failed, would retry exponentially", "subnetport", req.NamespacedName) - deleteFail(r, ctx, subnetPort, &err) - return common.ResultRequeue, err - } - log.Info("removed finalizer", "subnetport", req.NamespacedName) - deleteSuccess(r, ctx, subnetPort) - } else { - log.Info("finalizers cannot be recognized", "subnetport", req.NamespacedName) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) + subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) + if err := r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { + log.Error(err, "deletion failed, would retry exponentially", "SubnetPort", req.NamespacedName) + deleteFail(r, ctx, subnetPort, &err) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, subnetPort) } - return ctrl.Result{}, nil + return common.ResultNormal, nil } func subnetPortNamespaceVMIndexFunc(obj client.Object) []string { @@ -191,6 +179,29 @@ func addressBindingNamespaceVMIndexFunc(obj client.Object) []string { } } +func (r *SubnetPortReconciler) deleteSubnetPortByName(ctx context.Context, ns string, name string) error { + // When deleting SubnetPort by Name and Namespace, skip the SubnetPort belonging to the existed SubnetPort CR + nsxSubnetPorts := r.SubnetPortService.ListSubnetPortByName(ns, name) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) + if err != nil { + log.Error(err, "failed to list SubnetPort CRs") + return err + } + + for _, nsxSubnetPort := range nsxSubnetPorts { + if crSubnetPortIDsSet.Has(*nsxSubnetPort.Id) { + log.Info("skipping deletion, SubnetPort CR still exists in K8s", "ID", *nsxSubnetPort.Id) + continue + } + if err := r.SubnetPortService.DeleteSubnetPort(nsxSubnetPort); err != nil { + return err + } + } + log.Info("successfully deleted nsxSubnetPort", "namespace", ns, "name", name) + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &v1alpha1.SubnetPort{}, util.SubnetPortNamespaceVMIndexKey, subnetPortNamespaceVMIndexFunc); err != nil { @@ -201,18 +212,6 @@ func (r *SubnetPortReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.SubnetPort{}). - WithEventFilter( - predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - switch e.Object.(type) { - case *v1alpha1.AddressBinding: - return true - } - return false - }, - }, - ). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -290,24 +289,17 @@ func (r *SubnetPortReconciler) CollectGarbage(ctx context.Context) { if len(nsxSubnetPortSet) == 0 { return } - subnetPortList := &v1alpha1.SubnetPortList{} - err := r.Client.List(ctx, subnetPortList) + + crSubnetPortIDsSet, err := r.SubnetPortService.ListSubnetPortIDsFromCRs(ctx) if err != nil { - log.Error(err, "failed to list SubnetPort CR") return } - CRSubnetPortSet := sets.New[string]() - for _, subnetPort := range subnetPortList.Items { - subnetPortID := r.SubnetPortService.BuildSubnetPortId(&subnetPort.ObjectMeta) - CRSubnetPortSet.Insert(subnetPortID) - } - - diffSet := nsxSubnetPortSet.Difference(CRSubnetPortSet) + diffSet := nsxSubnetPortSet.Difference(crSubnetPortIDsSet) for elem := range diffSet { log.V(1).Info("GC collected SubnetPort CR", "UID", elem) metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - err = r.SubnetPortService.DeleteSubnetPort(elem) + err = r.SubnetPortService.DeleteSubnetPortById(elem) if err != nil { metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) } else { @@ -359,7 +351,7 @@ func (r *SubnetPortReconciler) UpdateSubnetPortStatusConditions(ctx context.Cont } } -func (r *SubnetPortReconciler) mergeSubnetPortStatusCondition(ctx context.Context, subnetPort *v1alpha1.SubnetPort, newCondition *v1alpha1.Condition) bool { +func (r *SubnetPortReconciler) mergeSubnetPortStatusCondition(_ context.Context, subnetPort *v1alpha1.SubnetPort, newCondition *v1alpha1.Condition) bool { matchedCondition := getExistingConditionOfType(newCondition.Type, subnetPort.Status.Conditions) if reflect.DeepEqual(matchedCondition, newCondition) { @@ -420,7 +412,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co _, err = r.SubnetService.GetSubnetByPath(subnetPath) if err != nil { log.Info("previous NSX subnet is deleted, deleting the stale subnet port", "subnetPort.UID", subnetPort.UID, "subnetPath", subnetPath) - if err = r.SubnetPortService.DeleteSubnetPort(subnetPortID); err != nil { + if err = r.SubnetPortService.DeleteSubnetPortById(subnetPortID); err != nil { log.Error(err, "failed to delete the stale subnetport", "subnetport.UID", subnetPort.UID) return } @@ -439,7 +431,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co log.Error(err, "subnet CR not found", "subnet CR", namespacedName) return } - if subnet != nil && !subnet.DeletionTimestamp.IsZero() { + if !subnet.DeletionTimestamp.IsZero() { isStale = true err := fmt.Errorf("subnet %s is being deleted, cannot operate subnetport %s", namespacedName, subnetPort.Name) return true, "", err @@ -464,7 +456,7 @@ func (r *SubnetPortReconciler) CheckAndGetSubnetPathForSubnetPort(ctx context.Co log.Error(err, "subnetSet CR not found", "subnet CR", namespacedName) return } - if subnetSet != nil && !subnetSet.DeletionTimestamp.IsZero() { + if !subnetSet.DeletionTimestamp.IsZero() { isStale = true err = fmt.Errorf("subnetset %s is being deleted, cannot operate subnetport %s", namespacedName, subnetPort.Name) return diff --git a/pkg/controllers/subnetport/subnetport_controller_test.go b/pkg/controllers/subnetport/subnetport_controller_test.go index ec0dbc55c..3aa0d6940 100644 --- a/pkg/controllers/subnetport/subnetport_controller_test.go +++ b/pkg/controllers/subnetport/subnetport_controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -77,41 +78,37 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { }) defer patchesGetSubnetByPath.Reset() - // not found - errNotFound := errors.New("not found") + // fail to get + errFailToGet := errors.New("failed to get CR") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errFailToGet) + _, ret := r.Reconcile(ctx, req) + assert.Equal(t, errFailToGet, ret) + + // not found and deletion success + errNotFound := apierrors.NewNotFound(v1alpha1.Resource("subnetport"), "") k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) - _, err := r.Reconcile(ctx, req) - assert.Equal(t, err, errNotFound) - // update fails - sp := &v1alpha1.SubnetPort{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" + patchesDeleteSubnetPortByName := gomonkey.ApplyFunc((*SubnetPortReconciler).deleteSubnetPortByName, + func(r *SubnetPortReconciler, ctx context.Context, ns string, name string) error { return nil }) - subnet := &v1alpha1.Subnet{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), subnet).Return(nil).AnyTimes().Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - s := obj.(*v1alpha1.Subnet) - s.Name = sp.Spec.Subnet - return nil - }) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) - patchesSuccess := gomonkey.ApplyFunc(updateSuccess, - func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort) { - }) - defer patchesSuccess.Reset() - patchesUpdateFail := gomonkey.ApplyFunc(updateFail, - func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort, e *error) { + defer patchesDeleteSubnetPortByName.Reset() + _, ret = r.Reconcile(ctx, req) + assert.Equal(t, nil, ret) + + // not found and deletion failed + err := errors.New("Deletion failed") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + patchesDeleteSubnetPortByName = gomonkey.ApplyFunc((*SubnetPortReconciler).deleteSubnetPortByName, + func(r *SubnetPortReconciler, ctx context.Context, ns string, name string) error { + return err }) - defer patchesUpdateFail.Reset() - _, ret := r.Reconcile(ctx, req) + defer patchesDeleteSubnetPortByName.Reset() + _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) // both subnet and subnetset are configured + sp := &v1alpha1.SubnetPort{} k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -119,6 +116,10 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.Spec.SubnetSet = "subnetset2" return nil }) + patchesUpdateFail := gomonkey.ApplyFunc(updateFail, + func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort, e *error) { + }) + defer patchesUpdateFail.Reset() err = errors.New("subnet and subnetset should not be configured at the same time") _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) @@ -135,7 +136,6 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return requests }) defer patchesVmMapFunc.Reset() - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -155,8 +155,8 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { defer patchesCreateOrUpdateSubnetPort.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) + // happy path - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -184,6 +184,10 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { return portState, nil }) defer patchesCreateOrUpdateSubnetPort.Reset() + patchesSuccess := gomonkey.ApplyFunc(updateSuccess, + func(r *SubnetPortReconciler, c context.Context, o *v1alpha1.SubnetPort) { + }) + defer patchesSuccess.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, nil, ret) @@ -194,15 +198,14 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.Spec.Subnet = "subnet1" time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} return nil }) err = errors.New("DeleteSubnetPort failed") - patchesDeleteSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { return err }) - defer patchesDeleteSubnetPort.Reset() + defer patchesDeleteSubnetPortById.Reset() patchesCreateOrUpdateSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).CreateOrUpdateSubnetPort, func(s *subnetport.SubnetPortService, obj interface{}, nsxSubnet *model.VpcSubnet, contextID string, tags *map[string]string) (*model.SegmentPortState, error) { assert.FailNow(t, "should not be called") @@ -216,46 +219,7 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { _, ret = r.Reconcile(ctx, req) assert.Equal(t, err, ret) - // handle deletion event - update subnetport failed in deletion event - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} - return nil - }) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(err) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, - func(s *subnetport.SubnetPortService, uid string) error { - return nil - }) - defer patchesDeleteSubnetPort.Reset() - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, err, ret) - // handle deletion event - successfully deleted - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( - func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.SubnetPort) - v1sp.Spec.Subnet = "subnet1" - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.SubnetPortFinalizerName} - return nil - }) - k8sClient.EXPECT().Update(ctx, gomock.Any()).Return(nil) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, - func(s *subnetport.SubnetPortService, uid string) error { - return nil - }) - defer patchesDeleteSubnetPort.Reset() - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, nil, ret) - - // handle deletion event - unknown finalizers k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do( func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.SubnetPort) @@ -264,20 +228,22 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { v1sp.ObjectMeta.DeletionTimestamp = &time return nil }) - patchesDeleteSubnetPort = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById = gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { - assert.FailNow(t, "should not be called") return nil }) - defer patchesDeleteSubnetPort.Reset() + defer patchesDeleteSubnetPortById.Reset() _, ret = r.Reconcile(ctx, req) assert.Equal(t, nil, ret) } func TestSubnetPortReconciler_GarbageCollector(t *testing.T) { // gc collect item "2345", local store has more item than k8s cache + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) service := &subnetport.SubnetPortService{ Service: common.Service{ + Client: k8sClient, NSXConfig: &config.NSXOperatorConfig{ NsxConfig: &config.NsxConfig{ EnforcementPoint: "vmc-enforcementpoint", @@ -293,13 +259,12 @@ func TestSubnetPortReconciler_GarbageCollector(t *testing.T) { return a }) defer patchesListNSXSubnetPortIDForCR.Reset() - patchesDeleteSubnetPort := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPort, + patchesDeleteSubnetPortById := gomonkey.ApplyFunc((*subnetport.SubnetPortService).DeleteSubnetPortById, func(s *subnetport.SubnetPortService, uid string) error { return nil }) - defer patchesDeleteSubnetPort.Reset() - mockCtl := gomock.NewController(t) - k8sClient := mock_client.NewMockClient(mockCtl) + defer patchesDeleteSubnetPortById.Reset() + r := &SubnetPortReconciler{ Client: k8sClient, Scheme: nil, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 02a559e67..95bc89fa9 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -105,9 +105,7 @@ const ( NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" StaticRouteFinalizerName = "staticroute.crd.nsx.vmware.com/finalizer" - SubnetPortFinalizerName = "subnetport.crd.nsx.vmware.com/finalizer" NetworkInfoFinalizerName = "networkinfo.crd.nsx.vmware.com/finalizer" - PodFinalizerName = "pod.crd.nsx.vmware.com/finalizer" IPPoolFinalizerName = "ippool.crd.nsx.vmware.com/finalizer" IPAddressAllocationFinalizerName = "ipaddressallocation.crd.nsx.vmware.com/finalizer" diff --git a/pkg/nsx/services/subnetport/builder.go b/pkg/nsx/services/subnetport/builder.go index 0c8795f94..7ff5e29ce 100644 --- a/pkg/nsx/services/subnetport/builder.go +++ b/pkg/nsx/services/subnetport/builder.go @@ -51,9 +51,6 @@ func (service *SubnetPortService) buildSubnetPort(obj interface{}, nsxSubnet *mo return nil, err } nsxSubnetPortPath := fmt.Sprintf("%s/ports/%s", *nsxSubnet.Path, nsxSubnetPortID) - if err != nil { - return nil, err - } namespace := &corev1.Namespace{} namespacedName := types.NamespacedName{ Name: objNamespace, diff --git a/pkg/nsx/services/subnetport/store.go b/pkg/nsx/services/subnetport/store.go index 6a01e4d68..c5496a20e 100644 --- a/pkg/nsx/services/subnetport/store.go +++ b/pkg/nsx/services/subnetport/store.go @@ -67,6 +67,24 @@ func subnetPortIndexBySubnetID(obj interface{}) ([]string, error) { } } +func subnetPortIndexNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnetPort: + return filterTag(o.Tags, common.TagScopeVMNamespace), nil + default: + return nil, errors.New("subnetPortIndexNamespace doesn't support unknown type") + } +} + +func subnetPortIndexPodNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.VpcSubnetPort: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("subnetPortIndexPodNamespace doesn't support unknown type") + } +} + // SubnetPortStore is a store for SubnetPorts type SubnetPortStore struct { common.ResourceStore diff --git a/pkg/nsx/services/subnetport/subnetport.go b/pkg/nsx/services/subnetport/subnetport.go index 36c2d947a..8f0cd7031 100644 --- a/pkg/nsx/services/subnetport/subnetport.go +++ b/pkg/nsx/services/subnetport/subnetport.go @@ -52,6 +52,8 @@ func InitializeSubnetPort(service servicecommon.Service) (*SubnetPortService, er cache.Indexers{ servicecommon.TagScopeSubnetPortCRUID: subnetPortIndexByCRUID, servicecommon.TagScopePodUID: subnetPortIndexByPodUID, + servicecommon.TagScopeVMNamespace: subnetPortIndexNamespace, + servicecommon.TagScopeNamespace: subnetPortIndexPodNamespace, servicecommon.IndexKeySubnetID: subnetPortIndexBySubnetID, }), BindingType: model.VpcSubnetPortBindingType(), @@ -176,7 +178,7 @@ func (service *SubnetPortService) CheckSubnetPortState(obj interface{}, nsxSubne if realizestate.IsRealizeStateError(err) { log.Error(err, "the created subnet port is in error realization state, cleaning the resource", "subnetport", portID) // only recreate subnet port on RealizationErrorStateError. - if err := service.DeleteSubnetPort(portID); err != nil { + if err := service.DeleteSubnetPortById(portID); err != nil { log.Error(err, "cleanup error subnetport failed", "subnetport", portID) return nil, err } @@ -213,26 +215,30 @@ func (service *SubnetPortService) GetSubnetPortState(obj interface{}, nsxSubnetP return &nsxSubnetPortState, nil } -func (service *SubnetPortService) DeleteSubnetPort(portID string) error { - nsxSubnetPort := service.SubnetPortStore.GetByKey(portID) - if nsxSubnetPort == nil || nsxSubnetPort.Id == nil { - log.Info("NSX subnet port is not found in store, skip deleting it", "id", portID) - return nil - } +func (service *SubnetPortService) DeleteSubnetPort(nsxSubnetPort *model.VpcSubnetPort) error { nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID := nsxutil.ParseVPCPath(*nsxSubnetPort.Path) - err := service.NSXClient.PortClient.Delete(nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID, portID) + err := service.NSXClient.PortClient.Delete(nsxOrgID, nsxProjectID, nsxVPCID, nsxSubnetID, *nsxSubnetPort.Id) err = nsxutil.TransNSXApiError(err) if err != nil { log.Error(err, "failed to delete subnetport", "nsxSubnetPort.Path", *nsxSubnetPort.Path) return err } - if err = service.SubnetPortStore.Delete(portID); err != nil { + if err = service.SubnetPortStore.Delete(*nsxSubnetPort.Id); err != nil { return err } - log.Info("successfully deleted nsxSubnetPort", "nsxSubnetPortID", portID) + log.Info("successfully deleted nsxSubnetPort", "nsxSubnetPortID", *nsxSubnetPort.Id) return nil } +func (service *SubnetPortService) DeleteSubnetPortById(portID string) error { + nsxSubnetPort := service.SubnetPortStore.GetByKey(portID) + if nsxSubnetPort == nil || nsxSubnetPort.Id == nil { + log.Info("NSX subnet port is not found in store, skip deleting it", "id", portID) + return nil + } + return service.DeleteSubnetPort(nsxSubnetPort) +} + func (service *SubnetPortService) ListNSXSubnetPortIDForCR() sets.Set[string] { log.V(2).Info("listing subnet port CR UIDs") subnetPortSet := sets.New[string]() @@ -305,6 +311,46 @@ func (service *SubnetPortService) GetPortsOfSubnet(nsxSubnetID string) (ports [] return subnetPortList } +func (service *SubnetPortService) ListSubnetPortIDsFromCRs(ctx context.Context) (sets.Set[string], error) { + subnetPortList := &v1alpha1.SubnetPortList{} + err := service.Client.List(ctx, subnetPortList) + if err != nil { + log.Error(err, "failed to list SubnetPort CR") + return nil, err + } + + crSubnetPortIDsSet := sets.New[string]() + for _, subnetPort := range subnetPortList.Items { + subnetPortID := service.BuildSubnetPortId(&subnetPort.ObjectMeta) + crSubnetPortIDsSet.Insert(subnetPortID) + } + return crSubnetPortIDsSet, nil +} + +func (service *SubnetPortService) ListSubnetPortByName(ns string, name string) []*model.VpcSubnetPort { + var result []*model.VpcSubnetPort + subnetports := service.SubnetPortStore.GetByIndex(servicecommon.TagScopeVMNamespace, ns) + for _, subnetport := range subnetports { + tagname := nsxutil.FindTag(subnetport.Tags, servicecommon.TagScopeSubnetPortCRName) + if tagname == name { + result = append(result, subnetport) + } + } + return result +} + +func (service *SubnetPortService) ListSubnetPortByPodName(ns string, name string) []*model.VpcSubnetPort { + var result []*model.VpcSubnetPort + subnetports := service.SubnetPortStore.GetByIndex(servicecommon.TagScopeNamespace, ns) + for _, subnetport := range subnetports { + tagname := nsxutil.FindTag(subnetport.Tags, servicecommon.TagScopePodName) + if tagname == name { + result = append(result, subnetport) + } + } + return result +} + func (service *SubnetPortService) Cleanup(ctx context.Context) error { subnetPorts := service.SubnetPortStore.List() log.Info("cleanup subnetports", "count", len(subnetPorts)) @@ -314,7 +360,7 @@ func (service *SubnetPortService) Cleanup(ctx context.Context) error { case <-ctx.Done(): return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) default: - err := service.DeleteSubnetPort(subnetPortID) + err := service.DeleteSubnetPortById(subnetPortID) if err != nil { log.Error(err, "cleanup subnetport failed", "subnetPortID", subnetPortID) return err From b990849a930702ba08ae1fed1b17bb4cb55ed60c Mon Sep 17 00:00:00 2001 From: zhengxiexie Date: Sun, 13 Oct 2024 21:11:22 +0800 Subject: [PATCH 09/18] Fix bug which would create duplicated subnets when realization fails (#781) Subnets are allocated from subnet sets, it's id is suffixed with a random string, if the subnet failed to realize, and not being deleted, then a new subnet with different suffix will be created, thus leading a mount of duplicated subnets being left in the system. Signed-off-by: Xie Zheng --- pkg/nsx/services/subnet/subnet.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index caad39431..4a6617a94 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -139,6 +139,12 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet // For SubnetSets, since the ID includes a random value, the created NSX Subnet needs to be deleted and recreated. if err = realizeService.CheckRealizeState(backoff, *nsxSubnet.Path, "RealizedLogicalSwitch"); err != nil { log.Error(err, "failed to check subnet realization state", "ID", *nsxSubnet.Id) + // Delete the subnet if realization check fails, avoiding creating duplicate subnets continuously. + deleteErr := service.DeleteSubnet(*nsxSubnet) + if deleteErr != nil { + log.Error(deleteErr, "failed to delete subnet after realization check failure", "ID", *nsxSubnet.Id) + return "", fmt.Errorf("realization check failed: %v; deletion failed: %v", err, deleteErr) + } return "", err } if err = service.SubnetStore.Apply(nsxSubnet); err != nil { From d3d5d6e659c67deddeca454719a048ed4bf27957 Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Tue, 8 Oct 2024 10:12:30 +0800 Subject: [PATCH 10/18] Remove staticroute finalizer The finalizer may block the deletion since the NSX resource may take long time to delete. Remove the finalizer so the deleting operation will return immediately --- .../staticroute/staticroute_controller.go | 93 +++++----- .../staticroute_controller_test.go | 162 +++++++++++++----- pkg/nsx/services/common/types.go | 1 - pkg/nsx/services/staticroute/staticroute.go | 14 ++ pkg/nsx/services/staticroute/store.go | 15 +- 5 files changed, 202 insertions(+), 83 deletions(-) diff --git a/pkg/controllers/staticroute/staticroute_controller.go b/pkg/controllers/staticroute/staticroute_controller.go index 265ff29ad..b1855d5e8 100644 --- a/pkg/controllers/staticroute/staticroute_controller.go +++ b/pkg/controllers/staticroute/staticroute_controller.go @@ -11,6 +11,7 @@ import ( "strings" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -18,9 +19,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -72,28 +70,64 @@ func deleteSuccess(r *StaticRouteReconciler, _ context.Context, o *v1alpha1.Stat metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeStaticRoute) } +func (r *StaticRouteReconciler) listStaticRouteCRIDs() (sets.Set[string], error) { + staticRouteList := &v1alpha1.StaticRouteList{} + err := r.Client.List(context.Background(), staticRouteList) + if err != nil { + log.Error(err, "failed to list StaticRoute CRs") + return nil, err + } + + CRStaticRouteSet := sets.New[string]() + for _, staticroute := range staticRouteList.Items { + CRStaticRouteSet.Insert(string(staticroute.UID)) + } + return CRStaticRouteSet, nil +} + +func (r *StaticRouteReconciler) deleteStaticRouteByName(ns, name string) error { + CRPolicySet, err := r.listStaticRouteCRIDs() + if err != nil { + return err + } + nsxStaticRoutes := r.Service.ListStaticRouteByName(ns, name) + for _, item := range nsxStaticRoutes { + uid := util.FindTag(item.Tags, commonservice.TagScopeStaticRouteCRUID) + if CRPolicySet.Has(uid) { + log.Info("skipping deletion, StaticRoute CR still exists in K8s", "staticrouteUID", uid, "nsxStatciRouteId", *item.Id) + continue + } + + log.Info("deleting StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + path := strings.Split(*item.Path, "/") + if err := r.Service.DeleteStaticRouteByPath(path[2], path[4], path[6], *item.Id); err != nil { + log.Error(err, "failed to delete StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + return err + } + log.Info("successfully deleted StaticRoute", "StaticRouteUID", uid, "nsxStaticRouteId", *item.Id) + } + return nil +} + func (r *StaticRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { obj := &v1alpha1.StaticRoute{} log.Info("reconciling staticroute CR", "staticroute", req.NamespacedName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeStaticRoute) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteStaticRouteByName(req.Namespace, req.Name); err != nil { + return ResultRequeue, err + } else { + return ResultNormal, nil + } + } log.Error(err, "unable to fetch static route CR", "req", req.NamespacedName) - return ResultNormal, client.IgnoreNotFound(err) + return ResultRequeue, err } if obj.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeStaticRoute) - if !controllerutil.ContainsFinalizer(obj, commonservice.StaticRouteFinalizerName) { - controllerutil.AddFinalizer(obj, commonservice.StaticRouteFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "staticroute", req.NamespacedName) - updateFail(r, ctx, obj, &err) - return ResultRequeue, err - } - log.V(1).Info("added finalizer on staticroute CR", "staticroute", req.NamespacedName) - } - if err := r.Service.CreateOrUpdateStaticRoute(req.Namespace, obj); err != nil { updateFail(r, ctx, obj, &err) // TODO: if error is not retriable, not requeue @@ -105,27 +139,14 @@ func (r *StaticRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) } updateSuccess(r, ctx, obj) } else { - if controllerutil.ContainsFinalizer(obj, commonservice.StaticRouteFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeStaticRoute) - // TODO, update the value from 'default' to actual value， get OrgID, ProjectID, VPCID depending on obj.Namespace from vpc store - if err := r.Service.DeleteStaticRoute(obj); err != nil { - log.Error(err, "delete failed, would retry exponentially", "staticroute", req.NamespacedName) - deleteFail(r, ctx, obj, &err) - return ResultRequeue, err - } - controllerutil.RemoveFinalizer(obj, commonservice.StaticRouteFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - deleteFail(r, ctx, obj, &err) - return ResultRequeue, err - } - log.V(1).Info("removed finalizer", "staticroute", req.NamespacedName) - deleteSuccess(r, ctx, obj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "staticroute", req.NamespacedName) + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeStaticRoute) + if err := r.Service.DeleteStaticRoute(obj); err != nil { + log.Error(err, "delete failed, would retry exponentially", "staticroute", req.NamespacedName) + deleteFail(r, ctx, obj, &err) + return ResultRequeue, err } + deleteSuccess(r, ctx, obj) } - return ResultNormal, nil } @@ -198,12 +219,6 @@ func getExistingConditionOfType(conditionType v1alpha1.StaticRouteStatusConditio func (r *StaticRouteReconciler) setupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.StaticRoute{}). - WithEventFilter(predicate.Funcs{ - DeleteFunc: func(e event.DeleteEvent) bool { - // Suppress Delete events to avoid filtering them out in the Reconcile function - return false - }, - }). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), diff --git a/pkg/controllers/staticroute/staticroute_controller_test.go b/pkg/controllers/staticroute/staticroute_controller_test.go index 695f03ed4..110fa9762 100644 --- a/pkg/controllers/staticroute/staticroute_controller_test.go +++ b/pkg/controllers/staticroute/staticroute_controller_test.go @@ -18,6 +18,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -175,23 +176,14 @@ func TestStaticRouteReconciler_Reconcile(t *testing.T) { ctx := context.Background() req := controllerruntime.Request{NamespacedName: types.NamespacedName{Namespace: "dummy", Name: "dummy"}} - // not found - errNotFound := errors.New("not found") - k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errNotFound) + // get error + errUnknowError := errors.New("unknown error") + k8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errUnknowError) _, err := r.Reconcile(ctx, req) - assert.Equal(t, err, errNotFound) + assert.Equal(t, err, errUnknowError) - // DeletionTimestamp.IsZero = ture, client update failed sp := &v1alpha1.StaticRoute{} - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil) - err = errors.New("Update failed") - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(err) fakewriter := fakeStatusWriter{} - k8sClient.EXPECT().Status().Return(fakewriter) - _, ret := r.Reconcile(ctx, req) - assert.Equal(t, err, ret) - - // DeletionTimestamp.IsZero = false, Finalizers doesn't include util.FinalizerName k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) time := metav1.Now() @@ -200,74 +192,54 @@ func TestStaticRouteReconciler_Reconcile(t *testing.T) { }) patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { - assert.FailNow(t, "should not be called") return nil }) - k8sClient.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) - _, ret = r.Reconcile(ctx, req) - assert.Equal(t, ret, nil) - patch.Reset() - - // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName - k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { - v1sp := obj.(*v1alpha1.StaticRoute) - time := metav1.Now() - v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} - return nil - }) - patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { - return nil - }) - _, ret = r.Reconcile(ctx, req) + _, ret := r.Reconcile(ctx, req) assert.Equal(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = false, Finalizers include util.FinalizerName, DeleteStaticRoute fail + // DeletionTimestamp.IsZero = false, DeleteStaticRoute fail k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) time := metav1.Now() v1sp.ObjectMeta.DeletionTimestamp = &time - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRoute", func(_ *staticroute.StaticRouteService, obj *v1alpha1.StaticRoute) error { return errors.New("delete failed") }) - - k8sClient.EXPECT().Status().Times(2).Return(fakewriter) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.NotEqual(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateStaticRoute fail + // DeletionTimestamp.IsZero = true, CreateorUpdateStaticRoute fail k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) v1sp.ObjectMeta.DeletionTimestamp = nil - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateStaticRoute", func(_ *staticroute.StaticRouteService, namespace string, obj *v1alpha1.StaticRoute) error { return errors.New("create failed") }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.NotEqual(t, ret, nil) patch.Reset() - // DeletionTimestamp.IsZero = true, Finalizers include util.FinalizerName, CreateorUpdateStaticRoute succ + // DeletionTimestamp.IsZero = true, CreateorUpdateStaticRoute succ k8sClient.EXPECT().Get(ctx, gomock.Any(), sp).Return(nil).Do(func(_ context.Context, _ client.ObjectKey, obj client.Object, option ...client.GetOption) error { v1sp := obj.(*v1alpha1.StaticRoute) v1sp.ObjectMeta.DeletionTimestamp = nil - v1sp.Finalizers = []string{common.StaticRouteFinalizerName} return nil }) + k8sClient.EXPECT().Status().Times(1).Return(fakewriter) patch = gomonkey.ApplyMethod(reflect.TypeOf(service), "CreateOrUpdateStaticRoute", func(_ *staticroute.StaticRouteService, namespace string, obj *v1alpha1.StaticRoute) error { return nil }) - k8sClient.EXPECT().Status().Times(1).Return(fakewriter) _, ret = r.Reconcile(ctx, req) assert.Equal(t, ret, nil) patch.Reset() @@ -366,3 +338,113 @@ func TestStaticRouteReconciler_Start(t *testing.T) { err := r.Start(mgr) assert.NotEqual(t, err, nil) } + +func TestStaticRouteReconciler_listStaticRouteCRIDs(t *testing.T) { + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + r := &StaticRouteReconciler{ + Client: k8sClient, + Scheme: nil, + } + + ctx := context.Background() + + // list returns an error + errList := errors.New("list error") + k8sClient.EXPECT().List(ctx, gomock.Any()).Return(errList) + _, err := r.listStaticRouteCRIDs() + assert.Equal(t, err, errList) + + // list returns no error, but no items + k8sClient.EXPECT().List(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + staticRouteList := list.(*v1alpha1.StaticRouteList) + staticRouteList.Items = []v1alpha1.StaticRoute{} + return nil + }) + crIDs, err := r.listStaticRouteCRIDs() + assert.NoError(t, err) + assert.Equal(t, 0, crIDs.Len()) + + // list returns items + k8sClient.EXPECT().List(ctx, gomock.Any()).DoAndReturn(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + staticRouteList := list.(*v1alpha1.StaticRouteList) + staticRouteList.Items = []v1alpha1.StaticRoute{ + {ObjectMeta: metav1.ObjectMeta{UID: "uid1"}}, + {ObjectMeta: metav1.ObjectMeta{UID: "uid2"}}, + } + return nil + }) + crIDs, err = r.listStaticRouteCRIDs() + assert.NoError(t, err) + assert.Equal(t, 2, crIDs.Len()) + assert.True(t, crIDs.Has("uid1")) + assert.True(t, crIDs.Has("uid2")) +} + +func TestStaticRouteReconciler_deleteStaticRouteByName(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + + k8sClient := mock_client.NewMockClient(mockCtl) + mockStaticRouteClient := mocks.NewMockStaticRoutesClient(mockCtl) + + service := &staticroute.StaticRouteService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + StaticRouteClient: mockStaticRouteClient, + }, + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + + r := &StaticRouteReconciler{ + Client: k8sClient, + Scheme: nil, + Service: service, + } + + // listStaticRouteCRIDs returns an error + errList := errors.New("list error") + patch := gomonkey.ApplyPrivateMethod(reflect.TypeOf(r), "listStaticRouteCRIDs", func(_ *StaticRouteReconciler) (sets.Set[string], error) { + return nil, errList + }) + defer patch.Reset() + + err := r.deleteStaticRouteByName("dummy-name", "dummy-ns") + assert.Equal(t, err, errList) + + // listStaticRouteCRIDs returns items, and deletion fails + patch.Reset() + patch.ApplyPrivateMethod(reflect.TypeOf(r), "listStaticRouteCRIDs", func(_ *StaticRouteReconciler) (sets.Set[string], error) { + return sets.New[string]("uid1"), nil + }) + patch.ApplyMethod(reflect.TypeOf(service), "ListStaticRouteByName", func(_ *staticroute.StaticRouteService, _ string, _ string) []*model.StaticRoutes { + return []*model.StaticRoutes{ + { + Id: pointy.String("route-id-1"), + Path: pointy.String("/orgs/org123/projects/pro123/vpcs/vpc123/static-routes/route-id-1"), + Tags: []model.Tag{{Scope: pointy.String(common.TagScopeStaticRouteCRUID), Tag: pointy.String("uid1")}}, + }, + { + Id: pointy.String("route-id-2"), + Path: pointy.String("/orgs/org123/projects/pro123/vpcs/vpc123/static-routes/route-id-2"), + Tags: []model.Tag{{Scope: pointy.String(common.TagScopeStaticRouteCRUID), Tag: pointy.String("uid2")}}, + }, + } + }) + + patch.ApplyMethod(reflect.TypeOf(service), "DeleteStaticRouteByPath", func(_ *staticroute.StaticRouteService, orgId string, projectId string, vpcId string, uid string) error { + if uid == "route-id-2" { + return errors.New("delete failed") + } + return nil + }) + + err = r.deleteStaticRouteByName("dummy-name", "dummy-ns") + assert.Error(t, err) + patch.Reset() +} diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 95bc89fa9..e1e3c6d24 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -104,7 +104,6 @@ const ( NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" - StaticRouteFinalizerName = "staticroute.crd.nsx.vmware.com/finalizer" NetworkInfoFinalizerName = "networkinfo.crd.nsx.vmware.com/finalizer" IPPoolFinalizerName = "ippool.crd.nsx.vmware.com/finalizer" IPAddressAllocationFinalizerName = "ipaddressallocation.crd.nsx.vmware.com/finalizer" diff --git a/pkg/nsx/services/staticroute/staticroute.go b/pkg/nsx/services/staticroute/staticroute.go index d09b37c4a..4a007b538 100644 --- a/pkg/nsx/services/staticroute/staticroute.go +++ b/pkg/nsx/services/staticroute/staticroute.go @@ -40,6 +40,7 @@ func InitializeStaticRoute(commonService common.Service, vpcService common.VPCSe staticRouteStore := &StaticRouteStore{} staticRouteStore.Indexer = cache.NewIndexer(keyFunc, cache.Indexers{ common.TagScopeStaticRouteCRUID: indexFunc, + common.TagScopeNamespace: indexStaticRouteNamespace, }) staticRouteStore.BindingType = model.StaticRoutesBindingType() staticRouteService.StaticRouteStore = staticRouteStore @@ -148,6 +149,19 @@ func (service *StaticRouteService) DeleteStaticRoute(obj *v1alpha1.StaticRoute) return service.DeleteStaticRouteByPath(vpcResourceInfo.OrgID, vpcResourceInfo.ProjectID, vpcResourceInfo.VPCID, id) } +func (service *StaticRouteService) ListStaticRouteByName(ns, name string) []*model.StaticRoutes { + var result []*model.StaticRoutes + staticroutes := service.StaticRouteStore.GetByIndex(common.TagScopeNamespace, ns) + for _, staticroute := range staticroutes { + sr := staticroute.(*model.StaticRoutes) + tagname := nsxutil.FindTag(sr.Tags, common.TagScopeStaticRouteCRName) + if tagname == name { + result = append(result, staticroute.(*model.StaticRoutes)) + } + } + return result +} + func (service *StaticRouteService) ListStaticRoute() []*model.StaticRoutes { staticRoutes := service.StaticRouteStore.List() staticRouteSet := []*model.StaticRoutes{} diff --git a/pkg/nsx/services/staticroute/store.go b/pkg/nsx/services/staticroute/store.go index b4d13153d..1d64ce2a2 100644 --- a/pkg/nsx/services/staticroute/store.go +++ b/pkg/nsx/services/staticroute/store.go @@ -29,17 +29,26 @@ func indexFunc(obj interface{}) ([]string, error) { res := make([]string, 0, 5) switch v := obj.(type) { case *model.StaticRoutes: - return filterTag(v.Tags), nil + return filterTag(v.Tags, common.TagScopeStaticRouteCRUID), nil default: break } return res, nil } -var filterTag = func(v []model.Tag) []string { +func indexStaticRouteNamespace(obj interface{}) ([]string, error) { + switch o := obj.(type) { + case *model.StaticRoutes: + return filterTag(o.Tags, common.TagScopeNamespace), nil + default: + return nil, errors.New("indexByStaticRouteNamespace doesn't support unknown type") + } +} + +var filterTag = func(v []model.Tag, tagScope string) []string { res := make([]string, 0, 5) for _, tag := range v { - if *tag.Scope == common.TagScopeStaticRouteCRUID { + if *tag.Scope == tagScope { res = append(res, *tag.Tag) } } From f4e8e16ab43a7ef0bdfd152034f4ab5b76976366 Mon Sep 17 00:00:00 2001 From: wenqi Date: Wed, 16 Oct 2024 10:37:26 +0800 Subject: [PATCH 11/18] Remove NetworkInfo Finalizer (#778) This PR attempts to resolve the issue of a Namespace being stuck in the terminating state. Due to the constraints of the Finalizer mechanism, when a Namespace is created and then deleted, then if we stop the Pod or other abnormal conditions in the Pod, the NetworkInfo resource cannot be cleaned up, causing the Namespace to remain stuck in the terminating state. When a NetworkInfo delete event is received, the corresponding stale VPC in NSX will be searched based on the Namespace Name. If the Namespace UID no longer exists in Kubernetes, the corresponding stale VPC will be deleted. Signed-off-by: Wenqi Qiu --- .../samples/nsx_v1alpha1_networkinfo.yaml | 2 - .../networkinfo/networkinfo_controller.go | 508 ++++++++++-------- .../networkinfo_controller_test.go | 149 ++++- .../networkinfo/networkinfo_utils.go | 4 +- pkg/nsx/services/common/types.go | 1 - pkg/nsx/services/vpc/vpc.go | 6 +- pkg/util/utils.go | 26 +- 7 files changed, 439 insertions(+), 257 deletions(-) diff --git a/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml b/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml index 1368d2e6a..06752bcc6 100644 --- a/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml +++ b/build/yaml/samples/nsx_v1alpha1_networkinfo.yaml @@ -2,8 +2,6 @@ apiVersion: crd.nsx.vmware.com/v1alpha1 kind: NetworkInfo metadata: creationTimestamp: "2024-05-14T02:14:18Z" - finalizers: - - networkinfo.crd.nsx.vmware.com/finalizer generation: 2 name: kube-system namespace: kube-system diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index d3f856362..e85a1cfc8 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -7,8 +7,11 @@ import ( "context" "errors" "fmt" + "time" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -17,7 +20,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" @@ -45,224 +47,194 @@ type NetworkInfoReconciler struct { } func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - obj := &v1alpha1.NetworkInfo{} - log.Info("reconciling NetworkInfo CR", "NetworkInfo", req.NamespacedName) + startTime := time.Now() + defer func() { + log.Info("Finished reconciling NetworkInfo", "NetworkInfo", req.NamespacedName, "duration(ms)", time.Since(startTime).Milliseconds()) + }() metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, common.MetricResTypeNetworkInfo) - if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { - log.Error(err, "unable to fetch NetworkInfo CR", "req", req.NamespacedName) - return common.ResultNormal, client.IgnoreNotFound(err) - } - if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeNetworkInfo) - if !controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { - controllerutil.AddFinalizer(obj, commonservice.NetworkInfoFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - log.Error(err, "add finalizer", "NetworkInfo", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) + networkInfoCR := &v1alpha1.NetworkInfo{} + if err := r.Client.Get(ctx, req.NamespacedName, networkInfoCR); err != nil { + if apierrors.IsNotFound(err) { + if err := r.deleteVPCsByName(ctx, req.Namespace); err != nil { + log.Error(err, "Failed to delete stale NSX VPC", "NetworkInfo", req.NamespacedName) return common.ResultRequeue, err } - log.V(1).Info("added finalizer on NetworkInfo CR", "NetworkInfo", req.NamespacedName) - } - // TODO: - // 1. check whether the logic to get VPC network config can be replaced by GetVPCNetworkConfigByNamespace - // 2. sometimes the variable nc points to a VPCNetworkInfo, sometimes it's a VPCNetworkConfiguration, we need to distinguish between them. - ncName, err := r.Service.GetNetworkconfigNameFromNS(obj.Namespace) - if err != nil { - log.Error(err, "failed to get network config name for VPC when creating NSX VPC", "VPC", obj.Name) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - nc, _exist := r.Service.GetVPCNetworkConfig(ncName) - if !_exist { - message := fmt.Sprintf("failed to read network config %s when creating NSX VPC", ncName) - log.Info(message) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, errors.New(message) - } - log.Info("got network config from store", "NetworkConfig", ncName) - vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} - err = r.Client.Get(ctx, types.NamespacedName{Name: commonservice.SystemVPCNetworkConfigurationName}, vpcNetworkConfiguration) - if err != nil { - log.Error(err, "failed to get system VPCNetworkConfiguration") - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - gatewayConnectionReady, _, err := getGatewayConnectionStatus(ctx, vpcNetworkConfiguration) - if err != nil { - log.Error(err, "failed to get the gateway connection status", "req", req.NamespacedName) - return common.ResultRequeueAfter10sec, err - } - - gatewayConnectionReason := "" - if !gatewayConnectionReady { - if ncName == commonservice.SystemVPCNetworkConfigurationName { - gatewayConnectionReady, gatewayConnectionReason, err = r.Service.ValidateGatewayConnectionStatus(&nc) - log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "gatewayConnectionReason", gatewayConnectionReason) - if err != nil { - log.Error(err, "failed to validate the edge and gateway connection", "org", nc.Org, "project", nc.NSXProject) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, gatewayConnectionReason) - } else { - log.Info("skipping reconciling the network info because the system gateway connection is not ready", "NetworkInfo", req.NamespacedName) - return common.ResultRequeueAfter60sec, nil - } - } - lbProvider := r.Service.GetLBProvider() - createdVpc, err := r.Service.CreateOrUpdateVPC(obj, &nc, lbProvider) - if err != nil { - log.Error(err, "create vpc failed, would retry exponentially", "VPC", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err + return common.ResultNormal, nil } + log.Error(err, "Unable to fetch NetworkInfo CR", "NetworkInfo", req.NamespacedName) + return common.ResultRequeue, err + } - var privateIPs []string - var vpcConnectivityProfilePath string - var nsxLBSPath string - isPreCreatedVPC := vpc.IsPreCreatedVPC(nc) - if isPreCreatedVPC { - privateIPs = createdVpc.PrivateIps - vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile - // Retrieve NSX lbs path if Avi is not used with the pre-created VPC. - if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil || - !*createdVpc.LoadBalancerVpcEndpoint.Enabled { - nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(*createdVpc.Path) - if err != nil { - log.Error(err, "failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - } - } else { - privateIPs = nc.PrivateIPs - vpcConnectivityProfilePath = nc.VPCConnectivityProfile + // Check if the CR is marked for deletion + if !networkInfoCR.ObjectMeta.DeletionTimestamp.IsZero() { + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) + if err := r.deleteVPCsByID(ctx, networkInfoCR.GetNamespace(), string(networkInfoCR.UID)); err != nil { + deleteFail(r, ctx, networkInfoCR, &err, r.Client) + log.Error(err, "Failed to delete stale NSX VPC, retrying", "NetworkInfo", req.NamespacedName) + return common.ResultRequeue, err } + deleteSuccess(r, ctx, networkInfoCR) + return common.ResultNormal, nil + } + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, common.MetricResTypeNetworkInfo) + // TODO: + // 1. check whether the logic to get VPC network config can be replaced by GetVPCNetworkConfigByNamespace + // 2. sometimes the variable nc points to a VPCNetworkInfo, sometimes it's a VPCNetworkConfiguration, we need to distinguish between them. + ncName, err := r.Service.GetNetworkconfigNameFromNS(networkInfoCR.Namespace) + if err != nil { + log.Error(err, "Failed to get network config name for VPC when creating NSX VPC", "NetworkInfo", networkInfoCR.Name) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + nc, _exist := r.Service.GetVPCNetworkConfig(ncName) + if !_exist { + message := fmt.Sprintf("Failed to read network config %s when creating NSX VPC", ncName) + log.Info(message) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, errors.New(message) + } + log.Info("Fetched network config from store", "NetworkConfig", ncName) + vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} + err = r.Client.Get(ctx, types.NamespacedName{Name: commonservice.SystemVPCNetworkConfigurationName}, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get system VPCNetworkConfiguration") + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + gatewayConnectionReady, _, err := getGatewayConnectionStatus(ctx, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get the gateway connection status", "NetworkInfo", req.NamespacedName) + return common.ResultRequeueAfter10sec, err + } - snatIP, path, cidr := "", "", "" - - vpcConnectivityProfile, err := r.Service.GetVpcConnectivityProfile(&nc, vpcConnectivityProfilePath) - if err != nil { - log.Error(err, "get VpcConnectivityProfile failed, would retry exponentially", "VPC", req.NamespacedName) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - hasExternalIPs := true + gatewayConnectionReason := "" + if !gatewayConnectionReady { if ncName == commonservice.SystemVPCNetworkConfigurationName { - if len(vpcConnectivityProfile.ExternalIpBlocks) == 0 { - hasExternalIPs = false - log.Error(err, "there is no ExternalIPBlock in VPC ConnectivityProfile", "VPC", req.NamespacedName) - } - setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, hasExternalIPs) - } - // currently, auto snat is not exposed, and use default value True - // checking autosnat to support future extension in vpc configuration - autoSnatEnabled := r.Service.IsEnableAutoSNAT(vpcConnectivityProfile) - if autoSnatEnabled { - snatIP, err = r.Service.GetDefaultSNATIP(*createdVpc) + gatewayConnectionReady, gatewayConnectionReason, err = r.Service.ValidateGatewayConnectionStatus(&nc) + log.Info("got the gateway connection status", "gatewayConnectionReady", gatewayConnectionReady, "gatewayConnectionReason", gatewayConnectionReason) if err != nil { - log.Error(err, "failed to read default SNAT ip from VPC", "VPC", createdVpc.Id) - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: "", - LoadBalancerIPAddresses: "", - PrivateIPs: privateIPs, - } - updateFail(r, ctx, obj, &err, r.Client, state) + log.Error(err, "Failed to validate the edge and gateway connection", "Org", nc.Org, "Project", nc.NSXProject) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) return common.ResultRequeueAfter10sec, err } + setVPCNetworkConfigurationStatusWithGatewayConnection(ctx, r.Client, vpcNetworkConfiguration, gatewayConnectionReady, gatewayConnectionReason) + } else { + log.Info("Skipping reconciliation due to unready system gateway connection", "NetworkInfo", req.NamespacedName) + return common.ResultRequeueAfter60sec, nil } - if ncName == commonservice.SystemVPCNetworkConfigurationName { - vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} - err := r.Client.Get(ctx, types.NamespacedName{Name: ncName}, vpcNetworkConfiguration) - if err != nil { - log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName) - updateFail(r, ctx, obj, &err, r.Client, nil) - return common.ResultRequeueAfter10sec, err - } - log.Info("got the AutoSnat status", "autoSnatEnabled", autoSnatEnabled, "req", req.NamespacedName) - setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, autoSnatEnabled) - } + } + lbProvider := r.Service.GetLBProvider() + createdVpc, err := r.Service.CreateOrUpdateVPC(networkInfoCR, &nc, lbProvider) + if err != nil { + log.Error(err, "Failed to create or update VPC", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } - // if lb vpc enabled, read avi subnet path and cidr - // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back, - // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. - if lbProvider == vpc.AVILB && createdVpc.LoadBalancerVpcEndpoint != nil && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { - path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) + var privateIPs []string + var vpcConnectivityProfilePath, nsxLBSPath string + isPreCreatedVPC := vpc.IsPreCreatedVPC(nc) + if isPreCreatedVPC { + privateIPs = createdVpc.PrivateIps + vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile + // Retrieve NSX lbs path if Avi is not used with the pre-created VPC. + if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil || + !*createdVpc.LoadBalancerVpcEndpoint.Enabled { + nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(*createdVpc.Path) if err != nil { - log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id) - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: snatIP, - LoadBalancerIPAddresses: "", - PrivateIPs: privateIPs, - } - updateFail(r, ctx, obj, &err, r.Client, state) + log.Error(err, "Failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) return common.ResultRequeueAfter10sec, err } } + } else { + privateIPs = nc.PrivateIPs + vpcConnectivityProfilePath = nc.VPCConnectivityProfile + } - state := &v1alpha1.VPCState{ - Name: *createdVpc.DisplayName, - DefaultSNATIP: snatIP, - LoadBalancerIPAddresses: cidr, - PrivateIPs: privateIPs, - VPCPath: *createdVpc.Path, - } + snatIP, path, cidr := "", "", "" - if !isPreCreatedVPC { - nsxLBSPath = r.Service.GetDefaultNSXLBSPathByVPC(*createdVpc.Id) - } - // AKO needs to know the AVI subnet path created by NSX - setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, nsxLBSPath, *createdVpc.Path) - updateSuccess(r, ctx, obj, r.Client, state, nc.Name, path) - if ncName == commonservice.SystemVPCNetworkConfigurationName && (!gatewayConnectionReady || !autoSnatEnabled || !hasExternalIPs) { - log.Info("requeuing the NetworkInfo CR because VPCNetworkConfiguration system is not ready", "gatewayConnectionReason", gatewayConnectionReason, "autoSnatEnabled", autoSnatEnabled, "hasExternalIPs", hasExternalIPs, "req", req) - return common.ResultRequeueAfter60sec, nil + vpcConnectivityProfile, err := r.Service.GetVpcConnectivityProfile(&nc, vpcConnectivityProfilePath) + if err != nil { + log.Error(err, "Failed to get VPC connectivity profile", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + hasExternalIPs := true + if ncName == commonservice.SystemVPCNetworkConfigurationName { + if len(vpcConnectivityProfile.ExternalIpBlocks) == 0 { + hasExternalIPs = false + log.Error(err, "There is no ExternalIPBlock in VPC ConnectivityProfile", "NetworkInfo", req.NamespacedName) } - } else { - if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) - isShared, err := r.Service.IsSharedVPCNamespaceByNS(obj.GetNamespace()) - if err != nil { - log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) - return common.ResultRequeue, err - } - vpcs := r.Service.GetVPCsByNamespace(obj.GetNamespace()) - // if nsx resource do not exist, continue to remove finalizer, or the crd can not be removed - if len(vpcs) == 0 { - // when nsx vpc not found in vpc store, skip deleting NSX VPC - log.Info("can not find VPC in store, skip deleting NSX VPC, remove finalizer from NetworkInfo CR") - } else if !isShared { - for _, vpc := range vpcs { - // first delete vpc and then ipblock or else it will fail arguing it is being referenced by other objects - if err := r.Service.DeleteVPC(*vpc.Path); err != nil { - log.Error(err, "failed to delete nsx VPC, would retry exponentially", "NetworkInfo", req.NamespacedName) - deleteFail(r, ctx, obj, &err, r.Client) - return common.ResultRequeueAfter10sec, err - } - } + setVPCNetworkConfigurationStatusWithNoExternalIPBlock(ctx, r.Client, vpcNetworkConfiguration, hasExternalIPs) + } + // currently, auto snat is not exposed, and use default value True + // checking autosnat to support future extension in VPC configuration + autoSnatEnabled := r.Service.IsEnableAutoSNAT(vpcConnectivityProfile) + if autoSnatEnabled { + snatIP, err = r.Service.GetDefaultSNATIP(*createdVpc) + if err != nil { + log.Error(err, "Failed to read default SNAT IP from VPC", "VPC", createdVpc.Id) + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: "", + LoadBalancerIPAddresses: "", + PrivateIPs: privateIPs, } + updateFail(r, ctx, networkInfoCR, &err, r.Client, state) + return common.ResultRequeueAfter10sec, err + } + } + if ncName == commonservice.SystemVPCNetworkConfigurationName { + vpcNetworkConfiguration := &v1alpha1.VPCNetworkConfiguration{} + err := r.Client.Get(ctx, types.NamespacedName{Name: ncName}, vpcNetworkConfiguration) + if err != nil { + log.Error(err, "Failed to get VPCNetworkConfiguration", "Name", ncName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } + log.Info("Got the AutoSnat status", "autoSnatEnabled", autoSnatEnabled, "NetworkInfo", req.NamespacedName) + setVPCNetworkConfigurationStatusWithSnatEnabled(ctx, r.Client, vpcNetworkConfiguration, autoSnatEnabled) + } - controllerutil.RemoveFinalizer(obj, commonservice.NetworkInfoFinalizerName) - if err := r.Client.Update(ctx, obj); err != nil { - deleteFail(r, ctx, obj, &err, r.Client) - return common.ResultRequeue, err - } - ncName, err := r.Service.GetNetworkconfigNameFromNS(obj.Namespace) - if err != nil { - log.Error(err, "failed to get network config name for VPC when deleting NetworkInfo CR", "NetworkInfo", obj.Name) - return common.ResultRequeueAfter10sec, err + // if lb VPC enabled, read avi subnet path and cidr + // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this VPC back, + // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. + if lbProvider == vpc.AVILB && createdVpc.LoadBalancerVpcEndpoint != nil && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { + path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) + if err != nil { + log.Error(err, "Failed to read LB Subnet path and CIDR", "VPC", createdVpc.Id) + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: snatIP, + LoadBalancerIPAddresses: "", + PrivateIPs: privateIPs, } - log.V(1).Info("removed finalizer", "NetworkInfo", req.NamespacedName) - deleteVPCNetworkConfigurationStatus(ctx, r.Client, ncName, vpcs, r.Service.ListVPC()) - deleteSuccess(r, ctx, obj) - } else { - // only print a message because it's not a normal case - log.Info("finalizers cannot be recognized", "NetworkInfo", req.NamespacedName) + updateFail(r, ctx, networkInfoCR, &err, r.Client, state) + return common.ResultRequeueAfter10sec, err } } + + state := &v1alpha1.VPCState{ + Name: *createdVpc.DisplayName, + DefaultSNATIP: snatIP, + LoadBalancerIPAddresses: cidr, + PrivateIPs: privateIPs, + VPCPath: *createdVpc.Path, + } + + if !isPreCreatedVPC { + nsxLBSPath = r.Service.GetDefaultNSXLBSPathByVPC(*createdVpc.Id) + } + // AKO needs to know the AVI subnet path created by NSX + setVPCNetworkConfigurationStatusWithLBS(ctx, r.Client, ncName, state.Name, path, nsxLBSPath, *createdVpc.Path) + updateSuccess(r, ctx, networkInfoCR, r.Client, state, nc.Name, path) + if ncName == commonservice.SystemVPCNetworkConfigurationName && (!gatewayConnectionReady || !autoSnatEnabled || !hasExternalIPs) { + log.Info("Requeue NetworkInfo CR because VPCNetworkConfiguration system is not ready", "gatewayConnectionReason", gatewayConnectionReason, "autoSnatEnabled", autoSnatEnabled, "hasExternalIPs", hasExternalIPs, "req", req) + return common.ResultRequeueAfter60sec, nil + } + return common.ResultNormal, nil } @@ -274,10 +246,10 @@ func (r *NetworkInfoReconciler) setupWithManager(mgr ctrl.Manager) error { MaxConcurrentReconciles: common.NumReconcile(), }). Watches( - // For created/removed network config, add/remove from vpc network config cache, + // For created/removed network config, add/remove from VPC network config cache, // and update IPBlocksInfo. // For modified network config, currently only support appending ips to public ip blocks, - // update network config in cache and update nsx vpc object. + // update network config in cache and update nsx VPC object. &v1alpha1.VPCNetworkConfiguration{}, &VPCNetworkConfigurationHandler{ Client: mgr.GetClient(), @@ -297,45 +269,145 @@ func (r *NetworkInfoReconciler) Start(mgr ctrl.Manager) error { return nil } -// CollectGarbage logic for nsx-vpc is that: +func (r *NetworkInfoReconciler) listNamespaceCRsNameIDSet(ctx context.Context) (sets.Set[string], sets.Set[string], error) { + // read all Namespaces from K8s + namespaces := &corev1.NamespaceList{} + err := r.Client.List(ctx, namespaces) + if err != nil { + return nil, nil, err + } + nsSet := sets.Set[string]{} + idSet := sets.Set[string]{} + for _, ns := range namespaces.Items { + nsSet.Insert(ns.Name) + idSet.Insert(string(ns.UID)) + } + return nsSet, idSet, nil +} + +// CollectGarbage logic for NSX VPC is that: // 1. list all current existing namespace in kubernetes -// 2. list all the nsx-vpc in vpcStore -// 3. loop all the nsx-vpc to get its namespace, check if the namespace still exist -// 4. if ns do not exist anymore, delete the nsx-vpc resource +// 2. list all the NSX VPC in vpcStore +// 3. loop all the NSX VPC to get its namespace, check if the namespace still exist +// 4. if ns do not exist anymore, delete the NSX VPC resource // it implements the interface GarbageCollector method. func (r *NetworkInfoReconciler) CollectGarbage(ctx context.Context) { - log.Info("VPC garbage collector started") - // read all nsx-vpc from vpc store + startTime := time.Now() + defer func() { + log.Info("VPC garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) + }() + + // read all NSX VPC from VPC store nsxVPCList := r.Service.ListVPC() if len(nsxVPCList) == 0 { + log.Info("No NSX VPCs found in the store, skipping garbage collection") return } - // read all namespaces from k8s - namespaces := &corev1.NamespaceList{} - err := r.Client.List(ctx, namespaces) + _, idSet, err := r.listNamespaceCRsNameIDSet(ctx) if err != nil { - log.Error(err, "failed to list k8s namespaces") + log.Error(err, "Failed to list Kubernetes Namespaces for VPC garbage collection") return } - nsSet := sets.NewString() - for _, ns := range namespaces.Items { - nsSet.Insert(ns.Name) - } - for i := len(nsxVPCList) - 1; i >= 0; i-- { - nsxVPCNamespace := getNamespaceFromNSXVPC(&nsxVPCList[i]) - if nsSet.Has(nsxVPCNamespace) { + for i, nsxVPC := range nsxVPCList { + nsxVPCNamespaceName := filterTagFromNSXVPC(&nsxVPCList[i], commonservice.TagScopeNamespace) + nsxVPCNamespaceID := filterTagFromNSXVPC(&nsxVPCList[i], commonservice.TagScopeNamespaceUID) + if idSet.Has(nsxVPCNamespaceID) { continue } - elem := nsxVPCList[i] - log.Info("GC collected nsx VPC object", "ID", elem.Id, "Namespace", nsxVPCNamespace) + log.Info("Garbage collecting NSX VPC object", "VPC", nsxVPC.Id, "Namespace", nsxVPCNamespaceName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) - err = r.Service.DeleteVPC(*elem.Path) - if err != nil { + + if err = r.Service.DeleteVPC(*nsxVPC.Path); err != nil { + log.Error(err, "Failed to delete NSX VPC", "VPC", nsxVPC.Id, "Namespace", nsxVPCNamespaceName) metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeNetworkInfo) - } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeNetworkInfo) + continue + } + + metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeNetworkInfo) + log.Info("Successfully deleted NSX VPC", "VPC", nsxVPC.Id) + } +} + +func (r *NetworkInfoReconciler) fetchStaleVPCsByNamespace(ctx context.Context, ns string) ([]*model.Vpc, error) { + isShared, err := r.Service.IsSharedVPCNamespaceByNS(ns) + if err != nil { + return nil, fmt.Errorf("failed to check if Namespace is shared for NS %s: %w", ns, err) + } + if isShared { + log.Info("Shared Namespace, skipping deletion of NSX VPC", "Namespace", ns) + return nil, nil + } + + return r.Service.GetVPCsByNamespace(ns), nil +} + +func (r *NetworkInfoReconciler) deleteVPCsByName(ctx context.Context, ns string) error { + _, idSet, err := r.listNamespaceCRsNameIDSet(ctx) + if err != nil { + log.Error(err, "Failed to list Kubernetes Namespaces") + return fmt.Errorf("failed to list Kubernetes Namespaces while deleting VPCs: %v", err) + } + + staleVPCs, err := r.fetchStaleVPCsByNamespace(ctx, ns) + if err != nil { + return err + } + + var vpcToDelete []*model.Vpc + for _, nsxVPC := range staleVPCs { + namespaceIDofVPC := filterTagFromNSXVPC(nsxVPC, commonservice.TagScopeNamespaceUID) + if idSet.Has(namespaceIDofVPC) { + log.Info("Skipping deletion, Namespace still exists in K8s", "Namespace", ns) + continue + } + vpcToDelete = append(vpcToDelete, nsxVPC) + } + return r.deleteVPCs(ctx, vpcToDelete, ns) +} + +func (r *NetworkInfoReconciler) deleteVPCsByID(ctx context.Context, ns, id string) error { + staleVPCs, err := r.fetchStaleVPCsByNamespace(ctx, ns) + if err != nil { + return err + } + + var vpcToDelete []*model.Vpc + for _, nsxVPC := range staleVPCs { + namespaceIDofVPC := filterTagFromNSXVPC(nsxVPC, commonservice.TagScopeNamespaceUID) + if namespaceIDofVPC == id { + vpcToDelete = append(vpcToDelete, nsxVPC) } } + return r.deleteVPCs(ctx, vpcToDelete, ns) +} + +func (r *NetworkInfoReconciler) deleteVPCs(ctx context.Context, staleVPCs []*model.Vpc, ns string) error { + if len(staleVPCs) == 0 { + log.Info("There is no VPCs found in store, skipping deletion of NSX VPC", "Namespace", ns) + return nil + } + var deleteErrs []error + for _, nsxVPC := range staleVPCs { + if nsxVPC.Path == nil { + log.Error(nil, "VPC path is nil, skipping", "VPC", nsxVPC) + continue + } + if err := r.Service.DeleteVPC(*nsxVPC.Path); err != nil { + log.Error(err, "Failed to delete VPC in NSX", "VPC", nsxVPC.Path) + deleteErrs = append(deleteErrs, fmt.Errorf("failed to delete VPC %s: %w", *nsxVPC.Path, err)) + } + } + if len(deleteErrs) > 0 { + return fmt.Errorf("multiple errors occurred while deleting VPCs: %v", deleteErrs) + } + + // Update the VPCNetworkConfiguration Status + ncName, err := r.Service.GetNetworkconfigNameFromNS(ns) + if err != nil { + return fmt.Errorf("failed to get VPCNetworkConfiguration for Namespace when deleting stale VPCs %s: %w", ns, err) + } + deleteVPCNetworkConfigurationStatus(ctx, r.Client, ncName, staleVPCs, r.Service.ListVPC()) + return nil } diff --git a/pkg/controllers/networkinfo/networkinfo_controller_test.go b/pkg/controllers/networkinfo/networkinfo_controller_test.go index f89cb51c5..ed278fa34 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller_test.go +++ b/pkg/controllers/networkinfo/networkinfo_controller_test.go @@ -5,11 +5,14 @@ package networkinfo import ( "context" + "errors" + "fmt" "reflect" "testing" "github.com/agiledragon/gomonkey" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -21,7 +24,6 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" - servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) @@ -93,11 +95,28 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { wantErr bool }{ { - name: "Empty", - prepareFunc: nil, - args: requestArgs, - want: common.ResultNormal, - wantErr: false, + name: "Empty", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) (patches *gomonkey.Patches) { + patches = gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetNetworkconfigNameFromNS", func(_ *vpc.VPCService, _ string) (string, error) { + return "", nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + patches.ApplyFunc(deleteVPCNetworkConfigurationStatus, func(ctx context.Context, client client.Client, ncName string, staleVPCs []*model.Vpc, aliveVPCs []model.Vpc) { + return + }) + return patches + }, + args: requestArgs, + want: common.ResultNormal, + wantErr: false, }, { name: "GatewayConnectionReadyInSystemVPC", @@ -722,3 +741,121 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { }) } } + +func TestNetworkInfoReconciler_deleteStaleVPCs(t *testing.T) { + r := createNetworkInfoReconciler() + + ctx := context.TODO() + namespace := "test-ns" + + t.Run("shared namespace, skip deletion", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return true, nil + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) + + t.Run("non-shared namespace, no VPCs found", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + return nil + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) + + t.Run("failed to delete VPC", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + vpcPath := "/vpc/1" + return []*model.Vpc{{Path: &vpcPath}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return fmt.Errorf("delete failed") + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + assert.Error(t, err) + assert.Contains(t, err.Error(), "delete failed") + }) + + t.Run("successful deletion of VPCs", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "IsSharedVPCNamespaceByNS", func(_ *vpc.VPCService, _ string) (bool, error) { + return false, nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCsByNamespace", func(_ *vpc.VPCService, _ string) []*model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []*model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetNetworkconfigNameFromNS", func(_ *vpc.VPCService, _ string) (string, error) { + return "", nil + }) + patches.ApplyFunc(deleteVPCNetworkConfigurationStatus, func(ctx context.Context, client client.Client, ncName string, staleVPCs []*model.Vpc, aliveVPCs []model.Vpc) { + return + }) + defer patches.Reset() + + err := r.deleteVPCsByName(ctx, namespace) + require.NoError(t, err) + }) +} + +func TestNetworkInfoReconciler_CollectGarbage(t *testing.T) { + r := createNetworkInfoReconciler() + + ctx := context.TODO() + + t.Run("no VPCs found in the store", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + return nil + }) + defer patches.Reset() + + r.CollectGarbage(ctx) + // No errors expected + }) + + t.Run("successful garbage collection", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return nil + }) + defer patches.Reset() + + r.CollectGarbage(ctx) + }) + + t.Run("failed to delete VPC", func(t *testing.T) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service), "ListVPC", func(_ *vpc.VPCService) []model.Vpc { + vpcPath1 := "/vpc/1" + vpcPath2 := "/vpc/2" + return []model.Vpc{{Path: &vpcPath1}, {Path: &vpcPath2}} + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "DeleteVPC", func(_ *vpc.VPCService, _ string) error { + return errors.New("deletion error") + }) + defer patches.Reset() + r.CollectGarbage(ctx) + }) +} diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 2a033ed3e..4933f657f 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -237,10 +237,10 @@ func deleteVPCNetworkConfigurationStatus(ctx context.Context, client client.Clie log.Info("Deleted stale VPCNetworkConfiguration status", "Name", ncName, "nc.Status.VPCs", nc.Status.VPCs, "staleVPCs", staleVPCNames) } -func getNamespaceFromNSXVPC(nsxVPC *model.Vpc) string { +func filterTagFromNSXVPC(nsxVPC *model.Vpc, tagName string) string { tags := nsxVPC.Tags for _, tag := range tags { - if *tag.Scope == svccommon.TagScopeNamespace { + if *tag.Scope == tagName { return *tag.Tag } } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index e1e3c6d24..22b81f26b 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -104,7 +104,6 @@ const ( NSXServiceAccountFinalizerName = "nsxserviceaccount.nsx.vmware.com/finalizer" T1SecurityPolicyFinalizerName = "securitypolicy.nsx.vmware.com/finalizer" - NetworkInfoFinalizerName = "networkinfo.crd.nsx.vmware.com/finalizer" IPPoolFinalizerName = "ippool.crd.nsx.vmware.com/finalizer" IPAddressAllocationFinalizerName = "ipaddressallocation.crd.nsx.vmware.com/finalizer" diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index ed525790f..0495fe499 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -698,13 +698,13 @@ func (s *VPCService) IsLBProviderChanged(existingVPC *model.Vpc, lbProvider LBPr return false } func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPCNetworkConfigInfo, lbProvider LBProvider) (*model.Vpc, error) { - // check from VPC store if vpc already exist + // check from VPC store if VPC already exist ns := obj.Namespace updateVpc := false nsObj := &v1.Namespace{} - // get name obj + // get Namespace if err := s.Client.Get(ctx, types.NamespacedName{Name: obj.Namespace}, nsObj); err != nil { - log.Error(err, "unable to fetch namespace", "name", obj.Namespace) + log.Error(err, "unable to fetch Namespace", "Name", obj.Namespace) return nil, err } diff --git a/pkg/util/utils.go b/pkg/util/utils.go index d1eebde26..8d1e06b4e 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -30,9 +30,7 @@ import ( ) const ( - wcpSystemResource = "vmware-system-shared-t1" - SubnetTypeSubnet = "subnet" - SubnetTypeSubnetSet = "subnetset" + wcpSystemResource = "vmware-system-shared-t1" ) var ( @@ -312,24 +310,6 @@ func If(condition bool, trueVal, falseVal interface{}) interface{} { } } -func GetMapValues(in interface{}) []string { - if in == nil { - return make([]string, 0) - } - switch in.(type) { - case map[string]string: - ssMap := in.(map[string]string) - values := make([]string, 0, len(ssMap)) - for _, v := range ssMap { - values = append(values, v) - } - return values - default: - log.Info("Unsupported map format") - return nil - } -} - // the changes map contains key/value map that you want to change. // if giving empty value for a key in changes map like: "mykey":"", that means removing this annotation from k8s resource func UpdateK8sResourceAnnotation(client client.Client, ctx context.Context, k8sObj client.Object, changes map[string]string) error { @@ -423,10 +403,6 @@ func GenerateTruncName(limit int, resName string, prefix, suffix, project, clust return generateDisplayName(common.ConnectorUnderline, resName, prefix, suffix, project, cluster) } -func CombineNamespaceName(name, namespace string) string { - return fmt.Sprintf("%s/%s", namespace, name) -} - func BuildBasicTags(cluster string, obj interface{}, namespaceID types.UID) []model.Tag { tags := []model.Tag{ { From 1020f3683de28dd6fbb4e9babc04e029b87bfca4 Mon Sep 17 00:00:00 2001 From: wenqi Date: Thu, 17 Oct 2024 16:13:10 +0800 Subject: [PATCH 12/18] Improve Subnet/SubnetSet GC (#798) Improve Subnet/SubnetSet GC For Subnet garbage collection (GC): Retrieve the Subnet CRUID values of all Subnets from the store, and delete the corresponding stale Subnets for those whose UID is not found in the existing Subnet CRs in Kubernetes. For SubnetSet garbage collection (GC), two tasks need to be handled: For each existing SubnetSet CR, delete any Subnet under it that does not have a SubnetPort attached. Retrieve the SubnetSet CRUID values of all Subnets from the store, and delete the corresponding stale Subnets for those whose SubnetSet UID is not found in the existing SubnetSet CRs in Kubernetes. This PR also improved the NSX Subnet Cleanup process. Signed-off-by: Wenqi Qiu --- pkg/controllers/subnet/subnet_controller.go | 29 ++--- .../subnet/subnet_controller_test.go | 66 ++++++---- .../subnetset/subnetset_controller.go | 113 +++++++++--------- pkg/nsx/services/subnet/subnet.go | 73 ++++++----- 4 files changed, 151 insertions(+), 130 deletions(-) diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 80c44a010..3e7fa27d3 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -379,31 +379,20 @@ func (r *SubnetReconciler) collectGarbage(ctx context.Context) { log.Error(err, "Failed to list Subnet CRs") return } + crdSubnetIDsSet := sets.New[string](crdSubnetIDs...) - var nsxSubnetList []*model.VpcSubnet - for _, crdSubnetID := range crdSubnetIDs { - nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnet(crdSubnetID)...) - } - if len(nsxSubnetList) == 0 { - log.Info("No Subnets found in NSX, garbage collection complete") - return - } - - crdSubnetIDsSet := sets.NewString(crdSubnetIDs...) - for _, nsxSubnet := range nsxSubnetList { - uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetCRUID) - if crdSubnetIDsSet.Has(uid) { - continue - } - - log.Info("GC collected Subnet CR", "UID", uid) + subnetUIDs := r.SubnetService.ListSubnetIDsFromNSXSubnets() + subnetIDsToDelete := subnetUIDs.Difference(crdSubnetIDsSet) + for subnetID := range subnetIDsToDelete { + nsxSubnets := r.SubnetService.ListSubnetCreatedBySubnet(subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeSubnet) - if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { - log.Error(err, "Failed to delete NSX subnet", "NSX Subnet UID", uid) + log.Info("Subnet garbage collection, cleaning stale Subnets", "Count", len(nsxSubnets)) + if err := r.deleteSubnets(nsxSubnets); err != nil { + log.Error(err, "Subnet garbage collection, failed to delete NSX subnet", "SubnetUID", subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeSubnet) } else { - log.Info("Successfully deleted NSX subnet", "NSX Subnet UID", uid) + log.Info("Subnet garbage collection, successfully deleted NSX subnet", "SubnetUID", subnetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeSubnet) } } diff --git a/pkg/controllers/subnet/subnet_controller_test.go b/pkg/controllers/subnet/subnet_controller_test.go index aadfbc0e5..eccd24915 100644 --- a/pkg/controllers/subnet/subnet_controller_test.go +++ b/pkg/controllers/subnet/subnet_controller_test.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -29,7 +30,9 @@ import ( ) func TestSubnetReconciler_GarbageCollector(t *testing.T) { + subnetStore := &subnet.SubnetStore{} service := &subnet.SubnetService{ + SubnetStore: subnetStore, Service: common.Service{ NSXConfig: &config.NSXOperatorConfig{ NsxConfig: &config.NsxConfig{ @@ -38,36 +41,54 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { }, }, } + serviceSubnetPort := &subnetport.SubnetPortService{ + Service: common.Service{ + NSXConfig: &config.NSXOperatorConfig{ + NsxConfig: &config.NsxConfig{ + EnforcementPoint: "vmc-enforcementpoint", + }, + }, + }, + } + mockCtl := gomock.NewController(t) + k8sClient := mock_client.NewMockClient(mockCtl) + r := &SubnetReconciler{ + Client: k8sClient, + Scheme: nil, + SubnetService: service, + SubnetPortService: serviceSubnetPort, + } + // Subnet doesn't have TagScopeSubnetSetCRId (not belong to SubnetSet) - // gc collect item "2345", local store has more item than k8s cache - patch := gomonkey.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - tags1 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("2345")}} - tags2 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("1234")}} + // gc collect item "fake-id1", local store has more item than k8s cache + patch := gomonkey.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + res := sets.New[string]("fake-id1", "fake-id2") + return res + }) + patch.ApplyMethod(reflect.TypeOf(r.SubnetService.SubnetStore), "GetByIndex", func(_ *subnet.SubnetStore, _ string) []*model.VpcSubnet { + tags1 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-id1")}} + tags2 := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("fake-id2")}} var a []*model.VpcSubnet - id1 := "2345" + id1 := "fake-id1" a = append(a, &model.VpcSubnet{Id: &id1, Tags: tags1}) - id2 := "1234" + id2 := "fake-id2" a = append(a, &model.VpcSubnet{Id: &id2, Tags: tags2}) return a }) - patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { + patch.ApplyMethod(reflect.TypeOf(r.SubnetPortService), "GetPortsOfSubnet", func(_ *subnetport.SubnetPortService, _ string) (ports []*model.VpcSubnetPort) { + return nil + }) + patch.ApplyMethod(reflect.TypeOf(r.SubnetService), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { return nil }) - mockCtl := gomock.NewController(t) - k8sClient := mock_client.NewMockClient(mockCtl) - r := &SubnetReconciler{ - Client: k8sClient, - Scheme: nil, - SubnetService: service, - } ctx := context.Background() srList := &v1alpha1.SubnetList{} k8sClient.EXPECT().List(ctx, srList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { a := list.(*v1alpha1.SubnetList) a.Items = append(a.Items, v1alpha1.Subnet{}) a.Items[0].ObjectMeta = metav1.ObjectMeta{} - a.Items[0].UID = "1234" + a.Items[0].UID = "fake-id2" return nil }) @@ -75,12 +96,9 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { // local store has same item as k8s cache patch.Reset() - patch.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - tags := []model.Tag{{Scope: common.String(common.TagScopeSubnetCRUID), Tag: common.String("1234")}} - var a []*model.VpcSubnet - id := "1234" - a = append(a, &model.VpcSubnet{Id: &id, Tags: tags}) - return a + patch = gomonkey.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + res := sets.New[string]("fake-id2") + return res }) patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { assert.FailNow(t, "should not be called") @@ -90,15 +108,15 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { a := list.(*v1alpha1.SubnetList) a.Items = append(a.Items, v1alpha1.Subnet{}) a.Items[0].ObjectMeta = metav1.ObjectMeta{} - a.Items[0].UID = "1234" + a.Items[0].UID = "fake-id2" return nil }) r.collectGarbage(ctx) // local store has no item patch.Reset() - patch.ApplyMethod(reflect.TypeOf(service), "ListSubnetCreatedBySubnet", func(_ *subnet.SubnetService, uid string) []*model.VpcSubnet { - return []*model.VpcSubnet{} + patch = patch.ApplyMethod(reflect.TypeOf(&common.ResourceStore{}), "ListIndexFuncValues", func(_ *common.ResourceStore, _ string) sets.Set[string] { + return nil }) patch.ApplyMethod(reflect.TypeOf(service), "DeleteSubnet", func(_ *subnet.SubnetService, subnet model.VpcSubnet) error { assert.FailNow(t, "should not be called") diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 7c182c2d7..1f0f0b57d 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -67,12 +67,12 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } return ResultNormal, nil } - log.Error(err, "Unable to fetch SubnetSet CR", "req", req.NamespacedName) + log.Error(err, "Unable to fetch SubnetSet CR", "SubnetSet", req.NamespacedName) return ResultRequeue, err } if !subnetsetCR.ObjectMeta.DeletionTimestamp.IsZero() { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) - err := r.deleteSubnetForSubnetSet(*subnetsetCR, false) + err := r.deleteSubnetForSubnetSet(*subnetsetCR, false, false) if err != nil { log.Error(err, "Failed to delete NSX Subnet, retrying", "SubnetSet", req.NamespacedName) deleteFail(r, ctx, subnetsetCR, err.Error()) @@ -97,7 +97,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if subnetsetCR.Spec.IPv4SubnetSize == 0 { vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(subnetsetCR.Namespace) if vpcNetworkConfig == nil { - err := fmt.Errorf("failed to find VPCNetworkConfig for namespace %s", subnetsetCR.Namespace) + err := fmt.Errorf("failed to find VPCNetworkConfig for Namespace %s", subnetsetCR.Namespace) log.Error(err, "Operate failed, would retry exponentially", "SubnetSet", req.NamespacedName) updateFail(r, ctx, subnetsetCR, err.Error()) return ResultRequeue, err @@ -106,7 +106,7 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( specChanged = true } if !util.IsPowerOfTwo(subnetsetCR.Spec.IPv4SubnetSize) { - errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", subnetsetCR.Spec.IPv4SubnetSize) + errorMsg := fmt.Sprintf("ipv4SubnetSize has invalid size %d, which needs to be >= 16 and power of 2", subnetsetCR.Spec.IPv4SubnetSize) log.Error(nil, errorMsg, "SubnetSet", req.NamespacedName) updateFail(r, ctx, subnetsetCR, errorMsg) return ResultNormal, nil @@ -261,36 +261,34 @@ func (r *SubnetSetReconciler) CollectGarbage(ctx context.Context) { defer func() { log.Info("SubnetSet garbage collection completed", "duration(ms)", time.Since(startTime).Milliseconds()) }() - subnetSetList := &v1alpha1.SubnetSetList{} - err := r.Client.List(ctx, subnetSetList) + + crdSubnetSetList := &v1alpha1.SubnetSetList{} + err := r.Client.List(ctx, crdSubnetSetList) if err != nil { - log.Error(err, "Failed to list SubnetSet CR") - return - } - var nsxSubnetList []*model.VpcSubnet - for _, subnetSet := range subnetSetList.Items { - nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnetSet(string(subnetSet.UID))...) - } - if len(nsxSubnetList) == 0 { + log.Error(err, "Failed to list SubnetSet CRs") return } - subnetSetIDs := sets.New[string]() - for _, subnetSet := range subnetSetList.Items { - if err := r.deleteSubnetForSubnetSet(subnetSet, true); err != nil { + crdSubnetSetIDsSet := sets.New[string]() + for _, subnetSet := range crdSubnetSetList.Items { + crdSubnetSetIDsSet.Insert(string(subnetSet.UID)) + if err := r.deleteSubnetForSubnetSet(subnetSet, true, true); err != nil { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } - subnetSetIDs.Insert(string(subnetSet.UID)) } - for _, subnet := range nsxSubnetList { - if !r.SubnetService.IsOrphanSubnet(*subnet, subnetSetIDs) { - continue - } - if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { + + subnetSetIDs := r.SubnetService.ListSubnetSetIDsFromNSXSubnets() + subnetSetIDsToDelete := subnetSetIDs.Difference(crdSubnetSetIDsSet) + for subnetSetID := range subnetSetIDsToDelete { + nsxSubnets := r.SubnetService.ListSubnetCreatedBySubnetSet(subnetSetID) + log.Info("SubnetSet garbage collection, cleaning stale Subnets for SubnetSet", "Count", len(nsxSubnets)) + if _, err := r.deleteSubnets(nsxSubnets); err != nil { + log.Error(err, "SubnetSet garbage collection, failed to delete NSX subnet", "SubnetSetUID", subnetSetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { + log.Info("SubnetSet garbage collection, successfully deleted NSX subnet", "SubnetSetUID", subnetSetID) metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } } @@ -301,59 +299,60 @@ func (r *SubnetSetReconciler) deleteSubnetBySubnetSetName(ctx context.Context, s return r.deleteStaleSubnets(ctx, nsxSubnets) } -func (r *SubnetSetReconciler) deleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, updateStatus bool) error { - nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.GetUID())) - if err := r.deleteSubnets(nsxSubnets); err != nil { - return err - } +func (r *SubnetSetReconciler) deleteSubnetForSubnetSet(subnetSet v1alpha1.SubnetSet, updateStatus, ignoreStaleSubnetPort bool) error { + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(subnetSet.GetUID())) + hasStaleSubnetPort, deleteErr := r.deleteSubnets(nsxSubnets) if updateStatus { - err := r.SubnetService.UpdateSubnetSetStatus(&obj) - if err != nil { + if err := r.SubnetService.UpdateSubnetSetStatus(&subnetSet); err != nil { return err } } - return nil -} - -func (r *SubnetSetReconciler) listSubnetSetIDsFromCRs(ctx context.Context) ([]string, error) { - crdSubnetSetList := &v1alpha1.SubnetSetList{} - err := r.Client.List(ctx, crdSubnetSetList) - if err != nil { - return nil, err + if deleteErr != nil { + return deleteErr } - - crdSubnetSetIDs := make([]string, 0, len(crdSubnetSetList.Items)) - for _, sr := range crdSubnetSetList.Items { - crdSubnetSetIDs = append(crdSubnetSetIDs, string(sr.UID)) + if hasStaleSubnetPort && !ignoreStaleSubnetPort { + return fmt.Errorf("stale Subnet ports found while deleting Subnet for SubnetSet %s/%s", subnetSet.Name, subnetSet.Namespace) } - return crdSubnetSetIDs, nil + return nil } -func (r *SubnetSetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) error { +// deleteSubnets deletes all the specified NSX Subnets. +// If any of the Subnets have stale SubnetPorts, they are skipped. The final result returns true. +// If there is an error while deleting any NSX Subnet, it is skipped, and the final result returns an error. +func (r *SubnetSetReconciler) deleteSubnets(nsxSubnets []*model.VpcSubnet) (hasStalePort bool, err error) { + var deleteErrs []error for _, nsxSubnet := range nsxSubnets { + r.SubnetService.LockSubnet(nsxSubnet.Path) portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) if portNums > 0 { - return fmt.Errorf("fail to delete Subnet/%s from SubnetSet CR, there is stale ports", *nsxSubnet.Id) + r.SubnetService.UnlockSubnet(nsxSubnet.Path) + hasStalePort = true + log.Info("Skipped deleting NSX Subnet due to stale ports", "nsxSubnet", *nsxSubnet.Id) + continue } - r.SubnetService.LockSubnet(nsxSubnet.Path) - err := r.SubnetService.DeleteSubnet(*nsxSubnet) - if err != nil { + if err := r.SubnetService.DeleteSubnet(*nsxSubnet); err != nil { r.SubnetService.UnlockSubnet(nsxSubnet.Path) - return fmt.Errorf("fail to delete Subnet/%s from SubnetSet CR: %+v", *nsxSubnet.Id, err) + deleteErr := fmt.Errorf("failed to delete NSX Subnet/%s: %+v", *nsxSubnet.Id, err) + deleteErrs = append(deleteErrs, deleteErr) + log.Error(deleteErr, "Skipping to next Subnet") + continue } r.SubnetService.UnlockSubnet(nsxSubnet.Path) } - log.Info("Successfully deleted all Subnets", "subnetCount", len(nsxSubnets)) - return nil + if len(deleteErrs) > 0 { + err = fmt.Errorf("multiple errors occurred while deleting Subnets: %v", deleteErrs) + return + } + log.Info("Successfully deleted all specified NSX Subnets", "subnetCount", len(nsxSubnets)) + return } func (r *SubnetSetReconciler) deleteStaleSubnets(ctx context.Context, nsxSubnets []*model.VpcSubnet) error { - crdSubnetSetIDs, err := r.listSubnetSetIDsFromCRs(ctx) + crdSubnetSetIDsSet, err := r.SubnetService.ListSubnetSetID(ctx) if err != nil { - log.Error(err, "Failed to list Subnet CRs") + log.Error(err, "Failed to list SubnetSet CRs") return err } - crdSubnetSetIDsSet := sets.NewString(crdSubnetSetIDs...) nsxSubnetsToDelete := make([]*model.VpcSubnet, 0, len(nsxSubnets)) for _, nsxSubnet := range nsxSubnets { uid := nsxutil.FindTag(nsxSubnet.Tags, servicecommon.TagScopeSubnetSetCRUID) @@ -364,7 +363,11 @@ func (r *SubnetSetReconciler) deleteStaleSubnets(ctx context.Context, nsxSubnets nsxSubnetsToDelete = append(nsxSubnetsToDelete, nsxSubnet) } log.Info("Cleaning stale Subnets for SubnetSet", "Count", len(nsxSubnetsToDelete)) - return r.deleteSubnets(nsxSubnetsToDelete) + hasStaleSubnetPort, err := r.deleteSubnets(nsxSubnetsToDelete) + if err != nil || hasStaleSubnetPort { + return fmt.Errorf("failed to delete stale Subnets, error: %v, hasStaleSubnetPort: %t", err, hasStaleSubnetPort) + } + return nil } func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index 4a6617a94..9f6bf9bee 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -191,18 +191,18 @@ func (service *SubnetService) ListSubnetCreatedBySubnetSet(id string) []*model.V return service.SubnetStore.GetByIndex(common.TagScopeSubnetSetCRUID, id) } -func (service *SubnetService) ListSubnetSetID(ctx context.Context) sets.Set[string] { +func (service *SubnetService) ListSubnetSetID(ctx context.Context) (sets.Set[string], error) { crdSubnetSetList := &v1alpha1.SubnetSetList{} - subnetsetIDs := sets.New[string]() err := service.Client.List(ctx, crdSubnetSetList) if err != nil { - log.Error(err, "failed to list subnetset CR") - return subnetsetIDs + return nil, err } + + crdSubnetSetIDs := sets.New[string]() for _, subnetset := range crdSubnetSetList.Items { - subnetsetIDs.Insert(string(subnetset.UID)) + crdSubnetSetIDs.Insert(string(subnetset.UID)) } - return subnetsetIDs + return crdSubnetSetIDs, nil } func (service *SubnetService) ListSubnetByName(ns, name string) []*model.VpcSubnet { @@ -231,16 +231,6 @@ func (service *SubnetService) ListSubnetBySubnetSetName(ns, subnetSetName string return res } -// check if subnet belongs to a subnetset, if yes, check if that subnetset still exists -func (service *SubnetService) IsOrphanSubnet(subnet model.VpcSubnet, subnetsetIDs sets.Set[string]) bool { - for _, tag := range subnet.Tags { - if *tag.Scope == common.TagScopeSubnetSetCRUID && subnetsetIDs.Has(*tag.Tag) { - return false - } - } - return true -} - func (service *SubnetService) DeleteIPAllocation(orgID, projectID, vpcID, subnetID string) error { ipAllocations, err := service.NSXClient.IPAllocationClient.List(orgID, projectID, vpcID, subnetID, ipPoolID, nil, nil, nil, nil, nil, nil) @@ -352,26 +342,47 @@ func (service *SubnetService) GetSubnetByPath(path string) (*model.VpcSubnet, er return nsxSubnet, err } -func (service *SubnetService) ListSubnetID() sets.Set[string] { +func (service *SubnetService) ListSubnetSetIDsFromNSXSubnets() sets.Set[string] { + subnetSetIDs := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetSetCRUID) + return subnetSetIDs +} + +func (service *SubnetService) ListSubnetIDsFromNSXSubnets() sets.Set[string] { + subnetIDs := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetCRUID) + return subnetIDs +} + +// ListIndexFuncValues returns all the indexed values of the given index +// Index maps the indexed value to a set of keys in the store that match on that value: type Index map[string]sets.String +// see the getIndexValues function in k8s.io/client-go/tools/cache/thread_safe_store.go +func (service *SubnetService) ListAllSubnet() []*model.VpcSubnet { + var allNSXSubnets []*model.VpcSubnet + // ListSubnetCreatedBySubnet subnets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetCRUID) + for subnetID := range subnets { + nsxSubnets := service.ListSubnetCreatedBySubnet(subnetID) + allNSXSubnets = append(allNSXSubnets, nsxSubnets...) + } + // ListSubnetCreatedBySubnetSet subnetSets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetSetCRUID) - return subnets.Union(subnetSets) + for subnetSetID := range subnetSets { + nsxSubnets := service.ListSubnetCreatedBySubnetSet(subnetSetID) + allNSXSubnets = append(allNSXSubnets, nsxSubnets...) + } + return allNSXSubnets } func (service *SubnetService) Cleanup(ctx context.Context) error { - uids := service.ListSubnetID() - log.Info("cleaning up subnet", "count", len(uids)) - for uid := range uids { - nsxSubnets := service.SubnetStore.GetByIndex(common.TagScopeSubnetCRUID, string(uid)) - for _, nsxSubnet := range nsxSubnets { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSubnet(*nsxSubnet) - if err != nil { - return err - } + allNSXSubnets := service.ListAllSubnet() + log.Info("cleaning up Subnet", "Count", len(allNSXSubnets)) + for _, nsxSubnet := range allNSXSubnets { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + err := service.DeleteSubnet(*nsxSubnet) + if err != nil { + return err } } } From 6ec818d8957df0f12cd3fdfe4d43ad153f10c423 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Fri, 23 Aug 2024 14:50:20 +0800 Subject: [PATCH 13/18] Add predicate func for pod controller and node controller Update the node predicate func, filter all the conditions compare. Add pod predicate func, remove host network check in the beginning of reconcile. Signed-off-by: Xie Zheng Signed-off-by: Xie Zheng --- pkg/controllers/node/node_controller.go | 12 ++++---- pkg/controllers/pod/pod_controller.go | 38 ++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/pkg/controllers/node/node_controller.go b/pkg/controllers/node/node_controller.go index e1b25a7fd..c0197cb74 100644 --- a/pkg/controllers/node/node_controller.go +++ b/pkg/controllers/node/node_controller.go @@ -97,7 +97,6 @@ func (r *NodeReconciler) Start(mgr ctrl.Manager) error { return nil } -// PredicateFuncsNode filters out events where only resourceVersion, lastHeartbeatTime, or lastTransitionTime have changed var PredicateFuncsNode = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { oldNode, okOld := e.ObjectOld.(*v1.Node) @@ -106,13 +105,12 @@ var PredicateFuncsNode = predicate.Funcs{ return true } - // If only the heartbeat time, transition time and resource version has changed, and other properties unchanged, ignore the update + // If only the condition, resource version, allocatable, capacity have changed, and other properties unchanged, ignore the update if len(newNode.Status.Conditions) > 0 && len(oldNode.Status.Conditions) > 0 { - if newNode.ResourceVersion != oldNode.ResourceVersion && - newNode.Status.Conditions[0].LastHeartbeatTime != oldNode.Status.Conditions[0].LastHeartbeatTime && - newNode.Status.Conditions[0].LastTransitionTime != oldNode.Status.Conditions[0].LastTransitionTime { - oldNode.Status.Conditions[0].LastHeartbeatTime = newNode.Status.Conditions[0].LastHeartbeatTime - oldNode.Status.Conditions[0].LastTransitionTime = newNode.Status.Conditions[0].LastTransitionTime + if newNode.ResourceVersion != oldNode.ResourceVersion { + oldNode.Status.Allocatable = newNode.Status.Allocatable + oldNode.Status.Capacity = newNode.Status.Capacity + oldNode.Status.Conditions = newNode.Status.Conditions return !reflect.DeepEqual(oldNode.Status, newNode.Status) } } diff --git a/pkg/controllers/pod/pod_controller.go b/pkg/controllers/pod/pod_controller.go index 5c8797ef6..36a23a484 100644 --- a/pkg/controllers/pod/pod_controller.go +++ b/pkg/controllers/pod/pod_controller.go @@ -18,6 +18,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -64,10 +66,6 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R log.Error(err, "unable to fetch Pod", "Pod", req.NamespacedName) return common.ResultRequeue, err } - if pod.Spec.HostNetwork { - log.Info("skipping handling hostnetwork pod", "pod", req.NamespacedName) - return common.ResultNormal, nil - } if len(pod.Spec.NodeName) == 0 { log.Info("pod is not scheduled on node yet, skipping", "pod", req.NamespacedName) return common.ResultNormal, nil @@ -135,6 +133,7 @@ func (r *PodReconciler) GetNodeByName(nodeName string) (*model.HostTransportNode func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&v1.Pod{}). + WithEventFilter(PredicateFuncsPod). WithOptions( controller.Options{ MaxConcurrentReconciles: common.NumReconcile(), @@ -267,3 +266,34 @@ func (r *PodReconciler) deleteSubnetPortByPodName(ctx context.Context, ns string log.Info("successfully deleted nsxSubnetPort for Pod", "namespace", ns, "name", name) return nil } + +// PredicateFuncsPod filters out events where pod.Spec.HostNetwork is true +var PredicateFuncsPod = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldPod, okOld := e.ObjectOld.(*v1.Pod) + newPod, okNew := e.ObjectNew.(*v1.Pod) + if !okOld || !okNew { + return true + } + + if oldPod.Spec.HostNetwork && newPod.Spec.HostNetwork { + return false + } + return true + }, + CreateFunc: func(e event.CreateEvent) bool { + pod, ok := e.Object.(*v1.Pod) + if !ok { + return true + } + + if pod.Spec.HostNetwork { + return false + } + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + GenericFunc: func(e event.GenericEvent) bool { + return true + }, +} From 8b88eeffbbcf1191067d746da8a8aa1430b1d345 Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Mon, 14 Oct 2024 13:34:40 +0800 Subject: [PATCH 14/18] Add vpc attachment support For normal vpc, operator will get the VpcConnectivityProfile path from nc. Only for pre created vpc, operator need to fetch the vpc attachment from nsx side and get VpcConnectivityProfile from vpc attachment --- go.mod | 8 +- go.sum | 16 +-- .../networkinfo/networkinfo_controller.go | 38 +++++- .../networkinfo_controller_test.go | 109 ++++++++++++++++++ pkg/nsx/client.go | 3 + pkg/nsx/services/common/types.go | 31 ++--- pkg/nsx/services/common/wrap.go | 15 +++ pkg/nsx/services/ipblocksinfo/ipblocksinfo.go | 35 +++--- .../ipblocksinfo/ipblocksinfo_test.go | 10 +- pkg/nsx/services/ipblocksinfo/store.go | 47 ++++++-- pkg/nsx/services/ipblocksinfo/store_test.go | 96 ++++++++++----- pkg/nsx/services/vpc/builder.go | 13 ++- pkg/nsx/services/vpc/vpc.go | 25 +++- pkg/nsx/services/vpc/wrap.go | 12 +- pkg/nsx/services/vpc/wrap_test.go | 4 +- 15 files changed, 367 insertions(+), 95 deletions(-) diff --git a/go.mod b/go.mod index 4cfe125b5..e0a4ea581 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ replace ( github.com/vmware-tanzu/nsx-operator/pkg/apis => ./pkg/apis github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1 => ./pkg/apis/vpc/v1alpha1 github.com/vmware-tanzu/nsx-operator/pkg/client => ./pkg/client - github.com/vmware/vsphere-automation-sdk-go/lib => github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a - github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a + github.com/vmware/vsphere-automation-sdk-go/lib => github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/runtime => github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962 + github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp => github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962 ) require ( diff --git a/go.sum b/go.sum index 99645ef7d..dedc810d7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962 h1:+nbR8Qgl96Ba0/gMnCGQasrUb7F7XRj+OxdfLyl8r7w= +github.com/TaoZou1/vsphere-automation-sdk-go/lib v0.0.0-20241014012640-c5c5c9408962/go.mod h1:ADkX8BkdnvT1Kc9ZfqHaV4qzaaD+9L8Ok2+pxK4xoD8= +github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962 h1:WjS+j/Y6wWU74RKx6c0IBTNZ3g3fah5X8z/c4KXA2us= +github.com/TaoZou1/vsphere-automation-sdk-go/runtime v0.0.0-20241014012640-c5c5c9408962/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962 h1:b5AQfl/tH4rr0OCg/hD9ARka5vJwvC2l26zAQ4dGXMA= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt v0.0.0-20241014012640-c5c5c9408962/go.mod h1:NSjO9WqelbsTEDb3pVxpYYz4zjgX0XPp43dKNT4Y+9k= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962 h1:KIFv8/EpZd9DJPBeLcUjPfDNoF2EU72nJ7GbHei1XGg= +github.com/TaoZou1/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20241014012640-c5c5c9408962/go.mod h1:ugk9I4YM62SSAox57l5NAVBCRIkPQ1RNLb3URxyTADc= github.com/a8m/tree v0.0.0-20210115125333-10a5fd5b637d/go.mod h1:FSdwKX97koS5efgm8WevNf7XS3PqtyFkKDDXrz778cg= github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw= github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw= @@ -143,14 +151,6 @@ github.com/vmware-tanzu/vm-operator/api v1.8.2/go.mod h1:vauVboD3sQxP+pb28TnI9wf github.com/vmware/govmomi v0.27.4 h1:5kY8TAkhB20lsjzrjE073eRb8+HixBI29PVMG5lxq6I= github.com/vmware/govmomi v0.27.4/go.mod h1:daTuJEcQosNMXYJOeku0qdBJP9SOLLWB3Mqz8THtv6o= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a h1:nF3PigKL+lN4ECHkgVJIZgLbpLrV6U6wkKHnIHOU9kA= -github.com/yanjunz97/vsphere-automation-sdk-go/lib v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:ysW7/EqFugBY2TcbvlDeRGaYIoG7Cs0i4l4WsMI/RmQ= -github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a h1:b08LCEgSR6GSsvQzx2fxVbEXSKRnaGcMUqKjlgwR6xM= -github.com/yanjunz97/vsphere-automation-sdk-go/runtime v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:DzLetYAmw1+vj7bqElRWEpuy40WYE/woL3alsymYa/c= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a h1:XEgprSLuSKIxr7OPEzBWrlo39ra7pDaWFwAIjK0VV7s= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:aJtyfDKvGyuP1ieRHCLoYjo2XtNZ401XfS7lCd43Bqs= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a h1:4FmesihC1B7udmKl7B2giLydxibViMdyldSforV5qbU= -github.com/yanjunz97/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20240823072631-de1833ffcf2a/go.mod h1:FX8UiCgNEOxweA73VZsyKZvMLPFfc70GBc1d4dj0nXI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index e85a1cfc8..b90207da9 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -46,6 +46,29 @@ type NetworkInfoReconciler struct { Recorder record.EventRecorder } +func (r *NetworkInfoReconciler) GetVpcConnectivityProfilePathByVpcPath(vpcPath string) (string, error) { + // TODO, if needs to add a cache for it + VPCResourceInfo, err := commonservice.ParseVPCResourcePath(vpcPath) + if err != nil { + log.Error(err, "failed to parse VPC path", "VPC Path", vpcPath) + return "", err + } + // pre created VPC may have more than one attachment, list all the attachment and select the first one + vpcAttachmentsListResult, err := r.Service.NSXClient.VpcAttachmentClient.List(VPCResourceInfo.OrgID, VPCResourceInfo.ProjectID, VPCResourceInfo.VPCID, nil, nil, nil, nil, nil, nil) + if err != nil { + log.Error(err, "failed to list VPC attachment", "VPC Path", vpcPath) + return "", err + } + vpcAttachments := vpcAttachmentsListResult.Results + if len(vpcAttachments) > 0 { + log.V(1).Info("found VPC attachment", "VPC Path", vpcPath, "VPC connectivity profile", vpcAttachments[0].VpcConnectivityProfile) + return *vpcAttachments[0].VpcConnectivityProfile, nil + } else { + err := fmt.Errorf("no VPC attachment found") + log.Error(err, "list VPC attachment", "VPC Path", vpcPath) + return "", err + } +} func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { startTime := time.Now() defer func() { @@ -133,17 +156,24 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) } var privateIPs []string - var vpcConnectivityProfilePath, nsxLBSPath string + var vpcConnectivityProfilePath string + var nsxLBSPath string isPreCreatedVPC := vpc.IsPreCreatedVPC(nc) if isPreCreatedVPC { privateIPs = createdVpc.PrivateIps - vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile + vpcPath := *createdVpc.Path + vpcConnectivityProfilePath, err = r.GetVpcConnectivityProfilePathByVpcPath(vpcPath) + if err != nil { + log.Error(err, "failed to get VPC connectivity profile path", "path", vpcPath) + updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) + return common.ResultRequeueAfter10sec, err + } // Retrieve NSX lbs path if Avi is not used with the pre-created VPC. if createdVpc.LoadBalancerVpcEndpoint == nil || createdVpc.LoadBalancerVpcEndpoint.Enabled == nil || !*createdVpc.LoadBalancerVpcEndpoint.Enabled { - nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(*createdVpc.Path) + nsxLBSPath, err = r.Service.GetLBSsFromNSXByVPC(vpcPath) if err != nil { - log.Error(err, "Failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) + log.Error(err, "failed to get NSX LBS path with pre-created VPC", "VPC", createdVpc.Path) updateFail(r, ctx, networkInfoCR, &err, r.Client, nil) return common.ResultRequeueAfter10sec, err } diff --git a/pkg/controllers/networkinfo/networkinfo_controller_test.go b/pkg/controllers/networkinfo/networkinfo_controller_test.go index ed278fa34..2302e4b46 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller_test.go +++ b/pkg/controllers/networkinfo/networkinfo_controller_test.go @@ -50,12 +50,32 @@ func (c *fakeVPCConnectivityProfilesClient) Update(orgIdParam string, projectIdP return model.VpcConnectivityProfile{}, nil } +type fakeVpcAttachmentClient struct{} + +func (c *fakeVpcAttachmentClient) List(orgIdParam string, projectIdParam string, vpcIdParam string, cursorParam *string, includeMarkForDeleteObjectsParam *bool, includedFieldsParam *string, pageSizeParam *int64, sortAscendingParam *bool, sortByParam *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{}, nil +} + +func (c *fakeVpcAttachmentClient) Get(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string) (model.VpcAttachment, error) { + return model.VpcAttachment{}, nil +} + +func (c *fakeVpcAttachmentClient) Patch(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string, vpcAttachmentParam model.VpcAttachment) error { + return nil +} +func (c *fakeVpcAttachmentClient) Update(orgIdParam string, projectIdParam string, vpcIdParam string, vpcAttachmentIdParam string, vpcAttachmentParam model.VpcAttachment) (model.VpcAttachment, error) { + return model.VpcAttachment{}, nil +} + +var fakeAttachmentClient = &fakeVpcAttachmentClient{} + func createNetworkInfoReconciler() *NetworkInfoReconciler { service := &vpc.VPCService{ Service: servicecommon.Service{ Client: nil, NSXClient: &nsx.Client{ VPCConnectivityProfilesClient: &fakeVPCConnectivityProfilesClient{}, + VpcAttachmentClient: fakeAttachmentClient, }, NSXConfig: &config.NSXOperatorConfig{ @@ -264,6 +284,10 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { return "non-system", nil }) + patches.ApplyMethod(reflect.TypeOf(r), "GetVpcConnectivityProfilePathByVpcPath", func(_ *NetworkInfoReconciler, _ string) (string, error) { + return "connectivity_profile", nil + }) + patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCNetworkConfig", func(_ *vpc.VPCService, _ string) (servicecommon.VPCNetworkConfigInfo, bool) { return servicecommon.VPCNetworkConfigInfo{ VPCConnectivityProfile: "/orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default", @@ -678,6 +702,9 @@ func TestNetworkInfoReconciler_Reconcile(t *testing.T) { return "pre-vpc-nc", nil }) + patches.ApplyMethod(reflect.TypeOf(r), "GetVpcConnectivityProfilePathByVpcPath", func(_ *NetworkInfoReconciler, _ string) (string, error) { + return "connectivity_profile", nil + }) patches.ApplyMethod(reflect.TypeOf(r.Service), "GetVPCNetworkConfig", func(_ *vpc.VPCService, _ string) (servicecommon.VPCNetworkConfigInfo, bool) { return servicecommon.VPCNetworkConfigInfo{ Org: "default", @@ -859,3 +886,85 @@ func TestNetworkInfoReconciler_CollectGarbage(t *testing.T) { r.CollectGarbage(ctx) }) } +func TestNetworkInfoReconciler_GetVpcConnectivityProfilePathByVpcPath(t *testing.T) { + tests := []struct { + name string + vpcPath string + prepareFunc func(*testing.T, *NetworkInfoReconciler, context.Context) *gomonkey.Patches + want string + wantErr bool + }{ + { + name: "Invalid VPC Path", + vpcPath: "/invalid/path", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + return nil + }, + want: "", + wantErr: true, + }, + { + name: "Failed to list VPC attachment", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service.NSXClient.VpcAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{}, fmt.Errorf("list error") + }) + return patches + }, + want: "", + wantErr: true, + }, + { + name: "No VPC attachment found", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(r.Service.NSXClient.VpcAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{Results: []model.VpcAttachment{}}, nil + }) + return patches + }, + want: "", + wantErr: true, + }, + { + name: "Successful VPC attachment retrieval", + vpcPath: "/orgs/default/projects/project-quality/vpcs/fake-vpc", + prepareFunc: func(t *testing.T, r *NetworkInfoReconciler, ctx context.Context) *gomonkey.Patches { + patches := gomonkey.ApplyMethod(reflect.TypeOf(fakeAttachmentClient), "List", func(_ *fakeVpcAttachmentClient, _ string, _ string, _ string, _ *string, _ *bool, _ *string, _ *int64, _ *bool, _ *string) (model.VpcAttachmentListResult, error) { + return model.VpcAttachmentListResult{ + Results: []model.VpcAttachment{ + {VpcConnectivityProfile: servicecommon.String("/orgs/default/projects/project-quality/vpc-connectivity-profiles/default"), + ParentPath: servicecommon.String("/orgs/default/projects/project-quality/vpcs/fake-vpc"), + Path: servicecommon.String("/orgs/default/projects/project-quality/vpcs/fake-vpc/attachments/default")}, + }, + }, nil + }) + return patches + }, + want: "/orgs/default/projects/project-quality/vpc-connectivity-profiles/default", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := createNetworkInfoReconciler() + ctx := context.TODO() + if tt.prepareFunc != nil { + patches := tt.prepareFunc(t, r, ctx) + if patches != nil { + defer patches.Reset() + } + } + got, err := r.GetVpcConnectivityProfilePathByVpcPath(tt.vpcPath) + if (err != nil) != tt.wantErr { + t.Errorf("GetVpcConnectivityProfilePathByVpcPath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetVpcConnectivityProfilePathByVpcPath() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index c43f6b45c..d0922bc6d 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -92,6 +92,7 @@ type Client struct { VPCLBSClient vpcs.VpcLbsClient VpcLbVirtualServersClient vpcs.VpcLbVirtualServersClient VpcLbPoolsClient vpcs.VpcLbPoolsClient + VpcAttachmentClient vpcs.AttachmentsClient ProjectClient orgs.ProjectsClient TransitGatewayClient projects.TransitGatewaysClient TransitGatewayAttachmentClient transit_gateways.AttachmentsClient @@ -186,6 +187,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(restConnector(cluster)) vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(restConnector(cluster)) + vpcAttachmentClient := vpcs.NewAttachmentsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -234,6 +236,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { VPCLBSClient: vpcLBSClient, VpcLbVirtualServersClient: vpcLbVirtualServersClient, VpcLbPoolsClient: vpcLbPoolsClient, + VpcAttachmentClient: vpcAttachmentClient, ProjectClient: projectClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 22b81f26b..47984e2ca 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -112,20 +112,21 @@ const ( IndexKeyNodeName = "IndexKeyNodeName" GCValidationInterval uint16 = 720 - RuleIngress = "ingress" - RuleEgress = "egress" - RuleActionAllow = "allow" - RuleActionDrop = "isolation" - RuleActionReject = "reject" - RuleAnyPorts = "all" - DefaultProject = "default" - SecurityPolicyPrefix = "sp" - NetworkPolicyPrefix = "np" - TargetGroupSuffix = "scope" - SrcGroupSuffix = "src" - DstGroupSuffix = "dst" - IpSetGroupSuffix = "ipset" - ShareSuffix = "share" + RuleIngress = "ingress" + RuleEgress = "egress" + RuleActionAllow = "allow" + RuleActionDrop = "isolation" + RuleActionReject = "reject" + RuleAnyPorts = "all" + DefaultProject = "default" + DefaultVpcAttachmentId = "default" + SecurityPolicyPrefix = "sp" + NetworkPolicyPrefix = "np" + TargetGroupSuffix = "scope" + SrcGroupSuffix = "src" + DstGroupSuffix = "dst" + IpSetGroupSuffix = "ipset" + ShareSuffix = "share" ) var ( @@ -151,6 +152,7 @@ var ( ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" ResourceTypeLBService = "LBService" + ResourceTypeVpcAttachment = "VpcAttachment" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" @@ -158,6 +160,7 @@ var ( ResourceTypeChildRule = "ChildRule" ResourceTypeChildGroup = "ChildGroup" ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" + ResourceTypeChildVpcAttachment = "ChildVpcAttachment" ResourceTypeChildResourceReference = "ChildResourceReference" ResourceTypeTlsCertificate = "TlsCertificate" ResourceTypeLBHttpProfile = "LBHttpProfile" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go index a2c96d3a0..ddc0dfac5 100644 --- a/pkg/nsx/services/common/wrap.go +++ b/pkg/nsx/services/common/wrap.go @@ -82,3 +82,18 @@ func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, erro } return []*data.StructValue{dataValue.(*data.StructValue)}, nil } + +func (service *Service) WrapAttachment(attachment *model.VpcAttachment) ([]*data.StructValue, error) { + attachment.ResourceType = pointy.String(ResourceTypeVpcAttachment) + childVpcAttachment := model.ChildVpcAttachment{ + Id: attachment.Id, + MarkedForDelete: attachment.MarkedForDelete, + ResourceType: ResourceTypeChildVpcAttachment, + VpcAttachment: attachment, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpcAttachment, childVpcAttachment.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} diff --git a/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go b/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go index a84aa88d0..231ebadab 100644 --- a/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go +++ b/pkg/nsx/services/ipblocksinfo/ipblocksinfo.go @@ -208,36 +208,35 @@ func (s *IPBlocksInfoService) getIPBlockCIDRsByVPCConfig(vpcConfigList []v1alpha return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("default project not found, try later") } - // for all VPC path, get VPCConnectivityProfile from VPC - vpcStore := &VPCStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.VpcBindingType(), - }} - queryParam := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeVpc) - count, err := s.SearchResource(common.ResourceTypeVpc, queryParam, vpcStore, nil) + // for all VPC path, get VPCConnectivityProfile from VPC attachment + vpcAttachmentStore := NewVpcAttachmentStore() + queryParam := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeVpcAttachment) + count, err := s.SearchResource(common.ResourceTypeVpcAttachment, queryParam, vpcAttachmentStore, nil) if err != nil { + log.Error(err, "failed to query VPC attachment") return externalIPCIDRs, privateTGWIPCIDRs, err } - log.V(2).Info("successfully fetch all Vpc from NSX", "count", count) + log.V(2).Info("successfully fetch all VPC Attachment from NSX", "count", count) for vpcPath := range vpcs { vpcResInfo, err := common.ParseVPCResourcePath(vpcPath) if err != nil { return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("invalid VPC path %s", vpcPath) } - - obj := vpcStore.GetByKey(vpcPath) - if obj == nil { - return externalIPCIDRs, privateTGWIPCIDRs, fmt.Errorf("failed to get VPC %s from NSX", vpcPath) - } - vpc := obj.(*model.Vpc) - log.V(2).Info("successfully fetch VPC", "path", vpcPath) - // for pre-created vpc, mark as default for those under default project + // for pre-created VPC, mark as default for those under default project vpcProjectPath := fmt.Sprintf("/orgs/%s/projects/%s", vpcResInfo.OrgID, vpcResInfo.ProjectID) + vpcAttachments := vpcAttachmentStore.GetByVpcPath(vpcPath) + if len(vpcAttachments) == 0 { + err = fmt.Errorf("no VPC attachment found") + log.Error(err, "get VPC attachment", "VPC Path", vpcPath) + return externalIPCIDRs, privateTGWIPCIDRs, err + } + log.V(2).Info("successfully fetch VPC attachment", "path", vpcPath, "VPC Attachment", vpcAttachments[0]) + vpcConnectivityProfile := vpcAttachments[0].VpcConnectivityProfile if vpcProjectPath == s.defaultProject { - vpcConnectivityProfileProjectMap[*vpc.VpcConnectivityProfile] = true + vpcConnectivityProfileProjectMap[*vpcConnectivityProfile] = true } else { - vpcConnectivityProfileProjectMap[*vpc.VpcConnectivityProfile] = false + vpcConnectivityProfileProjectMap[*vpcConnectivityProfile] = false } } diff --git a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go index 0abdb424e..774529044 100644 --- a/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go +++ b/pkg/nsx/services/ipblocksinfo/ipblocksinfo_test.go @@ -39,6 +39,7 @@ var ( vpcConnectivityProfilePath1 = "/orgs/default/projects/default/vpc-connectivity-profiles/vpc-connectivity-profile-1" vpcConnectivityProfilePath2 = "/orgs/default/projects/default/vpc-connectivity-profiles/vpc-connectivity-profile-2" vpcPath = "/orgs/default/projects/default/vpcs/vpc-1" + vpcAttachmentPath = vpcPath + "/attachments/default" projectPath = "/orgs/default/projects/default" ) @@ -58,12 +59,13 @@ func createService(t *testing.T) (*IPBlocksInfoService, *gomock.Controller, *moc func fakeSearchResource(_ *common.Service, resourceTypeValue string, _ string, store common.Store, _ common.Filter) (uint64, error) { var count uint64 switch resourceTypeValue { - case common.ResourceTypeVpc: - vpc := &model.Vpc{ - Path: &vpcPath, + case common.ResourceTypeVpcAttachment: + vpcAttachment := &model.VpcAttachment{ + ParentPath: &vpcPath, + Path: &vpcAttachmentPath, VpcConnectivityProfile: &vpcConnectivityProfilePath2, } - store.Apply(vpc) + store.Apply(vpcAttachment) count = 1 case common.ResourceTypeVpcConnectivityProfile: vpcConnectivityProfile1 := &model.VpcConnectivityProfile{ diff --git a/pkg/nsx/services/ipblocksinfo/store.go b/pkg/nsx/services/ipblocksinfo/store.go index d09740c63..4fd050620 100644 --- a/pkg/nsx/services/ipblocksinfo/store.go +++ b/pkg/nsx/services/ipblocksinfo/store.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" ) @@ -17,6 +18,8 @@ func keyFunc(obj interface{}) (string, error) { return *v.Path, nil case *model.IpAddressBlock: return *v.Path, nil + case *model.VpcAttachment: + return *v.Path, nil default: return "", fmt.Errorf("keyFunc doesn't support unknown type %s", v) } @@ -72,27 +75,55 @@ func (is *IPBlockStore) Apply(i interface{}) error { return nil } -type VPCStore struct { +type VpcAttachmentStore struct { common.ResourceStore } -func (vs *VPCStore) Apply(i interface{}) error { +func NewVpcAttachmentStore() *VpcAttachmentStore { + return &VpcAttachmentStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.VpcAttachmentBindingType(), + }} +} + +func (vas *VpcAttachmentStore) Apply(i interface{}) error { if i == nil { return nil } - vpc := i.(*model.Vpc) - if vpc.MarkedForDelete != nil && *vpc.MarkedForDelete { - err := vs.Delete(vpc) - log.V(1).Info("delete VPC from store", "VPC", vpc) + attachment := i.(*model.VpcAttachment) + if attachment.MarkedForDelete != nil && *attachment.MarkedForDelete { + err := vas.Delete(attachment) + log.V(1).Info("delete VPC attachment from store", "VpcAttachment", attachment) if err != nil { return err } } else { - err := vs.Add(vpc) - log.V(1).Info("add VPC to store", "VPC", vpc) + err := vas.Add(attachment) + log.V(1).Info("add VPC attachment to store", "VpcAttachment", attachment) if err != nil { return err } } return nil } + +func (vas *VpcAttachmentStore) GetByKey(path string) *model.VpcAttachment { + obj := vas.ResourceStore.GetByKey(path) + if obj != nil { + attachment := obj.(*model.VpcAttachment) + return attachment + } + return nil +} + +func (vas *VpcAttachmentStore) GetByVpcPath(vpcPath string) []*model.VpcAttachment { + result := []*model.VpcAttachment{} + attachments := vas.ResourceStore.List() + for _, item := range attachments { + attachment := item.(*model.VpcAttachment) + if attachment != nil && *attachment.ParentPath == vpcPath { + result = append(result, attachment) + } + } + return result +} diff --git a/pkg/nsx/services/ipblocksinfo/store_test.go b/pkg/nsx/services/ipblocksinfo/store_test.go index 23a3ebee6..ec675bc48 100644 --- a/pkg/nsx/services/ipblocksinfo/store_test.go +++ b/pkg/nsx/services/ipblocksinfo/store_test.go @@ -11,10 +11,11 @@ import ( ) var ( - fakeVpcPath = "vpc-path" - fakeVpcProfilePath = "vpc-connectivity-profile-path" - fakeIpBlockPath = "ip-block-path" - fakeDeleted = true + fakeVpcPath = "vpc-path" + fakeVpcProfilePath = "vpc-connectivity-profile-path" + fakeIpBlockPath = "ip-block-path" + fakeVpcAttachmentPath = fakeVpcPath + "/attachments/defaults" + fakeDeleted = true ) func Test_KeyFunc(t *testing.T) { @@ -104,17 +105,17 @@ func TestVPCConnectivityProfileStore_Apply(t *testing.T) { } } -func TestVPCStore_Apply(t *testing.T) { - vpcStore := &VPCStore{ResourceStore: common.ResourceStore{ +func TestIPBlockStore_Apply(t *testing.T) { + ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.VpcBindingType(), + BindingType: model.IpAddressBlockBindingType(), }} - vpc1 := model.Vpc{ - Path: &fakeVpcPath, + ipblock1 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, } - vpc2 := model.Vpc{ - Path: &fakeVpcPath, + ipblock2 := model.IpAddressBlock{ + Path: &fakeIpBlockPath, MarkedForDelete: &fakeDeleted, } @@ -125,28 +126,26 @@ func TestVPCStore_Apply(t *testing.T) { name string args args }{ - {"Add", args{i: &vpc1}}, - {"Delete", args{i: &vpc2}}, + {"Add", args{i: &ipblock1}}, + {"Delete", args{i: &ipblock2}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := vpcStore.Apply(tt.args.i) + err := ipBlockStore.Apply(tt.args.i) assert.Nil(t, err) }) } } +func TestVpcAttachmentStore_Apply(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() -func TestIPBlockStore_Apply(t *testing.T) { - ipBlockStore := &IPBlockStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.IpAddressBlockBindingType(), - }} - - ipblock1 := model.IpAddressBlock{ - Path: &fakeIpBlockPath, + attachment1 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, } - ipblock2 := model.IpAddressBlock{ - Path: &fakeIpBlockPath, + attachment2 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, MarkedForDelete: &fakeDeleted, } @@ -157,13 +156,56 @@ func TestIPBlockStore_Apply(t *testing.T) { name string args args }{ - {"Add", args{i: &ipblock1}}, - {"Delete", args{i: &ipblock2}}, + {"Add", args{i: &attachment1}}, + {"Delete", args{i: &attachment2}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := ipBlockStore.Apply(tt.args.i) + err := vpcAttachmentStore.Apply(tt.args.i) assert.Nil(t, err) }) } } +func TestVpcAttachmentStore_GetByKey(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() + + attachment := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + } + + err := vpcAttachmentStore.Apply(&attachment) + assert.Nil(t, err) + + t.Run("GetByKey", func(t *testing.T) { + got := vpcAttachmentStore.GetByKey(fakeVpcAttachmentPath) + assert.NotNil(t, got) + assert.Equal(t, &attachment, got) + }) +} + +func TestVpcAttachmentStore_GetByVpcPath(t *testing.T) { + vpcAttachmentStore := NewVpcAttachmentStore() + + attachment1 := model.VpcAttachment{ + Path: &fakeVpcAttachmentPath, + ParentPath: &fakeVpcPath, + } + attachment2 := model.VpcAttachment{ + Path: common.String(fakeVpcAttachmentPath + "2"), + ParentPath: &fakeVpcPath, + } + + err := vpcAttachmentStore.Apply(&attachment1) + assert.Nil(t, err) + err = vpcAttachmentStore.Apply(&attachment2) + assert.Nil(t, err) + + t.Run("GetByVpcPath", func(t *testing.T) { + got := vpcAttachmentStore.GetByVpcPath(fakeVpcPath) + assert.NotNil(t, got) + assert.Equal(t, 2, len(got)) + assert.Contains(t, got, &attachment1) + assert.Contains(t, got, &attachment2) + }) +} diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index c05038a47..2ac9aaac6 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -82,10 +82,6 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe Scope: common.String(common.TagScopeVPCManagedBy), Tag: common.String(common.AutoCreatedVPCTagValue)}) } - if nc.VPCConnectivityProfile != "" { - vpc.VpcConnectivityProfile = &nc.VPCConnectivityProfile - } - vpc.PrivateIps = nc.PrivateIPs return vpc, nil } @@ -106,3 +102,12 @@ func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSiz lbs.RelaxScaleValidation = relaxScaleValidation return lbs, nil } + +func buildVpcAttachment(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster string, vpcconnectiveprofile string) (*model.VpcAttachment, error) { + attachment := &model.VpcAttachment{} + attachment.VpcConnectivityProfile = &vpcconnectiveprofile + attachment.DisplayName = common.String(common.DefaultVpcAttachmentId) + attachment.Id = common.String(common.DefaultVpcAttachmentId) + attachment.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + return attachment, nil +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 0495fe499..42fa0d564 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -773,7 +773,8 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) } // build HAPI request - orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NSXProject, createdVpc, createdLBS) + createdAttachment, _ := buildVpcAttachment(obj, nsObj, s.NSXConfig.Cluster, nc.VPCConnectivityProfile) + orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NSXProject, createdVpc, createdLBS, createdAttachment) if err != nil { log.Error(err, "failed to build HAPI request") return nil, err @@ -849,6 +850,28 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC } } + // Check VpcAttachment realization + if createdAttachment != nil { + newAttachment, err := s.NSXClient.VpcAttachmentClient.Get(nc.Org, nc.NSXProject, *createdVpc.Id, *createdAttachment.Id) + if err != nil || newAttachment.VpcConnectivityProfile == nil { + log.Error(err, "failed to read VPC attachment object after creating or updating", "VpcAttachment", createdAttachment.Id) + return nil, err + } + log.V(2).Info("check VPC attachment realization state", "VpcAttachment", *createdAttachment.Id) + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(util.NSXTLBVSDefaultRetry, *newAttachment.Path, ""); err != nil { + log.Error(err, "failed to check VPC attachment realization state", "VpcAttachment", *createdAttachment.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created VPC attachment is in error realization state, cleaning the resource", "VpcAttachment", *createdAttachment.Id) + // delete the nsx vpc object and re-create it in the next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, err + } + } + return nil, err + } + } return &newVpc, nil } diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go index 8b16e771b..0322173d4 100644 --- a/pkg/nsx/services/vpc/wrap.go +++ b/pkg/nsx/services/vpc/wrap.go @@ -5,9 +5,9 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" ) -func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) { +func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService, attachment *model.VpcAttachment) (*model.OrgRoot, error) { + var vpcChildren []*data.StructValue if lbs != nil { - var vpcChildren []*data.StructValue childrenLBS, err := s.WrapLBS(lbs) if err != nil { return nil, err @@ -15,6 +15,14 @@ func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, l vpcChildren = append(vpcChildren, childrenLBS...) vpc.Children = vpcChildren } + if attachment != nil { + childrenAttachment, err := s.WrapAttachment(attachment) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, childrenAttachment...) + vpc.Children = vpcChildren + } var projectChildren []*data.StructValue childrenVPC, err := s.WrapVPC(vpc) if err != nil { diff --git a/pkg/nsx/services/vpc/wrap_test.go b/pkg/nsx/services/vpc/wrap_test.go index b51338705..9720f2188 100644 --- a/pkg/nsx/services/vpc/wrap_test.go +++ b/pkg/nsx/services/vpc/wrap_test.go @@ -15,6 +15,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { nsxtProject string vpc *model.Vpc lbs *model.LBService + attachment *model.VpcAttachment } tests := []struct { name string @@ -30,6 +31,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { nsxtProject: "testproject", vpc: &model.Vpc{}, lbs: &model.LBService{}, + attachment: &model.VpcAttachment{}, }, want: &model.OrgRoot{ResourceType: pointy.String("OrgRoot")}, wantChildren: 1, @@ -39,7 +41,7 @@ func TestVPCService_WrapHierarchyVPC(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &VPCService{} - got, err := s.WrapHierarchyVPC(tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs) + got, err := s.WrapHierarchyVPC(tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs, tt.args.attachment) if !tt.wantErr(t, err, fmt.Sprintf("WrapHierarchyVPC(%v, %v, %v, %v)", tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs)) { return } From 2496f109a52a9568fba170fed3cbee8d0484fcf0 Mon Sep 17 00:00:00 2001 From: Ran Gu Date: Mon, 21 Oct 2024 10:49:59 +0800 Subject: [PATCH 15/18] [AddressBinding] Add admission webhook to validate CREATE/UPDATE request (#782) Signed-off-by: gran --- build/yaml/webhook/certificate.yaml | 4 +- build/yaml/webhook/manifests.yaml | 22 ++- build/yaml/webhook/service.yaml | 2 +- cmd/main.go | 17 ++- cmd/webhookcert/main.go | 4 +- .../subnetport/addressbinding_webhook.go | 64 +++++++++ .../subnetport/addressbinding_webhook_test.go | 134 ++++++++++++++++++ .../subnetport/subnetport_controller.go | 13 +- .../subnetset/subnetset_controller.go | 16 +-- 9 files changed, 253 insertions(+), 23 deletions(-) create mode 100644 pkg/controllers/subnetport/addressbinding_webhook.go create mode 100644 pkg/controllers/subnetport/addressbinding_webhook_test.go diff --git a/build/yaml/webhook/certificate.yaml b/build/yaml/webhook/certificate.yaml index 4f584df41..f6a796072 100644 --- a/build/yaml/webhook/certificate.yaml +++ b/build/yaml/webhook/certificate.yaml @@ -31,8 +31,8 @@ metadata: spec: # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize dnsNames: - - subnetset.vmware-system-nsx.svc - - subnetset.vmware-system-nsx.svc.cluster.local + - vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc + - vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc.cluster.local issuerRef: kind: Issuer name: selfsigned-issuer diff --git a/build/yaml/webhook/manifests.yaml b/build/yaml/webhook/manifests.yaml index 71bc17b0b..65cd628fa 100644 --- a/build/yaml/webhook/manifests.yaml +++ b/build/yaml/webhook/manifests.yaml @@ -9,7 +9,7 @@ webhooks: - v1 clientConfig: service: - name: subnetset + name: vmware-system-nsx-operator-webhook-service namespace: vmware-system-nsx # kubebuilder webhookpath. path: /validate-nsx-vmware-com-v1alpha1-subnetset @@ -30,3 +30,23 @@ webhooks: resources: - subnetsets sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: vmware-system-nsx-operator-webhook-service + namespace: vmware-system-nsx + path: /validate-crd-nsx-vmware-com-v1alpha1-addressbinding + failurePolicy: Fail + name: addressbinding.validating.crd.nsx.vmware.com + rules: + - apiGroups: + - crd.nsx.vmware.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - addressbindings + sideEffects: None diff --git a/build/yaml/webhook/service.yaml b/build/yaml/webhook/service.yaml index 2a02eb376..5d71bb2fc 100644 --- a/build/yaml/webhook/service.yaml +++ b/build/yaml/webhook/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/created-by: nsx-operator app.kubernetes.io/part-of: nsx-operator app.kubernetes.io/managed-by: kustomize - name: subnetset + name: vmware-system-nsx-operator-webhook-service namespace: vmware-system-nsx spec: ports: diff --git a/cmd/main.go b/cmd/main.go index cb72b6190..428f05015 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" crdv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" @@ -213,18 +214,26 @@ func startServiceController(mgr manager.Manager, nsxClient *nsx.Client) { if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil { os.Exit(1) } - enableWebhook := true + var hookServer webhook.Server if _, err := os.Stat(config.WebhookCertDir); errors.Is(err, os.ErrNotExist) { log.Error(err, "server cert not found, disabling webhook server", "cert", config.WebhookCertDir) - enableWebhook = false + } else { + hookServer = webhook.NewServer(webhook.Options{ + Port: config.WebhookServerPort, + CertDir: config.WebhookCertDir, + }) + if err := mgr.Add(hookServer); err != nil { + log.Error(err, "failed to add hook server") + os.Exit(1) + } } - if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, enableWebhook); err != nil { + if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService, hookServer); err != nil { os.Exit(1) } node.StartNodeController(mgr, nodeService) staticroutecontroller.StartStaticRouteController(mgr, staticRouteService) - subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService) + subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService, hookServer) pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService) StartIPAddressAllocationController(mgr, ipAddressAllocationService, vpcService) networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService) diff --git a/cmd/webhookcert/main.go b/cmd/webhookcert/main.go index e2ac603c8..d550cdbc2 100644 --- a/cmd/webhookcert/main.go +++ b/cmd/webhookcert/main.go @@ -98,8 +98,8 @@ func generateWebhookCerts() error { Bytes: caBytes, }) - dnsNames := []string{"subnetset", "subnetset.vmware-system-nsx", "subnetset.vmware-system-nsx.svc"} - commonName := "subnetset.vmware-system-nsx.svc" + dnsNames := []string{"vmware-system-nsx-operator-webhook-service", "vmware-system-nsx-operator-webhook-service.vmware-system-nsx", "vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc"} + commonName := "vmware-system-nsx-operator-webhook-service.vmware-system-nsx.svc" serialNumber, err = rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) if err != nil { diff --git a/pkg/controllers/subnetport/addressbinding_webhook.go b/pkg/controllers/subnetport/addressbinding_webhook.go new file mode 100644 index 000000000..c00db77bd --- /dev/null +++ b/pkg/controllers/subnetport/addressbinding_webhook.go @@ -0,0 +1,64 @@ +package subnetport + +import ( + "context" + "fmt" + "net/http" + "reflect" + + admissionv1 "k8s.io/api/admission/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +// Create validator instead of using the existing one in controller-runtime because the existing one can't +// inspect admission.Request in Handle function. + +//+kubebuilder:webhook:path=/validate-crd-nsx-vmware-com-v1alpha1-addressbinding,mutating=false,failurePolicy=fail,sideEffects=None,groups=crd.nsx.vmware.com,resources=addressbindings,verbs=create;update,versions=v1alpha1,name=addressbinding.validating.crd.nsx.vmware.com,admissionReviewVersions=v1 + +type AddressBindingValidator struct { + Client client.Client + decoder admission.Decoder +} + +// Handle handles admission requests. +func (v *AddressBindingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + ab := &v1alpha1.AddressBinding{} + if req.Operation == admissionv1.Delete { + return admission.Allowed("") + } else { + err := v.decoder.Decode(req, ab) + if err != nil { + log.Error(err, "error while decoding AddressBinding", "AddressBinding", req.Namespace+"/"+req.Name) + return admission.Errored(http.StatusBadRequest, err) + } + } + switch req.Operation { + case admissionv1.Create: + existingAddressBindingList := &v1alpha1.AddressBindingList{} + abIndexValue := fmt.Sprintf("%s/%s", ab.Namespace, ab.Spec.VMName) + err := v.Client.List(context.TODO(), existingAddressBindingList, client.MatchingFields{util.AddressBindingNamespaceVMIndexKey: abIndexValue}) + if err != nil { + log.Error(err, "failed to list AddressBinding from cache", "indexValue", abIndexValue) + return admission.Errored(http.StatusInternalServerError, err) + } + for _, existingAddressBinding := range existingAddressBindingList.Items { + if ab.Name != existingAddressBinding.Name && ab.Spec.InterfaceName == existingAddressBinding.Spec.InterfaceName { + return admission.Denied("interface already has AddressBinding") + } + } + case admissionv1.Update: + oldAddressBinding := &v1alpha1.AddressBinding{} + if err := v.decoder.DecodeRaw(req.OldObject, oldAddressBinding); err != nil { + log.Error(err, "error while decoding AddressBinding", "AddressBinding", req.Namespace+"/"+req.Name) + return admission.Errored(http.StatusBadRequest, err) + } + if !reflect.DeepEqual(ab.Spec, oldAddressBinding.Spec) { + return admission.Denied("update AddressBinding is not allowed") + } + } + return admission.Allowed("") +} diff --git a/pkg/controllers/subnetport/addressbinding_webhook_test.go b/pkg/controllers/subnetport/addressbinding_webhook_test.go new file mode 100644 index 000000000..9fa8f7791 --- /dev/null +++ b/pkg/controllers/subnetport/addressbinding_webhook_test.go @@ -0,0 +1,134 @@ +package subnetport + +import ( + "context" + "fmt" + "net/http" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/assert" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/util" +) + +func TestAddressBindingValidator_Handle(t *testing.T) { + req1, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab1", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf1", + }, + }) + req1New, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab1", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf1new", + }, + }) + req2, _ := json.Marshal(&v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab2", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf2", + }, + }) + type args struct { + req admission.Request + } + tests := []struct { + name string + args args + prepareFunc func(*testing.T, client.Client, context.Context) *gomonkey.Patches + want admission.Response + }{ + { + name: "delete", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Delete}}}, + want: admission.Allowed(""), + }, + { + name: "create decode error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create}}}, + want: admission.Errored(http.StatusBadRequest, fmt.Errorf("there is no content to decode")), + }, + { + name: "create", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req1}}}}, + want: admission.Allowed(""), + }, + { + name: "create list error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req1}}}}, + prepareFunc: func(t *testing.T, client client.Client, ctx context.Context) *gomonkey.Patches { + return gomonkey.ApplyMethodSeq(client, "List", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("mock error")}, + Times: 1, + }}) + }, + want: admission.Errored(http.StatusInternalServerError, fmt.Errorf("mock error")), + }, + { + name: "create dup", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Create, Object: runtime.RawExtension{Raw: req2}}}}, + want: admission.Denied("interface already has AddressBinding"), + }, + { + name: "update decode error", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Update, Object: runtime.RawExtension{Raw: req1}}}}, + want: admission.Errored(http.StatusBadRequest, fmt.Errorf("there is no content to decode")), + }, + { + name: "update changed", + args: args{req: admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{Operation: admissionv1.Update, Object: runtime.RawExtension{Raw: req1New}, OldObject: runtime.RawExtension{Raw: req1}}}}, + want: admission.Denied("update AddressBinding is not allowed"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := clientgoscheme.Scheme + v1alpha1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&v1alpha1.AddressBinding{}).WithIndex(&v1alpha1.AddressBinding{}, util.AddressBindingNamespaceVMIndexKey, addressBindingNamespaceVMIndexFunc).Build() + decoder := admission.NewDecoder(scheme) + ctx := context.TODO() + client.Create(ctx, &v1alpha1.AddressBinding{ + ObjectMeta: v1.ObjectMeta{ + Namespace: "ns1", + Name: "ab2a", + }, + Spec: v1alpha1.AddressBindingSpec{ + VMName: "vm1", + InterfaceName: "inf2", + }, + }) + if tt.prepareFunc != nil { + patches := tt.prepareFunc(t, client, ctx) + defer patches.Reset() + } + v := &AddressBindingValidator{ + Client: client, + decoder: decoder, + } + assert.Equalf(t, tt.want, v.Handle(ctx, tt.args.req), "Handle()") + }) + } +} diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index 468995878..fa82c15d8 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -27,6 +27,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" @@ -256,7 +258,7 @@ func (r *SubnetPortReconciler) vmMapFunc(_ context.Context, vm client.Object) [] return requests } -func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService *subnet.SubnetService, vpcService *vpc.VPCService) { +func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService *subnet.SubnetService, vpcService *vpc.VPCService, hookServer webhook.Server) { subnetPortReconciler := SubnetPortReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -269,6 +271,15 @@ func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.S log.Error(err, "failed to create controller", "controller", "SubnetPort") os.Exit(1) } + if hookServer != nil { + hookServer.Register("/validate-crd-nsx-vmware-com-v1alpha1-addressbinding", + &webhook.Admission{ + Handler: &AddressBindingValidator{ + Client: mgr.GetClient(), + decoder: admission.NewDecoder(mgr.GetScheme()), + }, + }) + } go common.GenericGarbageCollector(make(chan bool), servicecommon.GCInterval, subnetPortReconciler.CollectGarbage) } diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 1f0f0b57d..7688053f0 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -22,7 +22,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" - "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" @@ -372,7 +371,7 @@ func (r *SubnetSetReconciler) deleteStaleSubnets(ctx context.Context, nsxSubnets func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider, - enableWebhook bool, + hookServer webhook.Server, ) error { subnetsetReconciler := &SubnetSetReconciler{ Client: mgr.GetClient(), @@ -382,7 +381,7 @@ func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetServ VPCService: vpcService, Recorder: mgr.GetEventRecorderFor("subnetset-controller"), } - if err := subnetsetReconciler.Start(mgr, enableWebhook); err != nil { + if err := subnetsetReconciler.Start(mgr, hookServer); err != nil { log.Error(err, "Failed to create controller", "controller", "SubnetSet") return err } @@ -391,19 +390,12 @@ func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetServ } // Start setup manager -func (r *SubnetSetReconciler) Start(mgr ctrl.Manager, enableWebhook bool) error { +func (r *SubnetSetReconciler) Start(mgr ctrl.Manager, hookServer webhook.Server) error { err := r.setupWithManager(mgr) if err != nil { return err } - if enableWebhook { - hookServer := webhook.NewServer(webhook.Options{ - Port: config.WebhookServerPort, - CertDir: config.WebhookCertDir, - }) - if err := mgr.Add(hookServer); err != nil { - return err - } + if hookServer != nil { hookServer.Register("/validate-nsx-vmware-com-v1alpha1-subnetset", &webhook.Admission{ Handler: &SubnetSetValidator{ From 7a7caf193509581680b424ce5c66a993ceedfb8a Mon Sep 17 00:00:00 2001 From: zhengxiexie Date: Mon, 21 Oct 2024 12:00:10 +0800 Subject: [PATCH 16/18] Fix the bug where the namespace cannot be deleted due to the username of the namespace deletion. (#810) Use global log. Signed-off-by: Xie Zheng Signed-off-by: Xie Zheng --- .../namespace/namespace_controller.go | 32 ++++++++++++++++++- .../subnetset/subnetset_webhook.go | 12 +++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/controllers/namespace/namespace_controller.go b/pkg/controllers/namespace/namespace_controller.go index a46267963..e7b198373 100644 --- a/pkg/controllers/namespace/namespace_controller.go +++ b/pkg/controllers/namespace/namespace_controller.go @@ -132,6 +132,33 @@ func (r *NamespaceReconciler) createDefaultSubnetSet(ns string, defaultSubnetSiz return nil } +func (r *NamespaceReconciler) deleteDefaultSubnetSet(ns string) error { + subnetSets := []string{ + types.DefaultVMSubnetSet, + types.DefaultPodSubnetSet, + } + for _, name := range subnetSets { + if err := retry.OnError(retry.DefaultRetry, func(err error) bool { + return err != nil + }, func() error { + obj := &v1alpha1.SubnetSet{} + err := r.Client.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: name, + }, obj) + if err != nil { + return client.IgnoreNotFound(err) + } + log.Info("delete default SubnetSet", "Namespace", ns, "Name", name) + return r.Client.Delete(context.Background(), obj) + }); err != nil { + log.Error(err, "failed to delete SubnetSet", "Namespace", ns, "Name", name) + return err + } + } + return nil +} + func (r *NamespaceReconciler) namespaceError(ctx context.Context, k8sObj client.Object, msg string, err error) { logErr := util.If(err == nil, errors.New(msg), err).(error) log.Error(logErr, msg) @@ -237,9 +264,12 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } return common.ResultNormal, nil } else { - log.Info("skip ns deletion event for ns", "Namespace", ns) metrics.CounterInc(r.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNamespace) r.VPCService.UnRegisterNamespaceNetworkconfigBinding(obj.GetNamespace()) + // actively delete default subnet set, so that subnetset webhook can admit the delete request + if err := r.deleteDefaultSubnetSet(ns); err != nil { + return common.ResultRequeueAfter10sec, nil + } return common.ResultNormal, nil } } diff --git a/pkg/controllers/subnetset/subnetset_webhook.go b/pkg/controllers/subnetset/subnetset_webhook.go index 396a211d1..2fc074cd8 100644 --- a/pkg/controllers/subnetset/subnetset_webhook.go +++ b/pkg/controllers/subnetset/subnetset_webhook.go @@ -10,16 +10,12 @@ import ( admissionv1 "k8s.io/api/admission/v1" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" ) -// log is for logging in this package. -var subnetsetlog = logf.Log.WithName("subnetset-webhook") - var NSXOperatorSA = "system:serviceaccount:vmware-system-nsx:ncp-svc-account" // Create validator instead of using the existing one in controller-runtime because the existing one can't @@ -63,17 +59,17 @@ func (v *SubnetSetValidator) Handle(ctx context.Context, req admission.Request) if req.Operation == admissionv1.Delete { err := v.decoder.DecodeRaw(req.OldObject, subnetSet) if err != nil { - subnetsetlog.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) + log.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) return admission.Errored(http.StatusBadRequest, err) } } else { err := v.decoder.Decode(req, subnetSet) if err != nil { - subnetsetlog.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) + log.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) return admission.Errored(http.StatusBadRequest, err) } } - subnetsetlog.Info("request user-info", "name", req.UserInfo.Username) + log.V(1).Info("request user-info", "name", req.UserInfo.Username) switch req.Operation { case admissionv1.Create: if !isDefaultSubnetSet(subnetSet) { @@ -86,7 +82,7 @@ func (v *SubnetSetValidator) Handle(ctx context.Context, req admission.Request) case admissionv1.Update: oldSubnetSet := &v1alpha1.SubnetSet{} if err := v.decoder.DecodeRaw(req.OldObject, oldSubnetSet); err != nil { - subnetsetlog.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) + log.Error(err, "error while decoding SubnetSet", "SubnetSet", req.Namespace+"/"+req.Name) return admission.Errored(http.StatusBadRequest, err) } if defaultSubnetSetLabelChanged(oldSubnetSet, subnetSet) { From 65d6bede7c6bffc0595247001e823cf595f216de Mon Sep 17 00:00:00 2001 From: zhengxiexie Date: Mon, 21 Oct 2024 12:00:30 +0800 Subject: [PATCH 17/18] Upgrade golang from 1.22.5 to 1.22.7 to avoid the CVEs (#811) Signed-off-by: Xie Zheng Signed-off-by: Xie Zheng --- build/image/photon/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/image/photon/Dockerfile b/build/image/photon/Dockerfile index 9926ed12b..4988a9203 100644 --- a/build/image/photon/Dockerfile +++ b/build/image/photon/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.5 as golang-build +FROM golang:1.22.7 as golang-build WORKDIR /source From 97377180228c70aaa26882ffe84cd12a8314e3d5 Mon Sep 17 00:00:00 2001 From: zhengxiexie Date: Mon, 21 Oct 2024 12:00:53 +0800 Subject: [PATCH 18/18] Update subnetset webhook apiGroup (#813) Signed-off-by: Xie Zheng Signed-off-by: Xie Zheng --- build/yaml/webhook/manifests.yaml | 6 +++--- pkg/controllers/subnetset/subnetset_controller.go | 2 +- pkg/controllers/subnetset/subnetset_webhook.go | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/build/yaml/webhook/manifests.yaml b/build/yaml/webhook/manifests.yaml index 65cd628fa..95ded4a1d 100644 --- a/build/yaml/webhook/manifests.yaml +++ b/build/yaml/webhook/manifests.yaml @@ -12,15 +12,15 @@ webhooks: name: vmware-system-nsx-operator-webhook-service namespace: vmware-system-nsx # kubebuilder webhookpath. - path: /validate-nsx-vmware-com-v1alpha1-subnetset + path: /validate-crd-nsx-vmware-com-v1alpha1-subnetset failurePolicy: Fail - name: default.subnetset.validating.nsx.vmware.com + name: default.subnetset.validating.crd.nsx.vmware.com objectSelector: matchExpressions: - { key: nsxoperator.vmware.com/default-subnetset-for, operator: In, values: ["Pod", "VirtualMachine"] } rules: - apiGroups: - - nsx.vmware.com + - crd.nsx.vmware.com apiVersions: - v1alpha1 operations: diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 7688053f0..c9502cc78 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -396,7 +396,7 @@ func (r *SubnetSetReconciler) Start(mgr ctrl.Manager, hookServer webhook.Server) return err } if hookServer != nil { - hookServer.Register("/validate-nsx-vmware-com-v1alpha1-subnetset", + hookServer.Register("/validate-crd-nsx-vmware-com-v1alpha1-subnetset", &webhook.Admission{ Handler: &SubnetSetValidator{ Client: mgr.GetClient(), diff --git a/pkg/controllers/subnetset/subnetset_webhook.go b/pkg/controllers/subnetset/subnetset_webhook.go index 2fc074cd8..fe0b0c691 100644 --- a/pkg/controllers/subnetset/subnetset_webhook.go +++ b/pkg/controllers/subnetset/subnetset_webhook.go @@ -21,7 +21,9 @@ var NSXOperatorSA = "system:serviceaccount:vmware-system-nsx:ncp-svc-account" // Create validator instead of using the existing one in controller-runtime because the existing one can't // inspect admission.Request in Handle function. -//+kubebuilder:webhook:path=/validate-nsx-vmware-com-v1alpha1-subnetset,mutating=false,failurePolicy=fail,sideEffects=None,groups=nsx.vmware.com.nsx.vmware.com,resources=subnetsets,verbs=create;update,versions=v1alpha1,name=default.subnetset.validating.nsx.vmware.com,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-crd-nsx-vmware-com-v1alpha1-subnetset,mutating=false,failurePolicy=fail,sideEffects=None, +//groups=crd.nsx.vmware.com,resources=subnetsets,verbs=create;update,versions=v1alpha1, +//name=default.subnetset.validating.crd.nsx.vmware.com,admissionReviewVersions=v1 type SubnetSetValidator struct { Client client.Client