Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create skeleton code for degraded mode procedure #1952

Merged
merged 2 commits into from
Mar 8, 2023
Merged

Create skeleton code for degraded mode procedure #1952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6778265
Create ValidateAndAddEndpoints to validate and add endpoints to maps.
sawsa307 Mar 6, 2023
155f260
Create skeleton for degraded mode procedure
sawsa307 Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions pkg/neg/syncers/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,72 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter negtypes.
return zoneNetworkEndpointMap, networkEndpointPodMap, dupCount, nil
}

func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGetter negtypes.ZoneGetter, podLister cache.Indexer, servicePortName string, networkEndpointType negtypes.NetworkEndpointType) (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap, int, error) {
targetMap := map[string]negtypes.NetworkEndpointSet{}
endpointPodMap := negtypes.EndpointPodMap{}
var dupCount int
for _, ed := range eds {
matchPort := ""
for _, port := range ed.Ports {
if port.Name == servicePortName {
matchPort = strconv.Itoa(int(port.Port))
break
}
}
if len(matchPort) == 0 {
continue
}
for _, endpointAddress := range ed.Addresses {
if endpointAddress.TargetRef == nil {
klog.V(2).Infof("Endpoint %q in Endpoints %s/%s does not have an associated pod. Skipping", endpointAddress.Addresses, ed.Meta.Namespace, ed.Meta.Name)
continue
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add a log to indicate no matching targetRef

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added. Thanks!

}
dupCount += validateAndAddEndpoints(endpointAddress, zoneGetter, podLister, matchPort, networkEndpointType, targetMap, endpointPodMap)
}
}
return targetMap, endpointPodMap, dupCount, nil
}

// validateAndAddEndpoints fills in missing information and creates network endpoint for each endpoint addresss
func validateAndAddEndpoints(ep negtypes.AddressData, zoneGetter negtypes.ZoneGetter, podLister cache.Indexer, matchPort string, endpointType negtypes.NetworkEndpointType, targetMap map[string]negtypes.NetworkEndpointSet, endpointPodMap negtypes.EndpointPodMap) int {
var dupCount int
for _, address := range ep.Addresses {
key := fmt.Sprintf("%s/%s", ep.TargetRef.Namespace, ep.TargetRef.Name)
obj, exists, err := podLister.GetByKey(key)
if err != nil || !exists {
klog.V(2).Infof("Endpoint %q does not correspond to an existing pod. Skipping", address)
continue
sawsa307 marked this conversation as resolved.
Show resolved Hide resolved
}
pod, ok := obj.(*apiv1.Pod)
if !ok {
klog.V(2).Infof("Endpoint %q does not correspond to a valid pod resource. Skipping", address)
continue
sawsa307 marked this conversation as resolved.
Show resolved Hide resolved
}
nodeName := pod.Spec.NodeName
zone, err := zoneGetter.GetZoneForNode(nodeName)
if err != nil {
klog.V(2).Infof("Endpoint %q does not have valid zone information. Skipping", address)
continue
sawsa307 marked this conversation as resolved.
Show resolved Hide resolved
}

if endpointType == negtypes.NonGCPPrivateEndpointType {
// Non-GCP network endpoints don't have associated nodes.
nodeName = ""
sawsa307 marked this conversation as resolved.
Show resolved Hide resolved
}
networkEndpoint := negtypes.NetworkEndpoint{IP: address, Port: matchPort, Node: nodeName}
if targetMap[zone] == nil {
targetMap[zone] = negtypes.NewNetworkEndpointSet()
}
targetMap[zone].Insert(networkEndpoint)
// increment the count for duplicated endpoint
if _, contains := endpointPodMap[networkEndpoint]; contains {
dupCount += 1
}
endpointPodMap[networkEndpoint] = types.NamespacedName{Namespace: ep.TargetRef.Namespace, Name: ep.TargetRef.Name}
}
return dupCount
}

// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map
func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter negtypes.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud, version meta.Version, mode negtypes.EndpointsCalculatorMode) (map[string]negtypes.NetworkEndpointSet, error) {
// Include zones that have non-candidate nodes currently. It is possible that NEGs were created in those zones previously and the endpoints now became non-candidates.
Expand Down
286 changes: 286 additions & 0 deletions pkg/neg/syncers/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ import (

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/cache"
negv1beta1 "k8s.io/ingress-gce/pkg/apis/svcneg/v1beta1"
"k8s.io/ingress-gce/pkg/composite"
negtypes "k8s.io/ingress-gce/pkg/neg/types"
Expand Down Expand Up @@ -1190,6 +1192,290 @@ func TestNEGRecreate(t *testing.T) {
}
}

func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) {
swetharepakula marked this conversation as resolved.
Show resolved Hide resolved
t.Parallel()

fakeZoneGetter := negtypes.NewFakeZoneGetter()
podLister := negtypes.NewTestContext().PodInformer.GetIndexer()
addPodsToLister(podLister)

testNonExistPort := "non-exists"
testEmptyNamedPort := ""
testNamedPort := "named-Port"

testCases := []struct {
desc string
portName string
expectedEndpointMap map[string]negtypes.NetworkEndpointSet
expectedPodMap negtypes.EndpointPodMap
networkEndpointType negtypes.NetworkEndpointType
}{
{
desc: "non exist target port",
portName: testNonExistPort,
expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{},
expectedPodMap: negtypes.EndpointPodMap{},
networkEndpointType: negtypes.VmIpPortEndpointType,
},
{
desc: "empty named port",
portName: testEmptyNamedPort,
expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{
negtypes.TestZone1: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"),
networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"),
networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"),
networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"),
networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80")),
negtypes.TestZone2: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.3.1||instance3||80")),
},
expectedPodMap: negtypes.EndpointPodMap{
networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod1"},
networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod2"},
networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod3"},
networkEndpointFromEncodedEndpoint("10.100.3.1||instance3||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod4"},
networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod5"},
networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod6"},
},
networkEndpointType: negtypes.VmIpPortEndpointType,
},
{
desc: "named target port",
portName: testNamedPort,
expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{
negtypes.TestZone1: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.2.2||instance2||81")),
negtypes.TestZone2: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.4.1||instance4||81"),
networkEndpointFromEncodedEndpoint("10.100.3.2||instance3||8081"),
networkEndpointFromEncodedEndpoint("10.100.4.2||instance4||8081"),
networkEndpointFromEncodedEndpoint("10.100.4.3||instance4||81"),
networkEndpointFromEncodedEndpoint("10.100.4.4||instance4||8081")),
},
expectedPodMap: negtypes.EndpointPodMap{
networkEndpointFromEncodedEndpoint("10.100.2.2||instance2||81"): types.NamespacedName{Namespace: testNamespace, Name: "pod7"},
networkEndpointFromEncodedEndpoint("10.100.4.1||instance4||81"): types.NamespacedName{Namespace: testNamespace, Name: "pod8"},
networkEndpointFromEncodedEndpoint("10.100.4.3||instance4||81"): types.NamespacedName{Namespace: testNamespace, Name: "pod9"},
networkEndpointFromEncodedEndpoint("10.100.3.2||instance3||8081"): types.NamespacedName{Namespace: testNamespace, Name: "pod10"},
networkEndpointFromEncodedEndpoint("10.100.4.2||instance4||8081"): types.NamespacedName{Namespace: testNamespace, Name: "pod11"},
networkEndpointFromEncodedEndpoint("10.100.4.4||instance4||8081"): types.NamespacedName{Namespace: testNamespace, Name: "pod12"},
},
networkEndpointType: negtypes.VmIpPortEndpointType,
},
{
desc: "Non-GCP network endpoints",
portName: testEmptyNamedPort,
expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{
negtypes.TestZone1: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.1.1||||80"),
networkEndpointFromEncodedEndpoint("10.100.1.2||||80"),
networkEndpointFromEncodedEndpoint("10.100.2.1||||80"),
networkEndpointFromEncodedEndpoint("10.100.1.3||||80"),
networkEndpointFromEncodedEndpoint("10.100.1.4||||80")),
negtypes.TestZone2: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.3.1||||80")),
},
expectedPodMap: negtypes.EndpointPodMap{
networkEndpointFromEncodedEndpoint("10.100.1.1||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod1"},
networkEndpointFromEncodedEndpoint("10.100.1.2||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod2"},
networkEndpointFromEncodedEndpoint("10.100.2.1||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod3"},
networkEndpointFromEncodedEndpoint("10.100.3.1||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod4"},
networkEndpointFromEncodedEndpoint("10.100.1.3||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod5"},
networkEndpointFromEncodedEndpoint("10.100.1.4||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod6"},
},
networkEndpointType: negtypes.NonGCPPrivateEndpointType,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
testEndpointSlices := getTestEndpointSlices(testService, testNamespace)

targetMap, endpointPodMap, _, err := toZoneNetworkEndpointMapDegradedMode(negtypes.EndpointsDataFromEndpointSlices(testEndpointSlices), fakeZoneGetter, podLister, tc.portName, tc.networkEndpointType)
if err != nil {
t.Errorf("expected error=nil, but got %v", err)
}

if !reflect.DeepEqual(targetMap, tc.expectedEndpointMap) {
t.Errorf("degraded mode endpoint set is not calculated correctly:\ngot %+v,\n expected %+v", targetMap, tc.expectedEndpointMap)
}
if !reflect.DeepEqual(endpointPodMap, tc.expectedPodMap) {
t.Errorf("degraded mode endpoint map is not calculated correctly:\ngot %+v,\n expected %+v", endpointPodMap, tc.expectedPodMap)
}
})
}
}

func TestValidateAndAddEndpoints(t *testing.T) {
t.Parallel()
matchPort := strconv.Itoa(int(80))
emptyNodeName := ""
ready := true
instance1 := negtypes.TestInstance1
endpointMap := map[string]negtypes.NetworkEndpointSet{
negtypes.TestZone1: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80")),
}
podMap := negtypes.EndpointPodMap{
networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod1"},
}

fakeZoneGetter := negtypes.NewFakeZoneGetter()
podLister := negtypes.NewTestContext().PodInformer.GetIndexer()
addPodsToLister(podLister)

testCases := []struct {
desc string
ep negtypes.AddressData
endpointType negtypes.NetworkEndpointType
expectedEndpointMap map[string]negtypes.NetworkEndpointSet
expectedPodMap negtypes.EndpointPodMap
}{
{
desc: "endpoint with nodeName",
ep: negtypes.AddressData{
Addresses: []string{"10.100.1.1"},
NodeName: &instance1,
TargetRef: &corev1.ObjectReference{
Namespace: testNamespace,
Name: "pod1",
},
Ready: ready,
},
endpointType: negtypes.VmIpPortEndpointType,
expectedEndpointMap: endpointMap,
expectedPodMap: podMap,
},
{
desc: "endpoint without nodeName",
ep: negtypes.AddressData{
Addresses: []string{"10.100.1.1"},
NodeName: nil,
TargetRef: &corev1.ObjectReference{
Namespace: testNamespace,
Name: "pod1",
},
Ready: ready,
},
endpointType: negtypes.VmIpPortEndpointType,
expectedEndpointMap: endpointMap,
expectedPodMap: podMap,
},
{
desc: "endpoint with empty nodeName",
ep: negtypes.AddressData{
Addresses: []string{"10.100.1.1"},
NodeName: &emptyNodeName,
TargetRef: &corev1.ObjectReference{
Namespace: testNamespace,
Name: "pod1",
},
Ready: ready,
},
endpointType: negtypes.VmIpPortEndpointType,
expectedEndpointMap: endpointMap,
expectedPodMap: podMap,
},
{
desc: "Non-GCP network endpoint",
ep: negtypes.AddressData{
TargetRef: &corev1.ObjectReference{
Namespace: testNamespace,
Name: "pod1",
},
NodeName: &emptyNodeName,
Addresses: []string{"10.100.1.1"},
Ready: ready,
},

endpointType: negtypes.NonGCPPrivateEndpointType,
expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{
negtypes.TestZone1: negtypes.NewNetworkEndpointSet(
networkEndpointFromEncodedEndpoint("10.100.1.1||||80")),
},
expectedPodMap: negtypes.EndpointPodMap{
networkEndpointFromEncodedEndpoint("10.100.1.1||||80"): types.NamespacedName{Namespace: testNamespace, Name: "pod1"}},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
targetMap := map[string]negtypes.NetworkEndpointSet{}
endpointPodMap := negtypes.EndpointPodMap{}
validateAndAddEndpoints(tc.ep, fakeZoneGetter, podLister, matchPort, tc.endpointType, targetMap, endpointPodMap)

if !reflect.DeepEqual(targetMap, tc.expectedEndpointMap) {
t.Errorf("degraded mode endpoint set is not calculated correctly:\ngot %+v,\n expected %+v", targetMap, tc.expectedEndpointMap)
}
if !reflect.DeepEqual(endpointPodMap, tc.expectedPodMap) {
t.Errorf("degraded mode endpoint map is not calculated correctly:\ngot %+v,\n expected %+v", endpointPodMap, tc.expectedPodMap)
}
})
}
}

func addPodsToLister(podLister cache.Indexer) {
// add all pods in default endpoint into podLister
for i := 1; i <= 6; i++ {
podLister.Add(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: fmt.Sprintf("pod%v", i),
},
Spec: corev1.PodSpec{
NodeName: testInstance1,
},
})
}
for i := 7; i <= 12; i++ {
podLister.Add(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: fmt.Sprintf("pod%v", i),
},
Spec: corev1.PodSpec{
NodeName: testInstance4,
},
})
}

podLister.Update(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: "pod3",
},
Spec: corev1.PodSpec{
NodeName: testInstance2,
},
})
podLister.Update(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: "pod4",
},
Spec: corev1.PodSpec{
NodeName: testInstance3,
},
})
podLister.Update(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: "pod7",
},
Spec: corev1.PodSpec{
NodeName: testInstance2,
},
})
podLister.Update(&corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: testNamespace,
Name: "pod10",
},
Spec: corev1.PodSpec{
NodeName: testInstance3,
},
})
}

// numToIP converts the given number to an IP address.
// assumes that the input is smaller than 2^32.
func numToIP(input int) string {
Expand Down