Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
connectivity: add full egress gateway test suite
Browse files Browse the repository at this point in the history
Signed-off-by: Gilberto Bertin <[email protected]>
jibi committed May 26, 2023
1 parent 3b74510 commit 9274730
Showing 9 changed files with 386 additions and 19 deletions.
7 changes: 7 additions & 0 deletions connectivity/check/context.go
Original file line number Diff line number Diff line change
@@ -821,6 +821,13 @@ func (ct *ConnectivityTest) PingCommand(peer TestPeer, ipFam IPFamily) []string
return cmd
}

func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam IPFamily) []string {
cmd := []string{"dig", "+time=2", "kubernetes"}

cmd = append(cmd, fmt.Sprintf("@%s", peer.Address(ipFam)))
return cmd
}

func (ct *ConnectivityTest) RandomClientPod() *Pod {
for _, p := range ct.clientPods {
return &p
12 changes: 12 additions & 0 deletions connectivity/check/deployment.go
Original file line number Diff line number Diff line change
@@ -1304,10 +1304,22 @@ func (ct *ConnectivityTest) patchEchoServicesWithExternalIPs(ctx context.Context

ensureFrontend := func() error {
for _, client := range ct.clients.clients() {
fmt.Println("client:", client)

for _, ciliumPod := range ct.CiliumPods() {
for _, service := range patchedServices {
for _, ip := range service.Spec.ExternalIPs {
cmd := []string{"sh", "-c",
fmt.Sprintf("cilium bpf lb list --frontends")}

out, err := client.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd)
if err != nil {
return err
}

fmt.Println(out.String())

cmd = []string{"sh", "-c",
fmt.Sprintf("cilium bpf lb list --frontends | grep %s:%d", ip, service.Spec.Ports[0].Port)}

fmt.Println(strings.Join(cmd, " "))
15 changes: 15 additions & 0 deletions connectivity/check/test.go
Original file line number Diff line number Diff line change
@@ -452,6 +452,21 @@ func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test {

// Set the egress gateway node
pl[i].Spec.EgressGateway.NodeSelector.MatchLabels["kubernetes.io/hostname"] = egressGatewayNode

// Set the excluded CIDR
if len(pl[i].Spec.ExcludedCIDRs) != 0 {
excludedCIDRs := []v2.IPv4CIDR{}
for _, nodeWithoutCilium := range t.NodesWithoutCilium() {
node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{})
if err != nil {
t.Fatalf("Can't get node")
}

excludedCIDRs = append(excludedCIDRs, v2.IPv4CIDR(fmt.Sprintf("%s/32", node.Status.Addresses[0].Address)))
}

pl[i].Spec.ExcludedCIDRs = excludedCIDRs
}
}

if err := t.addCEGPs(pl...); err != nil {
18 changes: 18 additions & 0 deletions connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
apiVersion: cilium.io/v2
kind: CiliumEgressGatewayPolicy
metadata:
name: cegp-sample-excluded-cidrs
spec:
selectors:
- podSelector:
matchLabels:
io.kubernetes.pod.namespace: cilium-test
kind: client
destinationCIDRs:
- 0.0.0.0/0
excludedCIDRs:
- NODE_WITHOUT_CILIUM_PLACEHOLDER/32
egressGateway:
nodeSelector:
matchLabels:
kubernetes.io/hostname: NODE_NAME_PLACEHOLDER
16 changes: 16 additions & 0 deletions connectivity/manifests/egress-gateway-policy.yaml
Original file line number Diff line number Diff line change
@@ -14,3 +14,19 @@ spec:
nodeSelector:
matchLabels:
kubernetes.io/hostname: NODE_NAME_PLACEHOLDER
---
apiVersion: cilium.io/v2
kind: CiliumEgressGatewayPolicy
metadata:
name: cegp-sample-echo-service
spec:
selectors:
- podSelector:
matchLabels:
kind: echo
destinationCIDRs:
- 0.0.0.0/0
egressGateway:
nodeSelector:
matchLabels:
kubernetes.io/hostname: NODE_NAME_PLACEHOLDER
15 changes: 15 additions & 0 deletions connectivity/suite.go
Original file line number Diff line number Diff line change
@@ -11,6 +11,8 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium/pkg/versioncheck"

"github.com/cilium/cilium-cli/connectivity/check"
"github.com/cilium/cilium-cli/connectivity/manifests/template"
"github.com/cilium/cilium-cli/connectivity/tests"
@@ -157,6 +159,9 @@ var (

//go:embed manifests/egress-gateway-policy.yaml
egressGatewayPolicyYAML string

//go:embed manifests/egress-gateway-policy-excluded-cidrs.yaml
egressGatewayPolicyExcludedCIDRsYAML string
)

var (
@@ -734,6 +739,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch
tests.EgressGateway(),
)

if versioncheck.MustCompile(">=1.14.0")(ct.CiliumVersion) {
ct.NewTest("egress-gateway-excluded-cidrs").
WithCiliumEgressGatewayPolicy(egressGatewayPolicyExcludedCIDRsYAML).
WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway),
check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)).
WithScenarios(
tests.EgressGatewayExcludedCIDRs(),
)
}

// The following tests have DNS redirect policies. They should be executed last.

ct.NewTest("north-south-loadbalancing-with-l7-policy").
169 changes: 169 additions & 0 deletions connectivity/tests/egressgateway-excluded-cirds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package tests

import (
"context"
"encoding/json"
"fmt"
"net"
"strings"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium-cli/connectivity/check"
"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/internal/utils"
)

// EgressGatewayExcludedCIDRs is a test case which, given the cegp-sample
// CiliumEgressGatewayExcludedCIDRsPolicy targeting:
// - a couple of client pods (kind=client) as source
// - the 0.0.0.0/0 destination CIDR
// - kind-worker2 as gateway node
//
// ensures that traffic from both clients reaches the echo-external service with
// the egress IP of the gateway node.
func EgressGatewayExcludedCIDRs() check.Scenario {
return &egressGatewayExcludedCIDRs{}
}

type egressGatewayExcludedCIDRs struct {
egressGatewayNode string
}

func (s *egressGatewayExcludedCIDRs) Name() string {
return "egress-gateway-excluded-cidrs"
}

func (s *egressGatewayExcludedCIDRs) Run(ctx context.Context, t *check.Test) {
ct := t.Context()

s.egressGatewayNode = t.EgressGatewayNode()
if s.egressGatewayNode == "" {
t.Fatal("Cannot get egress gateway node")
}

s.waitForBpfPolicyEntries(ctx, t)

// Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service)
i := 0
for _, client := range ct.ClientPods() {
client := client

for _, externalEcho := range ct.ExternalEchoPods() {
t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4))
clientIP := extractClientIPFromResponse(a.CmdOutput())

if !clientIP.Equal(net.ParseIP(client.Pod.Status.HostIP)) {
t.Fatal("Request reached external echo service with wrong source IP")
}
})
i++
}
}
}

// bpfEgressGatewayExcludedCIDRsPolicyEntry represents an entry in the BPF egress gateway
// policy map
type bpfEgressGatewayExcludedCIDRsPolicyEntry struct {
SourceIP string
DestCIDR string
EgressIP string
GatewayIP string
}

// matches is an helper used to compare the receiver bpfEgressGatewayExcludedCIDRsPolicyEntry
// with another entry
func (e *bpfEgressGatewayExcludedCIDRsPolicyEntry) matches(t bpfEgressGatewayExcludedCIDRsPolicyEntry) bool {
return t.SourceIP == e.SourceIP &&
t.DestCIDR == e.DestCIDR &&
t.EgressIP == e.EgressIP &&
t.GatewayIP == e.GatewayIP
}

// waitForBpfPolicyEntries waits for the egress gateway policy maps on each node
// to be populated with the entries for the cegp-sample CiliumEgressGatewayExcludedCIDRsPolicy
func (s *egressGatewayExcludedCIDRs) waitForBpfPolicyEntries(ctx context.Context, t *check.Test) {
ct := t.Context()

w := utils.NewWaitObserver(ctx, utils.WaitParameters{Timeout: 10 * time.Second})
defer w.Cancel()

ensureBpfPolicyEntries := func() error {
gatewayNodeInternalIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode)
if gatewayNodeInternalIP == nil {
t.Fatalf("Cannot retrieve internal IP of gateway node")
}

for _, ciliumPod := range ct.CiliumPods() {
for _, nodeWithoutCilium := range t.NodesWithoutCilium() {
node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{})
if err != nil {
t.Fatalf("Cannot retrieve external node")
}

egressIP := "0.0.0.0"
if ciliumPod.Pod.Spec.NodeName == s.egressGatewayNode {
egressIP = gatewayNodeInternalIP.String()
}

targetEntries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{}
for _, client := range ct.ClientPods() {
targetEntries = append(targetEntries,
bpfEgressGatewayExcludedCIDRsPolicyEntry{
SourceIP: client.Pod.Status.PodIP,
DestCIDR: "0.0.0.0/0",
EgressIP: egressIP,
GatewayIP: gatewayNodeInternalIP.String(),
})

targetEntries = append(targetEntries,
bpfEgressGatewayExcludedCIDRsPolicyEntry{
SourceIP: client.Pod.Status.PodIP,
DestCIDR: fmt.Sprintf("%s/32", node.Status.Addresses[0].Address),
EgressIP: egressIP,
GatewayIP: "Excluded CIDR",
})
}

cmd := strings.Split("cilium bpf egress list -o json", " ")
stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd)
if err != nil {
t.Fatal("failed to run cilium bpf egress list command: %w", err)
}

entries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{}
json.Unmarshal(stdout.Bytes(), &entries)

nextTargetEntry:
for _, targetEntry := range targetEntries {
for _, entry := range entries {
if targetEntry.matches(entry) {
continue nextTargetEntry
}
}

return fmt.Errorf("Could not find egress gateway policy entry matching %+v", targetEntry)
}
}
}

return nil
}

for {
if err := ensureBpfPolicyEntries(); err != nil {
if err := w.Retry(err); err != nil {
t.Fatal("Failed to ensure egress gateway policy map is properly populated:", err)
}

continue
}

return
}
}
149 changes: 130 additions & 19 deletions connectivity/tests/egressgateway.go
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@ import (
"github.com/cilium/cilium-cli/connectivity/check"
"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/internal/utils"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// EgressGateway is a test case which, given the cegp-sample
@@ -44,14 +46,47 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) {
t.Fatal("Cannot get egress gateway node")
}

egressIP := s.getGatewayNodeInternalIP(ct)
egressIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode)

s.waitForBpfPolicyEntries(ctx, t)

// Ping hosts (pod to host connectivity)
i := 0
for _, client := range ct.ClientPods() {
client := client

for _, dst := range ct.HostNetNSPodsByNode() {
dst := dst

t.NewAction(s, fmt.Sprintf("ping-%d", i), &client, &dst, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.PingCommand(dst, check.IPFamilyV4))
})
i++
}
}

// DNS query (pod to service connectivity)
i = 0
for _, client := range ct.ClientPods() {
client := client

kubeDNSService, err := ct.K8sClient().GetService(ctx, "kube-system", "kube-dns", metav1.GetOptions{})
if err != nil {
t.Fatal("Cannot get kube-dns service")
}
kubeDNSServicePeer := check.Service{Service: kubeDNSService}

t.NewAction(s, fmt.Sprintf("dig-%d", i), &client, kubeDNSServicePeer, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.DigCommand(kubeDNSServicePeer, check.IPFamilyV4))
})
i++
}

// Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service)
i = 0
for _, client := range ct.ClientPods() {
client := client

for _, externalEcho := range ct.ExternalEchoPods() {
t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4))
@@ -64,30 +99,72 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) {
i++
}
}
}

// getGatewayNodeInternalIP returns the k8s internal IP of the node acting as
// gateway for this test
func (s *egressGateway) getGatewayNodeInternalIP(ct *check.ConnectivityTest) net.IP {
gatewayNode, ok := ct.Nodes()[s.egressGatewayNode]
if !ok {
return nil
}
// When connecting from outside the cluster to a nodeport service whose pods are selected by an egress policy,
// the reply traffic should not be SNATed with the egress IP
i = 0
for _, client := range ct.ExternalEchoPods() {
client := client

for _, addr := range gatewayNode.Status.Addresses {
if addr.Type != "InternalIP" {
continue
for _, echo := range ct.EchoServices() {
// convert the service to a ServiceExternalIP as we want to access it through its external IP
echo := echo.ToExternalIPService()

t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4))
})
i++
}
}

ip := net.ParseIP(addr.Address)
if ip == nil || ip.To4() == nil {
continue
if status, ok := ct.Feature(check.FeatureTunnel); ok && !status.Enabled {
// When connecting from outside the cluster directly to a pod which is selected by an egress policy, the
// reply traffic should not be SNATed with the egress IP (only connections originating from these pods
// should go through egress gateway).
//
// This test is executed only when Cilium is running in direct routing mode, since we can simply add a
// route on the node outside the cluster to direct pod's traffic to the node where the pod is running
// (while in tunneling mode we would need the external node to send the traffic over the tunnel)

for _, echoPod := range ct.EchoPods() {
targetPodHostIP := echoPod.Pod.Status.HostIP
targetPodIP := echoPod.Pod.Status.PodIP

for _, externalNode := range ct.NodesWithoutCilium() {
for node, hostNetNSPod := range ct.HostNetNSPodsByNode() {
if node != externalNode {
continue
}

cmd := []string{"ip", "route", "add", targetPodIP, "via", targetPodHostIP}
_, err := hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd)
if err != nil {
t.Fatalf("failed to add ip route: %w", err)
}

defer func(hostNetNSPod check.Pod) {
cmd = []string{"ip", "route", "del", targetPodIP, "via", targetPodHostIP}
_, err = hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd)
if err != nil {
t.Fatalf("failed to delete ip route: %w", err)
}
}(hostNetNSPod)
}
}
}

return ip
}
i = 0
for _, client := range ct.ExternalEchoPods() {
client := client

return nil
for _, echo := range ct.EchoPods() {
t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) {
a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4))
})
i++
}
}
}
}

// bpfEgressGatewayPolicyEntry represents an entry in the BPF egress gateway
@@ -117,7 +194,7 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te
defer w.Cancel()

ensureBpfPolicyEntries := func() error {
gatewayNodeInternalIP := s.getGatewayNodeInternalIP(ct)
gatewayNodeInternalIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode)
if gatewayNodeInternalIP == nil {
t.Fatalf("Cannot retrieve internal IP of gateway node")
}
@@ -139,6 +216,16 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te
})
}

for _, echo := range ct.EchoPods() {
targetEntries = append(targetEntries,
bpfEgressGatewayPolicyEntry{
SourceIP: echo.Pod.Status.PodIP,
DestCIDR: "0.0.0.0/0",
EgressIP: egressIP,
GatewayIP: gatewayNodeInternalIP.String(),
})
}

cmd := strings.Split("cilium bpf egress list -o json", " ")
stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd)
if err != nil {
@@ -176,6 +263,30 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te
}
}

// getGatewayNodeInternalIP returns the k8s internal IP of the node acting as
// gateway for this test
func getGatewayNodeInternalIP(ct *check.ConnectivityTest, egressGatewayNode string) net.IP {
gatewayNode, ok := ct.Nodes()[egressGatewayNode]
if !ok {
return nil
}

for _, addr := range gatewayNode.Status.Addresses {
if addr.Type != "InternalIP" {
continue
}

ip := net.ParseIP(addr.Address)
if ip == nil || ip.To4() == nil {
continue
}

return ip
}

return nil
}

// extractClientIPFromResponse extracts the client IP from the response of the
// echo-external service
func extractClientIPFromResponse(res string) net.IP {
4 changes: 4 additions & 0 deletions k8s/client.go
Original file line number Diff line number Diff line change
@@ -673,6 +673,10 @@ func (c *Client) ListCiliumEnvoyConfigs(ctx context.Context, namespace string, o
return c.CiliumClientset.CiliumV2().CiliumEnvoyConfigs(namespace).List(ctx, options)
}

func (c *Client) GetNode(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Node, error) {
return c.Clientset.CoreV1().Nodes().Get(ctx, name, opts)
}

func (c *Client) ListNodes(ctx context.Context, options metav1.ListOptions) (*corev1.NodeList, error) {
return c.Clientset.CoreV1().Nodes().List(ctx, options)
}

0 comments on commit 9274730

Please sign in to comment.