From 1e249fd50f82d08ad0f60720417648b373420b02 Mon Sep 17 00:00:00 2001 From: Yusuke Suzuki Date: Tue, 7 May 2024 15:58:50 +0900 Subject: [PATCH] connectivity: add egress-gateway-with-l7-policy test egress-gateway-with-l7-policy checks if traffic from Pods that are selected by both Egress Gateway Policy and L7 Network Policy is properly SNATed with an Egress IP. Signed-off-by: Yusuke Suzuki --- CODEOWNERS | 1 + connectivity/builder/builder.go | 1 + .../builder/egress_gateway_with_l7_policy.go | 47 +++++++++++++++++++ .../builder/manifests/client-egress-icmp.yaml | 17 +++++++ .../client-egress-l7-http-external-node.yaml | 26 ++++++++++ connectivity/check/context.go | 30 +++++++----- connectivity/check/deployment.go | 21 +++++++++ connectivity/check/peer.go | 16 ++++++- connectivity/tests/egressgateway.go | 20 ++++++++ 9 files changed, 166 insertions(+), 13 deletions(-) create mode 100644 connectivity/builder/egress_gateway_with_l7_policy.go create mode 100644 connectivity/builder/manifests/client-egress-icmp.yaml create mode 100644 connectivity/builder/manifests/client-egress-l7-http-external-node.yaml diff --git a/CODEOWNERS b/CODEOWNERS index a732a6b741..99c98e3aa3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -45,6 +45,7 @@ /connectivity/builder/echo_ingress_mutual_auth_spiffe.go @cilium/sig-servicemesh /connectivity/builder/egress_gateway.go @cilium/egress-gateway /connectivity/builder/egress_gateway_excluded_cidrs.go @cilium/egress-gateway +/connectivity/builder/egress_gateway_with_l7_policy.go @cilium/egress-gateway /connectivity/builder/no_ipsec_xfrm_errors.go @cilium/sig-encryption /connectivity/builder/node_to_node_encryption.go @cilium/sig-encryption /connectivity/builder/pod_to_pod_encryption.go @cilium/sig-encryption diff --git a/connectivity/builder/builder.go b/connectivity/builder/builder.go index a186a39317..c3789d51c4 100644 --- a/connectivity/builder/builder.go +++ b/connectivity/builder/builder.go @@ -209,6 +209,7 @@ func concurrentTests(connTests []*check.ConnectivityTest) error { nodeToNodeEncryption{}, egressGateway{}, egressGatewayExcludedCidrs{}, + egressGatewayWithL7Policy{}, podToNodeCidrpolicy{}, northSouthLoadbalancingWithL7Policy{}, echoIngressL7{}, diff --git a/connectivity/builder/egress_gateway_with_l7_policy.go b/connectivity/builder/egress_gateway_with_l7_policy.go new file mode 100644 index 0000000000..8c1ecec204 --- /dev/null +++ b/connectivity/builder/egress_gateway_with_l7_policy.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package builder + +import ( + _ "embed" + + "github.com/cilium/cilium/pkg/versioncheck" + + "github.com/cilium/cilium-cli/connectivity/check" + "github.com/cilium/cilium-cli/connectivity/tests" + "github.com/cilium/cilium-cli/utils/features" +) + +//go:embed manifests/client-egress-icmp.yaml +var clientEgressICMPYAML string + +//go:embed manifests/client-egress-l7-http-external-node.yaml +var clientEgressL7HTTPExternalYAML string + +type egressGatewayWithL7Policy struct{} + +func (t egressGatewayWithL7Policy) build(ct *check.ConnectivityTest, _ map[string]string) { + newTest("egress-gateway-with-l7-policy", ct). + WithCondition(func() bool { + return versioncheck.MustCompile(">=1.16.0")(ct.CiliumVersion) && ct.Params().IncludeUnsafeTests + }). + WithCiliumPolicy(clientEgressICMPYAML). + WithCiliumPolicy(clientEgressOnlyDNSPolicyYAML). // DNS resolution only + WithCiliumPolicy(clientEgressL7HTTPExternalYAML). // L7 allow policy with HTTP introspection + WithCiliumEgressGatewayPolicy(check.CiliumEgressGatewayPolicyParams{ + Name: "cegp-sample-client", + PodSelectorKind: "client", + }). + WithCiliumEgressGatewayPolicy(check.CiliumEgressGatewayPolicyParams{ + Name: "cegp-sample-echo", + PodSelectorKind: "echo", + }). + WithIPRoutesFromOutsideToPodCIDRs(). + WithFeatureRequirements( + features.RequireEnabled(features.EgressGateway), + features.RequireEnabled(features.L7Proxy), + features.RequireEnabled(features.NodeWithoutCilium), + ). + WithScenarios(tests.EgressGateway()) +} diff --git a/connectivity/builder/manifests/client-egress-icmp.yaml b/connectivity/builder/manifests/client-egress-icmp.yaml new file mode 100644 index 0000000000..5b2031106b --- /dev/null +++ b/connectivity/builder/manifests/client-egress-icmp.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: client-egress-icmp +spec: + description: "Allow clients to send ICMP" + endpointSelector: + matchLabels: + kind: client + egress: + - icmps: + - fields: + - type: 8 + family: IPv4 + - type: 128 + family: IPv6 diff --git a/connectivity/builder/manifests/client-egress-l7-http-external-node.yaml b/connectivity/builder/manifests/client-egress-l7-http-external-node.yaml new file mode 100644 index 0000000000..b6f2d62d29 --- /dev/null +++ b/connectivity/builder/manifests/client-egress-l7-http-external-node.yaml @@ -0,0 +1,26 @@ +--- +# All clients are allowed to contact +# echo-external-node.cilium-test.svc.cluster.local/client-ip +# on port http-8080. +# The toFQDNs section relies on DNS introspection being performed by +# the client-egress-only-dns policy. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: client-egress-l7-http-external-node +spec: + description: "Allow GET echo-external-node.cilium-test.svc.cluster.local:8080/client-ip" + endpointSelector: + matchLabels: + any:kind: client + egress: + - toFQDNs: + - matchName: "echo-external-node.cilium-test.svc.cluster.local" + toPorts: + - ports: + - port: "8080" + protocol: TCP + rules: + http: + - method: GET + path: /client-ip diff --git a/connectivity/check/context.go b/connectivity/check/context.go index df99c83130..9099bfcf77 100644 --- a/connectivity/check/context.go +++ b/connectivity/check/context.go @@ -58,18 +58,19 @@ type ConnectivityTest struct { // Clients for source and destination clusters. clients *deploymentClients - ciliumPods map[string]Pod - echoPods map[string]Pod - echoExternalPods map[string]Pod - clientPods map[string]Pod - clientCPPods map[string]Pod - perfClientPods []Pod - perfServerPod []Pod - PerfResults []common.PerfSummary - echoServices map[string]Service - ingressService map[string]Service - k8sService Service - externalWorkloads map[string]ExternalWorkload + ciliumPods map[string]Pod + echoPods map[string]Pod + echoExternalPods map[string]Pod + clientPods map[string]Pod + clientCPPods map[string]Pod + perfClientPods []Pod + perfServerPod []Pod + PerfResults []common.PerfSummary + echoServices map[string]Service + echoExternalServices map[string]Service + ingressService map[string]Service + k8sService Service + externalWorkloads map[string]ExternalWorkload hostNetNSPodsByNode map[string]Pod secondaryNetworkNodeIPv4 map[string]string // node name => secondary ip @@ -207,6 +208,7 @@ func NewConnectivityTest(client *k8s.Client, p Parameters, version string, logge perfServerPod: []Pod{}, PerfResults: []common.PerfSummary{}, echoServices: make(map[string]Service), + echoExternalServices: make(map[string]Service), ingressService: make(map[string]Service), externalWorkloads: make(map[string]ExternalWorkload), hostNetNSPodsByNode: make(map[string]Pod), @@ -1131,6 +1133,10 @@ func (ct *ConnectivityTest) EchoServicesAll() map[string]Service { return ct.echoServices } +func (ct *ConnectivityTest) EchoExternalServices() map[string]Service { + return ct.echoExternalServices +} + func (ct *ConnectivityTest) ExternalEchoPods() map[string]Pod { return ct.echoExternalPods } diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go index f68f6a020f..4e16298f86 100644 --- a/connectivity/check/deployment.go +++ b/connectivity/check/deployment.go @@ -905,6 +905,16 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create deployment %s: %s", echoExternalNodeDeploymentName, err) } + + svc := newService(echoExternalNodeDeploymentName, + map[string]string{"name": echoExternalNodeDeploymentName, "kind": kindEchoExternalNodeName}, + map[string]string{"kind": kindEchoExternalNodeName}, "http", 8080) + svc.Spec.ClusterIP = corev1.ClusterIPNone + svc.Spec.Type = corev1.ServiceTypeClusterIP + _, err := ct.clients.src.CreateService(ctx, ct.params.TestNamespace, svc, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("unable to create service %s: %w", echoExternalNodeDeploymentName, err) + } } } else { ct.Infof("Skipping tests that require a node Without Cilium") @@ -1243,6 +1253,17 @@ func (ct *ConnectivityTest) validateDeployment(ctx context.Context) error { port: uint32(ct.Params().ExternalDeploymentPort), // listen port of the echo server inside the container } } + + echoExternalServices, err := ct.clients.dst.ListServices(ctx, ct.params.TestNamespace, metav1.ListOptions{LabelSelector: "kind=" + kindEchoExternalNodeName}) + if err != nil { + return fmt.Errorf("unable to list echo external services: %w", err) + } + + for _, echoExternalService := range echoExternalServices.Items { + ct.echoExternalServices[echoExternalService.Name] = Service{ + Service: echoExternalService.DeepCopy(), + } + } } for _, cp := range ct.clientPods { diff --git a/connectivity/check/peer.go b/connectivity/check/peer.go index 5eb9ebff13..ee5586a717 100644 --- a/connectivity/check/peer.go +++ b/connectivity/check/peer.go @@ -201,7 +201,7 @@ func (s Service) Path() string { // Address returns the network address of the Service. func (s Service) Address(family features.IPFamily) string { // If the cluster IP is empty (headless service case) or the IP family is set to any, return the service name - if s.Service.Spec.ClusterIP == "" || family == features.IPFamilyAny { + if s.Service.Spec.ClusterIP == "" || s.Service.Spec.ClusterIP == corev1.ClusterIPNone || family == features.IPFamilyAny { return fmt.Sprintf("%s.%s", s.Service.Name, s.Service.Namespace) } @@ -256,6 +256,12 @@ func (s Service) ToNodeportService(node *corev1.Node) NodeportService { } } +func (s Service) ToEchoIPService() EchoIPService { + return EchoIPService{ + Service: s, + } +} + // NodeportService wraps a Service and exposes it through its nodeport, acting as a peer in a connectivity test. // It implements interface TestPeer. type NodeportService struct { @@ -499,3 +505,11 @@ type EchoIPPod struct { func (p EchoIPPod) Path() string { return p.path + "/client-ip" } + +type EchoIPService struct { + Service +} + +func (s EchoIPService) Path() string { + return s.URLPath + "/client-ip" +} diff --git a/connectivity/tests/egressgateway.go b/connectivity/tests/egressgateway.go index fbd1cb97d9..d6548855be 100644 --- a/connectivity/tests/egressgateway.go +++ b/connectivity/tests/egressgateway.go @@ -235,6 +235,26 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) { i++ } + // Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service using DNS) + i = 0 + for _, client := range ct.ClientPods() { + client := client + + for _, externalEchoSvc := range ct.EchoExternalServices() { + externalEcho := externalEchoSvc.ToEchoIPService() + + t.NewAction(s, fmt.Sprintf("curl-external-echo-service-%d", i), &client, externalEcho, features.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlCommandWithOutput(externalEcho, features.IPFamilyV4, "-4")) + clientIP := extractClientIPFromResponse(a.CmdOutput()) + + if !clientIP.Equal(egressGatewayNodeInternalIP) { + t.Fatal("Request reached external echo service with wrong source IP") + } + }) + i++ + } + } + // Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service) i = 0 for _, client := range ct.ClientPods() {