From 60ae1d5fc54dedb1efbdcdf506a20fab299725bf Mon Sep 17 00:00:00 2001 From: Birol Bilgin Date: Tue, 10 Jan 2023 14:43:42 +0100 Subject: [PATCH] connectivity: Make pod to world test configurable A configurable domain allows us to run the PodToWord test in an environment where external traffic only allow for specific domains. This also gives us the ability to swap the domain in test environments to better handle flaky tests. Signed-off-by: Birol Bilgin --- connectivity/check/check.go | 1 + .../client-egress-l7-http-named-port.yaml | 8 ++-- .../manifests/client-egress-l7-http.yaml | 8 ++-- ...lient-egress-to-fqdns-one-one-one-one.yaml | 4 +- connectivity/suite.go | 42 +++++++++++++------ connectivity/tests/world.go | 16 +++---- internal/cli/cmd/connectivity.go | 1 + internal/utils/template.go | 21 ++++++++++ 8 files changed, 71 insertions(+), 30 deletions(-) create mode 100644 internal/utils/template.go diff --git a/connectivity/check/check.go b/connectivity/check/check.go index 8ceab7ed20..d4ab91b889 100644 --- a/connectivity/check/check.go +++ b/connectivity/check/check.go @@ -50,6 +50,7 @@ type Parameters struct { DNSTestServerImage string Datapath bool AgentPodSelector string + ExternalTarget string K8sVersion string HelmChartDirectory string diff --git a/connectivity/manifests/client-egress-l7-http-named-port.yaml b/connectivity/manifests/client-egress-l7-http-named-port.yaml index 84de9507a1..27471366a4 100644 --- a/connectivity/manifests/client-egress-l7-http-named-port.yaml +++ b/connectivity/manifests/client-egress-l7-http-named-port.yaml @@ -1,5 +1,5 @@ --- -# client2 is allowed to contact one.one.one.one and the echo Pod +# client2 is allowed to contact {{.ExternalTarget}} and the echo Pod # on port http-8080. HTTP introspection is enabled for client2. # The toFQDNs section relies on DNS introspection being performed by # the client-egress-only-dns policy. @@ -8,7 +8,7 @@ kind: CiliumNetworkPolicy metadata: name: client-egress-l7-http-named-port spec: - description: "Allow GET one.one.one.one:80/ and GET :/ from client2" + description: "Allow GET {{.ExternalTarget}}:80/ and GET :/ from client2" endpointSelector: matchLabels: other: client @@ -25,9 +25,9 @@ spec: http: - method: "GET" path: "/" - # Allow GET / requests, only towards one.one.one.one. + # Allow GET / requests, only towards {{.ExternalTarget}}. - toFQDNs: - - matchName: "one.one.one.one" + - matchName: "{{.ExternalTarget}}" toPorts: - ports: - port: "80" diff --git a/connectivity/manifests/client-egress-l7-http.yaml b/connectivity/manifests/client-egress-l7-http.yaml index 4fdb89162c..e1d2b82c23 100644 --- a/connectivity/manifests/client-egress-l7-http.yaml +++ b/connectivity/manifests/client-egress-l7-http.yaml @@ -1,5 +1,5 @@ --- -# client2 is allowed to contact one.one.one.one/ on port 80 and the echo Pod +# client2 is allowed to contact {{.ExternalTarget}}/ on port 80 and the echo Pod # on port 8080. HTTP introspection is enabled for client2. # The toFQDNs section relies on DNS introspection being performed by # the client-egress-only-dns policy. @@ -8,7 +8,7 @@ kind: CiliumNetworkPolicy metadata: name: client-egress-l7-http spec: - description: "Allow GET one.one.one.one:80/ and GET :8080/ from client2" + description: "Allow GET {{.ExternalTarget}}:80/ and GET :8080/ from client2" endpointSelector: matchLabels: other: client @@ -25,9 +25,9 @@ spec: http: - method: "GET" path: "/" - # Allow GET / requests, only towards one.one.one.one. + # Allow GET / requests, only towards {{.ExternalTarget}}. - toFQDNs: - - matchName: "one.one.one.one" + - matchName: "{{.ExternalTarget}}" toPorts: - ports: - port: "80" diff --git a/connectivity/manifests/client-egress-to-fqdns-one-one-one-one.yaml b/connectivity/manifests/client-egress-to-fqdns-one-one-one-one.yaml index 4471c31011..3cf848ef07 100644 --- a/connectivity/manifests/client-egress-to-fqdns-one-one-one-one.yaml +++ b/connectivity/manifests/client-egress-to-fqdns-one-one-one-one.yaml @@ -1,7 +1,7 @@ apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy metadata: - name: client-egress-to-fqdns-one-one-one-one + name: client-egress-to-fqdns-{{.ExternalTarget}} spec: endpointSelector: matchLabels: @@ -16,7 +16,7 @@ spec: - method: "GET" path: "/" toFQDNs: - - matchName: "one.one.one.one" + - matchName: "{{.ExternalTarget}}" - toPorts: - ports: - port: "53" diff --git a/connectivity/suite.go b/connectivity/suite.go index 8fb4cfb1b9..62874ebf9f 100644 --- a/connectivity/suite.go +++ b/connectivity/suite.go @@ -6,6 +6,7 @@ package connectivity import ( "context" _ "embed" + "fmt" "github.com/blang/semver/v4" @@ -121,6 +122,21 @@ func Run(ctx context.Context, ct *check.ConnectivityTest) error { return err } + renderedTemplates := map[string]string{} + + // render templates, if any problems fail early + for key, temp := range map[string]string{ + "clientEgressL7HTTPPolicyYAML": clientEgressL7HTTPPolicyYAML, + "clientEgressL7HTTPNamedPortPolicyYAML": clientEgressL7HTTPNamedPortPolicyYAML, + "clientEgressToFQDNsCiliumIOPolicyYAML": clientEgressToFQDNsCiliumIOPolicyYAML, + } { + val, err := utils.RenderTemplate(temp, ct.Params()) + if err != nil { + return err + } + renderedTemplates[key] = val + } + var v semver.Version if assumeCiliumVersion := ct.Params().AssumeCiliumVersion; assumeCiliumVersion != "" { ct.Warnf("Assuming Cilium version %s for connectivity tests", assumeCiliumVersion) @@ -594,16 +610,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest) error { // Test L7 HTTP introspection using an egress policy on the clients. ct.NewTest("client-egress-l7"). WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureL7Proxy)). - WithPolicy(clientEgressOnlyDNSPolicyYAML). // DNS resolution only - WithPolicy(clientEgressL7HTTPPolicyYAML). // L7 allow policy with HTTP introspection + WithPolicy(clientEgressOnlyDNSPolicyYAML). // DNS resolution only + WithPolicy(renderedTemplates["clientEgressL7HTTPPolicyYAML"]). // L7 allow policy with HTTP introspection WithScenarios( tests.PodToPod(), tests.PodToWorld(), ). WithExpectations(func(a *check.Action) (egress, ingress check.Result) { if a.Source().HasLabel("other", "client") && // Only client2 is allowed to make HTTP calls. - // Outbound HTTP to one.one.one.one is L7-introspected and allowed. - (a.Destination().Port() == 80 && a.Destination().Address() == "one.one.one.one" || + // Outbound HTTP to set domain-name defaults to one.one.one.one is L7-introspected and allowed. + (a.Destination().Port() == 80 && a.Destination().Address() == ct.Params().ExternalTarget || a.Destination().Port() == 8080) { // 8080 is traffic to echo Pod. if a.Destination().Path() == "/" || a.Destination().Path() == "" { egress = check.ResultOK @@ -622,16 +638,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest) error { // Test L7 HTTP named port introspection using an egress policy on the clients. ct.NewTest("client-egress-l7-named-port"). WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureL7Proxy)). - WithPolicy(clientEgressOnlyDNSPolicyYAML). // DNS resolution only - WithPolicy(clientEgressL7HTTPNamedPortPolicyYAML). // L7 allow policy with HTTP introspection (named port) + WithPolicy(clientEgressOnlyDNSPolicyYAML). // DNS resolution only + WithPolicy(renderedTemplates["clientEgressL7HTTPNamedPortPolicyYAML"]). // L7 allow policy with HTTP introspection (named port) WithScenarios( tests.PodToPod(), tests.PodToWorld(), ). WithExpectations(func(a *check.Action) (egress, ingress check.Result) { if a.Source().HasLabel("other", "client") && // Only client2 is allowed to make HTTP calls. - // Outbound HTTP to one.one.one.one is L7-introspected and allowed. - (a.Destination().Port() == 80 && a.Destination().Address() == "one.one.one.one" || + // Outbound HTTP to domain-name, default one.one.one.one, is L7-introspected and allowed. + (a.Destination().Port() == 80 && a.Destination().Address() == ct.Params().ExternalTarget || a.Destination().Port() == 8080) { // named port http-8080 is traffic to echo Pod. if a.Destination().Path() == "/" || a.Destination().Path() == "" { egress = check.ResultOK @@ -652,14 +668,14 @@ func Run(ctx context.Context, ct *check.ConnectivityTest) error { WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureL7Proxy)). WithScenarios( tests.PodToPod(), // connects to other Pods directly, no DNS - tests.PodToWorld(), // resolves one.one.one.one + tests.PodToWorld(), // resolves set domain-name defaults to one.one.one.one ). WithExpectations(func(a *check.Action) (egress check.Result, ingress check.Result) { return check.ResultDropCurlTimeout, check.ResultNone }) - // This policy only allows port 80 to "one.one.one.one". DNS proxy enabled. - ct.NewTest("to-fqdns").WithPolicy(clientEgressToFQDNsCiliumIOPolicyYAML). + // This policy only allows port 80 to domain-name, default one.one.one.one,. DNS proxy enabled. + ct.NewTest("to-fqdns").WithPolicy(renderedTemplates["clientEgressToFQDNsCiliumIOPolicyYAML"]). WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureL7Proxy)). WithScenarios( tests.PodToWorld(), @@ -680,12 +696,12 @@ func Run(ctx context.Context, ct *check.ConnectivityTest) error { return check.ResultDNSOKDropCurlHTTPError, check.ResultNone } - if a.Destination().Port() == 80 && a.Destination().Address() == "one.one.one.one" { + if a.Destination().Port() == 80 && a.Destination().Address() == ct.Params().ExternalTarget { if a.Destination().Path() == "/" || a.Destination().Path() == "" { egress = check.ResultDNSOK egress.HTTP = check.HTTP{ Method: "GET", - URL: "http://one.one.one.one/", + URL: fmt.Sprintf("http://%s/", ct.Params().ExternalTarget), } return egress, check.ResultNone } diff --git a/connectivity/tests/world.go b/connectivity/tests/world.go index dbd5b2b4f9..35a7252af6 100644 --- a/connectivity/tests/world.go +++ b/connectivity/tests/world.go @@ -17,16 +17,18 @@ func PodToWorld() check.Scenario { } // podToWorld implements a Scenario. -type podToWorld struct{} +type podToWorld struct { +} func (s *podToWorld) Name() string { return "pod-to-world" } func (s *podToWorld) Run(ctx context.Context, t *check.Test) { - http := check.HTTPEndpoint("one-one-one-one-http", "http://one.one.one.one") - https := check.HTTPEndpoint("one-one-one-one-https", "https://one.one.one.one") - httpsindex := check.HTTPEndpoint("one-one-one-one-https-index", "https://one.one.one.one/index.html") + extTarget := t.Context().Params().ExternalTarget + http := check.HTTPEndpoint(extTarget+"-http", "http://"+extTarget) + https := check.HTTPEndpoint(extTarget+"-https", "https://"+extTarget) + httpsindex := check.HTTPEndpoint(extTarget+"-https-index", fmt.Sprintf("https://%s/index.html", extTarget)) fp := check.FlowParameters{ DNSRequired: true, @@ -40,19 +42,19 @@ func (s *podToWorld) Run(ctx context.Context, t *check.Test) { client := client // copy to avoid memory aliasing when using reference // With http, over port 80. - t.NewAction(s, fmt.Sprintf("http-to-one-one-one-one-%d", i), &client, http).Run(func(a *check.Action) { + t.NewAction(s, fmt.Sprintf("http-to-%s-%d", extTarget, i), &client, http).Run(func(a *check.Action) { a.ExecInPod(ctx, ct.CurlCommand(http)) a.ValidateFlows(ctx, client, a.GetEgressRequirements(fp)) }) // With https, over port 443. - t.NewAction(s, fmt.Sprintf("https-to-one-one-one-one-%d", i), &client, https).Run(func(a *check.Action) { + t.NewAction(s, fmt.Sprintf("https-to-%s-%d", extTarget, i), &client, https).Run(func(a *check.Action) { a.ExecInPod(ctx, ct.CurlCommand(https)) a.ValidateFlows(ctx, client, a.GetEgressRequirements(fp)) }) // With https, over port 443, index.html. - t.NewAction(s, fmt.Sprintf("https-to-one-one-one-one-index-%d", i), &client, httpsindex).Run(func(a *check.Action) { + t.NewAction(s, fmt.Sprintf("https-to-%s-index-%d", extTarget, i), &client, httpsindex).Run(func(a *check.Action) { a.ExecInPod(ctx, ct.CurlCommand(httpsindex)) a.ValidateFlows(ctx, client, a.GetEgressRequirements(fp)) }) diff --git a/internal/cli/cmd/connectivity.go b/internal/cli/cmd/connectivity.go index 259317deee..89de6af40d 100644 --- a/internal/cli/cmd/connectivity.go +++ b/internal/cli/cmd/connectivity.go @@ -128,6 +128,7 @@ func newCmdConnectivityTest() *cobra.Command { cmd.Flags().BoolVarP(¶ms.Debug, "debug", "d", false, "Show debug messages") cmd.Flags().BoolVarP(¶ms.Timestamp, "timestamp", "t", false, "Show timestamp in messages") cmd.Flags().BoolVarP(¶ms.PauseOnFail, "pause-on-fail", "p", false, "Pause execution on test failure") + cmd.Flags().StringVar(¶ms.ExternalTarget, "external-target", "one.one.one.one", "Domain name to use as external target in connectivity tests") cmd.Flags().BoolVar(¶ms.SkipIPCacheCheck, "skip-ip-cache-check", true, "Skip IPCache check") cmd.Flags().MarkHidden("skip-ip-cache-check") cmd.Flags().BoolVar(¶ms.Datapath, "datapath", false, "Run datapath conformance tests") diff --git a/internal/utils/template.go b/internal/utils/template.go new file mode 100644 index 0000000000..9dafcec3aa --- /dev/null +++ b/internal/utils/template.go @@ -0,0 +1,21 @@ +package utils + +import ( + "bytes" + "text/template" +) + +// RenderTemplate executes temp with data and returns the result +func RenderTemplate(temp string, data any) (string, error) { + tm, err := template.New("template").Parse(temp) + if err != nil { + return "", err + } + + buf := bytes.NewBuffer(nil) + if err := tm.Execute(buf, data); err != nil { + return "", err + } + + return buf.String(), nil +}