From 689056e3cf94848758fd4945ab0a5095160e7bc6 Mon Sep 17 00:00:00 2001 From: Sebastian Wicki Date: Thu, 1 Jun 2023 17:43:07 +0200 Subject: [PATCH] connectivity: Ability to add custom annotations to test deployment This commit adds the ability to add custom namespace and pod annotations to the `cilium connectivity test` deployment. The two flags are `--deployment-pod-annotations` and `--namespace-annotations`. For the pod annotations, we accept a JSON map which contains the deployment name as the key, and the annotations as a string-to-string map. For the namespace annotation, we simply expect a string-to-string map. We could have used Viper's `StringToString` map for the namespace annotations, but not for the pod annotations, since there we would need a "`StringToStringToString`" map. Therefore, to remain consistent between the two flags, both flags exclusively JSON syntax. The flags are currently hidden, since we are not fully commited to this command-line syntax yet and it might still change in the future. Signed-off-by: Sebastian Wicki --- connectivity/check/check.go | 43 ++++++++++++++++++++ connectivity/check/deployment.go | 68 ++++++++++++++++++-------------- internal/cli/cmd/connectivity.go | 4 ++ 3 files changed, 86 insertions(+), 29 deletions(-) diff --git a/connectivity/check/check.go b/connectivity/check/check.go index babf210571..6f644c51d8 100644 --- a/connectivity/check/check.go +++ b/connectivity/check/check.go @@ -4,6 +4,7 @@ package check import ( + "encoding/json" "fmt" "io" "regexp" @@ -51,6 +52,8 @@ type Parameters struct { IncludeUnsafeTests bool AgentPodSelector string NodeSelector map[string]string + DeploymentAnnotations annotationsMap + NamespaceAnnotations annotations ExternalTarget string ExternalCIDR string ExternalIP string @@ -87,6 +90,46 @@ type nodesWithoutCiliumIP struct { Mask int } +type annotations map[string]string + +func marshalMap[M ~map[K]V, K comparable, V any](m *M) string { + if m == nil || len(*m) == 0 { + return "{}" // avoids printing "null" for nil map + } + + b, err := json.Marshal(*m) + if err != nil { + return fmt.Sprintf("error: %s", err) + } + return string(b) +} + +func (a *annotations) String() string { + return marshalMap(a) +} + +func (a *annotations) Set(s string) error { + return json.Unmarshal([]byte(s), a) +} + +func (a *annotations) Type() string { + return "json" +} + +type annotationsMap map[string]annotations + +func (a *annotationsMap) String() string { + return marshalMap(a) +} + +func (a *annotationsMap) Set(s string) error { + return json.Unmarshal([]byte(s), a) +} + +func (a *annotationsMap) Type() string { + return "json" +} + func (p Parameters) ciliumEndpointTimeout() time.Duration { return 5 * time.Minute } diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go index d2b2bc1038..9bf02380fe 100644 --- a/connectivity/check/deployment.go +++ b/connectivity/check/deployment.go @@ -104,6 +104,7 @@ type deploymentParameters struct { NodeSelector map[string]string ReadinessProbe *corev1.Probe Labels map[string]string + Annotations map[string]string HostNetwork bool Tolerations []corev1.Toleration } @@ -132,6 +133,7 @@ func newDeployment(p deploymentParameters) *appsv1.Deployment { "name": p.Name, "kind": p.Kind, }, + Annotations: p.Annotations, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -393,7 +395,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { _, err := ct.clients.src.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{}) if err != nil { ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.src.ClusterName(), ct.params.TestNamespace) - meta := metav1.ObjectMeta{Name: ct.params.TestNamespace} + meta := metav1.ObjectMeta{Name: ct.params.TestNamespace, Annotations: ct.params.NamespaceAnnotations} _, err = ct.clients.src.CreateNamespace(ctx, meta, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err) @@ -443,7 +445,8 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "client": "role", }, - Command: []string{"/bin/bash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations[nm.ClientName()], + Command: []string{"/bin/bash", "-c", "sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -480,9 +483,10 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "server": "role", }, - Port: 5001, - Image: ct.params.PerformanceImage, - Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations[nm.ServerName()], + Port: 5001, + Image: ct.params.PerformanceImage, + Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -535,8 +539,9 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "client": "role", }, - Image: ct.params.PerformanceImage, - Command: []string{"/bin/bash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations[nm.ClientAcrossName()], + Image: ct.params.PerformanceImage, + Command: []string{"/bin/bash", "-c", "sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -584,7 +589,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { _, err = ct.clients.dst.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{}) if err != nil { ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.dst.ClusterName(), ct.params.TestNamespace) - meta := metav1.ObjectMeta{Name: ct.params.TestNamespace} + meta := metav1.ObjectMeta{Name: ct.params.TestNamespace, Annotations: ct.params.NamespaceAnnotations} _, err = ct.clients.src.CreateNamespace(ctx, meta, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err) @@ -658,13 +663,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { ct.Logf("✨ [%s] Deploying same-node deployment...", ct.clients.src.ClusterName()) containerPort := 8080 echoDeployment := newDeploymentWithDNSTestServer(deploymentParameters{ - Name: echoSameNodeDeploymentName, - Kind: kindEchoName, - Port: containerPort, - NamedPort: "http-8080", - HostPort: hostPort, - Image: ct.params.JSONMockImage, - Labels: map[string]string{"other": "echo"}, + Name: echoSameNodeDeploymentName, + Kind: kindEchoName, + Port: containerPort, + NamedPort: "http-8080", + HostPort: hostPort, + Image: ct.params.JSONMockImage, + Labels: map[string]string{"other": "echo"}, + Annotations: ct.params.DeploymentAnnotations[echoSameNodeDeploymentName], Affinity: &corev1.Affinity{ PodAffinity: &corev1.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -701,6 +707,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Port: 8080, Image: ct.params.CurlImage, Command: []string{"/bin/ash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations[clientDeploymentName], NodeSelector: ct.params.NodeSelector, }) _, err = ct.clients.src.CreateServiceAccount(ctx, ct.params.TestNamespace, k8s.NewServiceAccount(clientDeploymentName), metav1.CreateOptions{}) @@ -718,13 +725,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { if err != nil { ct.Logf("✨ [%s] Deploying %s deployment...", ct.clients.src.ClusterName(), client2DeploymentName) clientDeployment := newDeployment(deploymentParameters{ - Name: client2DeploymentName, - Kind: kindClientName, - NamedPort: "http-8080", - Port: 8080, - Image: ct.params.CurlImage, - Command: []string{"/bin/ash", "-c", "sleep 10000000"}, - Labels: map[string]string{"other": "client"}, + Name: client2DeploymentName, + Kind: kindClientName, + NamedPort: "http-8080", + Port: 8080, + Image: ct.params.CurlImage, + Command: []string{"/bin/ash", "-c", "sleep 10000000"}, + Labels: map[string]string{"other": "client"}, + Annotations: ct.params.DeploymentAnnotations[client2DeploymentName], Affinity: &corev1.Affinity{ PodAffinity: &corev1.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -774,13 +782,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { ct.Logf("✨ [%s] Deploying other-node deployment...", ct.clients.dst.ClusterName()) containerPort := 8080 echoOtherNodeDeployment := newDeploymentWithDNSTestServer(deploymentParameters{ - Name: echoOtherNodeDeploymentName, - Kind: kindEchoName, - NamedPort: "http-8080", - Port: containerPort, - HostPort: hostPort, - Image: ct.params.JSONMockImage, - Labels: map[string]string{"first": "echo"}, + Name: echoOtherNodeDeploymentName, + Kind: kindEchoName, + NamedPort: "http-8080", + Port: containerPort, + HostPort: hostPort, + Image: ct.params.JSONMockImage, + Labels: map[string]string{"first": "echo"}, + Annotations: ct.params.DeploymentAnnotations[echoOtherNodeDeploymentName], Affinity: &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -863,6 +872,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { HostPort: 8080, Image: ct.params.JSONMockImage, Labels: map[string]string{"external": "echo"}, + Annotations: ct.params.DeploymentAnnotations[echoExternalNodeDeploymentName], NodeSelector: map[string]string{"cilium.io/no-schedule": "true"}, ReadinessProbe: newLocalReadinessProbe(containerPort, "/"), HostNetwork: true, diff --git a/internal/cli/cmd/connectivity.go b/internal/cli/cmd/connectivity.go index 8f0d2321a2..e2b4bc0baa 100644 --- a/internal/cli/cmd/connectivity.go +++ b/internal/cli/cmd/connectivity.go @@ -120,6 +120,10 @@ func newCmdConnectivityTest(hooks Hooks) *cobra.Command { cmd.Flags().StringVar(¶ms.AgentDaemonSetName, "agent-daemonset-name", defaults.AgentDaemonSetName, "Name of cilium agent daemonset") cmd.Flags().StringVar(¶ms.AgentPodSelector, "agent-pod-selector", defaults.AgentPodSelector, "Label on cilium-agent pods to select with") cmd.Flags().StringToStringVar(¶ms.NodeSelector, "node-selector", map[string]string{}, "Restrict connectivity test pods to nodes matching this label") + cmd.Flags().Var(¶ms.NamespaceAnnotations, "namespace-annotations", "Add annotations to the connectivity test namespace, e.g. '{\"foo\":\"bar\"}'") + cmd.Flags().MarkHidden("namespace-annotations") + cmd.Flags().Var(¶ms.DeploymentAnnotations, "deployment-pod-annotations", "Add annotations to the connectivity test pods, e.g. '{\"client\":{\"foo\":\"bar\"}}'") + cmd.Flags().MarkHidden("deployment-pod-annotations") cmd.Flags().StringVar(¶ms.MultiCluster, "multi-cluster", "", "Test across clusters to given context") cmd.Flags().StringSliceVar(&tests, "test", []string{}, "Run tests that match one of the given regular expressions, skip tests by starting the expression with '!', target Scenarios with e.g. '/pod-to-cidr'") cmd.Flags().StringVar(¶ms.FlowValidation, "flow-validation", check.FlowValidationModeWarning, "Enable Hubble flow validation { disabled | warning | strict }")