From 0329c768c914e1e4b0da7b5657e374ff700f2865 Mon Sep 17 00:00:00 2001 From: Sebastian Wicki Date: Tue, 6 Jun 2023 15:36:08 +0200 Subject: [PATCH] connectivity: Ability to add custom annotations to test deployment This commit adds the ability to add custom namespace and pod annotations to the `cilium connectivity test` deployment. The two flags are `--deployment-pod-annotations` and `--namespace-annotations`. For the pod annotations, we accept a JSON map which contains the deployment name as the key, and the annotations as a string-to-string map. For the namespace annotation, we simply expect a string-to-string map. We could have used Viper's `StringToString` map for the namespace annotations, but not for the pod annotations, since there we would need a "`StringToStringToString`" map. Therefore, to remain consistent between the two flags, both flags exclusively JSON syntax. If the deployment name in the annotations map ends with a `*` symbol, then we perform a longest prefix match. The flags are currently hidden, since we are not fully commited to this command-line syntax yet and it might still change in the future. Example: ``` $ cilium connectivity test \ --namespace-annotations='{"foo":"bar"}' \ --deployment-pod-annotations='{"client":{"baz":"qux"},"echo-*":{"quux":"corge"}}' ``` Signed-off-by: Sebastian Wicki --- connectivity/check/check.go | 2 + connectivity/check/deployment.go | 78 ++++++++++++++++++++------------ internal/cli/cmd/connectivity.go | 4 ++ 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/connectivity/check/check.go b/connectivity/check/check.go index 837f786e62..7e1bc87c10 100644 --- a/connectivity/check/check.go +++ b/connectivity/check/check.go @@ -53,6 +53,8 @@ type Parameters struct { IncludeUnsafeTests bool AgentPodSelector string NodeSelector map[string]string + DeploymentAnnotations annotationsMap + NamespaceAnnotations annotations ExternalTarget string ExternalCIDR string ExternalIP string diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go index 231bd0b5d3..39a84a37e6 100644 --- a/connectivity/check/deployment.go +++ b/connectivity/check/deployment.go @@ -104,6 +104,7 @@ type deploymentParameters struct { NodeSelector map[string]string ReadinessProbe *corev1.Probe Labels map[string]string + Annotations map[string]string HostNetwork bool Tolerations []corev1.Toleration } @@ -132,6 +133,7 @@ func newDeployment(p deploymentParameters) *appsv1.Deployment { "name": p.Name, "kind": p.Kind, }, + Annotations: p.Annotations, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -393,7 +395,12 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { _, err := ct.clients.src.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{}) if err != nil { ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.src.ClusterName(), ct.params.TestNamespace) - namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ct.params.TestNamespace}} + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ct.params.TestNamespace, + Annotations: ct.params.NamespaceAnnotations, + }, + } _, err = ct.clients.src.CreateNamespace(ctx, namespace, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err) @@ -443,7 +450,8 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "client": "role", }, - Command: []string{"/bin/bash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations.Match(nm.ClientName()), + Command: []string{"/bin/bash", "-c", "sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -480,9 +488,10 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "server": "role", }, - Port: 5001, - Image: ct.params.PerformanceImage, - Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations.Match(nm.ServerName()), + Port: 5001, + Image: ct.params.PerformanceImage, + Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -535,8 +544,9 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Labels: map[string]string{ "client": "role", }, - Image: ct.params.PerformanceImage, - Command: []string{"/bin/bash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations.Match(nm.ClientAcrossName()), + Image: ct.params.PerformanceImage, + Command: []string{"/bin/bash", "-c", "sleep 10000000"}, Affinity: &corev1.Affinity{ NodeAffinity: &corev1.NodeAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ @@ -584,7 +594,12 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { _, err = ct.clients.dst.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{}) if err != nil { ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.dst.ClusterName(), ct.params.TestNamespace) - namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ct.params.TestNamespace}} + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ct.params.TestNamespace, + Annotations: ct.params.NamespaceAnnotations, + }, + } _, err = ct.clients.dst.CreateNamespace(ctx, namespace, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err) @@ -658,13 +673,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { ct.Logf("✨ [%s] Deploying same-node deployment...", ct.clients.src.ClusterName()) containerPort := 8080 echoDeployment := newDeploymentWithDNSTestServer(deploymentParameters{ - Name: echoSameNodeDeploymentName, - Kind: kindEchoName, - Port: containerPort, - NamedPort: "http-8080", - HostPort: hostPort, - Image: ct.params.JSONMockImage, - Labels: map[string]string{"other": "echo"}, + Name: echoSameNodeDeploymentName, + Kind: kindEchoName, + Port: containerPort, + NamedPort: "http-8080", + HostPort: hostPort, + Image: ct.params.JSONMockImage, + Labels: map[string]string{"other": "echo"}, + Annotations: ct.params.DeploymentAnnotations.Match(echoSameNodeDeploymentName), Affinity: &corev1.Affinity{ PodAffinity: &corev1.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -701,6 +717,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Port: 8080, Image: ct.params.CurlImage, Command: []string{"/bin/ash", "-c", "sleep 10000000"}, + Annotations: ct.params.DeploymentAnnotations.Match(clientDeploymentName), NodeSelector: ct.params.NodeSelector, }) _, err = ct.clients.src.CreateServiceAccount(ctx, ct.params.TestNamespace, k8s.NewServiceAccount(clientDeploymentName), metav1.CreateOptions{}) @@ -718,13 +735,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { if err != nil { ct.Logf("✨ [%s] Deploying %s deployment...", ct.clients.src.ClusterName(), client2DeploymentName) clientDeployment := newDeployment(deploymentParameters{ - Name: client2DeploymentName, - Kind: kindClientName, - NamedPort: "http-8080", - Port: 8080, - Image: ct.params.CurlImage, - Command: []string{"/bin/ash", "-c", "sleep 10000000"}, - Labels: map[string]string{"other": "client"}, + Name: client2DeploymentName, + Kind: kindClientName, + NamedPort: "http-8080", + Port: 8080, + Image: ct.params.CurlImage, + Command: []string{"/bin/ash", "-c", "sleep 10000000"}, + Labels: map[string]string{"other": "client"}, + Annotations: ct.params.DeploymentAnnotations.Match(client2DeploymentName), Affinity: &corev1.Affinity{ PodAffinity: &corev1.PodAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -774,13 +792,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { ct.Logf("✨ [%s] Deploying other-node deployment...", ct.clients.dst.ClusterName()) containerPort := 8080 echoOtherNodeDeployment := newDeploymentWithDNSTestServer(deploymentParameters{ - Name: echoOtherNodeDeploymentName, - Kind: kindEchoName, - NamedPort: "http-8080", - Port: containerPort, - HostPort: hostPort, - Image: ct.params.JSONMockImage, - Labels: map[string]string{"first": "echo"}, + Name: echoOtherNodeDeploymentName, + Kind: kindEchoName, + NamedPort: "http-8080", + Port: containerPort, + HostPort: hostPort, + Image: ct.params.JSONMockImage, + Labels: map[string]string{"first": "echo"}, + Annotations: ct.params.DeploymentAnnotations.Match(echoOtherNodeDeploymentName), Affinity: &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ @@ -863,6 +882,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { HostPort: 8080, Image: ct.params.JSONMockImage, Labels: map[string]string{"external": "echo"}, + Annotations: ct.params.DeploymentAnnotations.Match(echoExternalNodeDeploymentName), NodeSelector: map[string]string{"cilium.io/no-schedule": "true"}, ReadinessProbe: newLocalReadinessProbe(containerPort, "/"), HostNetwork: true, diff --git a/internal/cli/cmd/connectivity.go b/internal/cli/cmd/connectivity.go index 8f0d2321a2..e2b4bc0baa 100644 --- a/internal/cli/cmd/connectivity.go +++ b/internal/cli/cmd/connectivity.go @@ -120,6 +120,10 @@ func newCmdConnectivityTest(hooks Hooks) *cobra.Command { cmd.Flags().StringVar(¶ms.AgentDaemonSetName, "agent-daemonset-name", defaults.AgentDaemonSetName, "Name of cilium agent daemonset") cmd.Flags().StringVar(¶ms.AgentPodSelector, "agent-pod-selector", defaults.AgentPodSelector, "Label on cilium-agent pods to select with") cmd.Flags().StringToStringVar(¶ms.NodeSelector, "node-selector", map[string]string{}, "Restrict connectivity test pods to nodes matching this label") + cmd.Flags().Var(¶ms.NamespaceAnnotations, "namespace-annotations", "Add annotations to the connectivity test namespace, e.g. '{\"foo\":\"bar\"}'") + cmd.Flags().MarkHidden("namespace-annotations") + cmd.Flags().Var(¶ms.DeploymentAnnotations, "deployment-pod-annotations", "Add annotations to the connectivity test pods, e.g. '{\"client\":{\"foo\":\"bar\"}}'") + cmd.Flags().MarkHidden("deployment-pod-annotations") cmd.Flags().StringVar(¶ms.MultiCluster, "multi-cluster", "", "Test across clusters to given context") cmd.Flags().StringSliceVar(&tests, "test", []string{}, "Run tests that match one of the given regular expressions, skip tests by starting the expression with '!', target Scenarios with e.g. '/pod-to-cidr'") cmd.Flags().StringVar(¶ms.FlowValidation, "flow-validation", check.FlowValidationModeWarning, "Enable Hubble flow validation { disabled | warning | strict }")