Skip to content

Commit

Permalink
connectivity: Ability to add custom annotations to test deployment
Browse files Browse the repository at this point in the history 
This commit adds the ability to add custom namespace and pod annotations
to the `cilium connectivity test` deployment. The two flags are
`--deployment-pod-annotations` and `--namespace-annotations`.

For the pod annotations, we accept a JSON map which contains the
deployment name as the key, and the annotations as a string-to-string
map. For the namespace annotation, we simply expect a string-to-string
map.

We could have used Viper's `StringToString` map for the namespace
annotations, but not for the pod annotations, since there we would need
a "`StringToStringToString`" map. Therefore, to remain consistent
between the two flags, both flags exclusively JSON syntax.

The flags are currently hidden, since we are not fully commited to this
command-line syntax yet and it might still change in the future.

Example:

```
$ cilium connectivity test \
    --namespace-annotations '{"foo":"bar"}' \
    --deployment-pod-annotations='{"client":{"baz":"qux"},"echo-same-node":{"quux":"corge"}}'
```

Signed-off-by: Sebastian Wicki <[email protected]>
  • Loading branch information
@gandro
gandro committed Jun 5, 2023
1 parent a6a59d8 commit 90c0189
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 29 deletions.
43 changes: 43 additions & 0 deletions connectivity/check/check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package check

import (
"encoding/json"
"fmt"
"io"
"regexp"
Expand Down Expand Up @@ -51,6 +52,8 @@ type Parameters struct {
IncludeUnsafeTests bool
AgentPodSelector string
NodeSelector map[string]string
DeploymentAnnotations annotationsMap
NamespaceAnnotations annotations
ExternalTarget string
ExternalCIDR string
ExternalIP string
Expand Down Expand Up @@ -87,6 +90,46 @@ type nodesWithoutCiliumIP struct {
Mask int
}

type annotations map[string]string

func marshalMap[M ~map[K]V, K comparable, V any](m *M) string {
if m == nil || len(*m) == 0 {
return "{}" // avoids printing "null" for nil map
}

b, err := json.Marshal(*m)
if err != nil {
return fmt.Sprintf("error: %s", err)
}
return string(b)
}

func (a *annotations) String() string {
return marshalMap(a)
}

func (a *annotations) Set(s string) error {
return json.Unmarshal([]byte(s), a)
}

func (a *annotations) Type() string {
return "json"
}

type annotationsMap map[string]annotations

func (a *annotationsMap) String() string {
return marshalMap(a)
}

func (a *annotationsMap) Set(s string) error {
return json.Unmarshal([]byte(s), a)
}

func (a *annotationsMap) Type() string {
return "json"
}

func (p Parameters) ciliumEndpointTimeout() time.Duration {
return 5 * time.Minute
}
Expand Down
78 changes: 49 additions & 29 deletions connectivity/check/deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ type deploymentParameters struct {
NodeSelector map[string]string
ReadinessProbe *corev1.Probe
Labels map[string]string
Annotations map[string]string
HostNetwork bool
Tolerations []corev1.Toleration
}
Expand Down Expand Up @@ -132,6 +133,7 @@ func newDeployment(p deploymentParameters) *appsv1.Deployment {
"name": p.Name,
"kind": p.Kind,
},
Annotations: p.Annotations,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
Expand Down Expand Up @@ -393,7 +395,12 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
_, err := ct.clients.src.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{})
if err != nil {
ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.src.ClusterName(), ct.params.TestNamespace)
namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ct.params.TestNamespace}}
namespace := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: ct.params.TestNamespace,
Annotations: ct.params.NamespaceAnnotations,
},
}
_, err = ct.clients.src.CreateNamespace(ctx, namespace, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err)
Expand Down Expand Up @@ -443,7 +450,8 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
Labels: map[string]string{
"client": "role",
},
Command: []string{"/bin/bash", "-c", "sleep 10000000"},
Annotations: ct.params.DeploymentAnnotations[nm.ClientName()],
Command: []string{"/bin/bash", "-c", "sleep 10000000"},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
Expand Down Expand Up @@ -480,9 +488,10 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
Labels: map[string]string{
"server": "role",
},
Port: 5001,
Image: ct.params.PerformanceImage,
Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"},
Annotations: ct.params.DeploymentAnnotations[nm.ServerName()],
Port: 5001,
Image: ct.params.PerformanceImage,
Command: []string{"/bin/bash", "-c", "netserver;sleep 10000000"},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
Expand Down Expand Up @@ -535,8 +544,9 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
Labels: map[string]string{
"client": "role",
},
Image: ct.params.PerformanceImage,
Command: []string{"/bin/bash", "-c", "sleep 10000000"},
Annotations: ct.params.DeploymentAnnotations[nm.ClientAcrossName()],
Image: ct.params.PerformanceImage,
Command: []string{"/bin/bash", "-c", "sleep 10000000"},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{
Expand Down Expand Up @@ -584,7 +594,12 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
_, err = ct.clients.dst.GetNamespace(ctx, ct.params.TestNamespace, metav1.GetOptions{})
if err != nil {
ct.Logf("✨ [%s] Creating namespace %s for connectivity check...", ct.clients.dst.ClusterName(), ct.params.TestNamespace)
namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ct.params.TestNamespace}}
namespace := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: ct.params.TestNamespace,
Annotations: ct.params.NamespaceAnnotations,
},
}
_, err = ct.clients.dst.CreateNamespace(ctx, namespace, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("unable to create namespace %s: %s", ct.params.TestNamespace, err)
Expand Down Expand Up @@ -658,13 +673,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
ct.Logf("✨ [%s] Deploying same-node deployment...", ct.clients.src.ClusterName())
containerPort := 8080
echoDeployment := newDeploymentWithDNSTestServer(deploymentParameters{
Name: echoSameNodeDeploymentName,
Kind: kindEchoName,
Port: containerPort,
NamedPort: "http-8080",
HostPort: hostPort,
Image: ct.params.JSONMockImage,
Labels: map[string]string{"other": "echo"},
Name: echoSameNodeDeploymentName,
Kind: kindEchoName,
Port: containerPort,
NamedPort: "http-8080",
HostPort: hostPort,
Image: ct.params.JSONMockImage,
Labels: map[string]string{"other": "echo"},
Annotations: ct.params.DeploymentAnnotations[echoSameNodeDeploymentName],
Affinity: &corev1.Affinity{
PodAffinity: &corev1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
Expand Down Expand Up @@ -701,6 +717,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
Port: 8080,
Image: ct.params.CurlImage,
Command: []string{"/bin/ash", "-c", "sleep 10000000"},
Annotations: ct.params.DeploymentAnnotations[clientDeploymentName],
NodeSelector: ct.params.NodeSelector,
})
_, err = ct.clients.src.CreateServiceAccount(ctx, ct.params.TestNamespace, k8s.NewServiceAccount(clientDeploymentName), metav1.CreateOptions{})
Expand All @@ -718,13 +735,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
if err != nil {
ct.Logf("✨ [%s] Deploying %s deployment...", ct.clients.src.ClusterName(), client2DeploymentName)
clientDeployment := newDeployment(deploymentParameters{
Name: client2DeploymentName,
Kind: kindClientName,
NamedPort: "http-8080",
Port: 8080,
Image: ct.params.CurlImage,
Command: []string{"/bin/ash", "-c", "sleep 10000000"},
Labels: map[string]string{"other": "client"},
Name: client2DeploymentName,
Kind: kindClientName,
NamedPort: "http-8080",
Port: 8080,
Image: ct.params.CurlImage,
Command: []string{"/bin/ash", "-c", "sleep 10000000"},
Labels: map[string]string{"other": "client"},
Annotations: ct.params.DeploymentAnnotations[client2DeploymentName],
Affinity: &corev1.Affinity{
PodAffinity: &corev1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
Expand Down Expand Up @@ -774,13 +792,14 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
ct.Logf("✨ [%s] Deploying other-node deployment...", ct.clients.dst.ClusterName())
containerPort := 8080
echoOtherNodeDeployment := newDeploymentWithDNSTestServer(deploymentParameters{
Name: echoOtherNodeDeploymentName,
Kind: kindEchoName,
NamedPort: "http-8080",
Port: containerPort,
HostPort: hostPort,
Image: ct.params.JSONMockImage,
Labels: map[string]string{"first": "echo"},
Name: echoOtherNodeDeploymentName,
Kind: kindEchoName,
NamedPort: "http-8080",
Port: containerPort,
HostPort: hostPort,
Image: ct.params.JSONMockImage,
Labels: map[string]string{"first": "echo"},
Annotations: ct.params.DeploymentAnnotations[echoOtherNodeDeploymentName],
Affinity: &corev1.Affinity{
PodAntiAffinity: &corev1.PodAntiAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
Expand Down Expand Up @@ -863,6 +882,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
HostPort: 8080,
Image: ct.params.JSONMockImage,
Labels: map[string]string{"external": "echo"},
Annotations: ct.params.DeploymentAnnotations[echoExternalNodeDeploymentName],
NodeSelector: map[string]string{"cilium.io/no-schedule": "true"},
ReadinessProbe: newLocalReadinessProbe(containerPort, "/"),
HostNetwork: true,
Expand Down
4 changes: 4 additions & 0 deletions internal/cli/cmd/connectivity.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,10 @@ func newCmdConnectivityTest(hooks Hooks) *cobra.Command {
cmd.Flags().StringVar(&params.AgentDaemonSetName, "agent-daemonset-name", defaults.AgentDaemonSetName, "Name of cilium agent daemonset")
cmd.Flags().StringVar(&params.AgentPodSelector, "agent-pod-selector", defaults.AgentPodSelector, "Label on cilium-agent pods to select with")
cmd.Flags().StringToStringVar(&params.NodeSelector, "node-selector", map[string]string{}, "Restrict connectivity test pods to nodes matching this label")
cmd.Flags().Var(&params.NamespaceAnnotations, "namespace-annotations", "Add annotations to the connectivity test namespace, e.g. '{\"foo\":\"bar\"}'")
cmd.Flags().MarkHidden("namespace-annotations")
cmd.Flags().Var(&params.DeploymentAnnotations, "deployment-pod-annotations", "Add annotations to the connectivity test pods, e.g. '{\"client\":{\"foo\":\"bar\"}}'")
cmd.Flags().MarkHidden("deployment-pod-annotations")
cmd.Flags().StringVar(&params.MultiCluster, "multi-cluster", "", "Test across clusters to given context")
cmd.Flags().StringSliceVar(&tests, "test", []string{}, "Run tests that match one of the given regular expressions, skip tests by starting the expression with '!', target Scenarios with e.g. '/pod-to-cidr'")
cmd.Flags().StringVar(&params.FlowValidation, "flow-validation", check.FlowValidationModeWarning, "Enable Hubble flow validation { disabled | warning | strict }")
Expand Down

0 comments on commit 90c0189

Please sign in to comment.