From 00c8c68520813235b34baca6620b8f59c26c03fc Mon Sep 17 00:00:00 2001 From: Andrew Sauber <2046750+asauber@users.noreply.github.com> Date: Thu, 18 May 2023 08:26:38 -0400 Subject: [PATCH] helm mode: autodetect service parameters for clustermesh Classic mode uses the Kubernetes flavor to automatically set the clustermesh-apiserver Service type and LoadBalancer annotations if needed. This patch reimplements this logic for Helm mode. Signed-off-by: Andrew Sauber --- .../in-cluster-test-scripts/multicluster.sh | 11 ++- clustermesh/clustermesh.go | 74 +++++++++++++++++-- internal/cli/cmd/clustermesh.go | 2 +- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/.github/in-cluster-test-scripts/multicluster.sh b/.github/in-cluster-test-scripts/multicluster.sh index 2c01cc7d3b..09a1de371f 100644 --- a/.github/in-cluster-test-scripts/multicluster.sh +++ b/.github/in-cluster-test-scripts/multicluster.sh @@ -47,8 +47,9 @@ cilium --context "${CONTEXT1}" status --wait cilium --context "${CONTEXT2}" status --wait # Enable cluster mesh -cilium --context "${CONTEXT1}" clustermesh enable --service-type=LoadBalancer -cilium --context "${CONTEXT2}" clustermesh enable --service-type=LoadBalancer +# Test autodetection of service parameters for GKE +cilium --context "${CONTEXT1}" clustermesh enable +cilium --context "${CONTEXT2}" clustermesh enable # Copy the clustermesh secrets # TODO(ajs): Patch the connect command to expect the Helm secret name @@ -68,6 +69,12 @@ fi cilium --context "${CONTEXT1}" clustermesh status --wait cilium --context "${CONTEXT2}" clustermesh status --wait +# Print clustermesh Service annotations +printf "Service annotations for Cluster 1 %s\n" \ + $(kubectl --context "${CONTEXT1}" get svc -n kube-system clustermesh-apiserver -o jsonpath='{.metadata.annotations}') +printf "Service annotations for Cluster 2 %s\n" \ + $(kubectl --context "${CONTEXT2}" get svc -n kube-system clustermesh-apiserver -o jsonpath='{.metadata.annotations}') + # Connect clusters cilium --context "${CONTEXT1}" clustermesh connect --destination-context "${CONTEXT2}" diff --git a/clustermesh/clustermesh.go b/clustermesh/clustermesh.go index 3efffad58e..17d1f03ff9 100644 --- a/clustermesh/clustermesh.go +++ b/clustermesh/clustermesh.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net" + "os" "regexp" "strconv" "strings" @@ -1783,13 +1784,74 @@ func (k *K8sClusterMesh) ExternalWorkloadStatus(ctx context.Context, names []str return err } -func EnableWithHelm(ctx context.Context, k8sClient *k8s.Client, params Parameters) error { - helmStrValues := []string{ - "clustermesh.useAPIServer=true", - fmt.Sprintf("clustermesh.apiserver.service.type=%s", params.ServiceType), - fmt.Sprintf("externalWorkloads.enabled=%t", params.EnableExternalWorkloads), +func log(format string, a ...interface{}) { + // TODO (ajs): make logger configurable + fmt.Fprintf(os.Stdout, format+"\n", a...) +} + +func generateEnableHelmValues(params Parameters, flavor k8s.Flavor) (map[string]interface{}, error) { + helmVals := map[string]interface{}{ + "clustermesh": map[string]interface{}{ + "useAPIServer": true, + }, + "externalWorkloads": map[string]interface{}{ + "enabled": params.EnableExternalWorkloads, + }, } - vals, err := helm.ParseVals(helmStrValues) + + if params.ServiceType == "" { + switch flavor.Kind { + case k8s.KindGKE: + log("🔮 Auto-exposing service within GCP VPC (cloud.google.com/load-balancer-type=Internal)") + helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{ + "service": map[string]interface{}{ + "type": "LoadBalancer", + "annotations": map[string]interface{}{ + "cloud.google.com/load-balancer-type": "Internal", + // Allows cross-region access + "networking.gke.io/internal-load-balancer-allow-global-access": "true", + }, + }, + } + case k8s.KindAKS: + log("🔮 Auto-exposing service within Azure VPC (service.beta.kubernetes.io/azure-load-balancer-internal)") + helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{ + "service": map[string]interface{}{ + "type": "LoadBalancer", + "annotations": map[string]interface{}{ + "service.beta.kubernetes.io/azure-load-balancer-internal": "true", + }, + }, + } + case k8s.KindEKS: + log("🔮 Auto-exposing service within AWS VPC (service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0") + helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{ + "service": map[string]interface{}{ + "type": "LoadBalancer", + "annotations": map[string]interface{}{ + "service.beta.kubernetes.io/aws-load-balancer-internal": "0.0.0.0/0", + }, + }, + } + default: + return nil, fmt.Errorf("cannot auto-detect service type, please specify using '--service-type' option") + } + } else { + if params.ServiceType == "NodePort" { + log("⚠️ Using service type NodePort may fail when nodes are removed from the cluster!") + } + helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{ + "service": map[string]interface{}{ + "type": params.ServiceType, + }, + } + } + + return helmVals, nil +} + +func EnableWithHelm(ctx context.Context, k8sClient *k8s.Client, params Parameters) error { + vals, err := generateEnableHelmValues(params, k8sClient.AutodetectFlavor(ctx)) if err != nil { return err } diff --git a/internal/cli/cmd/clustermesh.go b/internal/cli/cmd/clustermesh.go index 0bd38c15c9..e25d969e03 100644 --- a/internal/cli/cmd/clustermesh.go +++ b/internal/cli/cmd/clustermesh.go @@ -367,8 +367,8 @@ func newCmdClusterMeshEnableWithHelm() *cobra.Command { }, } - cmd.Flags().StringVar(¶ms.ServiceType, "service-type", "NodePort", "Type of Kubernetes service to expose control plane { LoadBalancer | NodePort | ClusterIP }") cmd.Flags().BoolVar(¶ms.EnableExternalWorkloads, "enable-external-workloads", false, "Enable support for external workloads, such as VMs") + cmd.Flags().StringVar(¶ms.ServiceType, "service-type", "", "Type of Kubernetes service to expose control plane { LoadBalancer | NodePort | ClusterIP }") return cmd }