helm mode: autodetect service parameters for clustermesh

Classic mode uses the Kubernetes flavor to automatically set the clustermesh-apiserver Service type and LoadBalancer annotations if needed. This patch reimplements this logic for Helm mode. Signed-off-by: Andrew Sauber <[email protected]>
cilium · May 23, 2023 · 00c8c68 · 00c8c68
1 parent 0c6549e
commit 00c8c68
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 9 deletions.
diff --git a/.github/in-cluster-test-scripts/multicluster.sh b/.github/in-cluster-test-scripts/multicluster.sh
@@ -47,8 +47,9 @@ cilium --context "${CONTEXT1}" status --wait
 cilium --context "${CONTEXT2}" status --wait
 
 # Enable cluster mesh
-cilium --context "${CONTEXT1}" clustermesh enable --service-type=LoadBalancer
-cilium --context "${CONTEXT2}" clustermesh enable --service-type=LoadBalancer
+# Test autodetection of service parameters for GKE
+cilium --context "${CONTEXT1}" clustermesh enable
+cilium --context "${CONTEXT2}" clustermesh enable
 
 # Copy the clustermesh secrets
 # TODO(ajs): Patch the connect command to expect the Helm secret name
@@ -68,6 +69,12 @@ fi
 cilium --context "${CONTEXT1}" clustermesh status --wait
 cilium --context "${CONTEXT2}" clustermesh status --wait
 
+# Print clustermesh Service annotations
+printf "Service annotations for Cluster 1 %s\n" \
+    $(kubectl --context "${CONTEXT1}" get svc -n kube-system clustermesh-apiserver -o jsonpath='{.metadata.annotations}')
+printf "Service annotations for Cluster 2 %s\n" \
+    $(kubectl --context "${CONTEXT2}" get svc -n kube-system clustermesh-apiserver -o jsonpath='{.metadata.annotations}')
+
 # Connect clusters
 cilium --context "${CONTEXT1}" clustermesh connect --destination-context "${CONTEXT2}"
 

diff --git a/clustermesh/clustermesh.go b/clustermesh/clustermesh.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 	"regexp"
 	"strconv"
 	"strings"
@@ -1783,13 +1784,74 @@ func (k *K8sClusterMesh) ExternalWorkloadStatus(ctx context.Context, names []str
 	return err
 }
 
-func EnableWithHelm(ctx context.Context, k8sClient *k8s.Client, params Parameters) error {
-	helmStrValues := []string{
-		"clustermesh.useAPIServer=true",
-		fmt.Sprintf("clustermesh.apiserver.service.type=%s", params.ServiceType),
-		fmt.Sprintf("externalWorkloads.enabled=%t", params.EnableExternalWorkloads),
+func log(format string, a ...interface{}) {
+	// TODO (ajs): make logger configurable
+	fmt.Fprintf(os.Stdout, format+"\n", a...)
+}
+
+func generateEnableHelmValues(params Parameters, flavor k8s.Flavor) (map[string]interface{}, error) {
+	helmVals := map[string]interface{}{
+		"clustermesh": map[string]interface{}{
+			"useAPIServer": true,
+		},
+		"externalWorkloads": map[string]interface{}{
+			"enabled": params.EnableExternalWorkloads,
+		},
 	}
-	vals, err := helm.ParseVals(helmStrValues)
+
+	if params.ServiceType == "" {
+		switch flavor.Kind {
+		case k8s.KindGKE:
+			log("🔮 Auto-exposing service within GCP VPC (cloud.google.com/load-balancer-type=Internal)")
+			helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{
+				"service": map[string]interface{}{
+					"type": "LoadBalancer",
+					"annotations": map[string]interface{}{
+						"cloud.google.com/load-balancer-type": "Internal",
+						// Allows cross-region access
+						"networking.gke.io/internal-load-balancer-allow-global-access": "true",
+					},
+				},
+			}
+		case k8s.KindAKS:
+			log("🔮 Auto-exposing service within Azure VPC (service.beta.kubernetes.io/azure-load-balancer-internal)")
+			helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{
+				"service": map[string]interface{}{
+					"type": "LoadBalancer",
+					"annotations": map[string]interface{}{
+						"service.beta.kubernetes.io/azure-load-balancer-internal": "true",
+					},
+				},
+			}
+		case k8s.KindEKS:
+			log("🔮 Auto-exposing service within AWS VPC (service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0")
+			helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{
+				"service": map[string]interface{}{
+					"type": "LoadBalancer",
+					"annotations": map[string]interface{}{
+						"service.beta.kubernetes.io/aws-load-balancer-internal": "0.0.0.0/0",
+					},
+				},
+			}
+		default:
+			return nil, fmt.Errorf("cannot auto-detect service type, please specify using '--service-type' option")
+		}
+	} else {
+		if params.ServiceType == "NodePort" {
+			log("⚠️  Using service type NodePort may fail when nodes are removed from the cluster!")
+		}
+		helmVals["clustermesh"].(map[string]interface{})["apiserver"] = map[string]interface{}{
+			"service": map[string]interface{}{
+				"type": params.ServiceType,
+			},
+		}
+	}
+
+	return helmVals, nil
+}
+
+func EnableWithHelm(ctx context.Context, k8sClient *k8s.Client, params Parameters) error {
+	vals, err := generateEnableHelmValues(params, k8sClient.AutodetectFlavor(ctx))
 	if err != nil {
 		return err
 	}

diff --git a/internal/cli/cmd/clustermesh.go b/internal/cli/cmd/clustermesh.go
@@ -367,8 +367,8 @@ func newCmdClusterMeshEnableWithHelm() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&params.ServiceType, "service-type", "NodePort", "Type of Kubernetes service to expose control plane { LoadBalancer | NodePort | ClusterIP }")
 	cmd.Flags().BoolVar(&params.EnableExternalWorkloads, "enable-external-workloads", false, "Enable support for external workloads, such as VMs")
+	cmd.Flags().StringVar(&params.ServiceType, "service-type", "", "Type of Kubernetes service to expose control plane { LoadBalancer | NodePort | ClusterIP }")
 
 	return cmd
 }