From 69e2dd2796a2a9856bdadbce93072228fefe2a26 Mon Sep 17 00:00:00 2001
From: Greg Haynes <greg.haynes@autodesk.com>
Date: Mon, 28 Oct 2024 17:08:48 +0000
Subject: [PATCH] Add vcluster-multi-env stack

The vcluster-multi-env stack configures a set of vclusters managed by
the CNOE ArgoCD which are then enrolled in the CNOE ArgoCD. This enables
developing multi-environment workflows on top of CNOE.

Signed-off-by: Greg Haynes <greg.haynes@autodesk.com>
---
 vcluster-multi-env/README.md                  | 41 ++++++++++
 vcluster-multi-env/add-vclusters.sh           | 48 ++++++++++++
 .../vcluster/application-vcluster.yaml        | 39 ++++++++++
 vcluster-multi-env/vcluster/ingress.yaml      | 27 +++++++
 .../vcluster/kustomization.yaml               |  3 +
 vcluster-multi-env/vclusters.yaml             | 78 +++++++++++++++++++
 6 files changed, 236 insertions(+)
 create mode 100644 vcluster-multi-env/README.md
 create mode 100755 vcluster-multi-env/add-vclusters.sh
 create mode 100644 vcluster-multi-env/vcluster/application-vcluster.yaml
 create mode 100644 vcluster-multi-env/vcluster/ingress.yaml
 create mode 100644 vcluster-multi-env/vcluster/kustomization.yaml
 create mode 100644 vcluster-multi-env/vclusters.yaml

diff --git a/vcluster-multi-env/README.md b/vcluster-multi-env/README.md
new file mode 100644
index 0000000..42dceba
--- /dev/null
+++ b/vcluster-multi-env/README.md
@@ -0,0 +1,41 @@
+# IDP Builder Multi-Environment
+
+Multi-environment emulation on top of CNOE.
+
+# Configuring Clusters
+
+By default, this stack creates two vclusters (staging and production). If you
+desire a different configuration you can edit the following list in
+`vclusters.yaml`:
+
+```yaml
+  generators:
+  - list:
+      elements:
+      - name: staging
+      - name: production
+```
+
+# Running
+
+```bash
+# Create CNOE deployment with vcluster-multi-env stack
+idpbuilder create -p vcluster-multi-env
+
+# Enroll vclusters in ArgoCD
+./vcluster-multi-env/add-vclusters.sh
+```
+
+# Using
+
+Your CNOE ArgoCD should now have a cluster enrolled for each configured
+vcluster (staging and production by default). These clusters will have the
+following labels for your use:
+
+```yaml
+    cnoe.io/vclusterMultiEnv/clusterClass: "app-runtime"
+    cnoe.io/vclusterMultiEnv/clusterName: "${cluster_name}"
+```
+
+You may now target them using, for example, an ArgoCD ApplicationSet cluster
+generator which matches these labels.
diff --git a/vcluster-multi-env/add-vclusters.sh b/vcluster-multi-env/add-vclusters.sh
new file mode 100755
index 0000000..78d6d96
--- /dev/null
+++ b/vcluster-multi-env/add-vclusters.sh
@@ -0,0 +1,48 @@
+#! /bin/bash
+
+set -eu
+
+vcluster_app_names=$(kubectl get application -A -l cnoe.io/applicationName=vcluster-package,cnoe.io/stackName=vcluster-multi-env --no-headers -o custom-columns=":metadata.name")
+environments=$(echo "$vcluster_app_names" | cut -f 1 -d '-')
+
+for env in $environments; do
+    cluster_name=$env
+
+    echo "Checking readiness for ${cluster_name} vcluster..."
+
+    until kubectl get secret -n ${cluster_name}-vcluster vc-${cluster_name}-vcluster-helm &> /dev/null; do
+      echo "Waiting for ${cluster_name} vcluster secret to be ready..."
+      sleep 10
+    done
+
+    echo "${cluster_name} vcluster is ready. Retrieving credentials..."
+    client_key=$(kubectl get secret -n ${cluster_name}-vcluster vc-${cluster_name}-vcluster-helm --template='{{index .data "client-key" }}')
+    client_certificate=$(kubectl get secret -n ${cluster_name}-vcluster vc-${cluster_name}-vcluster-helm --template='{{index .data "client-certificate" }}')
+    certificate_authority=$(kubectl get secret -n ${cluster_name}-vcluster vc-${cluster_name}-vcluster-helm --template='{{index .data "certificate-authority" }}')
+
+    kubectl apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${cluster_name}-vcluster-secret
+  namespace: argocd
+  labels:
+    argocd.argoproj.io/secret-type: cluster
+    cnoe.io/vclusterMultiEnv/clusterClass: "app-runtime"
+    cnoe.io/vclusterMultiEnv/clusterName: "${cluster_name}"
+type: Opaque
+stringData:
+  name: ${cluster_name}-vcluster
+  server: https://${cluster_name}-vcluster.cnoe.localtest.me:443
+  config: |
+    {
+      "tlsClientConfig": {
+        "insecure": false,
+        "caData": "${certificate_authority}",
+        "certData": "${client_certificate}",
+        "keyData": "${client_key}"
+      }
+    }
+EOF
+
+done
diff --git a/vcluster-multi-env/vcluster/application-vcluster.yaml b/vcluster-multi-env/vcluster/application-vcluster.yaml
new file mode 100644
index 0000000..11fb265
--- /dev/null
+++ b/vcluster-multi-env/vcluster/application-vcluster.yaml
@@ -0,0 +1,39 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: unpatched-vcluster
+  namespace: argocd
+  labels:
+    cnoe.io/stackName: vcluster-multi-env
+    cnoe.io/applicationName: vcluster-helm
+spec:
+  project: default
+  source:
+    chart: vcluster
+    repoURL: https://charts.loft.sh
+    targetRevision: 0.20.0
+    helm:
+      valuesObject:
+        sync:
+          fromHost:
+            nodes:
+              enabled: true # Required for virtualScheduler
+        controlPlane:
+          advanced:
+            virtualScheduler: # For Keptn support: https://keptn.sh/stable/docs/installation/configuration/vcluster/
+              enabled: true
+          proxy:
+            extraSANs:
+            - unpatched-vcluster-hostname.cnoe.localtest.me
+          statefulSet:
+            scheduling:
+              podManagementPolicy: OrderedReady
+        exportKubeConfig:
+          server: https://unpatched-vcluster-hostname.cnoe.localtest.me:443
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: unpatched-vcluster
+  syncPolicy:
+    automated: {}
+    syncOptions:
+    - CreateNamespace=true
\ No newline at end of file
diff --git a/vcluster-multi-env/vcluster/ingress.yaml b/vcluster-multi-env/vcluster/ingress.yaml
new file mode 100644
index 0000000..3d89fe1
--- /dev/null
+++ b/vcluster-multi-env/vcluster/ingress.yaml
@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    # We need the ingress to pass through ssl traffic to the vCluster
+    # This only works for the nginx-ingress (enable via --enable-ssl-passthrough
+    # https://kubernetes.github.io/ingress-nginx/user-guide/tls/#ssl-passthrough )
+    # for other ingress controllers please check their respective documentation.
+    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+  name: vcluster-ingress
+  labels:
+    cnoe.io/stackName: vcluster-multi-env
+spec:
+  ingressClassName: nginx # use your ingress class name
+  rules:
+  - host: unpatched-vcluster-hostname.cnoe.localtest.me
+    http:
+      paths:
+      - backend:
+          service:
+            name: unpatched-vcluster-services
+            port:
+              number: 443
+        path: /
+        pathType: ImplementationSpecific
\ No newline at end of file
diff --git a/vcluster-multi-env/vcluster/kustomization.yaml b/vcluster-multi-env/vcluster/kustomization.yaml
new file mode 100644
index 0000000..f0be1b3
--- /dev/null
+++ b/vcluster-multi-env/vcluster/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- application-vcluster.yaml
+- ingress.yaml
\ No newline at end of file
diff --git a/vcluster-multi-env/vclusters.yaml b/vcluster-multi-env/vclusters.yaml
new file mode 100644
index 0000000..09fee3f
--- /dev/null
+++ b/vcluster-multi-env/vclusters.yaml
@@ -0,0 +1,78 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: vclusters
+  namespace: argocd
+spec:
+  goTemplate: true
+  goTemplateOptions: ["missingkey=error"]
+  generators:
+  - list:
+      elements:
+      - name: staging
+      - name: production
+  template:
+    metadata:
+      name: '{{.name}}-vcluster'
+      labels:
+        cnoe.io/stackName: vcluster-multi-env
+        cnoe.io/applicationName: vcluster-package
+      finalizers:
+      - resources-finalizer.argocd.argoproj.io # enabling cascading deletion
+    spec:
+      project: 'default'
+      source:
+        repoURL: cnoe://
+        targetRevision: HEAD
+        path: vcluster
+        kustomize:
+          patches:
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Ingress
+            patch: |-
+              - op: replace
+                path: /spec/rules/0/host
+                value: {{.name}}-vcluster.cnoe.localtest.me
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Ingress
+            patch: |-
+              - op: replace
+                path: /spec/rules/0/http/paths/0/backend/service/name
+                value: {{.name}}-vcluster-helm
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Application
+            patch: |-
+              - op: replace
+                path: /metadata/name
+                value: {{.name}}-vcluster-helm
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Application
+            patch: |-
+              - op: replace
+                path: /spec/source/helm/valuesObject/controlPlane/proxy/extraSANs/0
+                value: {{.name}}-vcluster.cnoe.localtest.me
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Application
+            patch: |-
+              - op: replace
+                path: /spec/source/helm/valuesObject/exportKubeConfig/server
+                value: https://{{.name}}-vcluster.cnoe.localtest.me:443
+          - target:
+              labelSelector: cnoe.io/stackName=vcluster-multi-env
+              kind: Application
+            patch: |-
+              - op: replace
+                path: /spec/destination/namespace
+                value: {{.name}}-vcluster
+      destination:
+        server: "https://kubernetes.default.svc"
+        namespace: '{{.name}}-vcluster'
+      syncPolicy:
+        automated: {}
+        syncOptions:
+        - CreateNamespace=true
\ No newline at end of file