feat: ♻️ Refactor talos deployment

snoopy82481 · Oct 19, 2024 · e44a5a7 · e44a5a7
1 parent 2c88ef8
commit e44a5a7
Show file tree

Hide file tree

Showing 34 changed files with 877 additions and 620 deletions.
diff --git a/.taskfiles/bootstrap/Taskfile.yaml b/.taskfiles/bootstrap/Taskfile.yaml
@@ -0,0 +1,114 @@
+---
+# yaml-language-server: $schema=https://taskfile.dev/schema.json
+version: "3"
+
+vars:
+  GITHUB_DEPLOY_KEY_FILE: "{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml"
+  TALHELPER_CLUSTER_DIR: "{{.KUBERNETES_DIR}}/bootstrap/talos/clusterconfig"
+  TALHELPER_SECRET_FILE: "{{.KUBERNETES_DIR}}/bootstrap/talos/talsecret.sops.yaml"
+  TALHELPER_CONFIG_FILE: "{{.KUBERNETES_DIR}}/bootstrap/talos/talconfig.yaml"
+
+env:
+  TALOSCONFIG: "{{.TALHELPER_CLUSTER_DIR}}/talosconfig"
+
+tasks:
+  talos:
+    desc: Bootstrap the Talos cluster
+    dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
+    cmds:
+      - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}talhelper gensecret > {{.TALHELPER_SECRET_FILE}}{{end}}'
+      - '{{if eq .TALHELPER_SECRET_EXISTS "false"}}sops --encrypt --in-place {{.TALHELPER_SECRET_FILE}}{{end}}'
+      - talhelper genconfig --config-file {{.TALHELPER_CONFIG_FILE}} --secret-file {{.TALHELPER_SECRET_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}}
+      - talhelper gencommand apply --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="--insecure" | bash
+      - until talhelper gencommand bootstrap --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} | bash; do sleep 10; done
+      - until talhelper gencommand kubeconfig --config-file {{.TALHELPER_CONFIG_FILE}} --out-dir {{.TALHELPER_CLUSTER_DIR}} --extra-flags="{{.ROOT_DIR}} --force" | bash; do sleep 10; done
+      - task: apps
+      - talosctl health --server=false
+    vars:
+      TALHELPER_SECRET_EXISTS:
+        sh: test -f {{.TALHELPER_SECRET_FILE}} && echo true || echo false
+    preconditions:
+      - msg: Missing talhelper config file
+        sh: test -f {{.TALHELPER_CONFIG_FILE}}
+      - msg: Missing Sops config file
+        sh: test -f {{.SOPS_CONFIG_FILE}}
+      - msg: Missing Sops Age key file
+        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+  flux:
+    desc: Bootstrap Flux into the Talos cluster
+    cmds:
+      - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply --filename -{{end}}'
+      - '{{if eq .GITHUB_DEPLOY_KEY_EXISTS "true"}}sops exec-file {{.GITHUB_DEPLOY_KEY_FILE}} "kubectl apply --server-side --filename {}"{{end}}'
+      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux
+      - '{{if eq .SOPS_SECRET_EXISTS "false"}}cat {{.SOPS_AGE_KEY_FILE}} | kubectl --namespace flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin{{end}}'
+      - sops exec-file {{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml "kubectl apply --server-side --filename {}"
+      - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml
+      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config
+    vars:
+      GITHUB_DEPLOY_KEY_EXISTS:
+        sh: test -f {{.GITHUB_DEPLOY_KEY_FILE}} && echo true || echo false
+      SOPS_SECRET_EXISTS:
+        sh: kubectl --namespace flux-system get secret sops-age &>/dev/null && echo true || echo false
+    preconditions:
+      - msg: Missing kubeconfig
+        sh: test -f {{.KUBECONFIG}}
+      - msg: Missing Sops Age key file
+        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+  apps:
+    internal: true
+    dir: "{{.KUBERNETES_DIR}}/bootstrap/talos"
+    cmds:
+      - until kubectl wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
+      - helmfile --file {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+      - until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
+    preconditions:
+      - msg: Missing kubeconfig
+        sh: test -f {{.KUBECONFIG}}
+      - msg: Missing helmfile
+        sh: test -f {{.KUBERNETES_DIR}}/bootstrap/helmfile.yaml
+
+  age-keygen:
+    desc: Bootstrap the Sops Age key
+    cmd: age-keygen --output {{.SOPS_AGE_KEY_FILE}}
+    status:
+      - test -f {{.SOPS_AGE_KEY_FILE}}
+
+  encrypt:
+    desc: Encrypt all Kubernetes SOPS secrets
+    cmd: |
+      find "{{.KUBERNETES_DIR}}" -type f -name "*.sops.*" | while read -r file; do
+          if sops filestatus "${file}" | jq --exit-status ".encrypted == false" &>/dev/null; then
+              sops --encrypt --in-place "${file}"
+          fi
+      done
+    preconditions:
+      - msg: Missing Sops config file
+        sh: test -f {{.SOPS_CONFIG_FILE}}
+      - msg: Missing Sops Age key file
+        sh: test -f {{.SOPS_AGE_KEY_FILE}}
+
+  clean:
+    desc: Clean files and directories no longer needed after cluster bootstrap
+    cmds:
+      # Create backup directory
+      - mkdir -p {{.ROOT_DIR}}/.private
+      # Clean up CI
+      - rm -rf {{.ROOT_DIR}}/.github/tests
+      - rm -rf {{.ROOT_DIR}}/.github/workflows/e2e.yaml
+      # Clean up devcontainer
+      - rm -rf {{.ROOT_DIR}}/.devcontainer/ci
+      - rm -rf {{.ROOT_DIR}}/.github/workflows/devcontainer.yaml
+      # Move bootstrap directory to gitignored directory
+      - mv {{.BOOTSTRAP_DIR}} {{.ROOT_DIR}}/.private/bootstrap-{{.TS}}
+      - mv {{.MAKEJINJA_CONFIG_FILE}} {{.ROOT_DIR}}/.private/makejinja-{{.TS}}.toml
+      # Update renovate.json5
+      - sed -i {{if eq OS "darwin"}}''{{end}} 's/(..\.j2)\?//g' {{.ROOT_DIR}}/.github/renovate.json5
+    vars:
+      TS: "{{now | unixEpoch}}"
+    preconditions:
+      - msg: Missing bootstrap directory
+        sh: test -d {{.BOOTSTRAP_DIR}}
+      - msg: Missing Renovate config file
+        sh: test -f {{.ROOT_DIR}}/.github/renovate.json5
diff --git a/.taskfiles/flux/Taskfile.yaml b/.taskfiles/flux/Taskfile.yaml
@@ -6,7 +6,7 @@ vars:
   # renovate: datasource=github-releases depName=prometheus-operator/prometheus-operator
   PROMETHEUS_OPERATOR_VERSION: v0.77.1
   # renovate: datasource=github-releases depName=kubernetes-sigs/gateway-api
-  GATEWAY_API_VERSION: v1.0.0
+  GATEWAY_API_VERSION: v1.2.0
   CLUSTER_SECRET_SOPS_FILE: "{{.KUBERNETES_DIR}}/flux/vars/cluster-secrets.sops.yaml"
   CLUSTER_SETTINGS_FILE: "{{.KUBERNETES_DIR}}/flux/vars/cluster-settings.yaml"
   GITHUB_DEPLOY_KEY_FILE: "{{.KUBERNETES_DIR}}/bootstrap/flux/github-deploy-key.sops.yaml"
@@ -20,7 +20,6 @@ tasks:
       - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{.PROMETHEUS_OPERATOR_VERSION}}/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
       - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{.PROMETHEUS_OPERATOR_VERSION}}/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
       - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://github.com/kubernetes-sigs/gateway-api/releases/download/{{.GATEWAY_API_VERSION}}/standard-install.yaml
-      - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{.GATEWAY_API_VERSION}}/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml
       - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --filename https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/{{.GATEWAY_API_VERSION}}/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml
       - kubectl apply --kubeconfig {{.KUBECONFIG_FILE}} --server-side --kustomize {{.KUBERNETES_DIR}}/bootstrap/flux
       - cat {{.AGE_FILE}} | kubectl -n flux-system create secret generic sops-age --from-file=age.agekey=/dev/stdin

diff --git a/.taskfiles/kubernetes/Taskfile.yaml b/.taskfiles/kubernetes/Taskfile.yaml
@@ -6,13 +6,48 @@ vars:
   KUBECONFORM_SCRIPT: "{{.SCRIPTS_DIR}}/kubeconform.sh"
 
 tasks:
+  apply-ks:
+    desc: Apply a Flux Kustomization resource for a cluster
+    summary: |-
+      PATH: Path to the Flux Kustomization resource from the apps base dir (required, e.g. network/echo-server)
+      NS: Namespace the Flux Kustomization exists in (default: flux-system)
+    cmd: >
+      flux build --namespace {{.NS}} ks {{base .PATH}}
+      --kustomization-file {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
+      --path {{.KUBERNETES_DIR}}/apps/{{.PATH}}
+      {{- if contains "not found" .KS }}--dry-run \{{ end }}
+      | yq 'with(select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization"); .metadata.namespace = "{{.NS}}")' -
+      | kubectl apply --server-side --field-manager=kustomize-controller -f -
+    requires:
+      vars: [PATH]
+    vars:
+      NS: '{{.NS | default "flux-system"}}'
+      KS:
+        sh: flux --namespace {{.NS}} get kustomizations {{base .PATH}} 2>&1
+    preconditions:
+      - test -f {{.KUBERNETES_DIR}}/apps/{{.PATH}}/ks.yaml
+
+  reconcile:
+    desc: Force update Flux to pull in changes from your Git repository
+    cmd: flux reconcile --namespace flux-system kustomization cluster --with-source
+    preconditions:
+      - msg: Missing kubeconfig
+        sh: test -f {{.KUBECONFIG}}
+
+  kubeconform:
+    desc: Validate Kubernetes manifests with kubeconform
+    cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
+    preconditions:
+      - msg: Missing kubeconform script
+        sh: test -f {{.KUBECONFORM_SCRIPT}}
+
   resources:
     desc: Gather common resources in your cluster, useful when asking for support
     cmds:
-      - for: {var: resource}
+      - for: {var: RESOURCE}
         cmd: kubectl get {{.ITEM}} {{.CLI_ARGS | default "-A"}}
     vars:
-      resource: >-
+      RESOURCE: >-
         nodes
         gitrepositories
         kustomizations
@@ -22,14 +57,6 @@ tasks:
         certificaterequests
         ingresses
         pods
-
-  kubeconform:
-    desc: Validate Kubernetes manifests with kubeconform
-    cmd: bash {{.KUBECONFORM_SCRIPT}} {{.KUBERNETES_DIR}}
     preconditions:
-      - msg: "Missing kubeconform script"
-        sh: "test -f {{.KUBECONFORM_SCRIPT}}"
-
-  .reset:
-    internal: true
-    cmd: rm -rf {{.KUBERNETES_DIR}}
+      - msg: Missing kubeconfig
+        sh: test -f {{.KUBECONFIG}}
diff --git a/.taskfiles/talos/Taskfile.yaml b/.taskfiles/talos/Taskfile.yaml
@@ -13,71 +13,6 @@ env:
   TALOSCONFIG: "{{.TALOS_DIR}}/clusterconfig/talosconfig"
 
 tasks:
-  bootstrap:
-    desc: Bootstrap the Talos cluster
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - task: bootstrap-gensecret
-      - task: bootstrap-genconfig
-      - task: bootstrap-apply
-      - task: bootstrap-install
-      - task: fetch-kubeconfig
-      - task: bootstrap-apps
-      - talosctl health --server=false
-
-  bootstrap-gensecret:
-    desc: Generate the Talos secrets
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - talhelper gensecret > {{.TALHELPER_SECRET_FILE}}
-      - task: :sops:.encrypt-file
-        vars:
-          file: "{{.TALHELPER_SECRET_FILE}}"
-    preconditions:
-      - msg: "Missing talhelper config file"
-        sh: "test -f {{.TALHELPER_CONFIG_FILE}}"
-
-    status:
-      - test -f "{{.TALHELPER_SECRET_FILE}}"
-
-  bootstrap-genconfig:
-    desc: Generate the Talos configs
-    dir: "{{.TALOS_DIR}}"
-    cmd: talhelper genconfig
-    preconditions:
-      - msg: "Missing talhelper config file"
-        sh: "test -f {{.TALHELPER_CONFIG_FILE}}"
-
-  bootstrap-apply:
-    desc: Apply the Talos config on a node
-    dir: "{{.TALOS_DIR}}"
-    cmd: talhelper gencommand apply --extra-flags=--insecure | bash
-    preconditions:
-      - msg: "Missing talhelper config file"
-        sh: "test -f {{.TALHELPER_CONFIG_FILE}}"
-
-  bootstrap-install:
-    desc: Install the Talos cluster
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - echo "Installing Talos... ignore the errors and be patient"
-      - until talhelper gencommand bootstrap | bash; do sleep 10; done
-      - sleep 10
-    preconditions:
-      - msg: "Missing talhelper config file"
-        sh: "test -f {{.TALHELPER_CONFIG_FILE}}"
-
-  bootstrap-apps:
-    desc: Bootstrap core apps needed for Talos
-    dir: "{{.TALOS_DIR}}"
-    cmds:
-      - until kubectl --kubeconfig {{.KUBECONFIG_FILE}} wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
-      - helmfile --file ./apps/helmfile.yaml apply --skip-diff-on-install --suppress-diff
-      - until kubectl --kubeconfig {{.KUBECONFIG_FILE}} wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
-    preconditions:
-      - msg: "Missing kubeconfig"
-        sh: "test -f {{.KUBECONFIG_FILE}}"
-
   upgrade-talos:
     desc: Upgrade talos on a node
     cmd: talosctl --nodes {{.node}} upgrade --image {{.image}} --preserve=true --reboot-mode=default

diff --git a/.taskfiles/workstation/Brewfile b/.taskfiles/workstation/Brewfile
@@ -1,6 +1,6 @@
 tap "fluxcd/tap"
 tap "go-task/tap"
-tap "siderolabs/talos"
+tap "siderolabs/tap"
 brew "age"
 brew "cloudflared"
 brew "direnv"
@@ -13,8 +13,8 @@ brew "kubeconform"
 brew "kubernetes-cli"
 brew "kustomize"
 brew "moreutils"
+brew "siderolabs/tap/talosctl"
 brew "sops"
 brew "stern"
 brew "talhelper"
-brew "talosctl"
 brew "yq"