add metrics agents (#461)

## Summary by CodeRabbit - **New Features** - Introduced new HelmRelease configurations for cert-manager, monitoring agents, and Victoria Metrics Operator in Kubernetes. - Added resource specifications for `vmselect` in the VMCluster configuration. - Enhanced resource management for `vmselect` with defined limits and requests for memory and CPU. - **Bug Fixes** - Adjusted resource limits for Redis failover memory allocation. - **Documentation** - Updated README and release notes for various components, enhancing clarity and usability. - **Chores** - Updated image versions across multiple components for consistency and performance improvements. - Modified migration scripts to facilitate transitions and manage resources effectively.  --------- Co-authored-by: Andrei Kvapil <[email protected]>
aenix-io · Nov 4, 2024 · 3c27a1e · 3c27a1e
1 parent f06f653
commit 3c27a1e
Show file tree

Hide file tree

Showing 234 changed files with 53,124 additions and 1,766 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
       exclude: '^.*templates/.*\.yaml$'
       args: [--unsafe]
 - repo: https://github.com/igorshubovych/markdownlint-cli
-  rev: v0.41.0
+  rev: v0.42.0
   hooks:
     - id: markdownlint
       args: [--fix, --disable, MD013, MD041, --]

diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -179,7 +179,7 @@ talosctl apply -f controlplane.yaml -n 192.168.123.13 -e 192.168.123.13 -i
 timeout 60 sh -c 'until nc -nzv 192.168.123.11 50000 && nc -nzv 192.168.123.12 50000 && nc -nzv 192.168.123.13 50000; do sleep 1; done'
 
 # Bootstrap
-talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11
+timeout 10 sh -c 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done'
 
 # Wait for etcd
 timeout 180 sh -c 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep "rpc error"; do sleep 1; done'
@@ -190,7 +190,7 @@ export KUBECONFIG=$PWD/kubeconfig
 
 # Wait for kubernetes nodes appear
 timeout 60 sh -c 'until [ $(kubectl get node -o name | wc -l) = 3 ]; do sleep 1; done'
-kubectl create ns cozy-system
+kubectl create ns cozy-system -o yaml | kubectl apply -f -
 kubectl create -f - <<\EOT
 apiVersion: v1
 kind: ConfigMap

diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml
@@ -0,0 +1,54 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-cert-manager-crds
+  labels:
+    cozystack.io/repository: system
+    coztstack.io/target-cluster-name: {{ .Release.Name }}
+spec:
+  interval: 5m
+  releaseName: cert-manager-crds
+  chart:
+    spec:
+      chart: cozy-cert-manager-crds
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: HelmRepository
+        name: cozystack-system
+        namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-kubeconfig
+  targetNamespace: cozy-cert-manager-crds
+  storageNamespace: cozy-cert-manager-crds
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  {{- if .Values.addons.certManager.valuesOverride }}
+  valuesFrom:
+  - kind: Secret
+    name: {{ .Release.Name }}-cert-manager-crds-values-override
+    valuesKey: values
+  {{- end }}
+
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-cilium
+    namespace: {{ .Release.Namespace }}
+{{- if .Values.addons.certManager.valuesOverride }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-cert-manager-crds-values-override
+stringData:
+  values: |
+    {{- toYaml .Values.addons.certManager.valuesOverride | nindent 4 }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml
@@ -43,6 +43,8 @@ spec:
   {{- end }}
   - name: {{ .Release.Name }}-cilium
     namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Name }}-cert-manager-crds
+    namespace: {{ .Release.Namespace }}
 {{- end }}
 {{- if .Values.addons.certManager.valuesOverride }}
 ---

diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml
@@ -0,0 +1,105 @@
+{{- $myNS := lookup "v1" "Namespace" "" .Release.Namespace }}
+{{- $targetTenant := index $myNS.metadata.annotations "namespace.cozystack.io/monitoring" }}
+{{- if .Values.addons.monitoringAgents.enabled }}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-monitoring-agents
+  labels:
+    cozystack.io/repository: system
+    coztstack.io/target-cluster-name: {{ .Release.Name }}
+spec:
+  interval: 5m
+  releaseName: cozy-monitoring-agents
+  chart:
+    spec:
+      chart: cozy-monitoring-agents
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: HelmRepository
+        name: cozystack-system
+        namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-kubeconfig
+  targetNamespace: cozy-monitoring-agents
+  storageNamespace: cozy-monitoring-agents
+  install:
+    createNamespace: true
+    timeout: "300s"
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-cilium
+    namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Name }}-cozy-victoria-metrics-operator
+    namespace: {{ .Release.Namespace }}
+  values:
+    vmagent:
+      externalLabels:
+        cluster: {{ .Release.Name }}
+        tenant: {{ $targetTenant }}
+      remoteWrite:
+        url: http://vminsert-shortterm.{{ $targetTenant }}.svc:8480/insert/0/prometheus
+
+    fluent-bit:
+      readinessProbe:
+        httpGet:
+          path: /
+      daemonSetVolumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+      daemonSetVolumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      config:
+        outputs: |
+          [OUTPUT]
+              Name http
+              Match kube.*
+              Host vlogs-generic.{{ $targetTenant }}.svc
+              port 9428
+              compress gzip
+              uri /insert/jsonline?_stream_fields=stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date
+              format json_lines
+              json_date_format iso8601
+              header AccountID 0
+              header ProjectID 0
+        filters: |
+          [FILTER]
+              Name kubernetes
+              Match kube.*
+              Merge_Log On
+              Keep_Log On
+              K8S-Logging.Parser On
+              K8S-Logging.Exclude On
+          [FILTER]
+              Name                nest
+              Match               *
+              Wildcard            pod_name
+              Operation lift
+              Nested_under kubernetes
+              Add_prefix   kubernetes_
+          [FILTER]
+              Name modify
+              Match *
+              Add tenant {{ $targetTenant }}
+          [FILTER]
+              Name modify
+              Match *
+              Add cluster {{ .Release.Name }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml
@@ -0,0 +1,41 @@
+{{- if .Values.addons.monitoringAgents.enabled }}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ .Release.Name }}-cozy-victoria-metrics-operator
+  labels:
+    cozystack.io/repository: system
+    coztstack.io/target-cluster-name: {{ .Release.Name }}
+spec:
+  interval: 5m
+  releaseName: cozy-victoria-metrics-operator
+  chart:
+    spec:
+      chart: cozy-victoria-metrics-operator
+      reconcileStrategy: Revision
+      sourceRef:
+        kind: HelmRepository
+        name: cozystack-system
+        namespace: cozy-system
+  kubeConfig:
+    secretRef:
+      name: {{ .Release.Name }}-kubeconfig
+  targetNamespace: cozy-victoria-metrics-operator
+  storageNamespace: cozy-victoria-metrics-operator
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  dependsOn:
+  {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }}
+  - name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
+  {{- end }}
+  - name: {{ .Release.Name }}-cilium
+    namespace: {{ .Release.Namespace }}
+  - name: {{ .Release.Name }}-cert-manager-crds
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json
@@ -75,8 +75,23 @@
                             "default": {}
                         }
                     }
+                },
+                "monitoringAgents": {
+                    "type": "object",
+                    "properties": {
+                        "enabled": {
+                            "type": "boolean",
+                            "description": "Enables MonitoringAgents (fluentbit, vmagents for sending logs and metrics to storage) if tenant monitoring enabled, send to tenant storage, else to root storage",
+                            "default": false
+                        },
+                        "valuesOverride": {
+                            "type": "object",
+                            "description": "Custom values to override",
+                            "default": {}
+                        }
+                    }
                 }
             }
         }
     }
-}
+}
diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml
@@ -60,3 +60,12 @@ addons:
     ##
     enabled: false
     valuesOverride: {}
+
+  ## MonitoringAgents
+  ##
+  monitoringAgents:
+    ## @param addons.monitoringAgents.enabled Enables MonitoringAgents (fluentbit, vmagents for sending logs and metrics to storage) if tenant monitoring enabled, send to tenant storage, else to root storage
+    ## @param addons.monitoringAgents.valuesOverride Custom values to override
+    ##
+    enabled: false
+    valuesOverride: {}
diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml
@@ -20,7 +20,6 @@ spec:
         cpu: 150m
         memory: 400Mi
       limits:
-        cpu: 2
         memory: 1000Mi
     {{- with .Values.size }}
     storage:
@@ -37,7 +36,7 @@ spec:
           storageClassName: {{ . }}
           {{- end }}
     {{- end }}
-    exporter: 
+    exporter:
       enabled: true
       image: oliver006/redis_exporter:v1.55.0-alpine
       args:

diff --git a/packages/core/platform/bundles/distro-full.yaml b/packages/core/platform/bundles/distro-full.yaml
@@ -31,11 +31,17 @@ releases:
       autoDirectNodeRoutes: true
       routingMode: native
 
+- name: cert-manager-crds
+  releaseName: cert-manager-crds
+  chart: cozy-cert-manager-crds
+  namespace: cozy-cert-manager
+  dependsOn: [cilium]
+
 - name: cert-manager
   releaseName: cert-manager
   chart: cozy-cert-manager
   namespace: cozy-cert-manager
-  dependsOn: [cilium]
+  dependsOn: [cert-manager-crds]
 
 - name: cert-manager-issuers
   releaseName: cert-manager-issuers
@@ -49,9 +55,9 @@ releases:
   namespace: cozy-victoria-metrics-operator
   dependsOn: [cilium,cert-manager]
 
-- name: monitoring
-  releaseName: monitoring
-  chart: cozy-monitoring
+- name: monitoring-agents
+  releaseName: monitoring-agents
+  chart: cozy-monitoring-agents
   namespace: cozy-monitoring
   privileged: true
   dependsOn: [cilium,victoria-metrics-operator]

diff --git a/packages/core/platform/bundles/distro-hosted.yaml b/packages/core/platform/bundles/distro-hosted.yaml
@@ -14,11 +14,17 @@ releases:
   namespace: cozy-fluxcd
   dependsOn: [fluxcd-operator]
 
+- name: cert-manager-crds
+  releaseName: cert-manager-crds
+  chart: cozy-cert-manager-crds
+  namespace: cozy-cert-manager
+  dependsOn: []
+
 - name: cert-manager
   releaseName: cert-manager
   chart: cozy-cert-manager
   namespace: cozy-cert-manager
-  dependsOn: []
+  dependsOn: [cert-manager-crds]
 
 - name: cert-manager-issuers
   releaseName: cert-manager-issuers
@@ -32,9 +38,9 @@ releases:
   namespace: cozy-victoria-metrics-operator
   dependsOn: [cert-manager]
 
-- name: monitoring
-  releaseName: monitoring
-  chart: cozy-monitoring
+- name: monitoring-agents
+  releaseName: monitoring-agents
+  chart: cozy-monitoring-agents
   namespace: cozy-monitoring
   privileged: true
   dependsOn: [victoria-metrics-operator]
@@ -99,7 +105,7 @@ releases:
   chart: cozy-external-dns
   namespace: cozy-external-dns
   optional: true
-  dependsOn: [] 
+  dependsOn: []
 
 - name: external-secrets-operator
   releaseName: external-secrets-operator