update

Signed-off-by: Weifeng Wang <[email protected]> update traces Signed-off-by: Weifeng Wang <[email protected]> kubernetes(read-write-mode):update mimir Signed-off-by: Weifeng Wang <[email protected]> kubernetes(read-write-mode):update mimir Signed-off-by: Weifeng Wang <[email protected]> kubernetes(read-write-mode):update mimir Signed-off-by: Weifeng Wang <[email protected]> kubernetes(monolithic-mode):update mimir Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Mar 5, 2024 · 716bfec · 716bfec
1 parent b7e6470
commit 716bfec
Show file tree

Hide file tree

Showing 69 changed files with 290,438 additions and 65,280 deletions.
diff --git a/Makefile b/Makefile
@@ -158,7 +158,6 @@ clean: ## Clean cluster
 .PHONY: manifests
 manifests: ## Generates k8s manifests
 manifests: $(KUSTOMIZE) manifests-common manifests-monolithic-mode manifests-read-write-mode manifests-microservices-mode
-	@$(KUSTOMIZE) build monitoring-mixins > monitoring-mixins/k8s-all-in-one.yaml
 
 manifests-common: $(KUSTOMIZE)
 	$(info ******************** generates manifests-common manifests ********************)
@@ -174,8 +173,6 @@ manifests-common: $(KUSTOMIZE)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/common/rancher-pushprox > kubernetes/common/rancher-pushprox/manifests/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build --enable-helm kubernetes/common/redis > kubernetes/common/redis/manifests/k8s-all-in-one.yaml
 
-
-.PHONY: manifests-monolithic-mode
 manifests-monolithic-mode: $(KUSTOMIZE)
 	$(info ******************** generates monolithic-mode manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/logs > kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml
@@ -184,23 +181,19 @@ manifests-monolithic-mode: $(KUSTOMIZE)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/traces > kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/all-in-one > kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml
 
-.PHONY: manifests-read-write-mode
 manifests-read-write-mode: $(KUSTOMIZE)
 	$(info ******************** generates read-write-mode manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/read-write-mode/logs > kubernetes/read-write-mode/logs/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build kubernetes/read-write-mode/metrics > kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml
 
-.PHONY: manifests-microservices-mode
 manifests-microservices-mode: $(KUSTOMIZE)
 	$(info ******************** generates microservices-mode manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/logs > kubernetes/microservices-mode/logs/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/metrics > kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/profiles > kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/traces > kubernetes/microservices-mode/traces/k8s-all-in-one.yaml
 
-
 # prometheus-operator-crds
-.PHONY: deploy-prometheus-operator-crds
 deploy-prometheus-operator-crds:
 	$(info ******************** deploy prometheus-operator-crds manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/common/prometheus-operator-crds | kubectl replace -f - || $(KUSTOMIZE) build --enable-helm kubernetes/common/prometheus-operator-crds | kubectl create -f -
@@ -271,14 +264,22 @@ delete-grafana:
 	@$(KUSTOMIZE) build --enable-helm kubernetes/common/grafana | kubectl delete --ignore-not-found -f -
 
 
+define config_changes_trigger_pod_restart
+	@kubectl rollout restart deployment -n gateway nginx
+	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
+	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s
+	@kubectl rollout restart deployment -n monitoring-system grafana
+	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
+endef
 
 # Kubernetes monolithic-mode
 .PHONY: deploy-monolithic-mode-metrics
 deploy-monolithic-mode-metrics: deploy-grafana ## Deploy monolithic-mode metrics
 	$(info ******************** deploy monolithic-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build kubernetes/monolithic-mode/metrics | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the metrics."
@@ -291,6 +292,7 @@ deploy-monolithic-mode-logs: deploy-grafana ## Deploy monolithic-mode logs
 	$(info ******************** deploy monolithic-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the logs."
@@ -303,6 +305,7 @@ deploy-monolithic-mode-profiles: deploy-grafana ## Deploy monolithic-mode profil
 	$(info ******************** deploy monolithic-mode profiles manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/profiles | kubectl apply -f -
 	kubectl rollout status -n profiles-system statefulset/pyroscope --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the profiles."
@@ -314,14 +317,8 @@ delete-monolithic-mode-profiles:
 deploy-monolithic-mode-traces: deploy-grafana ## Deploy monolithic-mode traces
 	$(info ******************** deploy monolithic-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/traces | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
-	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
-	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s
-	@kubectl rollout restart deployment -n monitoring-system grafana
-	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the traces."
@@ -333,14 +330,8 @@ delete-monolithic-mode-traces:
 deploy-monolithic-mode-all-in-one: deploy-grafana ## Deploy monolithic-mode all-in-one
 	$(info ******************** deploy monolithic-mode all-in-one manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/all-in-one | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
-	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
-	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s	
-	@kubectl rollout restart deployment -n monitoring-system grafana
-	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the all-in-one."
@@ -354,10 +345,8 @@ delete-monolithic-mode-all-in-one:
 deploy-read-write-mode-metrics: deploy-grafana ## Deploy read-write-mode metrics
 	$(info ******************** deploy read-write-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build kubernetes/read-write-mode/metrics | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir-write --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the metrics."
@@ -370,8 +359,7 @@ deploy-read-write-mode-logs: deploy-grafana ## Deploy read-write-mode logs
 	$(info ******************** deploy read-write-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/read-write-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki-write --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the logs."
@@ -387,8 +375,7 @@ deploy-microservices-mode-logs: deploy-grafana ## Deploy microservices-mode logs
 	$(info ******************** deploy microservices-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki-distributed-ingester --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the logs."
@@ -400,24 +387,20 @@ delete-microservices-mode-logs:
 deploy-microservices-mode-metrics: deploy-grafana ## Deploy microservices-mode metrics
 	$(info ******************** deploy microservices-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/metrics | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n monitoring-system statefulset/mimir-distributed-ingester --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the metrics."
 delete-microservices-mode-metrics:
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/metrics | kubectl delete -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl delete -f -
 
 
 .PHONY: deploy-microservices-mode-profiles
 deploy-microservices-mode-profiles: deploy-grafana ## Deploy microservices-mode profiles
 	$(info ******************** deploy microservices-mode profiles manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/profiles | kubectl apply -f -
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the profiles."
@@ -429,14 +412,8 @@ delete-microservices-mode-profiles:
 deploy-microservices-mode-traces: deploy-grafana ## Deploy microservices-mode traces
 	$(info ******************** deploy microservices-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/traces | kubectl apply -f -
-	@$(KUSTOMIZE) build monitoring-mixins | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo-distributed-ingester --watch --timeout=600s
-	@kubectl rollout restart deployment -n gateway nginx
-	kubectl rollout status -n gateway deployment/nginx --watch --timeout=600s
-	@kubectl rollout restart daemonset -n monitoring-system grafana-agent
-	kubectl rollout status -n monitoring-system daemonset/grafana-agent --watch --timeout=600s
-	@kubectl rollout restart deployment -n monitoring-system grafana
-	kubectl rollout status -n monitoring-system deployment/grafana --watch --timeout=600s
+	@$(call config_changes_trigger_pod_restart)
 	@echo ""
 	@echo "Demo is running."
 	@echo "Go to http://localhost:8080/explore for the traces."

diff --git a/kubernetes/common/gateway/kustomization.yaml b/kubernetes/common/gateway/kustomization.yaml
@@ -8,6 +8,10 @@ resources:
 - nginx/nginx-dep.yaml
 - nginx/nginx-svc.yaml
 
+replicas:
+- count: 2
+  name: nginx
+
 images:
 - name: nginx
   newName: nginxinc/nginx-unprivileged

diff --git a/kubernetes/common/gateway/manifests/k8s-all-in-one.yaml b/kubernetes/common/gateway/manifests/k8s-all-in-one.yaml
@@ -272,7 +272,7 @@ metadata:
   name: nginx
   namespace: gateway
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: nginx

diff --git a/kubernetes/common/grafana-agent/configs/modules/traces.river b/kubernetes/common/grafana-agent/configs/modules/traces.river
@@ -82,6 +82,10 @@ otelcol.receiver.otlp "default" {
 }
 
 otelcol.processor.batch "default" {
+	send_batch_size     = 16384
+	send_batch_max_size = 0
+	timeout             = "2s"
+
 	output {
 		metrics = [otelcol.processor.memory_limiter.default.input]
 		logs    = [otelcol.processor.memory_limiter.default.input]
@@ -141,6 +145,10 @@ loki.process "autologging" {
 }
 
 otelcol.processor.memory_limiter "default" {
+	check_interval         = "1s"
+	limit_percentage       = 50
+	spike_limit_percentage = 30
+
 	output {
 		metrics = [otelcol.processor.k8sattributes.default.input]
 		logs    = [otelcol.processor.k8sattributes.default.input]

diff --git a/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml b/kubernetes/common/grafana-agent/manifests/k8s-all-in-one.yaml
@@ -579,7 +579,8 @@ data:
     {\n\t\tendpoint = argument.otlp_http_endpoint.value\n\t}\n\n\toutput {\n\t\tmetrics
     = [otelcol.processor.batch.default.input]\n\t\tlogs    = [otelcol.processor.batch.default.input]\n\t\ttraces
     \ = [\n\t\t\totelcol.processor.batch.default.input,\n\t\t\totelcol.connector.spanlogs.autologging.input,\n\t\t]\n\t}\n}\n\notelcol.processor.batch
-    \"default\" {\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\tlogs
+    \"default\" {\n\tsend_batch_size     = 16384\n\tsend_batch_max_size = 0\n\ttimeout
+    \            = \"2s\"\n\n\toutput {\n\t\tmetrics = [otelcol.processor.memory_limiter.default.input]\n\t\tlogs
     \   = [otelcol.processor.memory_limiter.default.input]\n\t\ttraces  = [otelcol.processor.memory_limiter.default.input]\n\t}\n}\n\n//
     The OpenTelemetry spanlog connector processes incoming trace spans and extracts
     data from them ready\n// for logging.\notelcol.connector.spanlogs \"autologging\"
@@ -603,7 +604,8 @@ data:
     this as the source for the output\n\t// logline. In this case, it essentially
     turns it into logfmt.\n\tstage.output {\n\t\tsource = \"body\"\n\t}\n\n\tforward_to
     = [loki.process.traceslogs.receiver]\n}\n\notelcol.processor.memory_limiter \"default\"
-    {\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\tlogs
+    {\n\tcheck_interval         = \"1s\"\n\tlimit_percentage       = 50\n\tspike_limit_percentage
+    = 30\n\n\toutput {\n\t\tmetrics = [otelcol.processor.k8sattributes.default.input]\n\t\tlogs
     \   = [otelcol.processor.k8sattributes.default.input]\n\t\ttraces  = [otelcol.processor.k8sattributes.default.input]\n\t}\n}\n\notelcol.processor.k8sattributes
     \"default\" {\n\toutput {\n\t\tmetrics = [otelcol.exporter.prometheus.tracesmetrics.input]\n\t\tlogs
     \   = [otelcol.exporter.loki.traceslogs.input]\n\t\ttraces  = argument.traces_forward_to.value\n\t}\n}\n\notelcol.exporter.prometheus
@@ -613,7 +615,7 @@ data:
     = argument.logs_forward_to.value\n}\n"
 kind: ConfigMap
 metadata:
-  name: agent-modules-d5hkbb8bkf
+  name: agent-modules-kbkcg9d4th
   namespace: monitoring-system
 ---
 apiVersion: v1
@@ -837,7 +839,7 @@ spec:
           path: /var/log
         name: varlog
       - configMap:
-          name: agent-modules-d5hkbb8bkf
+          name: agent-modules-kbkcg9d4th
         name: agent-modules
 ---
 apiVersion: monitoring.coreos.com/v1

diff --git a/kubernetes/microservices-mode/logs/configs/config.river b/kubernetes/microservices-mode/logs/configs/config.river
@@ -0,0 +1,47 @@
+/*
+The following example shows using the default all logs processing module, for
+a single tenant and specifying the destination url/credentials via environment
+variables.
+*/
+logging {
+	level  = coalesce(env("AGENT_LOG_LEVEL"), "info")
+	format = "logfmt"
+}
+
+module.file "lgtmp" {
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/lgtmp.river"
+
+	arguments {
+		cluster           = coalesce(env("CLUSTER"), "k3d-k3s-codelab")
+		logs_endpoint     = coalesce(env("LOGS_ENDPOINT"), "http://nginx.gateway.svc:3100")
+		metrics_endpoint  = coalesce(env("METRICS_ENDPOINT"), "http://nginx.gateway.svc:8080")
+		profiles_endpoint = coalesce(env("PROFILES_ENDPOINT"), "http://nginx.gateway.svc:4040")
+		traces_endpoint   = coalesce(env("TRACES_ENDPOINT"), "nginx.gateway.svc:4317")
+	}
+}
+
+/********************************************
+ * Logs
+ ********************************************/
+module.git "logs_primary" {
+	repository = "https://github.com/grafana/agent-modules.git"
+	revision   = "main"
+	path       = "modules/kubernetes/logs/all.river"
+
+	arguments {
+		forward_to = [module.file.lgtmp.exports.logs_receiver]
+	}
+}
+
+/********************************************
+ * Agent Integrations
+ ********************************************/
+module.file "agent_integrations" {
+	filename = coalesce(env("AGENT_CONFIG_FOLDER"), "/etc/agent-modules") + "/integrations.river"
+
+	arguments {
+		name       = "agent-integrations"
+		namespace  = "monitoring-system"
+		forward_to = [module.file.lgtmp.exports.metrics_receiver]
+	}
+}
diff --git a/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml b/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml
@@ -287,6 +287,32 @@ metadata:
   namespace: logging-system
 ---
 apiVersion: v1
+data:
+  config.river: "/*\nThe following example shows using the default all logs processing
+    module, for\na single tenant and specifying the destination url/credentials via
+    environment\nvariables.\n*/\nlogging {\n\tlevel  = coalesce(env(\"AGENT_LOG_LEVEL\"),
+    \"info\")\n\tformat = \"logfmt\"\n}\n\nmodule.file \"lgtmp\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"),
+    \"/etc/agent-modules\") + \"/lgtmp.river\"\n\n\targuments {\n\t\tcluster           =
+    coalesce(env(\"CLUSTER\"), \"k3d-k3s-codelab\")\n\t\tlogs_endpoint     = coalesce(env(\"LOGS_ENDPOINT\"),
+    \"http://nginx.gateway.svc:3100\")\n\t\tmetrics_endpoint  = coalesce(env(\"METRICS_ENDPOINT\"),
+    \"http://nginx.gateway.svc:8080\")\n\t\tprofiles_endpoint = coalesce(env(\"PROFILES_ENDPOINT\"),
+    \"http://nginx.gateway.svc:4040\")\n\t\ttraces_endpoint   = coalesce(env(\"TRACES_ENDPOINT\"),
+    \"nginx.gateway.svc:4317\")\n\t}\n}\n\n/********************************************\n
+    * Logs\n ********************************************/\nmodule.git \"logs_primary\"
+    {\n\trepository = \"https://github.com/grafana/agent-modules.git\"\n\trevision
+    \  = \"main\"\n\tpath       = \"modules/kubernetes/logs/all.river\"\n\n\targuments
+    {\n\t\tforward_to = [module.file.lgtmp.exports.logs_receiver]\n\t}\n}\n\n/********************************************\n
+    * Agent Integrations\n ********************************************/\nmodule.file
+    \"agent_integrations\" {\n\tfilename = coalesce(env(\"AGENT_CONFIG_FOLDER\"),
+    \"/etc/agent-modules\") + \"/integrations.river\"\n\n\targuments {\n\t\tname       =
+    \"agent-integrations\"\n\t\tnamespace  = \"monitoring-system\"\n\t\tforward_to
+    = [module.file.lgtmp.exports.metrics_receiver]\n\t}\n}\n"
+kind: ConfigMap
+metadata:
+  name: agent-config
+  namespace: monitoring-system
+---
+apiVersion: v1
 data:
   datasources.yaml: |
     apiVersion: 1

diff --git a/kubernetes/microservices-mode/logs/kustomization.yaml b/kubernetes/microservices-mode/logs/kustomization.yaml
@@ -27,6 +27,13 @@ configMapGenerator:
   files:
   - datasources.yaml=configs/grafana-datasources-loki.yaml
 
+- name: agent-config
+  namespace: monitoring-system
+  options:
+    disableNameSuffixHash: true
+  files:
+  - configs/config.river
+
 - name: loki-distributed
   namespace: logging-system
   behavior: replace