From f43736a9a967da982c9fa395c7292894796214bd Mon Sep 17 00:00:00 2001 From: RAMathews Date: Thu, 18 Jan 2024 15:59:39 +0000 Subject: [PATCH 1/7] added prometheus configuration, managed prometheus scraper resoure and removed some of the prometheus scrape jobs from ADOT config --- modules/eks-monitoring/main.tf | 45 + .../templates/opentelemetrycollector.yaml | 1279 +--------------- modules/eks-monitoring/prom_config.yaml | 1286 +++++++++++++++++ 3 files changed, 1332 insertions(+), 1278 deletions(-) create mode 100644 modules/eks-monitoring/prom_config.yaml diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 0d6f201e..11fc2298 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -258,3 +258,48 @@ module "external_secrets" { depends_on = [resource.helm_release.grafana_operator] } + +resource "aws_prometheus_workspace" "this" { + + tags = { + AMPAgentlessScraper = "" + } +} + + +resource "aws_prometheus_scraper" "basic" { + alias = "managed-prometheus-scraper" + + source { + eks{ + cluster_arn = data.aws_eks_cluster.eks_cluster.arn + subnet_ids = data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids + + } + + } + + + scrape_configuration= templatefile("${path.module}/prom_config.yaml", + { global_scrape_interval = var.prometheus_config.global_scrape_interval, + global_scrape_timeout = var.prometheus_config.global_scrape_timeout, + enableAPIserver = var.enable_apiserver_monitoring, + eks_cluster_id = var.eks_cluster_id, + region = var.managed_prometheus_workspace_region, + accountID = local.context.aws_caller_identity_account_id + }) + + destination { + amp { + workspace_arn = "arn:aws:aps:${var.managed_prometheus_workspace_region}:${local.context.aws_caller_identity_account_id}:workspace/${var.managed_prometheus_workspace_id}" + } + } + + tags = { + CreatedBy = "Terraform" + Owner = "AWS Observability Accelerator" + } + +} + + diff --git a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml index a2aa43a6..76dcd796 100644 --- a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml +++ b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml @@ -41,1282 +41,6 @@ spec: scrape_interval: {{ .Values.globalScrapeInterval }} scrape_timeout: {{ .Values.globalScrapeTimeout }} scrape_configs: - - job_name: 'kubernetes-kubelet' - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc.cluster.local:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$${1}/proxy/metrics - - job_name: 'kubelet' - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc.cluster.local:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor - - {{ if .Values.enableAPIserver }} - - job_name: 'apiserver' - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: - [ - __meta_kubernetes_namespace, - __meta_kubernetes_service_name, - __meta_kubernetes_endpoint_port_name, - ] - action: keep - regex: default;kubernetes;https - metric_relabel_configs: - - action: keep - source_labels: [__name__] - - source_labels: [__name__, le] - separator: ; - regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) - replacement: $1 - action: drop - {{ end }} - - - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeInterval }} - scheme: http - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (prometheus-node-exporter);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_jobLabel] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-metrics - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - default - - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-prometheus);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] - separator: ; - regex: (true);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-web - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-web - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - default - - job_name: serviceMonitor/default/kube-prometheus-stack-operator/0 - honor_labels: true - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-operator);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: https - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: https - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - default - - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/2 - honor_labels: true - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - metrics_path: /metrics/probes - scheme: https - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: https-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_k8s_app] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: https-metrics - action: replace - - source_labels: [__metrics_path__] - separator: ; - regex: (.*) - target_label: metrics_path - replacement: $$1 - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - kube-system - - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/1 - honor_labels: true - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - metrics_path: /metrics/cadvisor - scheme: https - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: https-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_k8s_app] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: https-metrics - action: replace - - source_labels: [__metrics_path__] - separator: ; - regex: (.*) - target_label: metrics_path - replacement: $$1 - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - kube-system - - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/0 - honor_labels: true - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: https - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] - separator: ; - regex: (kubelet);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: https-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_k8s_app] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: https-metrics - action: replace - - source_labels: [__metrics_path__] - separator: ; - regex: (.*) - target_label: metrics_path - replacement: $$1 - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - kube-system - - job_name: serviceMonitor/default/kube-prometheus-stack-kube-state-metrics/0 - honor_labels: true - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_instance, __meta_kubernetes_service_labelpresent_app_kubernetes_io_instance] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] - separator: ; - regex: (kube-state-metrics);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - default - - job_name: serviceMonitor/default/kube-prometheus-stack-kube-scheduler/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-kube-scheduler);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_jobLabel] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-metrics - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - kube-system - - - job_name: 'kube-proxy' - honor_labels: true - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: keep - source_labels: - - __meta_kubernetes_namespace - - __meta_kubernetes_pod_name - separator: '/' - regex: 'kube-system/kube-proxy.+' - - source_labels: - - __address__ - action: replace - target_label: __address__ - regex: (.+?)(\\:\\d+)? - replacement: $$1:10249 - - - job_name: serviceMonitor/default/kube-prometheus-stack-kube-controller-manager/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-kube-controller-manager);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_jobLabel] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-metrics - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - kube-system - - job_name: serviceMonitor/default/kube-prometheus-stack-coredns/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - authorization: - type: Bearer - credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-coredns);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-metrics - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_jobLabel] - separator: ; - regex: (.+) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-metrics - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - namespaces: - own_namespace: false - names: - - kube-system - - job_name: serviceMonitor/default/kube-prometheus-stack-alertmanager/0 - honor_timestamps: true - scrape_interval: {{ .Values.globalScrapeInterval }} - scrape_timeout: {{ .Values.globalScrapeTimeout }} - scheme: http - follow_redirects: true - enable_http2: true - relabel_configs: - - source_labels: [job] - separator: ; - regex: (.*) - target_label: __tmp_prometheus_job_name - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] - separator: ; - regex: (kube-prometheus-stack-alertmanager);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] - separator: ; - regex: (kube-prometheus-stack);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] - separator: ; - regex: (true);true - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_port_name] - separator: ; - regex: http-web - replacement: $$1 - action: keep - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Node;(.*) - target_label: node - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] - separator: ; - regex: Pod;(.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_namespace] - separator: ; - regex: (.*) - target_label: namespace - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: service - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_name] - separator: ; - regex: (.*) - target_label: pod - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_pod_container_name] - separator: ; - regex: (.*) - target_label: container - replacement: $$1 - action: replace - - source_labels: [__meta_kubernetes_service_name] - separator: ; - regex: (.*) - target_label: job - replacement: $$1 - action: replace - - separator: ; - regex: (.*) - target_label: endpoint - replacement: http-web - action: replace - - source_labels: [__address__] - separator: ; - regex: (.*) - modulus: 1 - target_label: __tmp_hash - replacement: $$1 - action: hashmod - - source_labels: [__tmp_hash] - separator: ; - regex: "0" - replacement: $$1 - action: keep - kubernetes_sd_configs: - - role: endpoints - kubeconfig_file: "" - follow_redirects: true - enable_http2: true - namespaces: - own_namespace: false - names: - - default - - job_name: 'kube-state-metrics' - static_configs: - - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] - - job_name: 'node-exporter' - kubernetes_sd_configs: - - role: endpoints - ec2_sd_configs: - relabel_configs: - - source_labels: [ __address__ ] - action: keep - regex: '.*:9100$' - - action: replace - source_labels: [__meta_kubernetes_endpoint_node_name] - target_label: nodename {{ if .Values.enableCustomMetrics }} {{- range $k, $v := fromYaml .Values.customMetrics }} - job_name: "{{ $k }}" @@ -1385,7 +109,6 @@ spec: regex: 'jvm_gc_collection_seconds.*' action: drop {{ end }} - {{ if .Values.enableNginx }} - job_name: 'kubernetes-nginx' sample_limit: {{ .Values.nginxScrapeSampleLimit }} @@ -1418,7 +141,6 @@ spec: - regex: exported_host action: labeldrop {{ end }} - {{ if .Values.enableIstio }} - honor_labels: true job_name: kubernetes-istio @@ -1552,3 +274,4 @@ spec: address: 0.0.0.0:8888 level: basic {{ end }} + diff --git a/modules/eks-monitoring/prom_config.yaml b/modules/eks-monitoring/prom_config.yaml new file mode 100644 index 00000000..ae9a7e91 --- /dev/null +++ b/modules/eks-monitoring/prom_config.yaml @@ -0,0 +1,1286 @@ +global: + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + external_labels: + cluster: ${eks_cluster_id} + region : ${region} + account_id : ${accountID} +scrape_configs: + - job_name: 'kubernetes-kubelet' + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics + + - job_name: 'kubelet' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc.cluster.local:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor + + %{~ if enableAPIserver ~} + - job_name: 'apiserver' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: + [ + __meta_kubernetes_namespace, + __meta_kubernetes_service_name, + __meta_kubernetes_endpoint_port_name, + ] + action: keep + regex: default;kubernetes;https + metric_relabel_configs: + - action: keep + source_labels: [__name__] + - source_labels: [__name__, le] + separator: ; + regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50) + replacement: $1 + action: drop + %{~ endif ~} + + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus-node-exporter/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (prometheus-node-exporter);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-prometheus/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-prometheus);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-operator/0 + honor_labels: true + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-operator);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/2 + honor_labels: true + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + metrics_path: /metrics/probes + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/1 + honor_labels: true + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + metrics_path: /metrics/cadvisor + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kubelet/0 + honor_labels: true + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: https + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_k8s_app, __meta_kubernetes_service_labelpresent_k8s_app] + separator: ; + regex: (kubelet);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: https-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_k8s_app] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: https-metrics + action: replace + - source_labels: [__metrics_path__] + separator: ; + regex: (.*) + target_label: metrics_path + replacement: $$1 + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-state-metrics/0 + honor_labels: true + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_instance, __meta_kubernetes_service_labelpresent_app_kubernetes_io_instance] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name] + separator: ; + regex: (kube-state-metrics);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-scheduler/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-scheduler);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: 'kube-proxy' + honor_labels: true + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + separator: '/' + regex: 'kube-system/kube-proxy.+' + - source_labels: + - __address__ + action: replace + target_label: __address__ + regex: (.+?)(\\:\\d+)? + replacement: $$1:10249 + - job_name: serviceMonitor/default/kube-prometheus-stack-kube-controller-manager/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-kube-controller-manager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-coredns/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + authorization: + type: Bearer + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-coredns);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-metrics + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_jobLabel] + separator: ; + regex: (.+) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-metrics + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + namespaces: + own_namespace: false + names: + - kube-system + - job_name: serviceMonitor/default/kube-prometheus-stack-alertmanager/0 + honor_timestamps: true + scrape_interval: ${global_scrape_interval} + scrape_timeout: ${global_scrape_timeout} + scheme: http + follow_redirects: true + enable_http2: true + relabel_configs: + - source_labels: [job] + separator: ; + regex: (.*) + target_label: __tmp_prometheus_job_name + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_label_app, __meta_kubernetes_service_labelpresent_app] + separator: ; + regex: (kube-prometheus-stack-alertmanager);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_release, __meta_kubernetes_service_labelpresent_release] + separator: ; + regex: (kube-prometheus-stack);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_service_label_self_monitor, __meta_kubernetes_service_labelpresent_self_monitor] + separator: ; + regex: (true);true + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_port_name] + separator: ; + regex: http-web + replacement: $$1 + action: keep + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Node;(.*) + target_label: node + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name] + separator: ; + regex: Pod;(.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_name] + separator: ; + regex: (.*) + target_label: pod + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_pod_container_name] + separator: ; + regex: (.*) + target_label: container + replacement: $$1 + action: replace + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: job + replacement: $$1 + action: replace + - separator: ; + regex: (.*) + target_label: endpoint + replacement: http-web + action: replace + - source_labels: [__address__] + separator: ; + regex: (.*) + modulus: 1 + target_label: __tmp_hash + replacement: $$1 + action: hashmod + - source_labels: [__tmp_hash] + separator: ; + regex: "0" + replacement: $$1 + action: keep + kubernetes_sd_configs: + - role: endpoints + kubeconfig_file: "" + follow_redirects: true + enable_http2: true + namespaces: + own_namespace: false + names: + - default + + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: + - __address__ + action: keep + regex: '.*:9100$' + - action: replace + source_labels: + - __meta_kubernetes_endpoint_node_name + target_label: nodename + From bb9d276ab4d6e8a343be0df08d1e7a12203a0782 Mon Sep 17 00:00:00 2001 From: RAMathews Date: Mon, 29 Jan 2024 12:56:46 +0000 Subject: [PATCH 2/7] Added cluster role and iamidentitymapping for managed metric scraping --- modules/eks-monitoring/main.tf | 45 +++++++++++++------ .../.helmignore | 23 ++++++++++ .../Chart.yaml | 24 ++++++++++ .../templates/clusterrolebinding.yaml | 26 +++++++++++ .../values.yaml | 0 .../templates/opentelemetrycollector.yaml | 1 - modules/eks-monitoring/prom_config.yaml | 1 - 7 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 modules/eks-monitoring/managed-prometheus-scraper-config/.helmignore create mode 100644 modules/eks-monitoring/managed-prometheus-scraper-config/Chart.yaml create mode 100644 modules/eks-monitoring/managed-prometheus-scraper-config/templates/clusterrolebinding.yaml create mode 100644 modules/eks-monitoring/managed-prometheus-scraper-config/values.yaml diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 11fc2298..ae154cd8 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -262,32 +262,35 @@ module "external_secrets" { resource "aws_prometheus_workspace" "this" { tags = { - AMPAgentlessScraper = "" - } + AMPAgentlessScraper = "" + } } - +resource "helm_release" "managed_prometheus__role" { + name = "managed-prometheus-role" + chart = "${path.module}/managed-prometheus-scraper-config" +} resource "aws_prometheus_scraper" "basic" { alias = "managed-prometheus-scraper" source { - eks{ + eks { cluster_arn = data.aws_eks_cluster.eks_cluster.arn - subnet_ids = data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids + subnet_ids = data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids } } - scrape_configuration= templatefile("${path.module}/prom_config.yaml", - { global_scrape_interval = var.prometheus_config.global_scrape_interval, - global_scrape_timeout = var.prometheus_config.global_scrape_timeout, - enableAPIserver = var.enable_apiserver_monitoring, - eks_cluster_id = var.eks_cluster_id, - region = var.managed_prometheus_workspace_region, - accountID = local.context.aws_caller_identity_account_id - }) + scrape_configuration = templatefile("${path.module}/prom_config.yaml", + { global_scrape_interval = var.prometheus_config.global_scrape_interval, + global_scrape_timeout = var.prometheus_config.global_scrape_timeout, + enableAPIserver = var.enable_apiserver_monitoring, + eks_cluster_id = var.eks_cluster_id, + region = var.managed_prometheus_workspace_region, + accountID = local.context.aws_caller_identity_account_id + }) destination { amp { @@ -297,9 +300,23 @@ resource "aws_prometheus_scraper" "basic" { tags = { CreatedBy = "Terraform" - Owner = "AWS Observability Accelerator" + Owner = "AWS Observability Accelerator" } } + +resource "terraform_data" "managed-amp-scrapper-role" { + provisioner "local-exec" { + command = < Date: Wed, 31 Jan 2024 23:19:38 +0100 Subject: [PATCH 3/7] Lint code --- modules/eks-monitoring/README.md | 2 ++ modules/eks-monitoring/locals.tf | 7 ++++- modules/eks-monitoring/main.tf | 45 ++++++++++++-------------------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 2899667e..0c108620 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -51,10 +51,12 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this |------|------| | [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | | [aws_prometheus_rule_group_namespace.recording_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | +| [aws_prometheus_scraper.basic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_scraper) | resource | | [aws_prometheus_workspace.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | | [helm_release.fluxcd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.grafana_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.kube_state_metrics](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.managed_prometheus_role](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.prometheus_node_exporter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.adothealth_monitoring_dashboards](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | | [kubectl_manifest.api_server_dashboards](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | diff --git a/modules/eks-monitoring/locals.tf b/modules/eks-monitoring/locals.tf index 8b942520..c3408d26 100644 --- a/modules/eks-monitoring/locals.tf +++ b/modules/eks-monitoring/locals.tf @@ -13,6 +13,7 @@ locals { managed_prometheus_workspace_id = var.enable_managed_prometheus ? aws_prometheus_workspace.this[0].id : var.managed_prometheus_workspace_id managed_prometheus_workspace_region = coalesce(var.managed_prometheus_workspace_region, data.aws_region.current.name) managed_prometheus_workspace_endpoint = "https://aps-workspaces.${local.managed_prometheus_workspace_region}.amazonaws.com/workspaces/${local.managed_prometheus_workspace_id}/" + managed_prometheus_workspace_arn = "arn:aws:aps:${local.managed_prometheus_workspace_region}:${data.aws_caller_identity.current.account_id}:workspace/${local.managed_prometheus_workspace_id}" name = "adot-collector-kubeprometheus" kube_service_account_name = try(var.helm_config.service_account, local.name) @@ -22,6 +23,10 @@ locals { eks_cluster_endpoint = data.aws_eks_cluster.eks_cluster.endpoint eks_cluster_version = data.aws_eks_cluster.eks_cluster.version + tags = merge(var.tags, { + Source = "AWS Observability Accelerator" + }) + context = { aws_caller_identity_account_id = data.aws_caller_identity.current.account_id aws_caller_identity_arn = data.aws_caller_identity.current.arn @@ -31,7 +36,7 @@ locals { eks_cluster_id = var.eks_cluster_id eks_oidc_issuer_url = local.eks_oidc_issuer_url eks_oidc_provider_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.eks_oidc_issuer_url}" - tags = var.tags + tags = local.tags irsa_iam_role_path = var.irsa_iam_role_path irsa_iam_permissions_boundary = var.irsa_iam_permissions_boundary } diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 76f4ff1f..7be26638 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -2,7 +2,11 @@ resource "aws_prometheus_workspace" "this" { count = var.enable_managed_prometheus ? 1 : 0 alias = local.name - tags = var.tags + + # Agentless scraping require this tag on the workspace + tags = merge(local.tags, { + AMPAgentlessScraper = "" + }) } module "operator" { @@ -270,54 +274,38 @@ module "external_secrets" { depends_on = [resource.helm_release.grafana_operator] } -resource "aws_prometheus_workspace" "this" { - - tags = { - AMPAgentlessScraper = "" - } -} - -resource "helm_release" "managed_prometheus__role" { +resource "helm_release" "managed_prometheus_role" { name = "managed-prometheus-role" chart = "${path.module}/managed-prometheus-scraper-config" } resource "aws_prometheus_scraper" "basic" { alias = "managed-prometheus-scraper" - source { eks { cluster_arn = data.aws_eks_cluster.eks_cluster.arn subnet_ids = data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids - } - } - - scrape_configuration = templatefile("${path.module}/prom_config.yaml", - { global_scrape_interval = var.prometheus_config.global_scrape_interval, - global_scrape_timeout = var.prometheus_config.global_scrape_timeout, - enableAPIserver = var.enable_apiserver_monitoring, - eks_cluster_id = var.eks_cluster_id, - region = var.managed_prometheus_workspace_region, - accountID = local.context.aws_caller_identity_account_id + scrape_configuration = templatefile("${path.module}/prom_config.yaml", { + global_scrape_interval = var.prometheus_config.global_scrape_interval, + global_scrape_timeout = var.prometheus_config.global_scrape_timeout, + enableAPIserver = var.enable_apiserver_monitoring, + eks_cluster_id = local.context.eks_cluster_id, + region = local.managed_prometheus_workspace_region, + accountID = local.context.aws_caller_identity_account_id }) destination { amp { - workspace_arn = "arn:aws:aps:${var.managed_prometheus_workspace_region}:${local.context.aws_caller_identity_account_id}:workspace/${var.managed_prometheus_workspace_id}" + workspace_arn = local.managed_prometheus_workspace_arn } } - tags = { - CreatedBy = "Terraform" - Owner = "AWS Observability Accelerator" - } - + tags = local.tags } - - +/*TODO - use native resource providers for iamidentity mapping or provide an output command resource "terraform_data" "managed-amp-scrapper-role" { provisioner "local-exec" { command = < Date: Mon, 5 Feb 2024 11:41:22 +0100 Subject: [PATCH 4/7] Add alertmanager definition, drop dead code --- modules/eks-monitoring/README.md | 3 +-- modules/eks-monitoring/main.tf | 14 ++++++++++++++ modules/eks-monitoring/variables.tf | 12 ------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 0c108620..14ee4d14 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -49,6 +49,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | Name | Type | |------|------| +| [aws_prometheus_alert_manager_definition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_alert_manager_definition) | resource | | [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | | [aws_prometheus_rule_group_namespace.recording_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | | [aws_prometheus_scraper.basic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_scraper) | resource | @@ -109,7 +110,6 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace. Required if `enable_external_secrets = true` | `string` | `""` | no | | [grafana\_cluster\_dashboard\_url](#input\_grafana\_cluster\_dashboard\_url) | Dashboard URL for Cluster Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/cluster.json"` | no | | [grafana\_kubelet\_dashboard\_url](#input\_grafana\_kubelet\_dashboard\_url) | Dashboard URL for Kubelet Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json"` | no | -| [grafana\_kubeproxy\_dashboard\_url](#input\_grafana\_kubeproxy\_dashboard\_url) | Dashboard URL for kube-proxy Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/kube-proxy/kube-proxy.json"` | no | | [grafana\_namespace\_workloads\_dashboard\_url](#input\_grafana\_namespace\_workloads\_dashboard\_url) | Dashboard URL for Namespace Workloads Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/namespace-workloads.json"` | no | | [grafana\_node\_exporter\_dashboard\_url](#input\_grafana\_node\_exporter\_dashboard\_url) | Dashboard URL for Node Exporter Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/nodeexporter-nodes.json"` | no | | [grafana\_nodes\_dashboard\_url](#input\_grafana\_nodes\_dashboard\_url) | Dashboard URL for Nodes Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/nodes.json"` | no | @@ -126,7 +126,6 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      default = string
    })
  })
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection | 
object({
    cw_log_retention_days = number
  })
| 
{
  "cw_log_retention_days": 90
}
| no | | [managed\_prometheus\_cross\_account\_role](#input\_managed\_prometheus\_cross\_account\_role) | Amazon Managed Prometheus Workspace's Account Role Arn | `string` | `""` | no | -| [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | | [managed\_prometheus\_workspace\_region](#input\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus Workspace's Region | `string` | `null` | no | | [ne\_config](#input\_ne\_config) | Node exporter configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "prometheus-node-exporter",
  "helm_chart_version": "4.24.0",
  "helm_release_name": "prometheus-node-exporter",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "prometheus-node-exporter",
  "scrape_interval": "60s",
  "scrape_timeout": "60s"
}
| no | diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 7be26638..a98a9e48 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -9,6 +9,20 @@ resource "aws_prometheus_workspace" "this" { }) } +resource "aws_prometheus_alert_manager_definition" "this" { + count = var.enable_alertmanager ? 1 : 0 + + workspace_id = local.managed_prometheus_workspace_id + + definition = < Date: Mon, 5 Feb 2024 21:16:59 +0100 Subject: [PATCH 5/7] Update scraper config - Allow scraper to be created with eks subnets lie in multiple azs - identify metrics coming from the scraper vs collector - output aws-auth as a one liner command line --- modules/eks-monitoring/locals.tf | 2 +- modules/eks-monitoring/main.tf | 66 ++++++++++++++++++- .../templates/opentelemetrycollector.yaml | 3 + modules/eks-monitoring/outputs.tf | 5 ++ modules/eks-monitoring/prom_config.yaml | 1 + 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/modules/eks-monitoring/locals.tf b/modules/eks-monitoring/locals.tf index c3408d26..c87bb6c3 100644 --- a/modules/eks-monitoring/locals.tf +++ b/modules/eks-monitoring/locals.tf @@ -15,7 +15,7 @@ locals { managed_prometheus_workspace_endpoint = "https://aps-workspaces.${local.managed_prometheus_workspace_region}.amazonaws.com/workspaces/${local.managed_prometheus_workspace_id}/" managed_prometheus_workspace_arn = "arn:aws:aps:${local.managed_prometheus_workspace_region}:${data.aws_caller_identity.current.account_id}:workspace/${local.managed_prometheus_workspace_id}" - name = "adot-collector-kubeprometheus" + name = "adot-collector" kube_service_account_name = try(var.helm_config.service_account, local.name) namespace = try(var.helm_config.namespace, local.name) diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index a98a9e48..7f84c0d1 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -292,12 +292,27 @@ resource "helm_release" "managed_prometheus_role" { name = "managed-prometheus-role" chart = "${path.module}/managed-prometheus-scraper-config" } -resource "aws_prometheus_scraper" "basic" { + +data "aws_subnet" "helper" { + for_each = toset(data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids) + id = each.key +} + +locals { + eks_availability_zone_subnets = { + for subnet in data.aws_subnet.helper : subnet.availability_zone => subnet.id... + } +} + +resource "aws_prometheus_scraper" "this" { alias = "managed-prometheus-scraper" source { eks { cluster_arn = data.aws_eks_cluster.eks_cluster.arn - subnet_ids = data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids + + //ValidationException: Subnets provided must be in unique availability zones + // might not always work + subnet_ids = [for subnet_ids in local.eks_availability_zone_subnets : subnet_ids[0]] } } @@ -319,6 +334,53 @@ resource "aws_prometheus_scraper" "basic" { tags = local.tags } +# locals { +# aws_auth_yaml = <<-EOF +# rolearn: ${aws_prometheus_scraper.this.role_arn} +# username: aps-collector-user +# EOF +# } + +# data "kubernetes_config_map" "aws_auth" { +# metadata { +# name = "aws-auth" +# namespace = "kube-system" +# } +# } + +# resource "kubernetes_config_map_v1_data" "aws_auth" { +# force = true + +# metadata { +# name = "aws-auth" +# namespace = "kube-system" +# } + +# data = { +# # Convert to list, make distinict to remove duplicates, and convert to yaml as mapRoles is a yaml string. +# # replace() remove double quotes on "strings" in yaml output. +# # distinct() only apply the change once, not append every run. +# mapRoles = replace(yamlencode(distinct(concat(yamldecode(data.kubernetes_config_map.aws_auth.data.mapRoles), yamldecode(local.aws_auth_yaml)))), "\"", "") +# } + +# lifecycle { +# ignore_changes = [] +# } +# } + +# // requires authentication mode API_AND_CONFIG_MAP at least on the cluster +# // TODO: update doc +# resource "aws_eks_access_entry" "scraper" { +# cluster_name = local.context.eks_cluster_id +# principal_arn = aws_prometheus_scraper.this.role_arn +# user_name = "aps-collector-user" +# type = "STANDARD" +# } + +# // │ Error: creating EKS Access Entry (attractive-wardrobe-1706565893:arn:aws:iam::339743103717:role/aws-service-role/scraper.aps.amazonaws.com/AWSServiceRoleForAmazonPrometheusScraper_085aa6c7-fc8c-4): operation error EKS: CreateAccessEntry, https response error StatusCode: 400, RequestID: cdca4512-8fcc-40af-87f2-3f168ce58bd7, InvalidParameterException: The caller is not allowed to modify access entries with a principalArn value of a Service Linked Role + + + /*TODO - use native resource providers for iamidentity mapping or provide an output command resource "terraform_data" "managed-amp-scrapper-role" { provisioner "local-exec" { diff --git a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml index 8d32595d..bf08361b 100644 --- a/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml +++ b/modules/eks-monitoring/otel-config/templates/opentelemetrycollector.yaml @@ -244,6 +244,9 @@ spec: - key: account_id action: upsert value: {{ .Values.accountId }} + - key: collected_by + action: upsert + value: adot {{ if .Values.enableTracing }} batch/traces: timeout: {{ .Values.tracingTimeout }} diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index 38027fae..aec2981d 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -27,3 +27,8 @@ output "managed_prometheus_workspace_region" { description = "Amazon Managed Prometheus workspace region" value = local.managed_prometheus_workspace_region } + +output "scraper_aws_auth" { + description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" + value = "eksctl create iamidentitymapping --cluster ${var.eks_cluster_id} --region ${local.managed_prometheus_workspace_region} --arn ${aws_prometheus_scraper.this.role_arn} --username aps-collector-user" +} diff --git a/modules/eks-monitoring/prom_config.yaml b/modules/eks-monitoring/prom_config.yaml index ba143c9c..3231e0d6 100644 --- a/modules/eks-monitoring/prom_config.yaml +++ b/modules/eks-monitoring/prom_config.yaml @@ -5,6 +5,7 @@ global: cluster: ${eks_cluster_id} region : ${region} account_id : ${accountID} + collected_by: aws-scraper scrape_configs: - job_name: 'kubernetes-kubelet' scrape_interval: ${global_scrape_interval} From 1e1b90f0d201ffa932296d3f5a73421939475e02 Mon Sep 17 00:00:00 2001 From: Rodrigue Koffi Date: Mon, 5 Feb 2024 21:22:44 +0100 Subject: [PATCH 6/7] Cleanup --- modules/eks-monitoring/README.md | 4 +- modules/eks-monitoring/main.tf | 70 ++------------------------------ 2 files changed, 6 insertions(+), 68 deletions(-) diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 14ee4d14..5d9b1ff2 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -52,7 +52,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [aws_prometheus_alert_manager_definition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_alert_manager_definition) | resource | | [aws_prometheus_rule_group_namespace.alerting_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | | [aws_prometheus_rule_group_namespace.recording_rules](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_rule_group_namespace) | resource | -| [aws_prometheus_scraper.basic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_scraper) | resource | +| [aws_prometheus_scraper.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_scraper) | resource | | [aws_prometheus_workspace.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | | [helm_release.fluxcd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.grafana_operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | @@ -69,6 +69,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [aws_eks_cluster.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_subnet.helper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | ## Inputs @@ -147,4 +148,5 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus workspace region | +| [scraper\_aws\_auth](#output\_scraper\_aws\_auth) | Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use | diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 7f84c0d1..084ecf30 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -293,6 +293,8 @@ resource "helm_release" "managed_prometheus_role" { chart = "${path.module}/managed-prometheus-scraper-config" } +// These helpers solve the ValidationException error thrown by the scraper if +// eks subnets are not in unique availability zones. data "aws_subnet" "helper" { for_each = toset(data.aws_eks_cluster.eks_cluster.vpc_config[0].subnet_ids) id = each.key @@ -309,10 +311,7 @@ resource "aws_prometheus_scraper" "this" { source { eks { cluster_arn = data.aws_eks_cluster.eks_cluster.arn - - //ValidationException: Subnets provided must be in unique availability zones - // might not always work - subnet_ids = [for subnet_ids in local.eks_availability_zone_subnets : subnet_ids[0]] + subnet_ids = [for subnet_ids in local.eks_availability_zone_subnets : subnet_ids[0]] } } @@ -333,66 +332,3 @@ resource "aws_prometheus_scraper" "this" { tags = local.tags } - -# locals { -# aws_auth_yaml = <<-EOF -# rolearn: ${aws_prometheus_scraper.this.role_arn} -# username: aps-collector-user -# EOF -# } - -# data "kubernetes_config_map" "aws_auth" { -# metadata { -# name = "aws-auth" -# namespace = "kube-system" -# } -# } - -# resource "kubernetes_config_map_v1_data" "aws_auth" { -# force = true - -# metadata { -# name = "aws-auth" -# namespace = "kube-system" -# } - -# data = { -# # Convert to list, make distinict to remove duplicates, and convert to yaml as mapRoles is a yaml string. -# # replace() remove double quotes on "strings" in yaml output. -# # distinct() only apply the change once, not append every run. -# mapRoles = replace(yamlencode(distinct(concat(yamldecode(data.kubernetes_config_map.aws_auth.data.mapRoles), yamldecode(local.aws_auth_yaml)))), "\"", "") -# } - -# lifecycle { -# ignore_changes = [] -# } -# } - -# // requires authentication mode API_AND_CONFIG_MAP at least on the cluster -# // TODO: update doc -# resource "aws_eks_access_entry" "scraper" { -# cluster_name = local.context.eks_cluster_id -# principal_arn = aws_prometheus_scraper.this.role_arn -# user_name = "aps-collector-user" -# type = "STANDARD" -# } - -# // │ Error: creating EKS Access Entry (attractive-wardrobe-1706565893:arn:aws:iam::339743103717:role/aws-service-role/scraper.aps.amazonaws.com/AWSServiceRoleForAmazonPrometheusScraper_085aa6c7-fc8c-4): operation error EKS: CreateAccessEntry, https response error StatusCode: 400, RequestID: cdca4512-8fcc-40af-87f2-3f168ce58bd7, InvalidParameterException: The caller is not allowed to modify access entries with a principalArn value of a Service Linked Role - - - -/*TODO - use native resource providers for iamidentity mapping or provide an output command -resource "terraform_data" "managed-amp-scrapper-role" { - provisioner "local-exec" { - command = < Date: Mon, 5 Feb 2024 22:01:28 +0100 Subject: [PATCH 7/7] Fix scraper's output --- examples/eks-cross-account-with-central-amp/main.tf | 6 ++---- examples/eks-istio/README.md | 1 + examples/eks-istio/outputs.tf | 5 +++++ examples/existing-cluster-java/README.md | 1 + examples/existing-cluster-java/outputs.tf | 5 +++++ examples/existing-cluster-nginx/README.md | 1 + examples/existing-cluster-nginx/outputs.tf | 5 +++++ examples/existing-cluster-with-base-and-infra/README.md | 1 + examples/existing-cluster-with-base-and-infra/outputs.tf | 5 +++++ modules/eks-monitoring/outputs.tf | 2 +- 10 files changed, 27 insertions(+), 5 deletions(-) diff --git a/examples/eks-cross-account-with-central-amp/main.tf b/examples/eks-cross-account-with-central-amp/main.tf index d48f8cbc..1f06b119 100644 --- a/examples/eks-cross-account-with-central-amp/main.tf +++ b/examples/eks-cross-account-with-central-amp/main.tf @@ -43,7 +43,6 @@ module "eks_monitoring_one" { enable_managed_prometheus = false managed_prometheus_workspace_id = module.managed_service_prometheus.workspace_id - managed_prometheus_workspace_endpoint = module.managed_service_prometheus.workspace_prometheus_endpoint managed_prometheus_workspace_region = var.cluster_one.region managed_prometheus_cross_account_role = aws_iam_role.cross_account_amp_role.arn irsa_iam_additional_policies = [aws_iam_policy.irsa_assume_role_policy_one.arn] @@ -96,9 +95,8 @@ module "eks_monitoring_two" { # prevents the module to create a workspace enable_managed_prometheus = false - managed_prometheus_workspace_id = module.managed_service_prometheus.workspace_id - managed_prometheus_workspace_endpoint = module.managed_service_prometheus.workspace_prometheus_endpoint - managed_prometheus_workspace_region = var.cluster_two.region + managed_prometheus_workspace_id = module.managed_service_prometheus.workspace_id + managed_prometheus_workspace_region = var.cluster_two.region managed_prometheus_cross_account_role = aws_iam_role.cross_account_amp_role.arn irsa_iam_additional_policies = [aws_iam_policy.irsa_assume_role_policy_two.arn] diff --git a/examples/eks-istio/README.md b/examples/eks-istio/README.md index 3a88dd41..87c61684 100644 --- a/examples/eks-istio/README.md +++ b/examples/eks-istio/README.md @@ -54,4 +54,5 @@ View the full documentation for this example [here](https://aws-observability.gi | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | AWS Region | +| [scraper\_aws\_auth](#output\_scraper\_aws\_auth) | Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use | diff --git a/examples/eks-istio/outputs.tf b/examples/eks-istio/outputs.tf index e14427e3..15359e95 100644 --- a/examples/eks-istio/outputs.tf +++ b/examples/eks-istio/outputs.tf @@ -22,3 +22,8 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "scraper_aws_auth" { + description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" + value = module.eks_monitoring.scraper_aws_auth +} diff --git a/examples/existing-cluster-java/README.md b/examples/existing-cluster-java/README.md index 1b1e08b2..6f87ef92 100644 --- a/examples/existing-cluster-java/README.md +++ b/examples/existing-cluster-java/README.md @@ -237,4 +237,5 @@ terraform destroy -var-file=terraform.tfvars | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | AWS Region | +| [scraper\_aws\_auth](#output\_scraper\_aws\_auth) | Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use | diff --git a/examples/existing-cluster-java/outputs.tf b/examples/existing-cluster-java/outputs.tf index e14427e3..15359e95 100644 --- a/examples/existing-cluster-java/outputs.tf +++ b/examples/existing-cluster-java/outputs.tf @@ -22,3 +22,8 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "scraper_aws_auth" { + description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" + value = module.eks_monitoring.scraper_aws_auth +} diff --git a/examples/existing-cluster-nginx/README.md b/examples/existing-cluster-nginx/README.md index dad16832..1eb080c9 100644 --- a/examples/existing-cluster-nginx/README.md +++ b/examples/existing-cluster-nginx/README.md @@ -248,4 +248,5 @@ add this `managed_prometheus_region=xxx` and `managed_prometheus_workspace_id=ws | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | AWS Region | +| [scraper\_aws\_auth](#output\_scraper\_aws\_auth) | Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use | diff --git a/examples/existing-cluster-nginx/outputs.tf b/examples/existing-cluster-nginx/outputs.tf index e14427e3..15359e95 100644 --- a/examples/existing-cluster-nginx/outputs.tf +++ b/examples/existing-cluster-nginx/outputs.tf @@ -22,3 +22,8 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "scraper_aws_auth" { + description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" + value = module.eks_monitoring.scraper_aws_auth +} diff --git a/examples/existing-cluster-with-base-and-infra/README.md b/examples/existing-cluster-with-base-and-infra/README.md index a2a22630..c2ae5a58 100644 --- a/examples/existing-cluster-with-base-and-infra/README.md +++ b/examples/existing-cluster-with-base-and-infra/README.md @@ -65,4 +65,5 @@ View the full documentation for this example [here](https://aws-observability.gi | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | AWS Region | +| [scraper\_aws\_auth](#output\_scraper\_aws\_auth) | Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use | diff --git a/examples/existing-cluster-with-base-and-infra/outputs.tf b/examples/existing-cluster-with-base-and-infra/outputs.tf index e14427e3..15359e95 100644 --- a/examples/existing-cluster-with-base-and-infra/outputs.tf +++ b/examples/existing-cluster-with-base-and-infra/outputs.tf @@ -22,3 +22,8 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "scraper_aws_auth" { + description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" + value = module.eks_monitoring.scraper_aws_auth +} diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index aec2981d..3af53a0a 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -30,5 +30,5 @@ output "managed_prometheus_workspace_region" { output "scraper_aws_auth" { description = "Execute this command to grand access to the managed scrapers to gain permissions on your cluster. Mandatory for the first use" - value = "eksctl create iamidentitymapping --cluster ${var.eks_cluster_id} --region ${local.managed_prometheus_workspace_region} --arn ${aws_prometheus_scraper.this.role_arn} --username aps-collector-user" + value = "eksctl create iamidentitymapping --cluster ${var.eks_cluster_id} --region ${local.managed_prometheus_workspace_region} --arn ${replace(aws_prometheus_scraper.this.role_arn, "aws-service-role/scraper.aps.amazonaws.com/", "")} --username aps-collector-user" }