From e3f7af57199c80d74afe4326517abf2fb28430f2 Mon Sep 17 00:00:00 2001 From: Weifeng Wang Date: Mon, 29 Apr 2024 17:18:48 +0800 Subject: [PATCH] [kubenertes] enabled log in traces Signed-off-by: Weifeng Wang --- .../common/config/alloy/all-in-one.alloy | 2 +- docker-compose/common/config/alloy/logs.alloy | 2 +- .../common/config/alloy/metrics.alloy | 2 +- .../common/config/alloy/profiles.alloy | 2 +- .../common/config/alloy/traces.alloy | 2 +- kubernetes/common/alloy/configs/config.alloy | 2 +- kubernetes/common/alloy/kustomization.yaml | 6 +- .../alloy/manifests/k8s-all-in-one.yaml | 159 +- kubernetes/common/alloy/values-k3d-k3s.yaml | 2 + .../logs/k8s-all-in-one.yaml | 19 +- .../logs/kustomization.yaml | 1 - kubernetes/microservices-mode/logs/logs.alloy | 2 +- .../metrics/k8s-all-in-one.yaml | 19 +- .../metrics/kustomization.yaml | 1 - .../microservices-mode/metrics/metrics.alloy | 2 +- .../profiles/k8s-all-in-one.yaml | 19 +- .../profiles/kustomization.yaml | 1 - .../profiles/profiles.alloy | 2 +- .../traces/configs/gateway_loki.conf | 108 ++ .../traces/k8s-all-in-one.yaml | 575 ++++++- .../traces/kustomization.yaml | 3 +- .../microservices-mode/traces/traces.alloy | 27 +- .../all-in-one/all-in-one.alloy | 2 +- .../all-in-one/k8s-all-in-one.yaml | 19 +- .../all-in-one/kustomization.yaml | 1 - .../monolithic-mode/logs/k8s-all-in-one.yaml | 19 +- .../monolithic-mode/logs/kustomization.yaml | 1 - kubernetes/monolithic-mode/logs/logs.alloy | 2 +- .../metrics/k8s-all-in-one.yaml | 19 +- .../metrics/kustomization.yaml | 1 - .../monolithic-mode/metrics/metrics.alloy | 2 +- .../profiles/k8s-all-in-one.yaml | 159 +- .../monolithic-mode/profiles/profiles.alloy | 2 +- .../traces/configs/gateway_loki.conf | 108 ++ .../traces/k8s-all-in-one.yaml | 751 ++++++++- .../monolithic-mode/traces/kustomization.yaml | 8 + .../monolithic-mode/traces/traces.alloy | 79 +- .../read-write-mode/logs/k8s-all-in-one.yaml | 19 +- .../read-write-mode/logs/kustomization.yaml | 1 - kubernetes/read-write-mode/logs/logs.alloy | 2 +- .../metrics/k8s-all-in-one.yaml | 19 +- .../metrics/kustomization.yaml | 1 - .../read-write-mode/metrics/metrics.alloy | 2 +- monitoring-mixins/k8s-all-in-one.yaml | 1456 ----------------- monitoring-mixins/kustomization.yaml | 5 +- 45 files changed, 2018 insertions(+), 1618 deletions(-) create mode 100644 kubernetes/microservices-mode/traces/configs/gateway_loki.conf create mode 100644 kubernetes/monolithic-mode/traces/configs/gateway_loki.conf diff --git a/docker-compose/common/config/alloy/all-in-one.alloy b/docker-compose/common/config/alloy/all-in-one.alloy index ad74fa91..a4bec579 100644 --- a/docker-compose/common/config/alloy/all-in-one.alloy +++ b/docker-compose/common/config/alloy/all-in-one.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/docker-compose/common/config/alloy/logs.alloy b/docker-compose/common/config/alloy/logs.alloy index 57a9f305..90dffced 100644 --- a/docker-compose/common/config/alloy/logs.alloy +++ b/docker-compose/common/config/alloy/logs.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/docker-compose/common/config/alloy/metrics.alloy b/docker-compose/common/config/alloy/metrics.alloy index e5f5998a..3a9801b5 100644 --- a/docker-compose/common/config/alloy/metrics.alloy +++ b/docker-compose/common/config/alloy/metrics.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/docker-compose/common/config/alloy/profiles.alloy b/docker-compose/common/config/alloy/profiles.alloy index f8ee4f87..770deb8b 100644 --- a/docker-compose/common/config/alloy/profiles.alloy +++ b/docker-compose/common/config/alloy/profiles.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/docker-compose/common/config/alloy/traces.alloy b/docker-compose/common/config/alloy/traces.alloy index c2459a36..ab7b2eb4 100644 --- a/docker-compose/common/config/alloy/traces.alloy +++ b/docker-compose/common/config/alloy/traces.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/common/alloy/configs/config.alloy b/kubernetes/common/alloy/configs/config.alloy index 409243b1..e94fd4fb 100644 --- a/kubernetes/common/alloy/configs/config.alloy +++ b/kubernetes/common/alloy/configs/config.alloy @@ -1,4 +1,4 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/common/alloy/kustomization.yaml b/kubernetes/common/alloy/kustomization.yaml index 424b8d82..19cfef4b 100644 --- a/kubernetes/common/alloy/kustomization.yaml +++ b/kubernetes/common/alloy/kustomization.yaml @@ -13,6 +13,7 @@ helmCharts: valuesFile: values-k3d-k3s.yaml resources: +- configs/kubernetes/integrations - ../../../monitoring-mixins/alloy-mixin/deploy configMapGenerator: @@ -51,8 +52,3 @@ configMapGenerator: - configs/kubernetes/jobs/kube-state-metrics.alloy - configs/kubernetes/jobs/kubelet.alloy - configs/kubernetes/jobs/node-exporter.alloy - -secretGenerator: -- name: alloy-env - literals: - - ALLOY_LOG_LEVEL=warn diff --git a/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml b/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml index 75d77242..bb51854f 100644 --- a/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml +++ b/kubernetes/common/alloy/manifests/k8s-all-in-one.yaml @@ -985,11 +985,11 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n" kind: ConfigMap metadata: - name: alloy-config-ftt29f8k85 + name: alloy-config-b664fd27t6 namespace: monitoring-system --- apiVersion: v1 @@ -1547,6 +1547,126 @@ metadata: namespace: monitoring-system --- apiVersion: v1 +data: + MEMCACHED_K8S_SECRET_NAME: alloy-integrations-memcached + MYSQL_K8S_SECRET_NAME: alloy-integrations-mysql + REDIS_K8S_SECRET_NAME: alloy-integrations-redis + memcached.alloy: "/*\nModule Components: component_memcached\n*/\n\ndeclare \"memcached_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/memcached)\"\n\t\toptional = + true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret name + (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|memcached_commands_total|memcached_connections_total|memcached_current_bytes|memcached_current_connections|memcached_current_items|memcached_items_evicted_total|memcached_items_total|memcached_max_connections|memcached_read_bytes_total|memcached_up|memcached_uptime_seconds|memcached_version|memcached_written_bytes_total)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"memcached\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-memcached\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Memcached\n\t****************************************************************/\n\t// + https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.memcached/\n\tprometheus.exporter.memcached + \"integrations_memcached_exporter\" {\n\t\taddress = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-address\"])\n\t\ttimeout + = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-timeout\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_memcached_exporter\" {\n\t\ttargets = prometheus.exporter.memcached.integrations_memcached_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/memcached\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_memcached_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_memcached_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_memcached_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_memcached_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + mysql.alloy: "/*\nModule Components: component_mysql\n*/\n\ndeclare \"mysql_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"namespace\" {\n\t\tcomment = \"kubernetes secret namespace (default: monitoring-system)\"\n\t\toptional + = true\n\t}\n\n\targument \"name\" {\n\t\tcomment = \"kubernetes secret name + (default: alloy-integrations-mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: see + below)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" {\n\t\tcomment + \ = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"mysql\" {\n\t\tname = coalesce(argument.name.value, \"alloy-integrations-mysql\")\n\t\tnamespace + = coalesce(argument.namespace.value, \"monitoring-system\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Mysql\n\t****************************************************************/\n\tprometheus.exporter.mysql + \"integrations_mysqld_exporter\" {\n\t\tdata_source_name = nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-username\"]) + + \":\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-password\"]) + + \"@(\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-host\"]) + + \")/\"\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_mysqld_exporter\" {\n\t\ttargets = prometheus.exporter.mysql.integrations_mysqld_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/mysql\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label = + \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.mysql.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_mysqld_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_mysqld_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_mysqld_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_mysqld_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(up|instance:mysql_heartbeat_lag_seconds|instance:mysql_slave_lag_seconds|mysql_global_status_aborted_clients|mysql_global_status_aborted_connects|mysql_global_status_buffer_pool_pages|mysql_global_status_bytes_received|mysql_global_status_bytes_sent|mysql_global_status_commands_total|mysql_global_status_created_tmp_disk_tables|mysql_global_status_created_tmp_files|mysql_global_status_created_tmp_tables|mysql_global_status_handlers_total|mysql_global_status_innodb_log_waits|mysql_global_status_innodb_mem_adaptive_hash|mysql_global_status_innodb_mem_dictionary|mysql_global_status_innodb_num_open_files|mysql_global_status_innodb_page_size|mysql_global_status_max_used_connections|mysql_global_status_open_files|mysql_global_status_open_table_definitions|mysql_global_status_open_tables|mysql_global_status_opened_files|mysql_global_status_opened_table_definitions|mysql_global_status_opened_tables|mysql_global_status_qcache_free_memory|mysql_global_status_qcache_hits|mysql_global_status_qcache_inserts|mysql_global_status_qcache_lowmem_prunes|mysql_global_status_qcache_not_cached|mysql_global_status_qcache_queries_in_cache|mysql_global_status_queries|mysql_global_status_questions|mysql_global_status_select_full_join|mysql_global_status_select_full_range_join|mysql_global_status_select_range|mysql_global_status_select_range_check|mysql_global_status_select_scan|mysql_global_status_slow_queries|mysql_global_status_sort_merge_passes|mysql_global_status_sort_range|mysql_global_status_sort_rows|mysql_global_status_sort_scan|mysql_global_status_table_locks_immediate|mysql_global_status_table_locks_waited|mysql_global_status_table_open_cache_hits|mysql_global_status_table_open_cache_misses|mysql_global_status_table_open_cache_overflows|mysql_global_status_threads_cached|mysql_global_status_threads_connected|mysql_global_status_threads_created|mysql_global_status_threads_running|mysql_global_status_uptime|mysql_global_status_wsrep_local_recv_queue|mysql_global_status_wsrep_local_state|mysql_global_status_wsrep_ready|mysql_global_variables_innodb_additional_mem_pool_size|mysql_global_variables_innodb_buffer_pool_size|mysql_global_variables_innodb_log_buffer_size|mysql_global_variables_key_buffer_size|mysql_global_variables_max_connections|mysql_global_variables_open_files_limit|mysql_global_variables_query_cache_size|mysql_global_variables_table_definition_cache|mysql_global_variables_table_open_cache|mysql_global_variables_thread_cache_size|mysql_global_variables_tokudb_cache_size|mysql_global_variables_wsrep_desync|mysql_heartbeat_now_timestamp_seconds|mysql_heartbeat_stored_timestamp_seconds|mysql_info_schema_processlist_threads|mysql_slave_status_seconds_behind_master|mysql_slave_status_slave_io_running|mysql_slave_status_slave_sql_running|mysql_slave_status_sql_delay|mysql_up)\")\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + redis.alloy: "/*\nModule Components: component_redis_exporter\n*/\n\ndeclare \"redis_exporter_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/redis_exporter)\"\n\t\toptional + = true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret + name (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|redis_blocked_clients|redis_cluster_slots_fail|redis_cluster_slots_pfail|redis_cluster_state|redis_commands_duration_seconds_total|redis_commands_total|redis_connected_clients|redis_connected_slaves|redis_db_keys|redis_db_keys_expiring|redis_evicted_keys_total|redis_keyspace_hits_total|redis_keyspace_misses_total|redis_master_last_io_seconds_ago|redis_memory_fragmentation_ratio|redis_memory_max_bytes|redis_memory_used_bytes|redis_memory_used_rss_bytes|redis_total_system_memory_bytes|redis_up)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t\tdefault = \"60s\"\n\t}\n\n\targument + \"scrape_timeout\" {\n\t\tcomment = \"How long before a scrape times out (default: + 10s)\"\n\t\toptional = true\n\t\tdefault = \"10s\"\n\t}\n\n\tremote.kubernetes.secret + \"redis\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-redis\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Redis\n\t****************************************************************/\n\tprometheus.exporter.redis + \"integrations_redis_exporter\" {\n\t\tredis_addr = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-address\"])\n\t\tredis_password + = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-password\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_redis_exporter\" {\n\t\ttargets = prometheus.exporter.redis.integrations_redis_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/redis_exporter\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.redis.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_redis_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_redis_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = argument.scrape_interval.value\n\t\tscrape_timeout + \ = argument.scrape_timeout.value\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to + = [prometheus.relabel.integrations_redis_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_redis_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" +kind: ConfigMap +metadata: + name: alloy-integrations + namespace: monitoring-system +--- +apiVersion: v1 data: apiserver.alloy: "/*\nModule Components: apiserver\nDescription: kubernetes Apiserver Metrics Scrape\n\n*/\n\ndeclare \"apiserver_metrics_scrape\" {\n\n\t/********************************************\n\t* @@ -4135,10 +4255,35 @@ metadata: --- apiVersion: v1 data: - ALLOY_LOG_LEVEL: d2Fybg== + instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= + instance-name: cHJpbWFyeQ== + instance-timeout: NXM= +kind: Secret +metadata: + name: alloy-integrations-memcached + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-name: cHJpbWFyeQ== + mysql-host: bXlzcWwubXlzcWwtc3lzdGVtLnN2Yy5jbHVzdGVyLmxvY2Fs + mysql-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= + mysql-username: bGd0bXA= +kind: Secret +metadata: + name: alloy-integrations-mysql + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-address: cmVkaXMtbWFzdGVyLnJlZGlzLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo2Mzc5 + instance-name: cHJpbWFyeQ== + instance-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: alloy-env-f7k229g955 + name: alloy-integrations-redis namespace: monitoring-system type: Opaque --- @@ -4279,7 +4424,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -4306,6 +4451,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -4325,7 +4472,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-ftt29f8k85 + name: alloy-config-b664fd27t6 name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/common/alloy/values-k3d-k3s.yaml b/kubernetes/common/alloy/values-k3d-k3s.yaml index 46d10de8..3626940c 100644 --- a/kubernetes/common/alloy/values-k3d-k3s.yaml +++ b/kubernetes/common/alloy/values-k3d-k3s.yaml @@ -3,6 +3,8 @@ crds: configReloader: enabled: false alloy: + securityContext: + privileged: true enableReporting: false configMap: create: false diff --git a/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml b/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml index e6e1750e..f9108a1c 100644 --- a/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/logs/k8s-all-in-one.yaml @@ -1183,7 +1183,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1216,7 +1216,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc namespace: monitoring-system --- apiVersion: v1 @@ -4631,15 +4631,6 @@ metadata: type: Opaque --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -6508,7 +6499,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -6535,6 +6526,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -6554,7 +6547,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/microservices-mode/logs/kustomization.yaml b/kubernetes/microservices-mode/logs/kustomization.yaml index 518db9ea..94625eb6 100644 --- a/kubernetes/microservices-mode/logs/kustomization.yaml +++ b/kubernetes/microservices-mode/logs/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - loki # optional diff --git a/kubernetes/microservices-mode/logs/logs.alloy b/kubernetes/microservices-mode/logs/logs.alloy index 2ee169ac..35687f34 100644 --- a/kubernetes/microservices-mode/logs/logs.alloy +++ b/kubernetes/microservices-mode/logs/logs.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml b/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml index 9543df6b..28a40c7a 100644 --- a/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml @@ -1009,7 +1009,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1033,7 +1033,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b namespace: monitoring-system --- apiVersion: v1 @@ -4513,15 +4513,6 @@ metadata: namespace: monitoring-system --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -5885,7 +5876,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -5912,6 +5903,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -5931,7 +5924,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/microservices-mode/metrics/kustomization.yaml b/kubernetes/microservices-mode/metrics/kustomization.yaml index d6e03a1c..a398d979 100644 --- a/kubernetes/microservices-mode/metrics/kustomization.yaml +++ b/kubernetes/microservices-mode/metrics/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - mimir-distributed images: diff --git a/kubernetes/microservices-mode/metrics/metrics.alloy b/kubernetes/microservices-mode/metrics/metrics.alloy index 9285c9dd..e4dfd69d 100644 --- a/kubernetes/microservices-mode/metrics/metrics.alloy +++ b/kubernetes/microservices-mode/metrics/metrics.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml index 86de3e1a..17fdd68d 100644 --- a/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/profiles/k8s-all-in-one.yaml @@ -1069,7 +1069,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1099,7 +1099,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-m9k4kcgmdc + name: alloy-config-6h7tb697c9 namespace: monitoring-system --- apiVersion: v1 @@ -4554,15 +4554,6 @@ metadata: namespace: profiles-system --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -5622,7 +5613,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -5649,6 +5640,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -5668,7 +5661,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-m9k4kcgmdc + name: alloy-config-6h7tb697c9 name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/microservices-mode/profiles/kustomization.yaml b/kubernetes/microservices-mode/profiles/kustomization.yaml index 37720273..8ade1bf6 100644 --- a/kubernetes/microservices-mode/profiles/kustomization.yaml +++ b/kubernetes/microservices-mode/profiles/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - pyroscope # optional diff --git a/kubernetes/microservices-mode/profiles/profiles.alloy b/kubernetes/microservices-mode/profiles/profiles.alloy index 7988f425..32709f3b 100644 --- a/kubernetes/microservices-mode/profiles/profiles.alloy +++ b/kubernetes/microservices-mode/profiles/profiles.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/microservices-mode/traces/configs/gateway_loki.conf b/kubernetes/microservices-mode/traces/configs/gateway_loki.conf new file mode 100644 index 00000000..fcae3a32 --- /dev/null +++ b/kubernetes/microservices-mode/traces/configs/gateway_loki.conf @@ -0,0 +1,108 @@ +server { + listen 3100; + listen [::]:3100; + + location = / { + return 200 'OK'; + auth_basic off; + access_log off; + } + + proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid; + + # Distributor endpoints + location = /api/prom/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /distributor/ring { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + + # Ingester endpoints + location /flush { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location ^~ /ingester/ { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location = /ingester { + internal; # to suppress 301 + } + + # Ring + location = /ring { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + # MemberListKV + location = /memberlist { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + + # Ruler endpoints + location = /ruler/ring { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + + + # Compactor endpoints + location = /compactor/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/delete { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/cache/generation_numbers { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # IndexGateway endpoints + location = /indexgateway/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # Config endpoints + location = /config { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # QueryFrontend, Querier endpoints + location = /api/prom/tail { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location = /loki/api/v1/tail { + proxy_pass http://${LOKI_QUERIER_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location ~ /api/prom/.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + location ~ /loki/api/v1.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + } \ No newline at end of file diff --git a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml index 5f746ece..63f0205d 100644 --- a/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/microservices-mode/traces/k8s-all-in-one.yaml @@ -1,9 +1,33 @@ apiVersion: v1 kind: Namespace +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: logging-system +--- +apiVersion: v1 +kind: Namespace metadata: name: tracing-system --- apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +--- +apiVersion: v1 kind: ServiceAccount metadata: labels: @@ -143,6 +167,27 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-clusterrole +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -161,6 +206,25 @@ subjects: name: alloy namespace: monitoring-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: loki-clusterrole +subjects: +- kind: ServiceAccount + name: loki + namespace: logging-system +--- apiVersion: v1 data: TEMPO_COMPACTOR_HOST: tempo-distributed-compactor.tracing-system.svc.cluster.local @@ -174,6 +238,115 @@ metadata: --- apiVersion: v1 data: + gateway_loki.conf.template: |- + server { + listen 3100; + listen [::]:3100; + + location = / { + return 200 'OK'; + auth_basic off; + access_log off; + } + + proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid; + + # Distributor endpoints + location = /api/prom/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /distributor/ring { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + + # Ingester endpoints + location /flush { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location ^~ /ingester/ { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location = /ingester { + internal; # to suppress 301 + } + + # Ring + location = /ring { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + # MemberListKV + location = /memberlist { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + + # Ruler endpoints + location = /ruler/ring { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + + + # Compactor endpoints + location = /compactor/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/delete { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/cache/generation_numbers { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # IndexGateway endpoints + location = /indexgateway/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # Config endpoints + location = /config { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # QueryFrontend, Querier endpoints + location = /api/prom/tail { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location = /loki/api/v1/tail { + proxy_pass http://${LOKI_QUERIER_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location ~ /api/prom/.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + location ~ /loki/api/v1.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + } gateway_mimir.conf.template: "server {\n listen 8080;\n listen [::]:8080;\n\n \ location = / {\n return 200 'OK';\n auth_basic off;\n access_log off;\n }\n\n proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;\n\n @@ -223,6 +396,154 @@ metadata: namespace: gateway --- apiVersion: v1 +data: + config.yaml: |2 + + auth_enabled: false + + # -reporting.enabled=false + analytics: + reporting_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: info + log_format: json + + # https://grafana.com/docs/loki/latest/configure/#use-environment-variables-in-the-configuration + common: + compactor_address: http://loki.logging-system.svc.cluster.local:3100 + path_prefix: /var/loki + replication_factor: 1 + storage: + s3: + bucketnames: loki-data + endpoint: ${LOKI_S3_ENDPOINT:-minio.minio-system.svc.cluster.local:443} + access_key_id: ${LOKI_S3_ACCESS_KEY_ID:-lgtmp} + secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY:-supersecret} + insecure: ${LOKI_S3_INSECURE:-false} + s3forcepathstyle: true + http_config: + insecure_skip_verify: true + + bloom_gateway: + enabled: true + client: + addresses: "dns+loki.logging-system.svc.cluster.local:9095" + cache_results: true + results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + bloom_compactor: + enabled: true + ring: + kvstore: + store: memberlist + + index_gateway: + mode: simple + + compactor: + working_directory: /tmp/compactor + + memberlist: + join_members: + - loki-memberlist.logging-system.svc.cluster.local:7946 + + # https://github.com/grafana/loki/blob/main/docs/sources/configure/_index.md#query_range + query_range: + align_queries_with_step: true + + cache_results: true + results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + cache_index_stats_results: true + index_stats_results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + pattern_ingester: + enabled: true + + limits_config: + max_global_streams_per_user: 0 + ingestion_rate_mb: 50000 + ingestion_burst_size_mb: 50000 + volume_enabled: true + + ruler: + storage: + s3: + bucketnames: loki-ruler + type: s3 + + runtime_config: + file: /etc/loki/runtime-config/runtime-config.yaml + + schema_config: + configs: + - from: "2024-04-08" + index: + period: 24h + prefix: loki_index_ + object_store: s3 + schema: v13 + store: tsdb + + storage_config: + tsdb_shipper: + active_index_directory: /var/loki/index + cache_location: /var/loki/cache + index_gateway_client: + server_address: "dns+loki.logging-system.svc.cluster.local:9095" + + chunk_store_config: + chunk_cache_config: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-config-5468cb29k9 + namespace: logging-system +--- +apiVersion: v1 +data: + runtime-config.yaml: | + # This file can be used to set overrides or other runtime config. + overrides: + "fake": # limits for anonymous that the whole cluster enforces + ingestion_rate_mb: 1500000 + max_streams_per_user: 100000 + max_chunks_per_query: 100000 + "anonymous": # limits for anonymous that the whole cluster enforces + ingestion_rate_mb: 1500000 + max_streams_per_user: 100000 + max_chunks_per_query: 100000 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-runtime-9599m5k6h2 + namespace: logging-system +--- +apiVersion: v1 data: alloy-cluster-node.json: |- { @@ -1076,7 +1397,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1094,6 +1415,12 @@ data: \ = [provider.self_hosted_stack.kubernetes.traces_receiver]\n}\n\ntracing {\n\t// Write all spans. Don't do this in production!\n\tsampling_fraction = 1\n\n\t// Forward Alloy internal spans to traces process.\n\twrite_to = [traces.process_and_transform.kubernetes.alloy_traces_input]\n}\n\n/********************************************\n + * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename + = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.rules_to_loki + \"kubernetes\" { }\n\nlogs.kubernetes_cluster_events \"kubernetes\" {\n\tcluster + = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape + \"kubernetes\" {\n\tcluster = \"k3d-k3s-codelab\"\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster @@ -1112,7 +1439,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-6k989hdmh5 + name: alloy-config-gf274kh9b7 namespace: monitoring-system --- apiVersion: v1 @@ -4666,11 +4993,17 @@ metadata: --- apiVersion: v1 data: - ALLOY_LOG_LEVEL: d2Fybg== + LOKI_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-env-58m52b99kc + namespace: logging-system type: Opaque --- apiVersion: v1 @@ -4738,6 +5071,75 @@ type: Opaque --- apiVersion: v1 kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + ports: + - name: http-metrics + port: 3100 + - appProtocol: grpc + name: grpc + port: 9095 + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + prometheus.io/service-monitor: "false" + name: loki-headless + namespace: logging-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + prometheus.io/service-monitor: "false" + name: loki-memberlist + namespace: logging-system +spec: + clusterIP: None + ports: + - name: tcp-gossip-ring + port: 7946 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist +--- +apiVersion: v1 +kind: Service metadata: labels: app.kubernetes.io/instance: alloy @@ -5906,6 +6308,128 @@ spec: --- apiVersion: apps/v1 kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + serviceName: loki-headless + template: + metadata: + annotations: + profiles.grafana.com/cpu.port_name: http-metrics + profiles.grafana.com/cpu.scrape: "false" + profiles.grafana.com/goroutine.port_name: http-metrics + profiles.grafana.com/goroutine.scrape: "false" + profiles.grafana.com/memory.port_name: http-metrics + profiles.grafana.com/memory.scrape: "false" + pyroscope.io/service_name: loki + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + spec: + automountServiceAccountToken: true + containers: + - args: + - -config.file=/etc/loki/config/config.yaml + - -target=all + - -config.expand-env=true + envFrom: + - secretRef: + name: loki-env-58m52b99kc + image: docker.io/grafana/loki:3.0.0 + imagePullPolicy: IfNotPresent + name: loki + ports: + - containerPort: 3100 + name: http-metrics + protocol: TCP + - containerPort: 9095 + name: grpc + protocol: TCP + - containerPort: 7946 + name: http-memberlist + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: http-metrics + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: tmp + - mountPath: /etc/loki/config + name: config + - mountPath: /etc/loki/runtime-config + name: runtime-config + - mountPath: /var/loki + name: storage + enableServiceLinks: true + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + serviceAccountName: loki + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: tmp + - configMap: + items: + - key: config.yaml + path: config.yaml + name: loki-config-5468cb29k9 + name: config + - configMap: + name: loki-runtime-9599m5k6h2 + name: runtime-config + updateStrategy: + rollingUpdate: + partition: 0 + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: StatefulSet metadata: labels: app.kubernetes.io/instance: alloy @@ -5963,7 +6487,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -5990,6 +6514,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -6009,7 +6535,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-6k989hdmh5 + name: alloy-config-gf274kh9b7 name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f @@ -6267,6 +6793,41 @@ spec: --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + endpoints: + - interval: 15s + port: http-metrics + relabelings: + - action: replace + replacement: logging-system/loki + sourceLabels: + - job + targetLabel: job + scheme: http + namespaceSelector: + matchNames: + - logging-system + selector: + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + matchLabels: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor metadata: labels: app.kubernetes.io/instance: alloy diff --git a/kubernetes/microservices-mode/traces/kustomization.yaml b/kubernetes/microservices-mode/traces/kustomization.yaml index 99396cda..d0b91cc0 100644 --- a/kubernetes/microservices-mode/traces/kustomization.yaml +++ b/kubernetes/microservices-mode/traces/kustomization.yaml @@ -9,11 +9,11 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - tempo-distributed # optional - ../../monolithic-mode/metrics/mimir +- ../../monolithic-mode/logs/loki secretGenerator: - name: tempo-distributed-env @@ -52,6 +52,7 @@ configMapGenerator: files: - gateway_tempo.conf.template=configs/gateway_tempo.conf - gateway_mimir.conf.template=configs/gateway_mimir.conf + - gateway_loki.conf.template=configs/gateway_loki.conf - name: nginx-env namespace: gateway options: diff --git a/kubernetes/microservices-mode/traces/traces.alloy b/kubernetes/microservices-mode/traces/traces.alloy index c17e77bc..bad8b27d 100644 --- a/kubernetes/microservices-mode/traces/traces.alloy +++ b/kubernetes/microservices-mode/traces/traces.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } @@ -43,6 +43,31 @@ tracing { write_to = [traces.process_and_transform.kubernetes.alloy_traces_input] } +/******************************************** + * Logs + ********************************************/ +import.file "logs" { + filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/logs" +} + +logs.rules_to_loki "kubernetes" { } + +logs.kubernetes_cluster_events "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + +logs.annotations_scrape "kubernetes" { + cluster = "k3d-k3s-codelab" + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + +logs.keep_labels "kubernetes" { + forward_to = [provider.self_hosted_stack.kubernetes.logs_receiver] +} + /******************************************** * Metrics ********************************************/ diff --git a/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy b/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy index c68a19ef..5ac8b9fd 100644 --- a/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy +++ b/kubernetes/monolithic-mode/all-in-one/all-in-one.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml index 2e197e8a..97a45b34 100644 --- a/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/all-in-one/k8s-all-in-one.yaml @@ -1734,7 +1734,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1793,7 +1793,7 @@ data: \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// }\n" kind: ConfigMap metadata: - name: alloy-config-dmg2hdb585 + name: alloy-config-mt4f8cf687 namespace: monitoring-system --- apiVersion: v1 @@ -5395,15 +5395,6 @@ metadata: type: Opaque --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -6206,7 +6197,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -6233,6 +6224,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -6252,7 +6245,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-dmg2hdb585 + name: alloy-config-mt4f8cf687 name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/monolithic-mode/all-in-one/kustomization.yaml b/kubernetes/monolithic-mode/all-in-one/kustomization.yaml index f9cdd605..7f12c8ab 100644 --- a/kubernetes/monolithic-mode/all-in-one/kustomization.yaml +++ b/kubernetes/monolithic-mode/all-in-one/kustomization.yaml @@ -10,7 +10,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - ../logs/loki - ../traces/tempo - ../metrics/mimir diff --git a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml index aee362cd..20fe6c9c 100644 --- a/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/logs/k8s-all-in-one.yaml @@ -1209,7 +1209,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1242,7 +1242,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc namespace: monitoring-system --- apiVersion: v1 @@ -4663,15 +4663,6 @@ metadata: type: Opaque --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -5182,7 +5173,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -5209,6 +5200,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -5228,7 +5221,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/monolithic-mode/logs/kustomization.yaml b/kubernetes/monolithic-mode/logs/kustomization.yaml index 68c1431e..ba86bf5f 100644 --- a/kubernetes/monolithic-mode/logs/kustomization.yaml +++ b/kubernetes/monolithic-mode/logs/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - loki # optional diff --git a/kubernetes/monolithic-mode/logs/logs.alloy b/kubernetes/monolithic-mode/logs/logs.alloy index 2ee169ac..35687f34 100644 --- a/kubernetes/monolithic-mode/logs/logs.alloy +++ b/kubernetes/monolithic-mode/logs/logs.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml index 780bde69..9fbb5db7 100644 --- a/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/metrics/k8s-all-in-one.yaml @@ -997,7 +997,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1021,7 +1021,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b namespace: monitoring-system --- apiVersion: v1 @@ -4427,15 +4427,6 @@ metadata: namespace: monitoring-system --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -4755,7 +4746,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -4782,6 +4773,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -4801,7 +4794,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/monolithic-mode/metrics/kustomization.yaml b/kubernetes/monolithic-mode/metrics/kustomization.yaml index c1835fae..8a4ebd46 100644 --- a/kubernetes/monolithic-mode/metrics/kustomization.yaml +++ b/kubernetes/monolithic-mode/metrics/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - mimir configMapGenerator: diff --git a/kubernetes/monolithic-mode/metrics/metrics.alloy b/kubernetes/monolithic-mode/metrics/metrics.alloy index 9285c9dd..e4dfd69d 100644 --- a/kubernetes/monolithic-mode/metrics/metrics.alloy +++ b/kubernetes/monolithic-mode/metrics/metrics.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml index 5663015b..b4ee2981 100644 --- a/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/profiles/k8s-all-in-one.yaml @@ -1060,7 +1060,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1090,7 +1090,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-m9k4kcgmdc + name: alloy-config-6h7tb697c9 namespace: monitoring-system --- apiVersion: v1 @@ -1648,6 +1648,126 @@ metadata: namespace: monitoring-system --- apiVersion: v1 +data: + MEMCACHED_K8S_SECRET_NAME: alloy-integrations-memcached + MYSQL_K8S_SECRET_NAME: alloy-integrations-mysql + REDIS_K8S_SECRET_NAME: alloy-integrations-redis + memcached.alloy: "/*\nModule Components: component_memcached\n*/\n\ndeclare \"memcached_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/memcached)\"\n\t\toptional = + true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret name + (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|memcached_commands_total|memcached_connections_total|memcached_current_bytes|memcached_current_connections|memcached_current_items|memcached_items_evicted_total|memcached_items_total|memcached_max_connections|memcached_read_bytes_total|memcached_up|memcached_uptime_seconds|memcached_version|memcached_written_bytes_total)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"memcached\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-memcached\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Memcached\n\t****************************************************************/\n\t// + https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.memcached/\n\tprometheus.exporter.memcached + \"integrations_memcached_exporter\" {\n\t\taddress = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-address\"])\n\t\ttimeout + = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-timeout\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_memcached_exporter\" {\n\t\ttargets = prometheus.exporter.memcached.integrations_memcached_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/memcached\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_memcached_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_memcached_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_memcached_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_memcached_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + mysql.alloy: "/*\nModule Components: component_mysql\n*/\n\ndeclare \"mysql_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"namespace\" {\n\t\tcomment = \"kubernetes secret namespace (default: monitoring-system)\"\n\t\toptional + = true\n\t}\n\n\targument \"name\" {\n\t\tcomment = \"kubernetes secret name + (default: alloy-integrations-mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: see + below)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" {\n\t\tcomment + \ = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"mysql\" {\n\t\tname = coalesce(argument.name.value, \"alloy-integrations-mysql\")\n\t\tnamespace + = coalesce(argument.namespace.value, \"monitoring-system\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Mysql\n\t****************************************************************/\n\tprometheus.exporter.mysql + \"integrations_mysqld_exporter\" {\n\t\tdata_source_name = nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-username\"]) + + \":\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-password\"]) + + \"@(\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-host\"]) + + \")/\"\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_mysqld_exporter\" {\n\t\ttargets = prometheus.exporter.mysql.integrations_mysqld_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/mysql\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label = + \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.mysql.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_mysqld_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_mysqld_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_mysqld_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_mysqld_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(up|instance:mysql_heartbeat_lag_seconds|instance:mysql_slave_lag_seconds|mysql_global_status_aborted_clients|mysql_global_status_aborted_connects|mysql_global_status_buffer_pool_pages|mysql_global_status_bytes_received|mysql_global_status_bytes_sent|mysql_global_status_commands_total|mysql_global_status_created_tmp_disk_tables|mysql_global_status_created_tmp_files|mysql_global_status_created_tmp_tables|mysql_global_status_handlers_total|mysql_global_status_innodb_log_waits|mysql_global_status_innodb_mem_adaptive_hash|mysql_global_status_innodb_mem_dictionary|mysql_global_status_innodb_num_open_files|mysql_global_status_innodb_page_size|mysql_global_status_max_used_connections|mysql_global_status_open_files|mysql_global_status_open_table_definitions|mysql_global_status_open_tables|mysql_global_status_opened_files|mysql_global_status_opened_table_definitions|mysql_global_status_opened_tables|mysql_global_status_qcache_free_memory|mysql_global_status_qcache_hits|mysql_global_status_qcache_inserts|mysql_global_status_qcache_lowmem_prunes|mysql_global_status_qcache_not_cached|mysql_global_status_qcache_queries_in_cache|mysql_global_status_queries|mysql_global_status_questions|mysql_global_status_select_full_join|mysql_global_status_select_full_range_join|mysql_global_status_select_range|mysql_global_status_select_range_check|mysql_global_status_select_scan|mysql_global_status_slow_queries|mysql_global_status_sort_merge_passes|mysql_global_status_sort_range|mysql_global_status_sort_rows|mysql_global_status_sort_scan|mysql_global_status_table_locks_immediate|mysql_global_status_table_locks_waited|mysql_global_status_table_open_cache_hits|mysql_global_status_table_open_cache_misses|mysql_global_status_table_open_cache_overflows|mysql_global_status_threads_cached|mysql_global_status_threads_connected|mysql_global_status_threads_created|mysql_global_status_threads_running|mysql_global_status_uptime|mysql_global_status_wsrep_local_recv_queue|mysql_global_status_wsrep_local_state|mysql_global_status_wsrep_ready|mysql_global_variables_innodb_additional_mem_pool_size|mysql_global_variables_innodb_buffer_pool_size|mysql_global_variables_innodb_log_buffer_size|mysql_global_variables_key_buffer_size|mysql_global_variables_max_connections|mysql_global_variables_open_files_limit|mysql_global_variables_query_cache_size|mysql_global_variables_table_definition_cache|mysql_global_variables_table_open_cache|mysql_global_variables_thread_cache_size|mysql_global_variables_tokudb_cache_size|mysql_global_variables_wsrep_desync|mysql_heartbeat_now_timestamp_seconds|mysql_heartbeat_stored_timestamp_seconds|mysql_info_schema_processlist_threads|mysql_slave_status_seconds_behind_master|mysql_slave_status_slave_io_running|mysql_slave_status_slave_sql_running|mysql_slave_status_sql_delay|mysql_up)\")\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + redis.alloy: "/*\nModule Components: component_redis_exporter\n*/\n\ndeclare \"redis_exporter_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/redis_exporter)\"\n\t\toptional + = true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret + name (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|redis_blocked_clients|redis_cluster_slots_fail|redis_cluster_slots_pfail|redis_cluster_state|redis_commands_duration_seconds_total|redis_commands_total|redis_connected_clients|redis_connected_slaves|redis_db_keys|redis_db_keys_expiring|redis_evicted_keys_total|redis_keyspace_hits_total|redis_keyspace_misses_total|redis_master_last_io_seconds_ago|redis_memory_fragmentation_ratio|redis_memory_max_bytes|redis_memory_used_bytes|redis_memory_used_rss_bytes|redis_total_system_memory_bytes|redis_up)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t\tdefault = \"60s\"\n\t}\n\n\targument + \"scrape_timeout\" {\n\t\tcomment = \"How long before a scrape times out (default: + 10s)\"\n\t\toptional = true\n\t\tdefault = \"10s\"\n\t}\n\n\tremote.kubernetes.secret + \"redis\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-redis\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Redis\n\t****************************************************************/\n\tprometheus.exporter.redis + \"integrations_redis_exporter\" {\n\t\tredis_addr = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-address\"])\n\t\tredis_password + = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-password\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_redis_exporter\" {\n\t\ttargets = prometheus.exporter.redis.integrations_redis_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/redis_exporter\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.redis.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_redis_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_redis_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = argument.scrape_interval.value\n\t\tscrape_timeout + \ = argument.scrape_timeout.value\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to + = [prometheus.relabel.integrations_redis_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_redis_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" +kind: ConfigMap +metadata: + name: alloy-integrations + namespace: monitoring-system +--- +apiVersion: v1 data: apiserver.alloy: "/*\nModule Components: apiserver\nDescription: kubernetes Apiserver Metrics Scrape\n\n*/\n\ndeclare \"apiserver_metrics_scrape\" {\n\n\t/********************************************\n\t* @@ -4425,10 +4545,35 @@ metadata: --- apiVersion: v1 data: - ALLOY_LOG_LEVEL: d2Fybg== + instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= + instance-name: cHJpbWFyeQ== + instance-timeout: NXM= +kind: Secret +metadata: + name: alloy-integrations-memcached + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-name: cHJpbWFyeQ== + mysql-host: bXlzcWwubXlzcWwtc3lzdGVtLnN2Yy5jbHVzdGVyLmxvY2Fs + mysql-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= + mysql-username: bGd0bXA= +kind: Secret +metadata: + name: alloy-integrations-mysql + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-address: cmVkaXMtbWFzdGVyLnJlZGlzLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo2Mzc5 + instance-name: cHJpbWFyeQ== + instance-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= kind: Secret metadata: - name: alloy-env-f7k229g955 + name: alloy-integrations-redis namespace: monitoring-system type: Opaque --- @@ -4801,7 +4946,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -4828,6 +4973,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -4847,7 +4994,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-m9k4kcgmdc + name: alloy-config-6h7tb697c9 name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/monolithic-mode/profiles/profiles.alloy b/kubernetes/monolithic-mode/profiles/profiles.alloy index 7988f425..32709f3b 100644 --- a/kubernetes/monolithic-mode/profiles/profiles.alloy +++ b/kubernetes/monolithic-mode/profiles/profiles.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/monolithic-mode/traces/configs/gateway_loki.conf b/kubernetes/monolithic-mode/traces/configs/gateway_loki.conf new file mode 100644 index 00000000..fcae3a32 --- /dev/null +++ b/kubernetes/monolithic-mode/traces/configs/gateway_loki.conf @@ -0,0 +1,108 @@ +server { + listen 3100; + listen [::]:3100; + + location = / { + return 200 'OK'; + auth_basic off; + access_log off; + } + + proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid; + + # Distributor endpoints + location = /api/prom/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /distributor/ring { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + + # Ingester endpoints + location /flush { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location ^~ /ingester/ { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location = /ingester { + internal; # to suppress 301 + } + + # Ring + location = /ring { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + # MemberListKV + location = /memberlist { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + + # Ruler endpoints + location = /ruler/ring { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + + + # Compactor endpoints + location = /compactor/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/delete { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/cache/generation_numbers { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # IndexGateway endpoints + location = /indexgateway/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # Config endpoints + location = /config { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # QueryFrontend, Querier endpoints + location = /api/prom/tail { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location = /loki/api/v1/tail { + proxy_pass http://${LOKI_QUERIER_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location ~ /api/prom/.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + location ~ /loki/api/v1.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + } \ No newline at end of file diff --git a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml index 633cf5b3..e36f0b18 100644 --- a/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml +++ b/kubernetes/monolithic-mode/traces/k8s-all-in-one.yaml @@ -1,9 +1,33 @@ apiVersion: v1 kind: Namespace +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: logging-system +--- +apiVersion: v1 +kind: Namespace metadata: name: tracing-system --- apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +--- +apiVersion: v1 kind: ServiceAccount metadata: labels: @@ -143,6 +167,27 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-clusterrole +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -161,8 +206,136 @@ subjects: name: alloy namespace: monitoring-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-clusterrolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: loki-clusterrole +subjects: +- kind: ServiceAccount + name: loki + namespace: logging-system +--- apiVersion: v1 data: + gateway_loki.conf.template: |- + server { + listen 3100; + listen [::]:3100; + + location = / { + return 200 'OK'; + auth_basic off; + access_log off; + } + + proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid; + + # Distributor endpoints + location = /api/prom/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/push { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + location = /distributor/ring { + proxy_pass http://${LOKI_DISTRIBUTOR_HOST}:3100$request_uri; + } + + # Ingester endpoints + location /flush { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location ^~ /ingester/ { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + location = /ingester { + internal; # to suppress 301 + } + + # Ring + location = /ring { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + # MemberListKV + location = /memberlist { + proxy_pass http://${LOKI_INGESTER_HOST}:3100$request_uri; + } + + + # Ruler endpoints + location = /ruler/ring { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /api/prom/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /loki/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/alerts.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + location ~ /prometheus/api/v1/rules.* { + proxy_pass http://${LOKI_RULER_HOST}:3100$request_uri; + } + + + # Compactor endpoints + location = /compactor/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/delete { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + location = /loki/api/v1/cache/generation_numbers { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # IndexGateway endpoints + location = /indexgateway/ring { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # Config endpoints + location = /config { + proxy_pass http://${LOKI_COMPACTOR_HOST}:3100$request_uri; + } + + # QueryFrontend, Querier endpoints + location = /api/prom/tail { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location = /loki/api/v1/tail { + proxy_pass http://${LOKI_QUERIER_HOST}:3100$request_uri; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location ~ /api/prom/.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + location ~ /loki/api/v1.* { + proxy_pass http://${LOKI_QUERY_FRONTEND_HOST}:3100$request_uri; + } + } gateway_mimir.conf.template: "server {\n listen 8080;\n listen [::]:8080;\n\n \ location = / {\n return 200 'OK';\n auth_basic off;\n access_log off;\n }\n\n proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;\n\n @@ -212,6 +385,154 @@ metadata: namespace: gateway --- apiVersion: v1 +data: + config.yaml: |2 + + auth_enabled: false + + # -reporting.enabled=false + analytics: + reporting_enabled: false + + server: + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: info + log_format: json + + # https://grafana.com/docs/loki/latest/configure/#use-environment-variables-in-the-configuration + common: + compactor_address: http://loki.logging-system.svc.cluster.local:3100 + path_prefix: /var/loki + replication_factor: 1 + storage: + s3: + bucketnames: loki-data + endpoint: ${LOKI_S3_ENDPOINT:-minio.minio-system.svc.cluster.local:443} + access_key_id: ${LOKI_S3_ACCESS_KEY_ID:-lgtmp} + secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY:-supersecret} + insecure: ${LOKI_S3_INSECURE:-false} + s3forcepathstyle: true + http_config: + insecure_skip_verify: true + + bloom_gateway: + enabled: true + client: + addresses: "dns+loki.logging-system.svc.cluster.local:9095" + cache_results: true + results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + bloom_compactor: + enabled: true + ring: + kvstore: + store: memberlist + + index_gateway: + mode: simple + + compactor: + working_directory: /tmp/compactor + + memberlist: + join_members: + - loki-memberlist.logging-system.svc.cluster.local:7946 + + # https://github.com/grafana/loki/blob/main/docs/sources/configure/_index.md#query_range + query_range: + align_queries_with_step: true + + cache_results: true + results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + cache_index_stats_results: true + index_stats_results_cache: + cache: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" + + pattern_ingester: + enabled: true + + limits_config: + max_global_streams_per_user: 0 + ingestion_rate_mb: 50000 + ingestion_burst_size_mb: 50000 + volume_enabled: true + + ruler: + storage: + s3: + bucketnames: loki-ruler + type: s3 + + runtime_config: + file: /etc/loki/runtime-config/runtime-config.yaml + + schema_config: + configs: + - from: "2024-04-08" + index: + period: 24h + prefix: loki_index_ + object_store: s3 + schema: v13 + store: tsdb + + storage_config: + tsdb_shipper: + active_index_directory: /var/loki/index + cache_location: /var/loki/cache + index_gateway_client: + server_address: "dns+loki.logging-system.svc.cluster.local:9095" + + chunk_store_config: + chunk_cache_config: + memcached_client: + addresses: "dns+memcached.memcached-system.svc.cluster.local:11211" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-config-5468cb29k9 + namespace: logging-system +--- +apiVersion: v1 +data: + runtime-config.yaml: | + # This file can be used to set overrides or other runtime config. + overrides: + "fake": # limits for anonymous that the whole cluster enforces + ingestion_rate_mb: 1500000 + max_streams_per_user: 100000 + max_chunks_per_query: 100000 + "anonymous": # limits for anonymous that the whole cluster enforces + ingestion_rate_mb: 1500000 + max_streams_per_user: 100000 + max_chunks_per_query: 100000 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-runtime-9599m5k6h2 + namespace: logging-system +--- +apiVersion: v1 data: alloy-cluster-node.json: |- { @@ -1065,7 +1386,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1074,6 +1395,21 @@ data: = coalesce(env(\"SELF_HOSTED_METRICS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:8080/api/v1/push\")\n\tlogs_endpoint_url \ = coalesce(env(\"SELF_HOSTED_LOGS_ENDPOINT_URL\"), \"http://nginx.gateway.svc:3100/loki/api/v1/push\")\n\ttraces_endpoint_url \ = coalesce(env(\"SELF_HOSTED_TRACES_ENDPOINT_URL\"), \"http://nginx.gateway.svc:4318\")\n}\n\n/********************************************\n + * Beyla ebpf\n ********************************************/\n// beyla.ebpf \"default\" + {\n// \tattributes {\n// \t\tkubernetes {\n// \t\t\tenable = \"true\"\n// \t\t}\n// + \t}\n\n// \troutes {\n// \t\tpatterns = [\n// \t\t\t\"/api/v1/push\",\n// \t\t\t\"/prometheus\",\n// + \t\t\t\"/prometheus/api/v1/series\",\n// \t\t\t\"/prometheus/api/v1/query_range\",\n// + \t\t\t\"/prometheus/api/v1/query_exemplars\",\n// \t\t\t\"/loki/api/v1/push\",\n// + \t\t\t\"/loki/api/v1/query_range\",\n// \t\t\t\"/loki/api/v1/labels\",\n// \t\t\t\"/otlp/v1/traces\",\n// + \t\t\t\"/otlp/v1/metrics\",\n// \t\t\t\"/api/traces/{trace_id}\",\n// \t\t]\n// + \t\tunmatched = \"path\"\n// \t}\n\n// \tdiscovery {\n// \t\tservices {\n// \t\t\tnamespace + \ = \"gateway\"\n// \t\t\t// name = \"nginx\"\n// \t\t\texe_path = \"nginx\"\n// + \t\t\topen_ports = \"8080,3100,3200,4318\"\n// \t\t}\n// \t}\n\n// \toutput {\n// + \t\ttraces = [otelcol.exporter.logging.traces.input, traces.process_and_transform.kubernetes.alloy_traces_input]\n// + \t}\n// }\n\n// otelcol.exporter.logging \"traces\" {\n// verbosity = + \"detailed\"\n// sampling_initial = 1\n// sampling_thereafter = 1\n// + }\n\n// prometheus.scrape \"beyla\" {\n// \ttargets = beyla.ebpf.default.targets\n// + \tforward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n// }\n\n/********************************************\n * Traces\n ********************************************/\nimport.file \"traces\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/traces\"\n}\n\n// traces Processing And Transformation process_and_transform\ntraces.process_and_transform @@ -1083,6 +1419,12 @@ data: \ = [provider.self_hosted_stack.kubernetes.traces_receiver]\n}\n\ntracing {\n\t// Write all spans. Don't do this in production!\n\tsampling_fraction = 1\n\n\t// Forward Alloy internal spans to traces process.\n\twrite_to = [traces.process_and_transform.kubernetes.alloy_traces_input]\n}\n\n/********************************************\n + * Logs\n ********************************************/\nimport.file \"logs\" {\n\tfilename + = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/logs\"\n}\n\nlogs.rules_to_loki + \"kubernetes\" { }\n\nlogs.kubernetes_cluster_events \"kubernetes\" {\n\tcluster + = coalesce(env(\"CLUSTER_NAME\"), \"k3d-k3s-codelab\")\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.annotations_scrape + \"kubernetes\" {\n\tcluster = \"k3d-k3s-codelab\"\n\n\tforward_to = [logs.keep_labels.kubernetes.receiver]\n}\n\nlogs.keep_labels + \"kubernetes\" {\n\tforward_to = [provider.self_hosted_stack.kubernetes.logs_receiver]\n}\n\n/********************************************\n * Metrics\n ********************************************/\nimport.file \"metrics\" {\n\tfilename = coalesce(env(\"ALLOY_MODULES_FOLDER\"), \"/etc/alloy/modules\") + \"/kubernetes/metrics\"\n}\n\nmetrics.annotations_scrape \"kubernetes\" {\n\tcluster @@ -1101,7 +1443,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-6k989hdmh5 + name: alloy-config-77k9mhg2cb namespace: monitoring-system --- apiVersion: v1 @@ -1659,6 +2001,126 @@ metadata: namespace: monitoring-system --- apiVersion: v1 +data: + MEMCACHED_K8S_SECRET_NAME: alloy-integrations-memcached + MYSQL_K8S_SECRET_NAME: alloy-integrations-mysql + REDIS_K8S_SECRET_NAME: alloy-integrations-redis + memcached.alloy: "/*\nModule Components: component_memcached\n*/\n\ndeclare \"memcached_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/memcached)\"\n\t\toptional = + true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret name + (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|memcached_commands_total|memcached_connections_total|memcached_current_bytes|memcached_current_connections|memcached_current_items|memcached_items_evicted_total|memcached_items_total|memcached_max_connections|memcached_read_bytes_total|memcached_up|memcached_uptime_seconds|memcached_version|memcached_written_bytes_total)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment + \ = \"How long before a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"memcached\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-memcached\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Memcached\n\t****************************************************************/\n\t// + https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.memcached/\n\tprometheus.exporter.memcached + \"integrations_memcached_exporter\" {\n\t\taddress = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-address\"])\n\t\ttimeout + = nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-timeout\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_memcached_exporter\" {\n\t\ttargets = prometheus.exporter.memcached.integrations_memcached_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/memcached\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.memcached.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_memcached_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_memcached_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_memcached_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_memcached_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + mysql.alloy: "/*\nModule Components: component_mysql\n*/\n\ndeclare \"mysql_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"namespace\" {\n\t\tcomment = \"kubernetes secret namespace (default: monitoring-system)\"\n\t\toptional + = true\n\t}\n\n\targument \"name\" {\n\t\tcomment = \"kubernetes secret name + (default: alloy-integrations-mysql)\"\n\t\toptional = true\n\t}\n\n\targument + \"keep_metrics\" {\n\t\tcomment = \"A regex of metrics to keep (default: see + below)\"\n\t\toptional = true\n\t}\n\n\targument \"scrape_interval\" {\n\t\tcomment + \ = \"How often to scrape metrics from the targets (default: 60s)\"\n\t\toptional + = true\n\t}\n\n\targument \"scrape_timeout\" {\n\t\tcomment = \"How long before + a scrape times out (default: 10s)\"\n\t\toptional = true\n\t}\n\n\tremote.kubernetes.secret + \"mysql\" {\n\t\tname = coalesce(argument.name.value, \"alloy-integrations-mysql\")\n\t\tnamespace + = coalesce(argument.namespace.value, \"monitoring-system\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Mysql\n\t****************************************************************/\n\tprometheus.exporter.mysql + \"integrations_mysqld_exporter\" {\n\t\tdata_source_name = nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-username\"]) + + \":\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-password\"]) + + \"@(\" + nonsensitive(remote.kubernetes.secret.mysql.data[\"mysql-host\"]) + + \")/\"\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_mysqld_exporter\" {\n\t\ttargets = prometheus.exporter.mysql.integrations_mysqld_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/mysql\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label = + \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.mysql.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_mysqld_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_mysqld_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = coalesce(argument.scrape_interval.value, + \"60s\")\n\t\tscrape_timeout = coalesce(argument.scrape_timeout.value, \"10s\")\n\n\t\tclustering + {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to = [prometheus.relabel.integrations_mysqld_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_mysqld_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = coalesce(argument.keep_metrics.value, + \"(up|instance:mysql_heartbeat_lag_seconds|instance:mysql_slave_lag_seconds|mysql_global_status_aborted_clients|mysql_global_status_aborted_connects|mysql_global_status_buffer_pool_pages|mysql_global_status_bytes_received|mysql_global_status_bytes_sent|mysql_global_status_commands_total|mysql_global_status_created_tmp_disk_tables|mysql_global_status_created_tmp_files|mysql_global_status_created_tmp_tables|mysql_global_status_handlers_total|mysql_global_status_innodb_log_waits|mysql_global_status_innodb_mem_adaptive_hash|mysql_global_status_innodb_mem_dictionary|mysql_global_status_innodb_num_open_files|mysql_global_status_innodb_page_size|mysql_global_status_max_used_connections|mysql_global_status_open_files|mysql_global_status_open_table_definitions|mysql_global_status_open_tables|mysql_global_status_opened_files|mysql_global_status_opened_table_definitions|mysql_global_status_opened_tables|mysql_global_status_qcache_free_memory|mysql_global_status_qcache_hits|mysql_global_status_qcache_inserts|mysql_global_status_qcache_lowmem_prunes|mysql_global_status_qcache_not_cached|mysql_global_status_qcache_queries_in_cache|mysql_global_status_queries|mysql_global_status_questions|mysql_global_status_select_full_join|mysql_global_status_select_full_range_join|mysql_global_status_select_range|mysql_global_status_select_range_check|mysql_global_status_select_scan|mysql_global_status_slow_queries|mysql_global_status_sort_merge_passes|mysql_global_status_sort_range|mysql_global_status_sort_rows|mysql_global_status_sort_scan|mysql_global_status_table_locks_immediate|mysql_global_status_table_locks_waited|mysql_global_status_table_open_cache_hits|mysql_global_status_table_open_cache_misses|mysql_global_status_table_open_cache_overflows|mysql_global_status_threads_cached|mysql_global_status_threads_connected|mysql_global_status_threads_created|mysql_global_status_threads_running|mysql_global_status_uptime|mysql_global_status_wsrep_local_recv_queue|mysql_global_status_wsrep_local_state|mysql_global_status_wsrep_ready|mysql_global_variables_innodb_additional_mem_pool_size|mysql_global_variables_innodb_buffer_pool_size|mysql_global_variables_innodb_log_buffer_size|mysql_global_variables_key_buffer_size|mysql_global_variables_max_connections|mysql_global_variables_open_files_limit|mysql_global_variables_query_cache_size|mysql_global_variables_table_definition_cache|mysql_global_variables_table_open_cache|mysql_global_variables_thread_cache_size|mysql_global_variables_tokudb_cache_size|mysql_global_variables_wsrep_desync|mysql_heartbeat_now_timestamp_seconds|mysql_heartbeat_stored_timestamp_seconds|mysql_info_schema_processlist_threads|mysql_slave_status_seconds_behind_master|mysql_slave_status_slave_io_running|mysql_slave_status_slave_sql_running|mysql_slave_status_sql_delay|mysql_up)\")\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" + redis.alloy: "/*\nModule Components: component_redis_exporter\n*/\n\ndeclare \"redis_exporter_metrics_scrape\" + {\n\n\t/*****************************************************************\n\t* + ARGUMENTS\n\t*****************************************************************/\n\targument + \"forward_to\" {\n\t\tcomment = \"Must be a list(MetricssReceiver) where collected + metrics should be forwarded to\"\n\t}\n\n\targument \"job_label\" {\n\t\tcomment + \ = \"job label (default: integrations/kubernetes/redis_exporter)\"\n\t\toptional + = true\n\t}\n\n\targument \"namespace\" {\n\t\tcomment = \"kubernetes secret + name (default: monitoring-system)\"\n\t\toptional = true\n\t}\n\n\targument \"name\" + {\n\t\tcomment = \"kubernetes secret name (default: alloy-integrations-redis)\"\n\t\toptional + = true\n\t}\n\n\targument \"keep_metrics\" {\n\t\toptional = true\n\t\tdefault + \ = \"(up|redis_blocked_clients|redis_cluster_slots_fail|redis_cluster_slots_pfail|redis_cluster_state|redis_commands_duration_seconds_total|redis_commands_total|redis_connected_clients|redis_connected_slaves|redis_db_keys|redis_db_keys_expiring|redis_evicted_keys_total|redis_keyspace_hits_total|redis_keyspace_misses_total|redis_master_last_io_seconds_ago|redis_memory_fragmentation_ratio|redis_memory_max_bytes|redis_memory_used_bytes|redis_memory_used_rss_bytes|redis_total_system_memory_bytes|redis_up)\"\n\t}\n\n\targument + \"scrape_interval\" {\n\t\tcomment = \"How often to scrape metrics from the targets + (default: 60s)\"\n\t\toptional = true\n\t\tdefault = \"60s\"\n\t}\n\n\targument + \"scrape_timeout\" {\n\t\tcomment = \"How long before a scrape times out (default: + 10s)\"\n\t\toptional = true\n\t\tdefault = \"10s\"\n\t}\n\n\tremote.kubernetes.secret + \"redis\" {\n\t\tnamespace = coalesce(argument.namespace.value, \"monitoring-system\")\n\t\tname + \ = coalesce(argument.name.value, \"alloy-integrations-redis\")\n\t}\n\n\t/***************************************************************\n\t* + Integrations Redis\n\t****************************************************************/\n\tprometheus.exporter.redis + \"integrations_redis_exporter\" {\n\t\tredis_addr = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-address\"])\n\t\tredis_password + = nonsensitive(remote.kubernetes.secret.redis.data[\"instance-password\"])\n\t}\n\n\t/***************************************************************\n\t* + Discovery Relabelings (pre-scrape)\n\t****************************************************************/\n\tdiscovery.relabel + \"integrations_redis_exporter\" {\n\t\ttargets = prometheus.exporter.redis.integrations_redis_exporter.targets\n\n\t\trule + {\n\t\t\ttarget_label = \"job\"\n\t\t\treplacement = coalesce(argument.job_label.value, + \"integrations/kubernetes/redis_exporter\")\n\t\t}\n\n\t\trule {\n\t\t\ttarget_label + = \"instance\"\n\t\t\treplacement = coalesce(nonsensitive(remote.kubernetes.secret.redis.data[\"instance-name\"]), + constants.hostname)\n\t\t}\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Scrape Integrations Targets\n\t****************************************************************/\n\tprometheus.scrape + \"integrations_redis_exporter\" {\n\t\ttargets = concat(\n\t\t\tdiscovery.relabel.integrations_redis_exporter.output,\n\t\t)\n\n\t\tenable_protobuf_negotiation + = true\n\t\tscrape_classic_histograms = true\n\n\t\tscrape_interval = argument.scrape_interval.value\n\t\tscrape_timeout + \ = argument.scrape_timeout.value\n\n\t\tclustering {\n\t\t\tenabled = true\n\t\t}\n\n\t\tforward_to + = [prometheus.relabel.integrations_redis_exporter.receiver]\n\t}\n\n\t/***************************************************************\n\t* + Prometheus Metric Relabelings (post-scrape)\n\t****************************************************************/\n\tprometheus.relabel + \"integrations_redis_exporter\" {\n\t\tforward_to = argument.forward_to.value\n\n\t\t// + keep only metrics that match the keep_metrics regex\n\t\trule {\n\t\t\tsource_labels + = [\"__name__\"]\n\t\t\tregex = argument.keep_metrics.value\n\t\t\taction + \ = \"keep\"\n\t\t}\n\t}\n}\n" +kind: ConfigMap +metadata: + name: alloy-integrations + namespace: monitoring-system +--- +apiVersion: v1 data: apiserver.alloy: "/*\nModule Components: apiserver\nDescription: kubernetes Apiserver Metrics Scrape\n\n*/\n\ndeclare \"apiserver_metrics_scrape\" {\n\n\t/********************************************\n\t* @@ -4521,10 +4983,59 @@ metadata: --- apiVersion: v1 data: - ALLOY_LOG_LEVEL: d2Fybg== + LOKI_S3_SECRET_ACCESS_KEY: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= +kind: Secret +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki-env-58m52b99kc + namespace: logging-system +type: Opaque +--- +apiVersion: v1 +data: + ALLOY_LOG_LEVEL: aW5mbw== +kind: Secret +metadata: + name: alloy-env-7hg2fmmgdb + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= + instance-name: cHJpbWFyeQ== + instance-timeout: NXM= kind: Secret metadata: - name: alloy-env-f7k229g955 + name: alloy-integrations-memcached + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-name: cHJpbWFyeQ== + mysql-host: bXlzcWwubXlzcWwtc3lzdGVtLnN2Yy5jbHVzdGVyLmxvY2Fs + mysql-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= + mysql-username: bGd0bXA= +kind: Secret +metadata: + name: alloy-integrations-mysql + namespace: monitoring-system +type: Opaque +--- +apiVersion: v1 +data: + instance-address: cmVkaXMtbWFzdGVyLnJlZGlzLXN5c3RlbS5zdmMuY2x1c3Rlci5sb2NhbDo2Mzc5 + instance-name: cHJpbWFyeQ== + instance-password: VkQ1MzhPWXhTRWlHRDRJOW1tRmZxRk1DR3ExdklpR20= +kind: Secret +metadata: + name: alloy-integrations-redis namespace: monitoring-system type: Opaque --- @@ -4559,6 +5070,75 @@ type: Opaque --- apiVersion: v1 kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + ports: + - name: http-metrics + port: 3100 + - appProtocol: grpc + name: grpc + port: 9095 + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + prometheus.io/service-monitor: "false" + name: loki-headless + namespace: logging-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + prometheus.io/service-monitor: "false" + name: loki-memberlist + namespace: logging-system +spec: + clusterIP: None + ports: + - name: tcp-gossip-ring + port: 7946 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist +--- +apiVersion: v1 +kind: Service metadata: labels: app.kubernetes.io/instance: alloy @@ -4835,6 +5415,128 @@ spec: --- apiVersion: apps/v1 kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + serviceName: loki-headless + template: + metadata: + annotations: + profiles.grafana.com/cpu.port_name: http-metrics + profiles.grafana.com/cpu.scrape: "false" + profiles.grafana.com/goroutine.port_name: http-metrics + profiles.grafana.com/goroutine.scrape: "false" + profiles.grafana.com/memory.port_name: http-metrics + profiles.grafana.com/memory.scrape: "false" + pyroscope.io/service_name: loki + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: memberlist + spec: + automountServiceAccountToken: true + containers: + - args: + - -config.file=/etc/loki/config/config.yaml + - -target=all + - -config.expand-env=true + envFrom: + - secretRef: + name: loki-env-58m52b99kc + image: docker.io/grafana/loki:3.0.0 + imagePullPolicy: IfNotPresent + name: loki + ports: + - containerPort: 3100 + name: http-metrics + protocol: TCP + - containerPort: 9095 + name: grpc + protocol: TCP + - containerPort: 7946 + name: http-memberlist + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: http-metrics + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp + name: tmp + - mountPath: /etc/loki/config + name: config + - mountPath: /etc/loki/runtime-config + name: runtime-config + - mountPath: /var/loki + name: storage + enableServiceLinks: true + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + serviceAccountName: loki + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: tmp + - configMap: + items: + - key: config.yaml + path: config.yaml + name: loki-config-5468cb29k9 + name: config + - configMap: + name: loki-runtime-9599m5k6h2 + name: runtime-config + updateStrategy: + rollingUpdate: + partition: 0 + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/name: loki + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: StatefulSet metadata: labels: app.kubernetes.io/instance: alloy @@ -4892,7 +5594,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env-7hg2fmmgdb optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -4919,6 +5621,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -4938,7 +5642,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-6k989hdmh5 + name: alloy-config-77k9mhg2cb name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f @@ -5128,6 +5832,41 @@ spec: --- apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/component: loki + app.kubernetes.io/instance: loki-monolithic-mode + app.kubernetes.io/managed-by: Kustomize + app.kubernetes.io/name: loki + app.kubernetes.io/version: 3.0.0 + name: loki + namespace: logging-system +spec: + endpoints: + - interval: 15s + port: http-metrics + relabelings: + - action: replace + replacement: logging-system/loki + sourceLabels: + - job + targetLabel: job + scheme: http + namespaceSelector: + matchNames: + - logging-system + selector: + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + matchLabels: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor metadata: labels: app.kubernetes.io/instance: alloy diff --git a/kubernetes/monolithic-mode/traces/kustomization.yaml b/kubernetes/monolithic-mode/traces/kustomization.yaml index 2b355599..b4bfe973 100644 --- a/kubernetes/monolithic-mode/traces/kustomization.yaml +++ b/kubernetes/monolithic-mode/traces/kustomization.yaml @@ -13,6 +13,13 @@ resources: # optional - ../metrics/mimir +- ../logs/loki + +secretGenerator: +- name: alloy-env + namespace: monitoring-system + literals: + - ALLOY_LOG_LEVEL=info configMapGenerator: - name: alloy-config @@ -29,3 +36,4 @@ configMapGenerator: files: - gateway_tempo.conf.template=configs/gateway_tempo.conf - gateway_mimir.conf.template=configs/gateway_mimir.conf + - gateway_loki.conf.template=configs/gateway_loki.conf diff --git a/kubernetes/monolithic-mode/traces/traces.alloy b/kubernetes/monolithic-mode/traces/traces.alloy index c17e77bc..71e47735 100644 --- a/kubernetes/monolithic-mode/traces/traces.alloy +++ b/kubernetes/monolithic-mode/traces/traces.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } @@ -19,6 +19,58 @@ provider.self_hosted_stack "kubernetes" { traces_endpoint_url = coalesce(env("SELF_HOSTED_TRACES_ENDPOINT_URL"), "http://nginx.gateway.svc:4318") } +/******************************************** + * Beyla ebpf + ********************************************/ +// beyla.ebpf "default" { +// attributes { +// kubernetes { +// enable = "true" +// } +// } + +// routes { +// patterns = [ +// "/api/v1/push", +// "/prometheus", +// "/prometheus/api/v1/series", +// "/prometheus/api/v1/query_range", +// "/prometheus/api/v1/query_exemplars", +// "/loki/api/v1/push", +// "/loki/api/v1/query_range", +// "/loki/api/v1/labels", +// "/otlp/v1/traces", +// "/otlp/v1/metrics", +// "/api/traces/{trace_id}", +// ] +// unmatched = "path" +// } + +// discovery { +// services { +// namespace = "gateway" +// // name = "nginx" +// exe_path = "nginx" +// open_ports = "8080,3100,3200,4318" +// } +// } + +// output { +// traces = [otelcol.exporter.logging.traces.input, traces.process_and_transform.kubernetes.alloy_traces_input] +// } +// } + +// otelcol.exporter.logging "traces" { +// verbosity = "detailed" +// sampling_initial = 1 +// sampling_thereafter = 1 +// } + +// prometheus.scrape "beyla" { +// targets = beyla.ebpf.default.targets +// forward_to = [provider.self_hosted_stack.kubernetes.metrics_receiver] +// } + /******************************************** * Traces ********************************************/ @@ -43,6 +95,31 @@ tracing { write_to = [traces.process_and_transform.kubernetes.alloy_traces_input] } +/******************************************** + * Logs + ********************************************/ +import.file "logs" { + filename = coalesce(env("ALLOY_MODULES_FOLDER"), "/etc/alloy/modules") + "/kubernetes/logs" +} + +logs.rules_to_loki "kubernetes" { } + +logs.kubernetes_cluster_events "kubernetes" { + cluster = coalesce(env("CLUSTER_NAME"), "k3d-k3s-codelab") + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + +logs.annotations_scrape "kubernetes" { + cluster = "k3d-k3s-codelab" + + forward_to = [logs.keep_labels.kubernetes.receiver] +} + +logs.keep_labels "kubernetes" { + forward_to = [provider.self_hosted_stack.kubernetes.logs_receiver] +} + /******************************************** * Metrics ********************************************/ diff --git a/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml b/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml index d2a72c3a..9555efb0 100644 --- a/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml +++ b/kubernetes/read-write-mode/logs/k8s-all-in-one.yaml @@ -1216,7 +1216,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1249,7 +1249,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc namespace: monitoring-system --- apiVersion: v1 @@ -4664,15 +4664,6 @@ metadata: type: Opaque --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -5565,7 +5556,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -5592,6 +5583,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -5611,7 +5604,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-hfmb58htdm + name: alloy-config-2g5bc6hfmc name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/read-write-mode/logs/kustomization.yaml b/kubernetes/read-write-mode/logs/kustomization.yaml index 9cefdc8b..ee568861 100644 --- a/kubernetes/read-write-mode/logs/kustomization.yaml +++ b/kubernetes/read-write-mode/logs/kustomization.yaml @@ -9,7 +9,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - loki # optional diff --git a/kubernetes/read-write-mode/logs/logs.alloy b/kubernetes/read-write-mode/logs/logs.alloy index 2ee169ac..35687f34 100644 --- a/kubernetes/read-write-mode/logs/logs.alloy +++ b/kubernetes/read-write-mode/logs/logs.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml b/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml index 780ed5a3..9ad30a07 100644 --- a/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml +++ b/kubernetes/read-write-mode/metrics/k8s-all-in-one.yaml @@ -997,7 +997,7 @@ metadata: --- apiVersion: v1 data: - config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"info\")\n\tformat + config.alloy: "logging {\n\tlevel = coalesce(env(\"ALLOY_LOG_LEVEL\"), \"warn\")\n\tformat = \"logfmt\"\n}\n\n/********************************************\n * Grafana LGTMP Stack Receiver Provider\n ********************************************/\nimport.git \"provider\" {\n\trepository = \"https://github.com/qclaogui/codelab-monitoring.git\"\n\trevision @@ -1021,7 +1021,7 @@ data: = [provider.self_hosted_stack.kubernetes.metrics_receiver]\n}\n" kind: ConfigMap metadata: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b namespace: monitoring-system --- apiVersion: v1 @@ -4428,15 +4428,6 @@ metadata: namespace: monitoring-system --- apiVersion: v1 -data: - ALLOY_LOG_LEVEL: d2Fybg== -kind: Secret -metadata: - name: alloy-env-f7k229g955 - namespace: monitoring-system -type: Opaque ---- -apiVersion: v1 data: instance-address: bWVtY2FjaGVkLm1lbWNhY2hlZC1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWw6MTEyMTE= instance-name: cHJpbWFyeQ== @@ -4934,7 +4925,7 @@ spec: fieldPath: spec.nodeName envFrom: - secretRef: - name: alloy-env-f7k229g955 + name: alloy-env optional: true image: docker.io/grafana/alloy:v1.0.0 imagePullPolicy: IfNotPresent @@ -4961,6 +4952,8 @@ spec: scheme: HTTP initialDelaySeconds: 10 timeoutSeconds: 1 + securityContext: + privileged: true volumeMounts: - mountPath: /etc/alloy name: config @@ -4980,7 +4973,7 @@ spec: serviceAccountName: alloy volumes: - configMap: - name: alloy-config-5fh6g2bbbg + name: alloy-config-764m96t48b name: config - configMap: name: alloy-modules-kubernetes-metrics-db96446m8f diff --git a/kubernetes/read-write-mode/metrics/kustomization.yaml b/kubernetes/read-write-mode/metrics/kustomization.yaml index cbe3c705..ec209dc7 100644 --- a/kubernetes/read-write-mode/metrics/kustomization.yaml +++ b/kubernetes/read-write-mode/metrics/kustomization.yaml @@ -12,7 +12,6 @@ kind: Kustomization resources: - ../../common/alloy -- ../../common/alloy/configs/kubernetes/integrations - mimir configMapGenerator: diff --git a/kubernetes/read-write-mode/metrics/metrics.alloy b/kubernetes/read-write-mode/metrics/metrics.alloy index 9285c9dd..e4dfd69d 100644 --- a/kubernetes/read-write-mode/metrics/metrics.alloy +++ b/kubernetes/read-write-mode/metrics/metrics.alloy @@ -1,5 +1,5 @@ logging { - level = coalesce(env("ALLOY_LOG_LEVEL"), "info") + level = coalesce(env("ALLOY_LOG_LEVEL"), "warn") format = "logfmt" } diff --git a/monitoring-mixins/k8s-all-in-one.yaml b/monitoring-mixins/k8s-all-in-one.yaml index 0951e7cd..d48e9b2e 100644 --- a/monitoring-mixins/k8s-all-in-one.yaml +++ b/monitoring-mixins/k8s-all-in-one.yaml @@ -58546,1360 +58546,6 @@ metadata: namespace: monitoring-system --- apiVersion: v1 -data: - pyroscope-reads.json: |- - { - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "pyroscope" - ], - "targetBlank": false, - "title": "Pyroscope Dashboards", - "type": "dashboards" - } - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", - "format": "time_series", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "{{ route }} 99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "{{ route }} 50th percentile", - "refId": "B" - }, - { - "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|querier)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) ", - "format": "time_series", - "legendFormat": "{{ route }} Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Querier", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", - "format": "time_series", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "{{ route }} 99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le,route) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "{{ route }} 50th percentile", - "refId": "B" - }, - { - "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*merge.*|.*series.*|.*type.*\", cluster=~\"$cluster\"}) by (route) ", - "format": "time_series", - "legendFormat": "{{ route }} Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "pyroscope" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ ], - "query": "label_values(pyroscope_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(pyroscope_build_info{cluster=~\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Pyroscope / Reads", - "uid": "pyroscope-reads", - "version": 0 - } -kind: ConfigMap -metadata: - annotations: - grafana_dashboard_folder: /dashboards/Pyroscope - labels: - grafana_dashboard: "1" - name: pyroscope-reads.json - namespace: monitoring-system ---- -apiVersion: v1 -data: - pyroscope-writes.json: |- - { - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "pyroscope" - ], - "targetBlank": false, - "title": "Pyroscope Dashboards", - "type": "dashboards" - } - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", - "format": "time_series", - "legendFormat": "{{ type }} 99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", - "format": "time_series", - "legendFormat": "{{ type }} 50th percentile", - "refId": "B" - }, - { - "expr": "1 * sum(job_type_cluster:pyroscope_distributor_received_compressed_bytes_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) / sum(job_type_cluster:pyroscope_distributor_received_compressed_bytes_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) ", - "format": "time_series", - "legendFormat": "{{ type }} Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Compressed Size", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", - "format": "time_series", - "legendFormat": "{{ type }} 99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le,type) (job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"})) * 1", - "format": "time_series", - "legendFormat": "{{ type }} 50th percentile", - "refId": "B" - }, - { - "expr": "1 * sum(job_type_cluster:pyroscope_distributor_received_samples_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) / sum(job_type_cluster:pyroscope_distributor_received_samples_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", type=~\".*\", cluster=~\"$cluster\"}) by (type) ", - "format": "time_series", - "legendFormat": "{{ type }} Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Samples", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "count", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Distributor Profiles received", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*|.*ingest.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", - "format": "time_series", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "50th percentile", - "refId": "B" - }, - { - "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"}) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|distributor)\", route=~\".*push.*\", cluster=~\"$cluster\"})", - "format": "time_series", - "legendFormat": "Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Distributor Requests", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "OK": "#7EB26D", - "cancel": "#A9A9A9", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", - "format": "time_series", - "legendFormat": "{{status}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "99th percentile", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})) * 1e3", - "format": "time_series", - "legendFormat": "50th percentile", - "refId": "B" - }, - { - "expr": "1e3 * sum(job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"}) / sum(job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate{job=~\"($namespace)/(pyroscope|ingester)\", route=~\".*push.*|.*ingest.*\", cluster=~\"$cluster\"})", - "format": "time_series", - "legendFormat": "Average", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "### Head size in bytes per table type\nIngesters maintain a local Head per-tenant. Each\nHead maintains the active profiling series; Then\nthe head gets periodically compacted into a block\non disk. This panel shows the estimated size of\nthe Head in memory for all ingesters.\n\n", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(pyroscope_head_size_bytes{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\",}) by (type)", - "format": "time_series", - "legendFormat": "{{type}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Head size in bytes per table type", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "### Head size in bytes per pod\nIngesters maintain a local Head per-tenant. Each\nHead maintains the active profiling series; Then\nthe head gets periodically compacted into a block\non disk. This panel shows the estimated size of\nthe Head in memory for all ingesters.\n\n", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(pyroscope_head_size_bytes{cluster=~\"$cluster\",job=~\"($namespace)/(pyroscope|ingester)\",}) by (instance)", - "format": "time_series", - "legendFormat": "{{instance}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Head size in bytes per pod", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester - Head", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "pyroscope" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ ], - "query": "label_values(pyroscope_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(pyroscope_build_info{cluster=~\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Pyroscope / Writes", - "uid": "pyroscope-writes", - "version": 0 - } -kind: ConfigMap -metadata: - annotations: - grafana_dashboard_folder: /dashboards/Pyroscope - labels: - grafana_dashboard: "1" - name: pyroscope-writes.json - namespace: monitoring-system ---- -apiVersion: v1 data: scheduler.json: |- { @@ -65228,105 +63874,3 @@ spec: - expr: | sum by(cluster, namespace, pod) (rate(cortex_ingester_ingested_samples_total[1m])) record: cluster_namespace_pod:cortex_ingester_ingested_samples_total:rate1m ---- -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: pyroscope-mixin-rules - namespace: monitoring-system -spec: - groups: - - name: pyroscope_rules - rules: - - expr: histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, job, cluster)) - record: job_cluster:pyroscope_request_duration_seconds:99quantile - - expr: histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, job, cluster)) - record: job_cluster:pyroscope_request_duration_seconds:50quantile - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, cluster) - / sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, cluster) - record: job_cluster:pyroscope_request_duration_seconds:avg - - expr: sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, - cluster) - record: job_cluster:pyroscope_request_duration_seconds_bucket:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, cluster) - record: job_cluster:pyroscope_request_duration_seconds_sum:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, cluster) - record: job_cluster:pyroscope_request_duration_seconds_count:sum_rate - - expr: histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, job, route, cluster)) - record: job_route_cluster:pyroscope_request_duration_seconds:99quantile - - expr: histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, job, route, cluster)) - record: job_route_cluster:pyroscope_request_duration_seconds:50quantile - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, route, - cluster) / sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, - route, cluster) - record: job_route_cluster:pyroscope_request_duration_seconds:avg - - expr: sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, job, - route, cluster) - record: job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (job, route, - cluster) - record: job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_count[1m])) by (job, route, - cluster) - record: job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate - - expr: histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, namespace, job, route, cluster)) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds:99quantile - - expr: histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket[1m])) - by (le, namespace, job, route, cluster)) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds:50quantile - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (namespace, job, - route, cluster) / sum(rate(pyroscope_request_duration_seconds_count[1m])) - by (namespace, job, route, cluster) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds:avg - - expr: sum(rate(pyroscope_request_duration_seconds_bucket[1m])) by (le, namespace, - job, route, cluster) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds_bucket:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_sum[1m])) by (namespace, job, - route, cluster) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds_sum:sum_rate - - expr: sum(rate(pyroscope_request_duration_seconds_count[1m])) by (namespace, - job, route, cluster) - record: namespace_job_route_cluster:pyroscope_request_duration_seconds_count:sum_rate - - expr: histogram_quantile(0.99, sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) - by (le, job, type, cluster)) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes:99quantile - - expr: histogram_quantile(0.50, sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) - by (le, job, type, cluster)) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes:50quantile - - expr: sum(rate(pyroscope_distributor_received_compressed_bytes_sum[1m])) by - (job, type, cluster) / sum(rate(pyroscope_distributor_received_compressed_bytes_count[1m])) - by (job, type, cluster) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes:avg - - expr: sum(rate(pyroscope_distributor_received_compressed_bytes_bucket[1m])) - by (le, job, type, cluster) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes_bucket:sum_rate - - expr: sum(rate(pyroscope_distributor_received_compressed_bytes_sum[1m])) by - (job, type, cluster) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes_sum:sum_rate - - expr: sum(rate(pyroscope_distributor_received_compressed_bytes_count[1m])) by - (job, type, cluster) - record: job_type_cluster:pyroscope_distributor_received_compressed_bytes_count:sum_rate - - expr: histogram_quantile(0.99, sum(rate(pyroscope_distributor_received_samples_bucket[1m])) - by (le, job, type, cluster)) - record: job_type_cluster:pyroscope_distributor_received_samples:99quantile - - expr: histogram_quantile(0.50, sum(rate(pyroscope_distributor_received_samples_bucket[1m])) - by (le, job, type, cluster)) - record: job_type_cluster:pyroscope_distributor_received_samples:50quantile - - expr: sum(rate(pyroscope_distributor_received_samples_sum[1m])) by (job, type, - cluster) / sum(rate(pyroscope_distributor_received_samples_count[1m])) by - (job, type, cluster) - record: job_type_cluster:pyroscope_distributor_received_samples:avg - - expr: sum(rate(pyroscope_distributor_received_samples_bucket[1m])) by (le, job, - type, cluster) - record: job_type_cluster:pyroscope_distributor_received_samples_bucket:sum_rate - - expr: sum(rate(pyroscope_distributor_received_samples_sum[1m])) by (job, type, - cluster) - record: job_type_cluster:pyroscope_distributor_received_samples_sum:sum_rate - - expr: sum(rate(pyroscope_distributor_received_samples_count[1m])) by (job, type, - cluster) - record: job_type_cluster:pyroscope_distributor_received_samples_count:sum_rate diff --git a/monitoring-mixins/kustomization.yaml b/monitoring-mixins/kustomization.yaml index b237c112..11a5160d 100644 --- a/monitoring-mixins/kustomization.yaml +++ b/monitoring-mixins/kustomization.yaml @@ -9,10 +9,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# default enabled resources: - go-runtime-mixin/deploy - kubernetes-mixin/deploy -# - loki-mixin/deploy - mimir-mixin/deploy -- pyroscope-mixin/deploy +# - pyroscope-mixin/deploy # - tempo-mixin/deploy +# - loki-mixin/deploy