From 874035bc3ceb6f91fb0f0944f919bf304cd89987 Mon Sep 17 00:00:00 2001 From: Grace Wehner Date: Wed, 20 Jan 2021 11:10:22 -0800 Subject: [PATCH 1/4] add label and field selectors in config map for custom prom scraping --- .../conf/telegraf-prom-side-car.conf | 3 +++ .../scripts/tomlparser-prom-customconfig.rb | 23 +++++++++++++++---- kubernetes/container-azm-ms-agentconfig.yaml | 11 +++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/build/linux/installer/conf/telegraf-prom-side-car.conf b/build/linux/installer/conf/telegraf-prom-side-car.conf index 55c201458..5e199cc7a 100644 --- a/build/linux/installer/conf/telegraf-prom-side-car.conf +++ b/build/linux/installer/conf/telegraf-prom-side-car.conf @@ -742,6 +742,9 @@ ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation. ## - prometheus.io/port: If port is not 9102 use this annotation $AZMON_SIDECAR_PROM_MONITOR_PODS + monitor_kubernetes_pods_version = 2 + kubernetes_label_selector = $AZMON_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR + kubernetes_field_selector = $AZMON_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR fieldpass = $AZMON_SIDECAR_PROM_FIELDPASS fielddrop = $AZMON_SIDECAR_PROM_FIELDDROP diff --git a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb index b8e2cc15f..4c4ccc5fc 100644 --- a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb @@ -23,6 +23,8 @@ @defaultSidecarFieldPass = [] @defaultSidecarFieldDrop = [] @defaultSidecarMonitorPods = false +@defaultSidecarLabelSelectors = "" +@defaultSidecarFieldSelectors = "" #Configurations to be used for the auto-generated input prometheus plugins for namespace filtering @metricVersion = 2 @@ -67,17 +69,19 @@ def checkForType(variable, varType) end end -def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) +def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors) begin new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_MONITOR_PODS", ("monitor_kubernetes_pods = #{monitorKubernetesPods}")) new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_PLUGINS_WITH_NAMESPACE_FILTER", "") + new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR", ("kubernetes_label_selector = #{kubernetesLabelSelectors}")) + new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR", ("kubernetes_field_selector = #{kubernetesFieldSelectors}")) rescue => errorStr puts "Exception while replacing default pod monitor settings for sidecar: #{errorStr}" end return new_contents end -def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting) +def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting, kubernetesLabelSelectors, kubernetesFieldSelectors) begin new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_MONITOR_PODS", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_SIDECAR_PROM_MONITOR_PODS") pluginConfigsWithNamespaces = "" @@ -89,7 +93,10 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu pluginConfigsWithNamespaces += "\n[[inputs.prometheus]] interval = \"#{interval}\" monitor_kubernetes_pods = true + monitor_kubernetes_pods_version = 2 monitor_kubernetes_pods_namespace = \"#{namespace}\" + kubernetes_label_selector = #{kubernetesLabelSelectors} + kubernetes_field_selector = #{kubernetesFieldSelectors} fieldpass = #{fieldPassSetting} fielddrop = #{fieldDropSetting} metric_version = #{@metricVersion} @@ -106,6 +113,8 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu rescue => errorStr puts "Exception while creating prometheus input plugins to filter namespaces in sidecar: #{errorStr}, using defaults" replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) + puts "Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults" + replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors) end end @@ -203,6 +212,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) fieldDrop = parsedConfig[:prometheus_data_collection_settings][:cluster][:fielddrop] monitorKubernetesPods = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods] monitorKubernetesPodsNamespaces = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods_namespaces] + kubernetesLabelSelectors = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_label_selector] + kubernetesFieldSelectors = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_field_selector] # Check for the right datattypes to enforce right setting values if checkForType(interval, String) && @@ -215,6 +226,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) fieldPass = (fieldPass.nil?) ? @defaultSidecarFieldPass : fieldPass fieldDrop = (fieldDrop.nil?) ? @defaultSidecarFieldDrop : fieldDrop monitorKubernetesPods = (monitorKubernetesPods.nil?) ? @defaultSidecarMonitorPods : monitorKubernetesPods + kubernetesLabelSelectors = (kubernetesLabelSelectors.nil?) ? @defaultSidecarLabelSelectors : defaultKubernetesLabelSelectors + kubernetesFieldSelectors = (kubernetesFieldSelectors.nil?) ? @defaultSidecarFieldSelectors : defaultKubernetesFieldSelectors file_name = "/opt/telegraf-test-prom-side-car.conf" # Copy the telegraf config file to a temp file to run telegraf in test mode with this config @@ -233,10 +246,10 @@ def populateSettingValuesFromConfigMap(parsedConfig) # Adding nil check here as well since checkForTypeArray returns true even if setting is nil to accomodate for other settings to be able - # - to use defaults in case of nil settings if monitorKubernetesPods && !monitorKubernetesPodsNamespaces.nil? && checkForTypeArray(monitorKubernetesPodsNamespaces, String) - new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting) + new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting, kubernetesLabelSelectors, kubernetesFieldSelectors) monitorKubernetesPodsNamespacesLength = monitorKubernetesPodsNamespaces.length else - new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) + new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors) monitorKubernetesPodsNamespacesLength = 0 end @@ -251,6 +264,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export TELEMETRY_SIDECAR_PROM_FIELDDROP_LENGTH=\"#{fieldDrop.length}\"\n") file.write("export TELEMETRY_SIDECAR_PROM_MONITOR_PODS=\"#{monitorKubernetesPods}\"\n") file.write("export TELEMETRY_SIDECAR_PROM_MONITOR_PODS_NS_LENGTH=\"#{monitorKubernetesPodsNamespacesLength}\"\n") + file.write("export TELEMETRY_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR_LENGTH=\"#{kubernetesLabelSelectors.length}\"\n") + file.write("export TELEMETRY_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR_LENGTH=\"#{kubernetesFieldSelectors.length}\"\n") # Close file after writing all environment variables file.close diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index aec1bb456..e38d9b4ab 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -76,6 +76,17 @@ data: ## ex: monitor_kubernetes_pods_namespaces = ["default1", "default2", "default3"] # monitor_kubernetes_pods_namespaces = ["default1"] + ## Label selector to target pods which have the specified label + ## This will take effect when monitor_kubernetes_pods is set to true + ## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + # kubernetes_label_selector = "env=dev,app=nginx" + + ## Field selector to target pods which have the specified field + ## This will take effect when monitor_kubernetes_pods is set to true + ## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/ + ## eg. To scrape pods on a specific node + # kubernetes_field_selector = "spec.nodeName=$HOSTNAME" + [prometheus_data_collection_settings.node] # Node level scrape endpoint(s). These metrics will be scraped from agent's DaemonSet running in every node in the cluster # Any errors related to prometheus scraping can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to. From 49582371b7abef876908db955248de4ac9f09134 Mon Sep 17 00:00:00 2001 From: Grace Wehner Date: Wed, 20 Jan 2021 11:14:56 -0800 Subject: [PATCH 2/4] variable name fix --- build/linux/installer/scripts/tomlparser-prom-customconfig.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb index 4c4ccc5fc..9adfde5bb 100644 --- a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb @@ -226,8 +226,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) fieldPass = (fieldPass.nil?) ? @defaultSidecarFieldPass : fieldPass fieldDrop = (fieldDrop.nil?) ? @defaultSidecarFieldDrop : fieldDrop monitorKubernetesPods = (monitorKubernetesPods.nil?) ? @defaultSidecarMonitorPods : monitorKubernetesPods - kubernetesLabelSelectors = (kubernetesLabelSelectors.nil?) ? @defaultSidecarLabelSelectors : defaultKubernetesLabelSelectors - kubernetesFieldSelectors = (kubernetesFieldSelectors.nil?) ? @defaultSidecarFieldSelectors : defaultKubernetesFieldSelectors + kubernetesLabelSelectors = (kubernetesLabelSelectors.nil?) ? @defaultSidecarLabelSelectors : kubernetesLabelSelectors + kubernetesFieldSelectors = (kubernetesFieldSelectors.nil?) ? @defaultSidecarFieldSelectors : kubernetesFieldSelectors file_name = "/opt/telegraf-test-prom-side-car.conf" # Copy the telegraf config file to a temp file to run telegraf in test mode with this config From a1a7820a23eef3a8282a5daa7975b5579c5993d9 Mon Sep 17 00:00:00 2001 From: Grace Wehner Date: Wed, 20 Jan 2021 11:19:44 -0800 Subject: [PATCH 3/4] remove repeat code --- build/linux/installer/scripts/tomlparser-prom-customconfig.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb index 9adfde5bb..4dab37acb 100644 --- a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb @@ -111,8 +111,6 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_PLUGINS_WITH_NAMESPACE_FILTER", pluginConfigsWithNamespaces) return new_contents rescue => errorStr - puts "Exception while creating prometheus input plugins to filter namespaces in sidecar: #{errorStr}, using defaults" - replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) puts "Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults" replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors) end From e8e3065d6faf41267ca64fe8132da888cfcf0183 Mon Sep 17 00:00:00 2001 From: Grace Wehner Date: Wed, 20 Jan 2021 11:23:08 -0800 Subject: [PATCH 4/4] merge fixes --- build/linux/installer/conf/telegraf-prom-side-car.conf | 1 - build/linux/installer/scripts/tomlparser-prom-customconfig.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build/linux/installer/conf/telegraf-prom-side-car.conf b/build/linux/installer/conf/telegraf-prom-side-car.conf index 5e199cc7a..9b2931e94 100644 --- a/build/linux/installer/conf/telegraf-prom-side-car.conf +++ b/build/linux/installer/conf/telegraf-prom-side-car.conf @@ -751,7 +751,6 @@ metric_version = 2 url_tag = "scrapeUrl" - monitor_kubernetes_pods_version = 2 ## Kubernetes config file to create client from. # kube_config = "/path/to/kubernetes.config" diff --git a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb index 4dab37acb..e638d0bb4 100644 --- a/build/linux/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/linux/installer/scripts/tomlparser-prom-customconfig.rb @@ -111,7 +111,7 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_PLUGINS_WITH_NAMESPACE_FILTER", pluginConfigsWithNamespaces) return new_contents rescue => errorStr - puts "Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults" + puts "Exception while creating prometheus input plugins to filter namespaces in sidecar: #{errorStr}, using defaults" replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors) end end