Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add label and field selectors in config map for custom prom scraping #493

Merged
merged 4 commits into from
Jan 20, 2021
Merged

add label and field selectors in config map for custom prom scraping #493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
874035b
add label and field selectors in config map for custom prom scraping
gracewehner Jan 20, 2021
4958237
variable name fix
gracewehner Jan 20, 2021
a1a7820
remove repeat code
gracewehner Jan 20, 2021
e8e3065
merge fixes
gracewehner Jan 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion build/linux/installer/conf/telegraf-prom-side-car.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,13 +742,15 @@
## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
## - prometheus.io/port: If port is not 9102 use this annotation
$AZMON_SIDECAR_PROM_MONITOR_PODS
monitor_kubernetes_pods_version = 2
kubernetes_label_selector = $AZMON_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR
kubernetes_field_selector = $AZMON_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR

fieldpass = $AZMON_SIDECAR_PROM_FIELDPASS
fielddrop = $AZMON_SIDECAR_PROM_FIELDDROP

metric_version = 2
url_tag = "scrapeUrl"
monitor_kubernetes_pods_version = 2
## Kubernetes config file to create client from.
# kube_config = "/path/to/kubernetes.config"

Expand Down
23 changes: 18 additions & 5 deletions build/linux/installer/scripts/tomlparser-prom-customconfig.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
@defaultSidecarFieldPass = []
@defaultSidecarFieldDrop = []
@defaultSidecarMonitorPods = false
@defaultSidecarLabelSelectors = ""
@defaultSidecarFieldSelectors = ""

#Configurations to be used for the auto-generated input prometheus plugins for namespace filtering
@metricVersion = 2
Expand Down Expand Up @@ -67,17 +69,19 @@ def checkForType(variable, varType)
end
end

def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors)
begin
new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_MONITOR_PODS", ("monitor_kubernetes_pods = #{monitorKubernetesPods}"))
new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_PLUGINS_WITH_NAMESPACE_FILTER", "")
new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR", ("kubernetes_label_selector = #{kubernetesLabelSelectors}"))
new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR", ("kubernetes_field_selector = #{kubernetesFieldSelectors}"))
rescue => errorStr
puts "Exception while replacing default pod monitor settings for sidecar: #{errorStr}"
end
return new_contents
end

def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting)
def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting, kubernetesLabelSelectors, kubernetesFieldSelectors)
begin
new_contents = new_contents.gsub("$AZMON_SIDECAR_PROM_MONITOR_PODS", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_SIDECAR_PROM_MONITOR_PODS")
pluginConfigsWithNamespaces = ""
Expand All @@ -89,7 +93,10 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu
pluginConfigsWithNamespaces += "\n[[inputs.prometheus]]
interval = \"#{interval}\"
monitor_kubernetes_pods = true
monitor_kubernetes_pods_version = 2
monitor_kubernetes_pods_namespace = \"#{namespace}\"
kubernetes_label_selector = #{kubernetesLabelSelectors}
kubernetes_field_selector = #{kubernetesFieldSelectors}
fieldpass = #{fieldPassSetting}
fielddrop = #{fieldDropSetting}
metric_version = #{@metricVersion}
Expand All @@ -105,7 +112,7 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu
return new_contents
rescue => errorStr
puts "Exception while creating prometheus input plugins to filter namespaces in sidecar: #{errorStr}, using defaults"
replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors)
end
end

Expand Down Expand Up @@ -203,6 +210,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
fieldDrop = parsedConfig[:prometheus_data_collection_settings][:cluster][:fielddrop]
monitorKubernetesPods = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods]
monitorKubernetesPodsNamespaces = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods_namespaces]
kubernetesLabelSelectors = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_label_selector]
kubernetesFieldSelectors = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_field_selector]

# Check for the right datattypes to enforce right setting values
if checkForType(interval, String) &&
Expand All @@ -215,6 +224,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
fieldPass = (fieldPass.nil?) ? @defaultSidecarFieldPass : fieldPass
fieldDrop = (fieldDrop.nil?) ? @defaultSidecarFieldDrop : fieldDrop
monitorKubernetesPods = (monitorKubernetesPods.nil?) ? @defaultSidecarMonitorPods : monitorKubernetesPods
kubernetesLabelSelectors = (kubernetesLabelSelectors.nil?) ? @defaultSidecarLabelSelectors : kubernetesLabelSelectors
kubernetesFieldSelectors = (kubernetesFieldSelectors.nil?) ? @defaultSidecarFieldSelectors : kubernetesFieldSelectors

file_name = "/opt/telegraf-test-prom-side-car.conf"
# Copy the telegraf config file to a temp file to run telegraf in test mode with this config
Expand All @@ -233,10 +244,10 @@ def populateSettingValuesFromConfigMap(parsedConfig)
# Adding nil check here as well since checkForTypeArray returns true even if setting is nil to accomodate for other settings to be able -
# - to use defaults in case of nil settings
if monitorKubernetesPods && !monitorKubernetesPodsNamespaces.nil? && checkForTypeArray(monitorKubernetesPodsNamespaces, String)
new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting)
new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting, kubernetesLabelSelectors, kubernetesFieldSelectors)
monitorKubernetesPodsNamespacesLength = monitorKubernetesPodsNamespaces.length
else
new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods, kubernetesLabelSelectors, kubernetesFieldSelectors)
monitorKubernetesPodsNamespacesLength = 0
end

Expand All @@ -251,6 +262,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
file.write("export TELEMETRY_SIDECAR_PROM_FIELDDROP_LENGTH=\"#{fieldDrop.length}\"\n")
file.write("export TELEMETRY_SIDECAR_PROM_MONITOR_PODS=\"#{monitorKubernetesPods}\"\n")
file.write("export TELEMETRY_SIDECAR_PROM_MONITOR_PODS_NS_LENGTH=\"#{monitorKubernetesPodsNamespacesLength}\"\n")
file.write("export TELEMETRY_SIDECAR_PROM_KUBERNETES_LABEL_SELECTOR_LENGTH=\"#{kubernetesLabelSelectors.length}\"\n")
file.write("export TELEMETRY_SIDECAR_PROM_KUBERNETES_FIELD_SELECTOR_LENGTH=\"#{kubernetesFieldSelectors.length}\"\n")

# Close file after writing all environment variables
file.close
Expand Down
11 changes: 11 additions & 0 deletions kubernetes/container-azm-ms-agentconfig.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,17 @@ data:
## ex: monitor_kubernetes_pods_namespaces = ["default1", "default2", "default3"]
# monitor_kubernetes_pods_namespaces = ["default1"]

## Label selector to target pods which have the specified label
## This will take effect when monitor_kubernetes_pods is set to true
## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
# kubernetes_label_selector = "env=dev,app=nginx"

## Field selector to target pods which have the specified field
## This will take effect when monitor_kubernetes_pods is set to true
## Reference the docs at https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/
## eg. To scrape pods on a specific node
# kubernetes_field_selector = "spec.nodeName=$HOSTNAME"

[prometheus_data_collection_settings.node]
# Node level scrape endpoint(s). These metrics will be scraped from agent's DaemonSet running in every node in the cluster
# Any errors related to prometheus scraping can be found in the KubeMonAgentEvents table in the Log Analytics workspace that the cluster is sending data to.
Expand Down