From 965d704d99662c2c55c7a98db2b6a5c120d18c62 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 24 Sep 2020 18:46:51 -0700 Subject: [PATCH 1/9] turnoff mdm nonsupported cluster types --- source/plugins/ruby/out_mdm.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index c4cc46dd7..a53aaa126 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -61,8 +61,6 @@ def configure(conf) def start super begin - file = File.read(@@azure_json_path) - @data_hash = JSON.parse(file) aks_resource_id = ENV["AKS_RESOURCE_ID"] aks_region = ENV["AKS_REGION"] @@ -77,6 +75,12 @@ def start aks_region = aks_region.gsub(" ", "") end + if !aks_resource_id.to_s.empty? && !aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") + && !aks_resource_id.downcase.include?("microsoft.containerservice/managedclusters") + @log.info "MDM Metris not supported for this cluster type resource: #{aks_resource_id}" + @can_send_data_to_mdm = false + end + if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" @@ -106,6 +110,9 @@ def start @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token else + # azure json file only used for aks + file = File.read(@@azure_json_path) + @data_hash = JSON.parse(file) # Check to see if SP exists, if it does use SP. Else, use msi sp_client_id = @data_hash["aadClientId"] sp_client_secret = @data_hash["aadClientSecret"] From 7a3ced97db500cdfa59b5c2513a9697d81c71109 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 24 Sep 2020 18:49:53 -0700 Subject: [PATCH 2/9] enable validation of server cert for ai ruby http client --- .../channel/sender_base.rb | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb index 33ac49286..4a3027da3 100644 --- a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb +++ b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb @@ -1,9 +1,9 @@ -require 'yajl/json_gem' -require 'net/http' -require 'openssl' -require 'stringio' -require 'zlib' -require 'logger' +require "yajl/json_gem" +require "net/http" +require "openssl" +require "stringio" +require "zlib" +require "logger" module ApplicationInsights module Channel @@ -53,9 +53,9 @@ def initialize(service_endpoint_uri, proxy = {}) def send(data_to_send) uri = URI(@service_endpoint_uri) headers = { - 'Accept' => 'application/json', - 'Content-Type' => 'application/json; charset=utf-8', - 'Content-Encoding' => 'gzip' + "Accept" => "application/json", + "Content-Type" => "application/json; charset=utf-8", + "Content-Encoding" => "gzip", } request = Net::HTTP::Post.new(uri.path, headers) @@ -66,19 +66,19 @@ def send(data_to_send) request.body = compressed_data if @proxy.nil? || @proxy.empty? http = Net::HTTP.new uri.hostname, uri.port - else + else http = Net::HTTP.new(uri.hostname, uri.port, @proxy[:addr], @proxy[:port], @proxy[:user], @proxy[:pass]) end - if uri.scheme.downcase == 'https' + if uri.scheme.downcase == "https" http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.verify_mode = OpenSSL::SSL::VERIFY_PEER end response = http.request(request) http.finish if http.started? if !response.kind_of? Net::HTTPSuccess - @logger.warn('application_insights') { "Failed to send data: #{response.message}" } + @logger.warn("application_insights") { "Failed to send data: #{response.message}" } end end From bdc7ee233512374133f7c268d4944143359a84a3 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 24 Sep 2020 18:52:48 -0700 Subject: [PATCH 3/9] add kubelet operations total and total error metrics --- build/linux/installer/conf/telegraf.conf | 9 ++++----- kubernetes/linux/main.sh | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/build/linux/installer/conf/telegraf.conf b/build/linux/installer/conf/telegraf.conf index 013aa1af2..202ac9741 100644 --- a/build/linux/installer/conf/telegraf.conf +++ b/build/linux/installer/conf/telegraf.conf @@ -632,8 +632,7 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["$CADVISOR_METRICS_URL"] - ## Include "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC" when we add for support for 1.18 - fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC"] + fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC"] metric_version = 2 url_tag = "scrapeUrl" @@ -675,7 +674,7 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["$CADVISOR_METRICS_URL"] - + fieldpass = ["kubelet_running_pod_count","volume_manager_total_volumes", "kubelet_node_config_error", "process_resident_memory_bytes", "process_cpu_seconds_total"] metric_version = 2 @@ -690,7 +689,7 @@ ## Optional TLS Config tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true - + ## prometheus custom metrics [[inputs.prometheus]] @@ -731,7 +730,7 @@ #name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = $AZMON_INTEGRATION_NPM_METRICS_URL_LIST_NODE - + metric_version = 2 url_tag = "scrapeUrl" diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index d9fdc42e9..11972f0f4 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -300,11 +300,10 @@ fi echo "configured container runtime on kubelet is : "$CONTAINER_RUNTIME echo "export CONTAINER_RUNTIME="$CONTAINER_RUNTIME >> ~/.bashrc -# enable these metrics in next agent release -# export KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC="kubelet_runtime_operations_total" -# echo "export KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC="$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC >> ~/.bashrc -# export KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC="kubelet_runtime_operations_errors_total" -# echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC >> ~/.bashrc +export KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC="kubelet_runtime_operations_total" +echo "export KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC="$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC >> ~/.bashrc +export KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC="kubelet_runtime_operations_errors_total" +echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC >> ~/.bashrc # default to docker metrics export KUBELET_RUNTIME_OPERATIONS_METRIC="kubelet_docker_operations" From 33fea579c1d265c2fe1d55767a1a9167c982c3f8 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 24 Sep 2020 22:02:14 -0700 Subject: [PATCH 4/9] node selector label change --- .../templates/omsagent-daemonset-windows.yaml | 6 +++++- charts/azuremonitor-containers/values.yaml | 8 ++++++++ kubernetes/omsagent.yaml | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml index 7acd46c37..9c66b8bfe 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml @@ -25,8 +25,12 @@ spec: dockerProviderVersion: {{ .Values.omsagent.image.dockerProviderVersion }} schema-versions: "v1" spec: - nodeSelector: + nodeSelector: + {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} + kubernetes.io/os: windows + {{- else -}} beta.kubernetes.io/os: windows + {{- end }} {{- if .Values.omsagent.rbac }} serviceAccountName: omsagent {{- end }} diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 9c48cf9fb..5a6e19e59 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -59,7 +59,11 @@ omsagent: nodeSelectorTerms: - labelSelector: matchExpressions: + {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} + - key: kubernetes.io/os + {{- else -}} - key: beta.kubernetes.io/os + {{- end }} operator: In values: - linux @@ -74,7 +78,11 @@ omsagent: nodeSelectorTerms: - labelSelector: matchExpressions: + {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} + - key: kubernetes.io/os + {{- else -}} - key: beta.kubernetes.io/os + {{- end }} operator: In values: - linux diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 9c8f9de14..09e50b5a4 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -419,7 +419,8 @@ spec: nodeSelectorTerms: - labelSelector: matchExpressions: - - key: beta.kubernetes.io/os + # kubernetes.io/os label doesnt exist in k8s versions < 1.14 so make sure to choose label based on k8s version in aks yaml + - key: kubernetes.io/os operator: In values: - linux From e8211bd0e62575c86da85fe7406868a378680de3 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 25 Sep 2020 06:16:36 -0700 Subject: [PATCH 5/9] label update --- .../templates/omsagent-daemonset-windows.yaml | 9 +++--- charts/azuremonitor-containers/values.yaml | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml index 9c66b8bfe..72b09f6c1 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml @@ -25,12 +25,13 @@ spec: dockerProviderVersion: {{ .Values.omsagent.image.dockerProviderVersion }} schema-versions: "v1" spec: - nodeSelector: - {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} + nodeSelector: kubernetes.io/os: windows - {{- else -}} +{{- else }} + nodeSelector: beta.kubernetes.io/os: windows - {{- end }} +{{- end }} {{- if .Values.omsagent.rbac }} serviceAccountName: omsagent {{- end }} diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 5a6e19e59..1804d1197 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -59,11 +59,18 @@ omsagent: nodeSelectorTerms: - labelSelector: matchExpressions: - {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} - key: kubernetes.io/os - {{- else -}} + operator: In + values: + - linux + - key: type + operator: NotIn + values: + - virtual-kubelet + nodeSelectorTerms: + - labelSelector: + matchExpressions: - key: beta.kubernetes.io/os - {{- end }} operator: In values: - linux @@ -78,11 +85,22 @@ omsagent: nodeSelectorTerms: - labelSelector: matchExpressions: - {{- if semverCompare "^1.14-0" .Capabilities.KubeVersion.GitVersion -}} - key: kubernetes.io/os - {{- else -}} + operator: In + values: + - linux + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/role + operator: NotIn + values: + - master + nodeSelectorTerms: + - labelSelector: + matchExpressions: - key: beta.kubernetes.io/os - {{- end }} operator: In values: - linux From 4799edf3d1bd38f9cf434d10c5d11bb7e5553c82 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 25 Sep 2020 06:51:17 -0700 Subject: [PATCH 6/9] wip --- source/plugins/ruby/out_mdm.rb | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index a53aaa126..0f04e8ec0 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -67,6 +67,9 @@ def start if aks_resource_id.to_s.empty? @log.info "Environment Variable AKS_RESOURCE_ID is not set.. " @can_send_data_to_mdm = false + elsif !aks_resource_id.downcase.include?("/microsoft.kubernetes/connectedclusters/") && !aks_resource_id.downcase.include?("/microsoft.containerservice/managedclusters/") + @log.info "MDM Metris not supported for this cluster type resource: #{aks_resource_id}" + @can_send_data_to_mdm = false end if aks_region.to_s.empty? @log.info "Environment Variable AKS_REGION is not set.. " @@ -75,12 +78,6 @@ def start aks_region = aks_region.gsub(" ", "") end - if !aks_resource_id.to_s.empty? && !aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") - && !aks_resource_id.downcase.include?("microsoft.containerservice/managedclusters") - @log.info "MDM Metris not supported for this cluster type resource: #{aks_resource_id}" - @can_send_data_to_mdm = false - end - if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" From b7d2757db42dd224fe7447b3387bc6cf09fa68da Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 25 Sep 2020 06:56:11 -0700 Subject: [PATCH 7/9] wip --- source/plugins/ruby/out_mdm.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 0f04e8ec0..1018cfddc 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -67,10 +67,11 @@ def start if aks_resource_id.to_s.empty? @log.info "Environment Variable AKS_RESOURCE_ID is not set.. " @can_send_data_to_mdm = false - elsif !aks_resource_id.downcase.include?("/microsoft.kubernetes/connectedclusters/") && !aks_resource_id.downcase.include?("/microsoft.containerservice/managedclusters/") + elsif !aks_resource_id.downcase.include?("/microsoft.containerservice/managedclusters/") && !aks_resource_id.downcase.include?("/microsoft.kubernetes/connectedclusters/") @log.info "MDM Metris not supported for this cluster type resource: #{aks_resource_id}" @can_send_data_to_mdm = false end + if aks_region.to_s.empty? @log.info "Environment Variable AKS_REGION is not set.. " @can_send_data_to_mdm = false @@ -107,7 +108,7 @@ def start @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token else - # azure json file only used for aks + # azure json file only used for aks and doesnt exist non-azure envs file = File.read(@@azure_json_path) @data_hash = JSON.parse(file) # Check to see if SP exists, if it does use SP. Else, use msi From 87f4fb1ea2a3578913ed7fe38bf13f394d1263f6 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 25 Sep 2020 07:08:52 -0700 Subject: [PATCH 8/9] wip --- source/plugins/ruby/out_mdm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 1018cfddc..1c805255a 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -108,7 +108,7 @@ def start @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token else - # azure json file only used for aks and doesnt exist non-azure envs + # azure json file only used for aks and doesnt exist in non-azure envs file = File.read(@@azure_json_path) @data_hash = JSON.parse(file) # Check to see if SP exists, if it does use SP. Else, use msi From 37382bf70b632dd863d6429a62d6e3d99aa22223 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 25 Sep 2020 11:58:28 -0700 Subject: [PATCH 9/9] revert quotes --- .../channel/sender_base.rb | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb index 4a3027da3..bedbae4ee 100644 --- a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb +++ b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb @@ -1,9 +1,9 @@ -require "yajl/json_gem" -require "net/http" -require "openssl" -require "stringio" -require "zlib" -require "logger" +require 'yajl/json_gem' +require 'net/http' +require 'openssl' +require 'stringio' +require 'zlib' +require 'logger' module ApplicationInsights module Channel @@ -53,9 +53,9 @@ def initialize(service_endpoint_uri, proxy = {}) def send(data_to_send) uri = URI(@service_endpoint_uri) headers = { - "Accept" => "application/json", - "Content-Type" => "application/json; charset=utf-8", - "Content-Encoding" => "gzip", + 'Accept' => 'application/json', + 'Content-Type' => 'application/json; charset=utf-8', + 'Content-Encoding' => 'gzip' } request = Net::HTTP::Post.new(uri.path, headers) @@ -69,7 +69,7 @@ def send(data_to_send) else http = Net::HTTP.new(uri.hostname, uri.port, @proxy[:addr], @proxy[:port], @proxy[:user], @proxy[:pass]) end - if uri.scheme.downcase == "https" + if uri.scheme.downcase == 'https' http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_PEER end @@ -78,7 +78,7 @@ def send(data_to_send) http.finish if http.started? if !response.kind_of? Net::HTTPSuccess - @logger.warn("application_insights") { "Failed to send data: #{response.message}" } + @logger.warn('application_insights') { "Failed to send data: #{response.message}" } end end