Dev to prod merge release ciprod01312022

.github/workflows/pr-checker.yml

@@ -56,7 +56,7 @@ jobs:
           format: 'table'
           severity: 'CRITICAL,HIGH'
           vuln-type: 'os,library'
-          skip-dirs: 'opt/telegraf,usr/sbin/telegraf'
+          skip-dirs: '/usr/sbin'
           exit-code: '1'
           timeout: '5m0s'
   WINDOWS-build:
@@ -94,4 +94,3 @@ jobs:
             cd ./kubernetes/windows/ && docker build . --file Dockerfile -t $env:IMAGETAG --build-arg IMAGE_TAG=$env:IMAGETAG_TELEMETRY
       - name: List-docker-images
         run: docker images --digests --all
-

.github/workflows/run_unit_tests.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v2
       - name: install fluent
         run: |
-          sudo gem install fluentd -v "1.12.2" --no-document
+          sudo gem install fluentd -v "1.14.2" --no-document
           sudo fluentd --setup ./fluent
       - name: Run unit tests
         run: |

ReleaseNotes.md

@@ -11,6 +11,41 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 1/31/2022 -
+##### Version microsoft/oms:ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022 (linux)
+##### Version microsoft/oms:win-ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod01312022 (windows)
+##### Code change log
+- Linux Agent
+  - Configurable DB name via configmap for ADX (default DB name:containerinsights)
+  - Default to cAdvisor port to 10250 and container runtime to  Containerd
+  - Update AgentVersion annotation in yamls (omsagent and chart) with released MDSD agent version
+  - Incresing windows agent CPU limits from 200m to 500m
+  - Ignore new disk path that comes from containerd starting with k8s version >= 1.19.x, which was adding unnecessary InsightsMetrics logs and increasing cost
+  - Route the AI SDK logs to log file instead of stdout
+  - Telemetry to collect ContainerLog Records with empty Timestamp
+  - FluentBit version upgrade from 1.6.8 to 1.7.8
+- Windows Agent
+  - Update to use FluentBit for container log collection and removed FluentD dependency for container log collection
+  - Telemetry to track if any of the variable fields of windows container inventory records has field size >= 64KB
+  - Add windows os check in in_cadvisor_perf plugin to avoid making call in MDSD in MSI auth mode
+  - Bug fix for placeholder_hostname in telegraf metrics
+  - FluentBit version upgrade from 1.4.0 to 1.7.8
+- Common
+    - Upgrade FluentD gem version from 1.12.2 to 1.14.2
+    - Upgrade Telegraf version from 1.18.0 to 1.20.3
+    - Fix for exception in node allocatable
+    - Telemetry to track nodeCount & containerCount
+- Other changes
+   - Updates to Arc K8s Extension ARM Onboarding templates with GA API version
+   - Added ARM Templates for MSI Based Onboarding for AKS
+   - Conformance test updates relates to sidecar container
+   - Troubelshooting script to detect issues related to Arc K8s Extension onboarding
+   - Remove the dependency SP for CDPX since configured to use MSI
+   - Linux Agent Image build improvements
+   - Update msys2 version to fix windows agent build
+   - Add explicit exit code 1 across all the PS scripts
+
+
 ### 10/13/2021 -
 ##### Version microsoft/oms:ciprod10132021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10132021 (linux)
 ##### Version microsoft/oms:win-ciprod10132021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10132021 (windows)

build/common/installer/scripts/td-agent-bit-conf-customizer.rb

@@ -3,6 +3,11 @@
 
 @td_agent_bit_conf_path = "/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf"
 
+@os_type = ENV["OS_TYPE"]
+if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  @td_agent_bit_conf_path = "/etc/fluent-bit/fluent-bit.conf"
+end
+
 @default_service_interval = "15"
 @default_mem_buf_limit = "10"
 
@@ -20,14 +25,14 @@ def substituteFluentBitPlaceHolders
     bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"]
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
 
-    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
 
     tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil
 
-    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) ||  (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
+    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
       puts "config:warn buffer max size must be greater or equal to chunk size"
       tailBufferMaxSize = tailBufferChunkSize
     end

build/common/installer/scripts/tomlparser-agent-config.rb

@@ -0,0 +1,264 @@
+#!/usr/local/bin/ruby
+
+#this should be require relative in Linux and require in windows, since it is a gem install on windows
+@os_type = ENV["OS_TYPE"]
+if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  require "tomlrb"
+else
+  require_relative "tomlrb"
+end
+
+require_relative "ConfigParseErrorLogger"
+
+@configMapMountPath = "/etc/config/settings/agent-settings"
+@configSchemaVersion = ""
+@enable_health_model = false
+
+# 250 Node items (15KB per node) account to approximately 4MB
+@nodesChunkSize = 250
+# 1000 pods (10KB per pod) account to approximately 10MB
+@podsChunkSize = 1000
+# 4000 events (1KB per event) account to approximately 4MB
+@eventsChunkSize = 4000
+# roughly each deployment is 8k
+# 500 deployments account to approximately 4MB
+@deploymentsChunkSize = 500
+# roughly each HPA is 3k
+# 2000 HPAs account to approximately 6-7MB
+@hpaChunkSize = 2000
+# stream batch sizes to avoid large file writes
+# too low will consume higher disk iops
+@podsEmitStreamBatchSize = 200
+@nodesEmitStreamBatchSize = 100
+
+# higher the chunk size rs pod memory consumption higher and lower api latency
+# similarly lower the value, helps on the memory consumption but incurrs additional round trip latency
+# these needs to be tuned be based on the workload
+# nodes
+@nodesChunkSizeMin = 100
+@nodesChunkSizeMax = 400
+# pods
+@podsChunkSizeMin = 250
+@podsChunkSizeMax = 1500
+# events
+@eventsChunkSizeMin = 2000
+@eventsChunkSizeMax = 10000
+# deployments
+@deploymentsChunkSizeMin = 500
+@deploymentsChunkSizeMax = 1000
+# hpa
+@hpaChunkSizeMin = 500
+@hpaChunkSizeMax = 2000
+
+# emit stream sizes to prevent lower values which costs disk i/o
+# max will be upto the chunk size
+@podsEmitStreamBatchSizeMin = 50
+@nodesEmitStreamBatchSizeMin = 50
+
+# configmap settings related fbit config
+@fbitFlushIntervalSecs = 0
+@fbitTailBufferChunkSizeMBs = 0
+@fbitTailBufferMaxSizeMBs = 0
+@fbitTailMemBufLimitMBs = 0
+
+
+def is_number?(value)
+  true if Integer(value) rescue false
+end
+
+# Use parser to parse the configmap toml file to a ruby structure
+def parseConfigMap
+  begin
+    # Check to see if config map is created
+    if (File.file?(@configMapMountPath))
+      puts "config::configmap container-azm-ms-agentconfig for agent settings mounted, parsing values"
+      parsedConfig = Tomlrb.load_file(@configMapMountPath, symbolize_keys: true)
+      puts "config::Successfully parsed mounted config map"
+      return parsedConfig
+    else
+      puts "config::configmap container-azm-ms-agentconfig for agent settings not mounted, using defaults"
+      return nil
+    end
+  rescue => errorStr
+    ConfigParseErrorLogger.logError("Exception while parsing config map for agent settings : #{errorStr}, using defaults, please check config map for errors")
+    return nil
+  end
+end
+
+# Use the ruby structure created after config parsing to set the right values to be used as environment variables
+def populateSettingValuesFromConfigMap(parsedConfig)
+  begin
+    if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil?
+      if !parsedConfig[:agent_settings][:health_model].nil? && !parsedConfig[:agent_settings][:health_model][:enabled].nil?
+        @enable_health_model = parsedConfig[:agent_settings][:health_model][:enabled]
+        puts "enable_health_model = #{@enable_health_model}"
+      end
+      chunk_config = parsedConfig[:agent_settings][:chunk_config]
+      if !chunk_config.nil?
+        nodesChunkSize = chunk_config[:NODES_CHUNK_SIZE]
+        if !nodesChunkSize.nil? && is_number?(nodesChunkSize) && (@nodesChunkSizeMin..@nodesChunkSizeMax) === nodesChunkSize.to_i
+          @nodesChunkSize = nodesChunkSize.to_i
+          puts "Using config map value: NODES_CHUNK_SIZE = #{@nodesChunkSize}"
+        end
+
+        podsChunkSize = chunk_config[:PODS_CHUNK_SIZE]
+        if !podsChunkSize.nil? && is_number?(podsChunkSize) && (@podsChunkSizeMin..@podsChunkSizeMax) === podsChunkSize.to_i
+          @podsChunkSize = podsChunkSize.to_i
+          puts "Using config map value: PODS_CHUNK_SIZE = #{@podsChunkSize}"
+        end
+
+        eventsChunkSize = chunk_config[:EVENTS_CHUNK_SIZE]
+        if !eventsChunkSize.nil? && is_number?(eventsChunkSize) && (@eventsChunkSizeMin..@eventsChunkSizeMax) === eventsChunkSize.to_i
+          @eventsChunkSize = eventsChunkSize.to_i
+          puts "Using config map value: EVENTS_CHUNK_SIZE = #{@eventsChunkSize}"
+        end
+
+        deploymentsChunkSize = chunk_config[:DEPLOYMENTS_CHUNK_SIZE]
+        if !deploymentsChunkSize.nil? && is_number?(deploymentsChunkSize) && (@deploymentsChunkSizeMin..@deploymentsChunkSizeMax) === deploymentsChunkSize.to_i
+          @deploymentsChunkSize = deploymentsChunkSize.to_i
+          puts "Using config map value: DEPLOYMENTS_CHUNK_SIZE = #{@deploymentsChunkSize}"
+        end
+
+        hpaChunkSize = chunk_config[:HPA_CHUNK_SIZE]
+        if !hpaChunkSize.nil? && is_number?(hpaChunkSize) && (@hpaChunkSizeMin..@hpaChunkSizeMax) === hpaChunkSize.to_i
+          @hpaChunkSize = hpaChunkSize.to_i
+          puts "Using config map value: HPA_CHUNK_SIZE = #{@hpaChunkSize}"
+        end
+
+        podsEmitStreamBatchSize = chunk_config[:PODS_EMIT_STREAM_BATCH_SIZE]
+        if !podsEmitStreamBatchSize.nil? && is_number?(podsEmitStreamBatchSize) &&
+           podsEmitStreamBatchSize.to_i <= @podsChunkSize && podsEmitStreamBatchSize.to_i >= @podsEmitStreamBatchSizeMin
+          @podsEmitStreamBatchSize = podsEmitStreamBatchSize.to_i
+          puts "Using config map value: PODS_EMIT_STREAM_BATCH_SIZE = #{@podsEmitStreamBatchSize}"
+        end
+        nodesEmitStreamBatchSize = chunk_config[:NODES_EMIT_STREAM_BATCH_SIZE]
+        if !nodesEmitStreamBatchSize.nil? && is_number?(nodesEmitStreamBatchSize) &&
+           nodesEmitStreamBatchSize.to_i <= @nodesChunkSize && nodesEmitStreamBatchSize.to_i >= @nodesEmitStreamBatchSizeMin
+          @nodesEmitStreamBatchSize = nodesEmitStreamBatchSize.to_i
+          puts "Using config map value: NODES_EMIT_STREAM_BATCH_SIZE = #{@nodesEmitStreamBatchSize}"
+        end
+      end
+      # fbit config settings
+      fbit_config = parsedConfig[:agent_settings][:fbit_config]
+      if !fbit_config.nil?
+        fbitFlushIntervalSecs = fbit_config[:log_flush_interval_secs]
+        if !fbitFlushIntervalSecs.nil? && is_number?(fbitFlushIntervalSecs) && fbitFlushIntervalSecs.to_i > 0
+          @fbitFlushIntervalSecs = fbitFlushIntervalSecs.to_i
+          puts "Using config map value: log_flush_interval_secs = #{@fbitFlushIntervalSecs}"
+        end
+
+        fbitTailBufferChunkSizeMBs = fbit_config[:tail_buf_chunksize_megabytes]
+        if !fbitTailBufferChunkSizeMBs.nil? && is_number?(fbitTailBufferChunkSizeMBs) && fbitTailBufferChunkSizeMBs.to_i > 0
+          @fbitTailBufferChunkSizeMBs = fbitTailBufferChunkSizeMBs.to_i
+          puts "Using config map value: tail_buf_chunksize_megabytes  = #{@fbitTailBufferChunkSizeMBs}"
+        end
+
+        fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes]
+        if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0           
+          if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs
+            @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i
+            puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}"
+          else
+            # tail_buf_maxsize_megabytes has to be greater or equal to tail_buf_chunksize_megabytes
+            @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs
+            puts "config::warn: tail_buf_maxsize_megabytes must be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provided config value not valid"
+          end
+        end
+        # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash
+        if  @fbitTailBufferChunkSizeMBs > 0  && @fbitTailBufferMaxSizeMBs == 0
+          @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs
+          puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes"
+        end 
+
+        fbitTailMemBufLimitMBs = fbit_config[:tail_mem_buf_limit_megabytes]
+        if !fbitTailMemBufLimitMBs.nil? && is_number?(fbitTailMemBufLimitMBs) && fbitTailMemBufLimitMBs.to_i > 0
+          @fbitTailMemBufLimitMBs = fbitTailMemBufLimitMBs.to_i
+          puts "Using config map value: tail_mem_buf_limit_megabytes  = #{@fbitTailMemBufLimitMBs}"
+        end
+      end
+    end
+  rescue => errorStr
+    puts "config::error:Exception while reading config settings for agent configuration setting - #{errorStr}, using defaults"
+    @enable_health_model = false
+  end
+end
+
+@configSchemaVersion = ENV["AZMON_AGENT_CFG_SCHEMA_VERSION"]
+puts "****************Start Config Processing********************"
+if [email protected]? && [email protected]? && @configSchemaVersion.strip.casecmp("v1") == 0 #note v1 is the only supported schema version , so hardcoding it
+  configMapSettings = parseConfigMap
+  if !configMapSettings.nil?
+    populateSettingValuesFromConfigMap(configMapSettings)
+  end
+else
+  if (File.file?(@configMapMountPath))
+    ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version")
+  end
+  @enable_health_model = false
+end
+
+# Write the settings to file, so that they can be set as environment variables
+file = File.open("agent_config_env_var", "w")
+
+if !file.nil?
+  file.write("export AZMON_CLUSTER_ENABLE_HEALTH_MODEL=#{@enable_health_model}\n")
+  file.write("export NODES_CHUNK_SIZE=#{@nodesChunkSize}\n")
+  file.write("export PODS_CHUNK_SIZE=#{@podsChunkSize}\n")
+  file.write("export EVENTS_CHUNK_SIZE=#{@eventsChunkSize}\n")
+  file.write("export DEPLOYMENTS_CHUNK_SIZE=#{@deploymentsChunkSize}\n")
+  file.write("export HPA_CHUNK_SIZE=#{@hpaChunkSize}\n")
+  file.write("export PODS_EMIT_STREAM_BATCH_SIZE=#{@podsEmitStreamBatchSize}\n")
+  file.write("export NODES_EMIT_STREAM_BATCH_SIZE=#{@nodesEmitStreamBatchSize}\n")
+  # fbit settings
+  if @fbitFlushIntervalSecs > 0
+    file.write("export FBIT_SERVICE_FLUSH_INTERVAL=#{@fbitFlushIntervalSecs}\n")
+  end
+  if @fbitTailBufferChunkSizeMBs > 0
+    file.write("export FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSizeMBs}\n")
+  end
+  if @fbitTailBufferMaxSizeMBs > 0
+    file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n")
+  end 
+  if @fbitTailMemBufLimitMBs > 0
+    file.write("export FBIT_TAIL_MEM_BUF_LIMIT=#{@fbitTailMemBufLimitMBs}\n")
+  end 
+  # Close file after writing all environment variables
+  file.close
+else
+  puts "Exception while opening file for writing config environment variables"
+  puts "****************End Config Processing********************"
+end
+
+def get_command_windows(env_variable_name, env_variable_value)
+  return "[System.Environment]::SetEnvironmentVariable(\"#{env_variable_name}\", \"#{env_variable_value}\", \"Process\")" + "\n" + "[System.Environment]::SetEnvironmentVariable(\"#{env_variable_name}\", \"#{env_variable_value}\", \"Machine\")" + "\n"
+end
+
+if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  # Write the settings to file, so that they can be set as environment variables
+  file = File.open("setagentenv.ps1", "w")
+
+  if !file.nil?
+    if @fbitFlushIntervalSecs > 0
+      commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs)
+      file.write(commands)
+    end
+    if @fbitTailBufferChunkSizeMBs > 0
+      commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs)
+      file.write(commands)
+    end
+    if @fbitTailBufferMaxSizeMBs > 0
+      commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs)
+      file.write(commands)
+    end 
+    if @fbitTailMemBufLimitMBs > 0
+      commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs)
+      file.write(commands)
+    end 
+    # Close file after writing all environment variables
+    file.close
+    puts "****************End Config Processing********************"
+  else
+    puts "Exception while opening file for writing config environment variables for WINDOWS LOG"
+    puts "****************End Config Processing********************"
+  end
+end