add mutex to metric

cosmos/lib/cosmos/utilities/metric.rb

@@ -18,6 +18,7 @@
 # copyright holder
 
 require 'cosmos/models/metric_model'
+require 'thread'
 
 module Cosmos
   class Metric
@@ -48,6 +49,7 @@ def initialize(microservice:, scope:)
       @scope = scope
       @microservice = microservice
       @size = 5000
+      @mutex = Mutex.new
     end
 
     def add_sample(name:, value:, labels:)
@@ -69,15 +71,17 @@ def add_sample(name:, value:, labels:)
       # the value is added to @items and the count of the value is increased
       # if the count of the values exceed the size of the array it sets the
       # count back to zero and the array will over write older data.
-      key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
-      if not @items.has_key?(key)
-        Logger.debug("new data for #{@scope}, #{key}")
-        @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+      @mutex.synchronize do
+        key = "#{name}|" + labels.map { |k, v| "#{k}=#{v}" }.join(',')
+        if not @items.has_key?(key)
+          Logger.debug("new data for #{@scope}, #{key}")
+          @items[key] = { 'values' => Array.new(@size), 'count' => 0 }
+        end
+        count = @items[key]['count']
+        # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
+        @items[key]['values'][count] = value
+        @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
       end
-      count = @items[key]['count']
-      # Logger.info("adding data for #{@scope}, #{count} #{key}, #{value}")
-      @items[key]['values'][count] = value
-      @items[key]['count'] = count + 1 >= @size ? 0 : count + 1
     end
 
     def percentile(sorted_values, percentile)
@@ -106,24 +110,26 @@ def output
       # array. to store the array as the value with the metric name again joined
       # with the @microservice and @scope.
       Logger.debug("#{@microservice} #{@scope} sending metrics to redis, #{@items.length}") if @items.length > 0
-      @items.each do |key, values|
-        label_list = []
-        name, labels = key.split('|')
-        metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
-        sorted_values = values['values'].compact.sort
-        for percentile_value in [10, 50, 90, 95, 99]
-          percentile_result = percentile(sorted_values, percentile_value)
-          labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
-          labels['percentile'] = percentile_value
-          labels['metric__value'] = percentile_result
-          label_list.append(labels)
-        end
-        begin
-          Logger.debug("sending metrics summary to redis key: #{@microservice}")
-          metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
-          metric.create(force: true)
-        rescue RuntimeError
-          Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+      @mutex.synchronize do
+        @items.each do |key, values|
+          label_list = []
+          name, labels = key.split('|')
+          metric_labels = labels.nil? ? {} : labels.split(',').map { |x| x.split('=') }.map { |k, v| { k => v } }.reduce({}, :merge)
+          sorted_values = values['values'].compact.sort
+          for percentile_value in [10, 50, 90, 95, 99]
+            percentile_result = percentile(sorted_values, percentile_value)
+            labels = metric_labels.clone.merge({ 'scope' => @scope, 'microservice' => @microservice })
+            labels['percentile'] = percentile_value
+            labels['metric__value'] = percentile_result
+            label_list.append(labels)
+          end
+          begin
+            Logger.debug("sending metrics summary to redis key: #{@microservice}")
+            metric = MetricModel.new(name: @microservice, scope: @scope, metric_name: name, label_list: label_list)
+            metric.create(force: true)
+          rescue RuntimeError
+            Logger.error("failed attempt to update metric, #{key}, #{name} #{@scope}")
+          end
         end
       end
     end