Chargeback no longer depend on MetricRollup

app/models/chargeback.rb

@@ -10,105 +10,63 @@ class Chargeback < ActsAsArModel
     :fixed_compute_metric => :integer,
   )
 
-  VIRTUAL_COL_USES = {
-    "v_derived_cpu_total_cores_used" => "cpu_usage_rate_average"
-  }
-
   def self.build_results_for_report_chargeback(options)
     _log.info("Calculating chargeback costs...")
     @options = options = ReportOptions.new_from_h(options)
 
-    rates = RatesCache.new
-
-    base_rollup = MetricRollup.includes(
-      :resource           => [:hardware, :tenant, :tags, :vim_performance_states, :custom_attributes, {:container_image => :custom_attributes}],
-      :parent_host        => :tags,
-      :parent_ems_cluster => :tags,
-      :parent_storage     => :tags,
-      :parent_ems         => :tags)
-                              .select(*Metric::BASE_COLS).order("resource_id, timestamp")
-    perf_cols = MetricRollup.attribute_names
-    rate_cols = ChargebackRate.where(:default => true).flat_map do |rate|
-      rate.chargeback_rate_details.map(&:metric).select { |metric| perf_cols.include?(metric.to_s) }
-    end
-
-    rate_cols.map! { |x| VIRTUAL_COL_USES.include?(x) ? VIRTUAL_COL_USES[x] : x }.flatten!
-    base_rollup = base_rollup.select(*rate_cols)
-
-    timerange = options.report_time_range
     data = {}
+    rates = RatesCache.new
+    ConsumptionHistory.for_report(self, options) do |consumption|
+      rates_to_apply = rates.get(consumption)
 
-    interval_duration = options.duration_of_report_step
-
-    timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time|
-      records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => "hourly")
-      records = where_clause(records, options)
-      records = Metric::Helper.remove_duplicate_timestamps(records)
-      next if records.empty?
-      _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")
-
-      # we are building hash with grouped calculated values
-      # values are grouped by resource_id and timestamp (query_start_time...query_end_time)
-      records.group_by(&:resource_id).each do |_, metric_rollup_records|
-        metric_rollup_records = metric_rollup_records.select { |x| x.resource.present? }
-        consumption = Consumption.new(metric_rollup_records, query_start_time, query_end_time)
-        next if metric_rollup_records.empty?
-
-        # we need to select ChargebackRates for groups of MetricRollups records
-        # and rates are selected by first MetricRollup record
-        metric_rollup_record = metric_rollup_records.first
-        rates_to_apply = rates.get(metric_rollup_record)
-
-        key = report_row_key(metric_rollup_record)
-        data[key] ||= new(options, metric_rollup_record)
+      key = report_row_key(consumption)
+      data[key] ||= new(options, consumption)
 
-        chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
-        data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')
+      chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
+      data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')
 
-        # we are getting hash with metrics and costs for metrics defined for chargeback
-        data[key].calculate_costs(consumption, rates_to_apply)
-      end
+      # we are getting hash with metrics and costs for metrics defined for chargeback
+      data[key].calculate_costs(consumption, rates_to_apply)
     end
-
     _log.info("Calculating chargeback costs...Complete")
 
     [data.values]
   end
 
-  def self.report_row_key(metric_rollup_record)
-    ts_key = @options.start_of_report_step(metric_rollup_record.timestamp)
+  def self.report_row_key(consumption)
+    ts_key = @options.start_of_report_step(consumption.timestamp)
     if @options[:groupby_tag].present?
-      classification = classification_for_perf(metric_rollup_record)
+      classification = classification_for(consumption)
       classification_id = classification.present? ? classification.id : 'none'
       "#{classification_id}_#{ts_key}"
     else
-      default_key(metric_rollup_record, ts_key)
+      default_key(consumption, ts_key)
     end
   end
 
-  def self.default_key(metric_rollup_record, ts_key)
-    "#{metric_rollup_record.resource_id}_#{ts_key}"
+  def self.default_key(consumption, ts_key)
+    "#{consumption.resource_id}_#{ts_key}"
   end
 
-  def self.classification_for_perf(metric_rollup_record)
-    tag = metric_rollup_record.tag_names.split('|').find { |x| x.starts_with?(@options[:groupby_tag]) } # 'department/*'
+  def self.classification_for(consumption)
+    tag = consumption.tag_names.find { |x| x.starts_with?(@options[:groupby_tag]) } # 'department/*'
     tag = tag.split('/').second unless tag.blank? # 'department/finance' -> 'finance'
     @options.tag_hash[tag]
   end
 
-  def initialize(options, metric_rollup_record)
+  def initialize(options, consumption)
     @options = options
     super()
     if @options[:groupby_tag].present?
-      classification = self.class.classification_for_perf(metric_rollup_record)
+      classification = self.class.classification_for(consumption)
       self.tag_name = classification.present? ? classification.description : _('<Empty>')
     else
-      init_extra_fields(metric_rollup_record)
+      init_extra_fields(consumption)
     end
-    self.start_date, self.end_date, self.display_range = options.report_step_range(metric_rollup_record.timestamp)
+    self.start_date, self.end_date, self.display_range = options.report_step_range(consumption.timestamp)
     self.interval_name = options.interval
     self.chargeback_rates = ''
-    self.entity = metric_rollup_record.resource
+    self.entity = consumption.resource
   end
 
   def calculate_costs(consumption, rates)

app/models/chargeback/consumption.rb

@@ -1,10 +1,18 @@
 class Chargeback
   class Consumption
+    delegate :timestamp, :resource, :resource_id, :resource_name, :resource_type, :parent_ems,
+             :hash_features_affecting_rate, :tag_list_with_prefix, :parents_determining_rate,
+             :to => :first_metric_rollup_record
+
     def initialize(metric_rollup_records, start_time, end_time)
       @rollups = metric_rollup_records
       @start_time, @end_time = start_time, end_time
     end
 
+    def tag_names
+      first_metric_rollup_record.tag_names.split('|')
+    end
+
     def max(metric)
       values(metric).max
     end
@@ -32,5 +40,9 @@ def values(metric)
       @values ||= {}
       @values[metric] ||= @rollups.collect(&metric.to_sym).compact
     end
+
+    def first_metric_rollup_record
+      @fmrr ||= @rollups.first
+    end
   end
 end

app/models/chargeback/consumption_history.rb

@@ -0,0 +1,49 @@
+class Chargeback
+  class ConsumptionHistory
+    VIRTUAL_COL_USES = {
+      'v_derived_cpu_total_cores_used' => 'cpu_usage_rate_average'
+    }.freeze
+
+    def self.for_report(cb_class, options)
+      base_rollup = base_rollup_scope
+      timerange = options.report_time_range
+      interval_duration = options.duration_of_report_step
+
+      timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time|
+        records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
+        records = cb_class.where_clause(records, options)
+        records = Metric::Helper.remove_duplicate_timestamps(records)
+        next if records.empty?
+        _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")
+
+        # we are building hash with grouped calculated values
+        # values are grouped by resource_id and timestamp (query_start_time...query_end_time)
+        records.group_by(&:resource_id).each do |_, metric_rollup_records|
+          metric_rollup_records = metric_rollup_records.select { |x| x.resource.present? }
+          consumption = Consumption.new(metric_rollup_records, query_start_time, query_end_time)
+          next if metric_rollup_records.empty?
+          yield(consumption)
+        end
+      end
+    end
+
+    def self.base_rollup_scope
+      base_rollup = MetricRollup.includes(
+        :resource           => [:hardware, :tenant, :tags, :vim_performance_states, :custom_attributes,
+                                {:container_image => :custom_attributes}],
+        :parent_host        => :tags,
+        :parent_ems_cluster => :tags,
+        :parent_storage     => :tags,
+        :parent_ems         => :tags)
+                                .select(*Metric::BASE_COLS).order('resource_id, timestamp')
+
+      perf_cols = MetricRollup.attribute_names
+      rate_cols = ChargebackRate.where(:default => true).flat_map do |rate|
+        rate.chargeback_rate_details.map(&:metric).select { |metric| perf_cols.include?(metric.to_s) }
+      end
+      rate_cols.map! { |x| VIRTUAL_COL_USES[x] || x }.flatten!
+      base_rollup.select(*rate_cols)
+    end
+    private_class_method :base_rollup_scope
+  end
+end

app/models/chargeback/rates_cache.rb

@@ -1,22 +1,24 @@
 class Chargeback
   class RatesCache
-    def get(perf)
+    def get(consumption)
+      # we need to select ChargebackRates for groups of MetricRollups records
+      # and rates are selected by first MetricRollup record
       @rates ||= {}
-      @rates[perf.hash_features_affecting_rate] ||= rates(perf)
+      @rates[consumption.hash_features_affecting_rate] ||= rates(consumption)
     end
 
     private
 
-    def rates(metric_rollup_record)
-      rates = ChargebackRate.get_assigned_for_target(metric_rollup_record.resource,
-                                                     :tag_list => metric_rollup_record.tag_list_with_prefix,
-                                                     :parents  => metric_rollup_record.parents_determining_rate)
+    def rates(consumption)
+      rates = ChargebackRate.get_assigned_for_target(consumption.resource,
+                                                     :tag_list => consumption.tag_list_with_prefix,
+                                                     :parents  => consumption.parents_determining_rate)
 
-      if metric_rollup_record.resource_type == Container.name && rates.empty?
+      if consumption.resource_type == Container.name && rates.empty?
         rates = [ChargebackRate.find_by(:description => "Default Container Image Rate", :rate_type => "Compute")]
       end
 
-      metric_rollup_record_tags = metric_rollup_record.tag_names.split("|")
+      metric_rollup_record_tags = consumption.tag_names
 
       unique_rates_by_tagged_resources(rates, metric_rollup_record_tags)
     end

app/models/chargeback_container_image.rb

@@ -53,12 +53,12 @@ def self.default_key(metric_rollup_record, ts_key)
     @options[:groupby] == 'project' ? "#{project.id}_#{ts_key}" : "#{project.id}_#{image.id}_#{ts_key}"
   end
 
-  def self.image(perf)
-    @data_index.fetch_path(:container_image, :by_container_id, perf.resource_id)
+  def self.image(consumption)
+    @data_index.fetch_path(:container_image, :by_container_id, consumption.resource_id)
   end
 
-  def self.project(perf)
-    @data_index.fetch_path(:container_project, :by_container_id, perf.resource_id)
+  def self.project(consumption)
+    @data_index.fetch_path(:container_project, :by_container_id, consumption.resource_id)
   end
 
   def self.where_clause(records, _options)
@@ -87,13 +87,14 @@ def self.report_col_options
 
   private
 
-  def init_extra_fields(perf)
-    self.project_name  = self.class.project(perf).name
-    self.image_name    = self.class.image(perf).try(:full_name) || _('Deleted') # until image archiving is implemented
-    self.project_uid   = self.class.project(perf).ems_ref
-    self.provider_name = perf.parent_ems.try(:name)
-    self.provider_uid  = perf.parent_ems.try(:guid)
-    self.archived      = self.class.project(perf).archived? ? _('Yes') : _('No')
-    self.entity        = self.class.image(perf)
+  def init_extra_fields(consumption)
+    self.project_name  = self.class.project(consumption).name
+    # until image archiving is implemented
+    self.image_name    = self.class.image(consumption).try(:full_name) || _('Deleted')
+    self.project_uid   = self.class.project(consumption).ems_ref
+    self.provider_name = consumption.parent_ems.try(:name)
+    self.provider_uid  = consumption.parent_ems.try(:guid)
+    self.archived      = self.class.project(consumption).archived? ? _('Yes') : _('No')
+    self.entity        = self.class.image(consumption)
   end
 end # class ChargebackContainerImage

app/models/chargeback_container_project.rb

@@ -70,11 +70,11 @@ def self.report_col_options
 
   private
 
-  def init_extra_fields(perf)
-    self.project_name  = perf.resource_name
-    self.project_uid   = perf.resource.ems_ref
-    self.provider_name = perf.parent_ems.try(:name)
-    self.provider_uid  = perf.parent_ems.try(:guid)
-    self.archived      = perf.resource.archived? ? _('Yes') : _('No')
+  def init_extra_fields(consumption)
+    self.project_name  = consumption.resource_name
+    self.project_uid   = consumption.resource.ems_ref
+    self.provider_name = consumption.parent_ems.try(:name)
+    self.provider_uid  = consumption.parent_ems.try(:guid)
+    self.archived      = consumption.resource.archived? ? _('Yes') : _('No')
   end
 end # class Chargeback

app/models/chargeback_vm.rb

@@ -92,9 +92,9 @@ def self.report_col_options
     }
   end
 
-  def self.vm_owner(perf)
+  def self.vm_owner(consumption)
     @vm_owners ||= vms.each_with_object({}) { |vm, res| res[vm.id] = vm.evm_owner_name }
-    @vm_owners[perf.resource_id] ||= perf.resource.evm_owner_name
+    @vm_owners[consumption.resource_id] ||= consumption.resource.evm_owner_name
   end
 
   def self.vms
@@ -134,13 +134,13 @@ def self.vms
 
   private
 
-  def init_extra_fields(perf)
-    self.vm_id         = perf.resource_id
-    self.vm_name       = perf.resource_name
-    self.vm_uid        = perf.resource.ems_ref
-    self.vm_guid       = perf.resource.try(:guid)
-    self.owner_name    = self.class.vm_owner(perf)
-    self.provider_name = perf.parent_ems.try(:name)
-    self.provider_uid  = perf.parent_ems.try(:guid)
+  def init_extra_fields(consumption)
+    self.vm_id         = consumption.resource_id
+    self.vm_name       = consumption.resource_name
+    self.vm_uid        = consumption.resource.ems_ref
+    self.vm_guid       = consumption.resource.try(:guid)
+    self.owner_name    = self.class.vm_owner(consumption)
+    self.provider_name = consumption.parent_ems.try(:name)
+    self.provider_uid  = consumption.parent_ems.try(:guid)
   end
 end # class Chargeback