From b8d84ff73317252b73c463b293779f7fc3ddddd2 Mon Sep 17 00:00:00 2001 From: lpichler Date: Thu, 17 May 2018 10:33:40 +0200 Subject: [PATCH] Report chargeback from all regions --- app/models/chargeback.rb | 30 ++--- app/models/chargeback/consumption_history.rb | 8 +- app/models/chargeback_container_image.rb | 6 +- app/models/chargeback_container_project.rb | 6 +- app/models/chargeback_vm.rb | 24 ++-- spec/models/chargeback_vm_spec.rb | 112 ++++++++++++++++++- spec/support/chargeback_helper.rb | 3 +- 7 files changed, 153 insertions(+), 36 deletions(-) diff --git a/app/models/chargeback.rb b/app/models/chargeback.rb index 4142956193c..67d5d28a325 100644 --- a/app/models/chargeback.rb +++ b/app/models/chargeback.rb @@ -32,22 +32,26 @@ def self.build_results_for_report_chargeback(options) data = {} rates = RatesCache.new - ConsumptionHistory.for_report(self, options) do |consumption| - rates_to_apply = rates.get(consumption) - key = report_row_key(consumption) - data[key] ||= new(options, consumption) + MiqRegion.all.each do |region| + ConsumptionHistory.for_report(self, options, region.region) do |consumption| + rates_to_apply = rates.get(consumption) - chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description) - data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ') + key = report_row_key(consumption) + data[key] ||= new(options, consumption, region.region) - # we are getting hash with metrics and costs for metrics defined for chargeback - if Settings[:new_chargeback] - data[key].new_chargeback_calculate_costs(consumption, rates_to_apply) - else - data[key].calculate_costs(consumption, rates_to_apply) + chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description) + data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ') + + # we are getting hash with metrics and costs for metrics defined for chargeback + if Settings[:new_chargeback] + data[key].new_chargeback_calculate_costs(consumption, rates_to_apply) + else + data[key].calculate_costs(consumption, rates_to_apply) + end end end + _log.info("Calculating chargeback costs...Complete") [data.values] @@ -77,7 +81,7 @@ def self.groupby_label_value(consumption, groupby_label) nil end - def initialize(options, consumption) + def initialize(options, consumption, region) @options = options super() if @options[:groupby_tag].present? @@ -87,7 +91,7 @@ def initialize(options, consumption) label_value = self.class.groupby_label_value(consumption, options[:groupby_label]) self.label_name = label_value.present? ? label_value : _('') else - init_extra_fields(consumption) + init_extra_fields(consumption, region) end self.start_date, self.end_date, self.display_range = options.report_step_range(consumption.timestamp) self.interval_name = options.interval diff --git a/app/models/chargeback/consumption_history.rb b/app/models/chargeback/consumption_history.rb index e7f3f19334d..177994ca37b 100644 --- a/app/models/chargeback/consumption_history.rb +++ b/app/models/chargeback/consumption_history.rb @@ -1,11 +1,11 @@ class Chargeback class ConsumptionHistory - def self.for_report(cb_class, options) - base_rollup = base_rollup_scope + def self.for_report(cb_class, options, region) + base_rollup = base_rollup_scope.in_region(region) timerange = options.report_time_range interval_duration = options.duration_of_report_step - extra_resources = cb_class.try(:extra_resources_without_rollups) || [] + extra_resources = cb_class.try(:extra_resources_without_rollups, region) || [] timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time| extra_resources.each do |resource| consumption = ConsumptionWithoutRollups.new(resource, query_start_time, query_end_time) @@ -15,7 +15,7 @@ def self.for_report(cb_class, options) next unless options.include_metrics? records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly') - records = cb_class.where_clause(records, options) + records = cb_class.where_clause(records, options, region) records = Metric::Helper.remove_duplicate_timestamps(records) next if records.empty? _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}") diff --git a/app/models/chargeback_container_image.rb b/app/models/chargeback_container_image.rb index da760f7570b..0bb286deb29 100644 --- a/app/models/chargeback_container_image.rb +++ b/app/models/chargeback_container_image.rb @@ -83,8 +83,8 @@ def self.project(consumption) @data_index.fetch_path(:container_project, :by_container_id, consumption.resource_id) || @unknown_project end - def self.where_clause(records, _options) - records.where(:resource_type => Container.name, :resource_id => @containers.pluck(:id)) + def self.where_clause(records, _options, region) + records.where(:resource_type => Container.name, :resource_id => @containers.in_region(region).pluck(:id)) end def self.report_static_cols @@ -115,7 +115,7 @@ def self.report_col_options private - def init_extra_fields(consumption) + def init_extra_fields(consumption, _region) self.project_name = self.class.project(consumption).name self.image_name = self.class.image(consumption).try(:full_name) self.project_uid = self.class.project(consumption).ems_ref diff --git a/app/models/chargeback_container_project.rb b/app/models/chargeback_container_project.rb index 67e00ed0bb5..6cf30153cd8 100644 --- a/app/models/chargeback_container_project.rb +++ b/app/models/chargeback_container_project.rb @@ -46,8 +46,8 @@ def self.build_results_for_report_ChargebackContainerProject(options) @projects = nil end - def self.where_clause(records, _options) - records.where(:resource_type => ContainerProject.name, :resource_id => @projects.select(:id)) + def self.where_clause(records, _options, region) + records.where(:resource_type => ContainerProject.name, :resource_id => @projects.in_region(region).select(:id)) end def self.report_static_cols @@ -74,7 +74,7 @@ def self.report_col_options private - def init_extra_fields(consumption) + def init_extra_fields(consumption, _region) self.project_name = consumption.resource_name self.project_uid = consumption.resource.ems_ref self.provider_name = consumption.parent_ems.try(:name) diff --git a/app/models/chargeback_vm.rb b/app/models/chargeback_vm.rb index a17f5e67647..149a6d0fa68 100644 --- a/app/models/chargeback_vm.rb +++ b/app/models/chargeback_vm.rb @@ -80,25 +80,25 @@ def self.build_results_for_report_ChargebackVm(options) build_results_for_report_chargeback(options) end - def self.where_clause(records, options) + def self.where_clause(records, options, region) scope = records.where(:resource_type => "VmOrTemplate") if options[:tag] && (@report_user.nil? || !@report_user.self_service?) scope.for_tag_names(options[:tag].split("/")[2..-1]) else - scope.where(:resource => vms) + scope.where(:resource => vms(region)) end end - def self.extra_resources_without_rollups + def self.extra_resources_without_rollups(region) # support hyper-v for which we do not collect metrics yet (also when we are including metrics in calculations) - scope = @options.include_metrics? ? ManageIQ::Providers::Microsoft::InfraManager::Vm : vms + scope = @options.include_metrics? ? ManageIQ::Providers::Microsoft::InfraManager::Vm : vms(region) scope = scope.eager_load(:hardware, :taggings, :tags, :host, :ems_cluster, :storage, :ext_management_system, :tenant) if @options[:tag] && (@report_user.nil? || !@report_user.self_service?) scope.find_tagged_with(:any => @options[:tag], :ns => '*') else - scope.where(:id => vms) + scope.where(:id => vms(region)) end end @@ -143,13 +143,14 @@ def self.report_col_options }.merge(sub_metric_columns) end - def self.vm_owner(consumption) - @vm_owners ||= vms.each_with_object({}) { |vm, res| res[vm.id] = vm.evm_owner_name } + def self.vm_owner(consumption, region) + @vm_owners ||= vms(region).each_with_object({}) { |vm, res| res[vm.id] = vm.evm_owner_name } @vm_owners[consumption.resource_id] ||= consumption.resource.try(:evm_owner_name) end - def self.vms - @vms ||= + def self.vms(region) + @vms ||= {} + @vms[region] ||= begin # Find Vms by user or by tag if @options[:entity_id] @@ -167,6 +168,7 @@ def self.vms vms elsif @options[:tenant_id] tenant = Tenant.find(@options[:tenant_id]) + tenant = Tenant.in_region(region).find_by(:name => tenant.name) if tenant.nil? _log.error("Unable to find tenant '#{@options[:tenant_id]}'. Calculating chargeback costs aborted.") raise MiqException::Error, "Unable to find tenant '#{@options[:tenant_id]}'" @@ -187,12 +189,12 @@ def self.vms private - def init_extra_fields(consumption) + def init_extra_fields(consumption, region) self.vm_id = consumption.resource_id self.vm_name = consumption.resource_name self.vm_uid = consumption.resource.try(:ems_ref) self.vm_guid = consumption.resource.try(:guid) - self.owner_name = self.class.vm_owner(consumption) + self.owner_name = self.class.vm_owner(consumption, region) self.provider_name = consumption.parent_ems.try(:name) self.provider_uid = consumption.parent_ems.try(:guid) end diff --git a/spec/models/chargeback_vm_spec.rb b/spec/models/chargeback_vm_spec.rb index 4520c4f62c3..bb4a46b4412 100644 --- a/spec/models/chargeback_vm_spec.rb +++ b/spec/models/chargeback_vm_spec.rb @@ -856,7 +856,7 @@ def result_row_for_vm(vm) {'vm_name' => @vm1.name, 'owner_name' => admin.name, 'vm_uid' => 'ems_ref', 'vm_guid' => @vm1.guid, 'vm_id' => @vm1.id} end - subject { ChargebackVm.new(report_options, consumption).attributes } + subject { ChargebackVm.new(report_options, consumption, MiqRegion.my_region_number).attributes } before do ChargebackVm.instance_variable_set(:@vm_owners, vm_owners) @@ -933,6 +933,116 @@ def result_row_for_vm(vm) subject { ChargebackVm.build_results_for_report_ChargebackVm(options).first.first } + context "with global and remote regions" do + let(:options_tenant) { base_options.merge(:interval => 'monthly', :tenant_id => tenant_1.id).tap { |t| t.delete(:tag) } } + let(:vm_global) { FactoryGirl.create(:vm_vmware) } + let!(:region_1) { FactoryGirl.create(:miq_region) } + + def region_id_for(klass, region) + klass.id_in_region(klass.count + 1_000_000, region) + end + + def find_result_by_vm_name_and_region(chargeback_result, vm_name, region) + first_region_id, last_region_id = MiqRegion.region_to_array(region) + + chargeback_result.detect do |result| + result.vm_name == vm_name && result.vm_id.between?(first_region_id, last_region_id) + end + end + + let(:tenant_name_1) { "T1" } + let(:tenant_name_2) { "T2" } + let(:tenant_name_3) { "T3" } + + let(:vm_name_1) { "VM 1 T1" } + + # BUILD tenants and VMs structure for default region + # + # T1(vm_1, vm_2) -> + # T2(vm_1, vm_2) + # T3(vm_1, vm_2) + let!(:tenant_1) { FactoryGirl.create(:tenant, :parent => Tenant.root_tenant, :name => tenant_name_1, :description => tenant_name_1) } + let(:vm_1_t_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1, :name => vm_name_1) } + let(:vm_2_t_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1) } + + let(:tenant_2) { FactoryGirl.create(:tenant, :name => tenant_name_2, :parent => tenant_1, :description => tenant_name_2) } + let(:vm_1_t_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2) } + let(:vm_2_t_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2) } + + let(:tenant_3) { FactoryGirl.create(:tenant, :name => tenant_name_3, :parent => tenant_1, :description => tenant_name_3) } + let(:vm_1_t_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3) } + let(:vm_2_t_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3) } + + # BUILD tenants and VMs structure for region_1 + # + # T1(vm_1, vm_2) -> + # T2(vm_1, vm_2) + # T3(vm_1, vm_2) + # + let!(:root_tenant_region_1) do + tenant = FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region)) + tenant.parent = nil + tenant.save(:validate => false) # skip validate to set parent = nil + tenant + end + + let!(:tenant_1_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_1, :parent => root_tenant_region_1, :description => tenant_name_1) } + let(:vm_1_region_1_t_1) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1, :name => vm_name_1) } + let(:vm_2_region_1_t_1) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) } + + let!(:tenant_2_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_2, :parent => tenant_1_region_1, :description => tenant_name_2) } + let(:vm_1_region_1_t_2) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) } + let(:vm_2_region_1_t_2) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_2_region_1) } + + let!(:tenant_3_region_1) { FactoryGirl.create(:tenant, :id => region_id_for(Tenant, region_1.region), :name => tenant_name_3, :parent => tenant_1_region_1, :description => tenant_name_3) } + let(:vm_1_region_1_t_3) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_3_region_1) } + let(:vm_2_region_1_t_3) { FactoryGirl.create(:vm_vmware, :id => region_id_for(Vm, region_1.region), :tenant => tenant_3_region_1) } + + before do + # default region + add_metric_rollups_for(vm_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + add_metric_rollups_for(vm_2_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + add_metric_rollups_for(vm_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + add_metric_rollups_for(vm_2_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + add_metric_rollups_for(vm_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + add_metric_rollups_for(vm_2_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data) + + # region 1 + add_metric_rollups_for(vm_1_region_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + add_metric_rollups_for(vm_2_region_1_t_1, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + add_metric_rollups_for(vm_1_region_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + add_metric_rollups_for(vm_2_region_1_t_2, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + add_metric_rollups_for(vm_1_region_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + add_metric_rollups_for(vm_2_region_1_t_3, month_beginning...month_end, 12.hours, metric_rollup_params, :with_data, region_1.region) + end + + subject! { ChargebackVm.build_results_for_report_ChargebackVm(options_tenant).first } + + it "report from all regions and only for tenant_1" do + # report only VMs from tenant 1 + vm_ids = subject.map(&:vm_id) + vm_ids_from_tenant = [tenant_1, tenant_1_region_1].map { |t| t.subtree.map(&:vms).map(&:ids) }.flatten + expect(vm_ids).to match_array(vm_ids_from_tenant) + + # default region subject + default_region_chargeback = find_result_by_vm_name_and_region(subject, vm_name_1, MiqRegion.my_region_number) + used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, vm_1_t_1) + expect(default_region_chargeback.cpu_used_metric).to be_within(0.01).of(used_metric) + expect(default_region_chargeback.cpu_used_cost).to be_within(0.01).of(used_metric * hourly_rate * hours_in_month) + expect(default_region_chargeback.cpu_allocated_cost).to be_within(0.01).of(cpu_count * count_hourly_rate * hours_in_month) + expect(default_region_chargeback.cpu_allocated_metric).to eq(cpu_count) + + # region 1 + region_1_chargeback = find_result_by_vm_name_and_region(subject, vm_name_1, region_1.region) + used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, vm_1_region_1_t_1) + expect(region_1_chargeback.cpu_used_metric).to be_within(0.01).of(used_metric) + expect(region_1_chargeback.cpu_used_cost).to be_within(0.01).of(used_metric * hourly_rate * hours_in_month) + expect(region_1_chargeback.cpu_allocated_cost).to be_within(0.01).of(cpu_count * count_hourly_rate * hours_in_month) + + expect(region_1_chargeback.vm_id).to eq(vm_1_region_1_t_1.id) + end + end + it "cpu" do expect(subject.cpu_allocated_metric).to eq(cpu_count) used_metric = used_average_for(:cpu_usagemhz_rate_average, hours_in_month, @vm1) diff --git a/spec/support/chargeback_helper.rb b/spec/support/chargeback_helper.rb index 58ed5a0c7a2..e2d879998a4 100644 --- a/spec/support/chargeback_helper.rb +++ b/spec/support/chargeback_helper.rb @@ -12,9 +12,10 @@ def used_average_for(metric, hours_in_interval, resource) resource.metric_rollups.sum(&metric) / hours_in_interval end - def add_metric_rollups_for(resources, range, step, metric_rollup_params, trait = :with_data) + def add_metric_rollups_for(resources, range, step, metric_rollup_params, trait = :with_data, region = nil) range.step_value(step).each do |time| Array(resources).each do |resource| + metric_rollup_params[:id] = region_id_for(MetricRollup, region) if region metric_rollup_params[:timestamp] = time metric_rollup_params[:resource_id] = resource.id metric_rollup_params[:resource_name] = resource.name