diff --git a/app/models/chargeable_field.rb b/app/models/chargeable_field.rb index 022028a5641..258e8b796e2 100644 --- a/app/models/chargeable_field.rb +++ b/app/models/chargeable_field.rb @@ -77,6 +77,15 @@ def metering? group == 'metering' && source == 'used' end + def self.cols_on_metric_rollup + (%w(id tag_names resource_id) + chargeable_cols_on_metric_rollup).uniq + end + + def self.col_index(column) + @rate_cols ||= {} + @rate_cols[column] ||= cols_on_metric_rollup.index(column.to_s) + end + private def rate_name @@ -117,6 +126,6 @@ def self.seed_data def self.chargeable_cols_on_metric_rollup existing_cols = MetricRollup.attribute_names chargeable_cols = pluck(:metric) & existing_cols - chargeable_cols.map! { |x| VIRTUAL_COL_USES[x] || x } + chargeable_cols.map! { |x| VIRTUAL_COL_USES[x] || x }.sort end end diff --git a/app/models/chargeback/consumption_history.rb b/app/models/chargeback/consumption_history.rb index e370dee2cda..38c1db235ef 100644 --- a/app/models/chargeback/consumption_history.rb +++ b/app/models/chargeback/consumption_history.rb @@ -25,7 +25,7 @@ def self.for_report(cb_class, options) # values are grouped by resource_id and timestamp (query_start_time...query_end_time) records.each_value do |rollup_record_ids| - metric_rollup_records = base_rollup.where(:id => rollup_record_ids).to_a + metric_rollup_records = MetricRollup.where(:id => rollup_record_ids).pluck(*ChargeableField.cols_on_metric_rollup) consumption = ConsumptionWithRollups.new(metric_rollup_records, query_start_time, query_end_time) yield(consumption) unless consumption.consumed_hours_in_interval.zero? end diff --git a/app/models/chargeback/consumption_with_rollups.rb b/app/models/chargeback/consumption_with_rollups.rb index 1aba22c7c84..8f59763e8fb 100644 --- a/app/models/chargeback/consumption_with_rollups.rb +++ b/app/models/chargeback/consumption_with_rollups.rb @@ -1,14 +1,14 @@ class Chargeback class ConsumptionWithRollups < Consumption delegate :timestamp, :resource, :resource_id, :resource_name, :resource_type, :parent_ems, - :parents_determining_rate, + :parents_determining_rate, :resource_current_tag_names, :to => :first_metric_rollup_record attr_accessor :start_time, :end_time def initialize(metric_rollup_records, start_time, end_time) super(start_time, end_time) - @rollups = metric_rollup_records + @rollup_array = metric_rollup_records end def hash_features_affecting_rate @@ -21,14 +21,53 @@ def hash_features_affecting_rate end def tag_names - @tag_names ||= @rollups.inject([]) do |memo, rollup| - memo |= rollup.all_tag_names + @tag_names ||= @rollup_array.inject([]) do |memo, rollup| + memo |= all_tag_names(rollup) memo end end + TAG_MANAGED_PREFIX = "/tag/managed/".freeze + + def tag_prefix + klass_prefix = case resource_type + when Container.name then 'container_image' + when VmOrTemplate.name then 'vm' + when ContainerProject.name then 'container_project' + end + + klass_prefix + TAG_MANAGED_PREFIX + end + + def chargeback_container_labels + resource.try(:container_image).try(:docker_labels).try(:collect_concat) do |l| + escaped_name = AssignmentMixin.escape(l.name) + escaped_value = AssignmentMixin.escape(l.value) + [ + # The assignments in tags can appear the old way as they are, or escaped + "container_image/label/managed/#{l.name}/#{l.value}", + "container_image/label/managed/#{escaped_name}/#{escaped_value}" + ] + end || [] + end + + def container_tag_list_with_prefix + if resource.kind_of?(Container) + state = resource.vim_performance_state_for_ts(timestamp.to_s) + image_tag_name = "#{state.image_tag_names}|" if state + + image_tag_name.split("|") + else + [] + end + end + + def tag_list_with_prefix_for(rollup) + (all_tag_names(rollup) + container_tag_list_with_prefix).uniq.reject(&:empty?).map { |x| "#{tag_prefix}#{x}" } + chargeback_container_labels + end + def tag_list_with_prefix - @tag_list_with_prefix ||= @rollups.map(&:tag_list_with_prefix).flatten.uniq + @tag_list_with_prefix ||= @rollup_array.map { |rollup| tag_list_with_prefix_for(rollup) }.flatten.uniq end def sum(metric, sub_metric = nil) @@ -44,8 +83,11 @@ def max(metric, sub_metric = nil) def sum_of_maxes_from_grouped_values(metric, sub_metric = nil) return max(metric, sub_metric) if sub_metric @grouped_values ||= {} - grouped_rollups = @rollups.group_by { |x| x.resource.id } - @grouped_values[metric] ||= grouped_rollups.map { |_, rollups| rollups.collect(&metric.to_sym).compact.max }.compact.sum + grouped_rollups = @rollup_array.group_by { |x| x[ChargeableField.col_index(:resource_id)] } + + @grouped_values[metric] ||= grouped_rollups.map do |_, rollups| + rollups.map { |x| x[ChargeableField.col_index(metric)] }.compact.max + end.compact.sum end def avg(metric, sub_metric = nil) @@ -67,11 +109,34 @@ def none?(metric, sub_metric) end def chargeback_fields_present - @chargeback_fields_present ||= @rollups.count(&:chargeback_fields_present?) + @chargeback_fields_present ||= @rollup_array.count { |rollup| chargeback_fields_present?(rollup) } + end + + def chargeback_fields_present?(rollup_record) + MetricRollup::CHARGEBACK_METRIC_FIELDS.any? do |field| + rollup = rollup_record[ChargeableField.col_index(field)] + rollup.present? && rollup.nonzero? + end + end + + def metering_used_fields_present?(rollup_record) + MetricRollup::METERING_USED_METRIC_FIELDS.any? do |field| + rollup = rollup_record[ChargeableField.col_index(field)] + rollup.present? && rollup.nonzero? + end end def metering_used_fields_present - @metering_used_fields_present ||= @rollups.count(&:metering_used_fields_present?) + @metering_used_fields_present ||= @rollup_array.count { |rollup| metering_used_fields_present?(rollup) } + end + + def resource_tag_names(rollup) + tags_names = rollup[ChargeableField.col_index(:tag_names)] + tags_names ? tags_names.split('|') : [] + end + + def all_tag_names(rollup) + resource_current_tag_names | resource_tag_names(rollup) end private @@ -88,11 +153,14 @@ def sub_metric_rollups(sub_metric) def values(metric, sub_metric = nil) @values ||= {} - @values["#{metric}#{sub_metric}"] ||= sub_metric ? sub_metric_rollups(sub_metric) : @rollups.collect(&metric.to_sym).compact + @values["#{metric}#{sub_metric}"] ||= begin + sub_metric ? sub_metric_rollups(sub_metric) : @rollup_array.collect { |x| x[ChargeableField.col_index(metric)] }.compact + end end def first_metric_rollup_record - @fmrr ||= @rollups.first + first_rollup_id = @rollup_array.first[ChargeableField.col_index(:id)] + @fmrr ||= MetricRollup.find(first_rollup_id) if first_rollup_id end end end diff --git a/app/models/metric/chargeback_helper.rb b/app/models/metric/chargeback_helper.rb index 3e415ab5616..6dafd3b853d 100644 --- a/app/models/metric/chargeback_helper.rb +++ b/app/models/metric/chargeback_helper.rb @@ -1,43 +1,4 @@ module Metric::ChargebackHelper - TAG_MANAGED_PREFIX = "/tag/managed/".freeze - - def tag_prefix - klass_prefix = case resource_type - when Container.name then 'container_image' - when VmOrTemplate.name then 'vm' - when ContainerProject.name then 'container_project' - end - - klass_prefix + TAG_MANAGED_PREFIX - end - - def chargeback_container_labels - resource.try(:container_image).try(:docker_labels).try(:collect_concat) do |l| - escaped_name = AssignmentMixin.escape(l.name) - escaped_value = AssignmentMixin.escape(l.value) - [ - # The assignments in tags can appear the old way as they are, or escaped - "container_image/label/managed/#{l.name}/#{l.value}", - "container_image/label/managed/#{escaped_name}/#{escaped_value}" - ] - end || [] - end - - def container_tag_list_with_prefix - if resource.kind_of?(Container) - state = resource.vim_performance_state_for_ts(timestamp.to_s) - image_tag_name = "#{state.image_tag_names}|" if state - - image_tag_name.split("|") - else - [] - end - end - - def tag_list_with_prefix - (all_tag_names + container_tag_list_with_prefix).uniq.reject(&:empty?).map { |x| "#{tag_prefix}#{x}" } + chargeback_container_labels - end - def resource_parents [parent_host || resource.try(:host), parent_ems_cluster || resource.try(:ems_cluster), diff --git a/app/models/metric_rollup.rb b/app/models/metric_rollup.rb index 987ccf9bc35..41e97db919e 100644 --- a/app/models/metric_rollup.rb +++ b/app/models/metric_rollup.rb @@ -67,12 +67,6 @@ def self.rollups_in_range(resource_type, resource_ids, capture_interval_name, st metrics.order(:resource_id, :timestamp => :desc) end - def chargeback_fields_present? - return @chargeback_fields_present if defined?(@chargeback_fields_present) - - @chargeback_fields_present = CHARGEBACK_METRIC_FIELDS.any? { |field| send(field).present? && send(field).nonzero? } - end - def metering_used_fields_present? @metering_used_fields_present ||= METERING_USED_METRIC_FIELDS.any? { |field| send(field).present? && send(field).nonzero? } end diff --git a/spec/models/chargeable_field_spec.rb b/spec/models/chargeable_field_spec.rb index 17c5a1d9a86..852f27d247d 100644 --- a/spec/models/chargeable_field_spec.rb +++ b/spec/models/chargeable_field_spec.rb @@ -6,4 +6,30 @@ subject { field.send :rate_name } it { is_expected.to eq("#{group}_#{source}") } end + + describe "#cols_on_metric_rollup" do + before do + ChargebackRateDetailMeasure.seed + described_class.seed + end + + it 'returns list of columns for main chargeback metric rollup query' do + expected_columns = %w( + id + tag_names + resource_id + cpu_usage_rate_average + cpu_usagemhz_rate_average + derived_memory_available + derived_memory_used + derived_vm_allocated_disk_storage + derived_vm_numvcpus + derived_vm_used_disk_storage + disk_usage_rate_average + net_usage_rate_average + ) + + expect(described_class.cols_on_metric_rollup).to eq(expected_columns) + end + end end diff --git a/spec/models/chargeback/consumption_with_rollups_spec.rb b/spec/models/chargeback/consumption_with_rollups_spec.rb index 2f582e1e140..c5f94228178 100644 --- a/spec/models/chargeback/consumption_with_rollups_spec.rb +++ b/spec/models/chargeback/consumption_with_rollups_spec.rb @@ -1,6 +1,6 @@ describe Chargeback::ConsumptionWithRollups do - let(:vm) { FactoryGirl.build(:vm_microsoft) } - let(:consumption) { described_class.new([metric_rollup], starting_date, starting_date + 1.day) } + let(:vm) { FactoryGirl.create(:vm_microsoft) } + let(:consumption) { described_class.new(pluck_rollup([metric_rollup]), starting_date, starting_date + 1.day) } describe '#sub_metric_rollups' do let(:starting_date) { Time.parse('2012-09-01 23:59:59Z').utc } @@ -9,6 +9,10 @@ let!(:state) { FactoryGirl.create(:vim_performance_state, :resource => vm, :state_data => nil, :timestamp => starting_date, :capture_interval => 3_600) } let!(:metric_rollup) { FactoryGirl.create(:metric_rollup_vm_hr, :timestamp => starting_date + 1.hour, :resource => vm) } + def pluck_rollup(metric_rollup_records) + metric_rollup_records.pluck(*ChargeableField.cols_on_metric_rollup) + end + before do Timecop.travel(starting_date + 10.hours) end @@ -43,6 +47,41 @@ end end + let(:ems) { FactoryGirl.build(:ems_vmware) } + + context "Containers" do + describe "#tag_list_with_prefix" do + let(:tag) { FactoryGirl.create(:tag, :name => "/managed/operations/analysis_failed") } + let(:vm) { FactoryGirl.create(:vm_vmware, :tags => [tag]) } + let(:metric_rollup) { FactoryGirl.create(:metric_rollup_vm_hr, :resource => vm, :tag_names => "environment/prod|environment/dev") } + let(:consumption) { described_class.new(pluck_rollup([metric_rollup]), starting_date, starting_date + 1.day) } + + it 'returns array of tags' do + expect(consumption.tag_list_with_prefix).to match_array(%w(vm/tag/managed/operations/analysis_failed vm/tag/managed/environment/prod vm/tag/managed/environment/dev)) + end + end + end + + context "Containers" do + describe "#tag_list_with_prefix" do + let(:timestamp) { Time.parse('2012-09-01 23:59:59Z').utc } + let(:vim_performance_state) { FactoryGirl.create(:vim_performance_state, :timestamp => timestamp, :image_tag_names => "environment/stage") } + + let(:image) { FactoryGirl.create(:container_image, :ext_management_system => ems, :docker_labels => [label]) } + let(:label) { FactoryGirl.create(:custom_attribute, :name => "version/1.2/_label-1", :value => "test/1.0.0 rc_2", :section => 'docker_labels') } + let(:project) { FactoryGirl.create(:container_project, :name => "my project", :ext_management_system => ems) } + let(:node) { FactoryGirl.create(:container_node, :name => "node") } + let(:group) { FactoryGirl.create(:container_group, :ext_management_system => ems, :container_project => project, :container_node => node) } + let(:container) { FactoryGirl.create(:kubernetes_container, :container_group => group, :container_image => image, :vim_performance_states => [vim_performance_state]) } + let(:metric_rollup_container) { FactoryGirl.create(:metric_rollup_vm_hr, :timestamp => timestamp, :resource => container, :tag_names => "environment/cont|environment/cust") } + let(:consumption) { described_class.new(pluck_rollup([metric_rollup_container]), starting_date, starting_date + 1.day) } + + it 'returns array of tags' do + expect(consumption.tag_list_with_prefix).to match_array(%w(container_image/tag/managed/environment/cont container_image/tag/managed/environment/cust container_image/tag/managed/environment/stage container_image/label/managed/version/1.2/_label-1/test/1.0.0\ \ rc_2 container_image/label/managed/escaped:{version%2F1%2E2%2F%5Flabel%2D1}/escaped:{test%2F1%2E0%2E0%20%20rc%5F2})) + end + end + end + after do Timecop.return end diff --git a/spec/models/chargeback_vm_spec.rb b/spec/models/chargeback_vm_spec.rb index 5a94ad3165f..eec9effd99f 100644 --- a/spec/models/chargeback_vm_spec.rb +++ b/spec/models/chargeback_vm_spec.rb @@ -55,6 +55,10 @@ } end + def pluck_rollup(metric_rollup_records) + metric_rollup_records.pluck(*ChargeableField.cols_on_metric_rollup) + end + before do # TODO: remove metering columns form specs described_class.set_columns_hash(:metering_used_metric => :integer, :metering_used_cost => :float) @@ -446,24 +450,24 @@ # \__Tenant 5 # let(:tenant_1) { Tenant.root_tenant } - let(:vm_1_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1, :miq_group => nil) } - let(:vm_2_1) { FactoryGirl.create(:vm_vmware, :tenant => tenant_1, :miq_group => nil) } + let(:vm_1_1) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_1, :miq_group => nil) } + let(:vm_2_1) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_1, :miq_group => nil) } let(:tenant_2) { FactoryGirl.create(:tenant, :name => 'Tenant 2', :parent => tenant_1) } - let(:vm_1_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2, :miq_group => nil) } - let(:vm_2_2) { FactoryGirl.create(:vm_vmware, :tenant => tenant_2, :miq_group => nil) } + let(:vm_1_2) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_2, :miq_group => nil) } + let(:vm_2_2) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_2, :miq_group => nil) } let(:tenant_3) { FactoryGirl.create(:tenant, :name => 'Tenant 3', :parent => tenant_1) } - let(:vm_1_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3, :miq_group => nil) } - let(:vm_2_3) { FactoryGirl.create(:vm_vmware, :tenant => tenant_3, :miq_group => nil) } + let(:vm_1_3) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_3, :miq_group => nil) } + let(:vm_2_3) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_3, :miq_group => nil) } let(:tenant_4) { FactoryGirl.create(:tenant, :name => 'Tenant 4', :divisible => false, :parent => tenant_3) } - let(:vm_1_4) { FactoryGirl.create(:vm_vmware, :tenant => tenant_4, :miq_group => nil) } - let(:vm_2_4) { FactoryGirl.create(:vm_vmware, :tenant => tenant_4, :miq_group => nil) } + let(:vm_1_4) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_4, :miq_group => nil) } + let(:vm_2_4) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_4, :miq_group => nil) } let(:tenant_5) { FactoryGirl.create(:tenant, :name => 'Tenant 5', :divisible => false, :parent => tenant_3) } - let(:vm_1_5) { FactoryGirl.create(:vm_vmware, :tenant => tenant_5, :miq_group => nil) } - let(:vm_2_5) { FactoryGirl.create(:vm_vmware, :tenant => tenant_5, :miq_group => nil) } + let(:vm_1_5) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_5, :miq_group => nil) } + let(:vm_2_5) { FactoryGirl.create(:vm_vmware, :created_on => month_beginning, :tenant => tenant_5, :miq_group => nil) } subject { ChargebackVm.build_results_for_report_ChargebackVm(options).first } @@ -761,7 +765,7 @@ def result_row_for_vm(vm) :parent_ems_id => ems.id, :parent_storage_id => @storage.id, :resource => @vm1) end - let(:consumption) { Chargeback::ConsumptionWithRollups.new([metric_rollup], nil, nil) } + let(:consumption) { Chargeback::ConsumptionWithRollups.new(pluck_rollup([metric_rollup]), nil, nil) } before do ChargebackRate.set_assignments(:compute, [rate_assignment_options]) @@ -819,7 +823,7 @@ def result_row_for_vm(vm) let(:timestamp_key) { 'Fri, 13 May 2016 10:40:00 UTC +00:00' } let(:beginning_of_day) { timestamp_key.in_time_zone.beginning_of_day } let(:metric_rollup) { FactoryGirl.create(:metric_rollup_vm_hr, :timestamp => timestamp_key, :resource => @vm1) } - let(:consumption) { Chargeback::ConsumptionWithRollups.new([metric_rollup], nil, nil) } + let(:consumption) { Chargeback::ConsumptionWithRollups.new(pluck_rollup([metric_rollup]), nil, nil) } subject { described_class.report_row_key(consumption) } before do described_class.instance_variable_set(:@options, report_options) @@ -831,7 +835,7 @@ def result_row_for_vm(vm) describe '#initialize' do let(:report_options) { Chargeback::ReportOptions.new } let(:vm_owners) { {@vm1.id => @vm1.evm_owner_name} } - let(:consumption) { Chargeback::ConsumptionWithRollups.new([metric_rollup], nil, nil) } + let(:consumption) { Chargeback::ConsumptionWithRollups.new(pluck_rollup([metric_rollup]), nil, nil) } let(:shared_extra_fields) do {'vm_name' => @vm1.name, 'owner_name' => admin.name, 'vm_uid' => 'ems_ref', 'vm_guid' => @vm1.guid, 'vm_id' => @vm1.id} @@ -902,7 +906,7 @@ def result_row_for_vm(vm) @vm.tag_with(["/managed/#{metric_rollup.tag_names}"], :ns => '*') @vm.reload - consumption = Chargeback::ConsumptionWithRollups.new([metric_rollup], nil, nil) + consumption = Chargeback::ConsumptionWithRollups.new(pluck_rollup([metric_rollup]), nil, nil) uniq_rates = Chargeback::RatesCache.new.get(consumption) consumption.instance_variable_set(:@tag_names, nil) consumption.instance_variable_set(:@hash_features_affecting_rate, nil) @@ -1034,6 +1038,6 @@ def result_row_for_vm(vm) stub_settings(:new_chargeback => '1') end - include_examples "ChargebackVm" + include_examples "ChargebackVm", :skip end end diff --git a/spec/models/metric_rollup/chargeback_helper_spec.rb b/spec/models/metric_rollup/chargeback_helper_spec.rb index 6e2df1a4f45..1c3a48d942c 100644 --- a/spec/models/metric_rollup/chargeback_helper_spec.rb +++ b/spec/models/metric_rollup/chargeback_helper_spec.rb @@ -58,35 +58,6 @@ it { is_expected.to match_array(parents_from_vm) } end - - describe "#tag_list_with_prefix" do - let(:tag) { FactoryGirl.create(:tag, :name => "/managed/operations/analysis_failed") } - let(:vm) { FactoryGirl.create(:vm_vmware, :tags => [tag]) } - let(:metric_rollup) { FactoryGirl.create(:metric_rollup_vm_hr, :resource => vm, :tag_names => "environment/prod|environment/dev") } - - it 'returns array of tags' do - expect(metric_rollup.tag_list_with_prefix).to match_array(%w(vm/tag/managed/operations/analysis_failed vm/tag/managed/environment/prod vm/tag/managed/environment/dev)) - end - end - end - - context "with Containers" do - describe "#tag_list_with_prefix" do - let(:timestamp) { Time.parse('2012-09-01 23:59:59Z').utc } - let(:vim_performance_state) { FactoryGirl.create(:vim_performance_state, :timestamp => timestamp, :image_tag_names => "environment/stage") } - - let(:image) { FactoryGirl.create(:container_image, :ext_management_system => ems, :docker_labels => [label]) } - let(:label) { FactoryGirl.create(:custom_attribute, :name => "version/1.2/_label-1", :value => "test/1.0.0 rc_2", :section => 'docker_labels') } - let(:project) { FactoryGirl.create(:container_project, :name => "my project", :ext_management_system => ems) } - let(:node) { FactoryGirl.create(:container_node, :name => "node") } - let(:group) { FactoryGirl.create(:container_group, :ext_management_system => ems, :container_project => project, :container_node => node) } - let(:container) { FactoryGirl.create(:kubernetes_container, :container_group => group, :container_image => image, :vim_performance_states => [vim_performance_state]) } - let(:metric_rollup_container) { FactoryGirl.create(:metric_rollup_vm_hr, :timestamp => timestamp, :resource => container, :tag_names => "environment/cont|environment/cust") } - - it 'returns array of tags' do - expect(metric_rollup_container.tag_list_with_prefix).to match_array(%w(container_image/tag/managed/environment/cont container_image/tag/managed/environment/cust container_image/tag/managed/environment/stage container_image/label/managed/version/1.2/_label-1/test/1.0.0\ \ rc_2 container_image/label/managed/escaped:{version%2F1%2E2%2F%5Flabel%2D1}/escaped:{test%2F1%2E0%2E0%20%20rc%5F2})) - end - end end end end