Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve metrics saving #15976

Merged
merged 12 commits into from
Sep 26, 2017
Prev Previous commit
Pack transform_resources! in a helper method
Pack transform_resources! in a helper method
  • Loading branch information
Ladas committed Sep 26, 2017
commit ce5ebca691fdf9fa1a27fc0691f576a4206d62d7
18 changes: 11 additions & 7 deletions app/models/metric/ci_mixin/processing.rb
Original file line number Diff line number Diff line change
@@ -11,13 +11,7 @@ def perf_process(interval_name, start_time, end_time, counters_data)

affected_timestamps = []

# Fetch ActiveRecord object by 1 query per Model
grouped_resource_refs = counters_data.keys.each_with_object({}) { |x, obj| (obj[x.first] ||= []) << x.second }
fetched_records = grouped_resource_refs.keys.each_with_object({}) do |x, obj|
x.constantize.where(:id => grouped_resource_refs[x]).each { |rec| obj[[x, rec.id]] = rec }
end
# Transforming [Class, id] that were sent via the counters_data into the ActiveRecord objects
counters_data.transform_keys! { |x| fetched_records[x] }
transform_resources!(counters_data)
resources = counters_data.keys

_log.info("#{log_header} Processing for #{log_specific_targets(resources)}, for range [#{start_time} - #{end_time}]...")
@@ -108,6 +102,16 @@ def perf_process(interval_name, start_time, end_time, counters_data)

private

def transform_resources!(counters_data)
# Fetch ActiveRecord object by 1 query per Model
grouped_resource_refs = counters_data.keys.each_with_object({}) { |x, obj| (obj[x.first] ||= []) << x.second }
fetched_records = grouped_resource_refs.keys.each_with_object({}) do |x, obj|
x.constantize.where(:id => grouped_resource_refs[x]).each { |rec| obj[[x, rec.id]] = rec }
end
# Transforming [Class, id] that were sent via the counters_data into the ActiveRecord objects
counters_data.transform_keys! { |x| fetched_records[x] }
end

def transform_parameters_row_per_metric(_resources, interval_name, _start_time, _end_time, rt_rows)
rt_rows.flat_map do |ts, rt|
rt.merge!(Metric::Processing.process_derived_columns(rt[:resource], rt, interval_name == 'realtime' ? Metric::Helper.nearest_hourly_timestamp(ts) : nil))