Skip to content

Commit

Permalink
Merge pull request #13337 from Ladas/rename_dto_to_inventory
Browse files Browse the repository at this point in the history
Rename dto to inventory
  • Loading branch information
agrare authored Jan 9, 2017
2 parents aa69a4f + 416f07f commit 21e0f8f
Show file tree
Hide file tree
Showing 19 changed files with 1,128 additions and 1,065 deletions.
4 changes: 2 additions & 2 deletions app/models/ems_refresh/save_inventory.rb
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module EmsRefresh::SaveInventory
def save_ems_inventory(ems, hashes, target = nil)
if hashes && hashes[:_dto_collection]
hashes.delete(:_dto_collection)
if hashes && hashes[:_inventory_collection]
hashes.delete(:_inventory_collection)
ManagerRefresh::SaveInventory.save_inventory(ems, hashes)
return
end
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
class ManageIQ::Providers::CloudManager::RefreshParserDto < ::ManagerRefresh::RefreshParserDto
class ManageIQ::Providers::CloudManager::RefreshParserInventoryObject < ::ManagerRefresh::RefreshParserInventoryObject
private

def parse_key_pair(kp)
Expand Down
63 changes: 0 additions & 63 deletions app/models/manager_refresh/dto_lazy.rb

This file was deleted.

2 changes: 1 addition & 1 deletion app/models/manager_refresh/graph/topological_sort.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def make_topological_sort(original_nodes, edges)
# Topological sort of the graph of the DTO collections to find the right order of saving DTO collections and
# identify what DTO collections can be saved in parallel.
################################################################################################################
# The expected input here is the directed acyclic Graph G (dto_collections), consisting of Vertices(Nodes) V and
# The expected input here is the directed acyclic Graph G (inventory_collections), consisting of Vertices(Nodes) V and
# Edges E:
# G = (V, E)
#
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
module ManagerRefresh
class DtoCollection
class InventoryCollection
attr_accessor :saved

attr_reader :model_class, :strategy, :attributes_blacklist, :attributes_whitelist, :custom_save_block, :parent,
Expand Down Expand Up @@ -36,7 +36,7 @@ def initialize(model_class, manager_ref: nil, association: nil, parent: nil, str
blacklist_attributes!(attributes_blacklist) if attributes_blacklist.present?
whitelist_attributes!(attributes_whitelist) if attributes_whitelist.present?

validate_dto_collection!
validate_inventory_collection!
end

def to_a
Expand All @@ -61,8 +61,8 @@ def load_from_db

def process_strategy_local_db_cache_all
self.saved = true
# TODO(lsmola) selected need to contain also :keys used in other DtoCollections pointing to this one, once we
# get list of all keys for each DtoCollection ,we can uncomnent
# TODO(lsmola) selected need to contain also :keys used in other InventoryCollections pointing to this one, once
# we get list of all keys for each InventoryCollection ,we can uncomnent
# selected = [:id] + manager_ref.map { |x| model_class.reflect_on_association(x).try(:foreign_key) || x }
# selected << :type if model_class.new.respond_to? :type
# load_from_db.select(selected).find_each do |record|
Expand Down Expand Up @@ -99,12 +99,12 @@ def saveable?
dependencies.all?(&:saved?)
end

def <<(dto)
unless data_index[dto.manager_uuid]
data_index[dto.manager_uuid] = dto
data << dto
def <<(inventory_object)
unless data_index[inventory_object.manager_uuid]
data_index[inventory_object.manager_uuid] = inventory_object
data << inventory_object

actualize_dependencies(dto)
actualize_dependencies(inventory_object)
end
end

Expand Down Expand Up @@ -138,11 +138,11 @@ def find(manager_uuid)
end

def lazy_find(manager_uuid, key: nil, default: nil)
::ManagerRefresh::DtoLazy.new(self, manager_uuid, :key => key, :default => default)
::ManagerRefresh::InventoryObjectLazy.new(self, manager_uuid, :key => key, :default => default)
end

def new_dto(hash)
::ManagerRefresh::Dto.new(self, hash)
def new_inventory_object(hash)
::ManagerRefresh::InventoryObject.new(self, hash)
end

def filtered_dependency_attributes
Expand Down Expand Up @@ -181,31 +181,31 @@ def dependencies
filtered_dependency_attributes.values.map(&:to_a).flatten.uniq
end

def dependency_attributes_for(dto_collections)
def dependency_attributes_for(inventory_collections)
attributes = Set.new
dto_collections.each do |dto_collection|
attributes += filtered_dependency_attributes.select { |_key, value| value.include?(dto_collection) }.keys
inventory_collections.each do |inventory_collection|
attributes += filtered_dependency_attributes.select { |_key, value| value.include?(inventory_collection) }.keys
end
attributes
end

def blacklist_attributes!(attributes)
# The manager_ref attributes cannot be blacklisted, otherwise we will not be able to identify the dto object. We
# do not automatically remove attributes causing fixed dependencies, so beware that without them, you won't be
# able to create the record.
# The manager_ref attributes cannot be blacklisted, otherwise we will not be able to identify the
# inventory_object. We do not automatically remove attributes causing fixed dependencies, so beware that without
# them, you won't be able to create the record.
self.attributes_blacklist += attributes - (fixed_attributes + internal_attributes)
end

def whitelist_attributes!(attributes)
# The manager_ref attributes always needs to be in the white list, otherwise we will not be able to identify the
# dto object. We do not automatically add attributes causing fixed dependencies, so beware that without them, you
# won't be able to create the record.
# inventory_object. We do not automatically add attributes causing fixed dependencies, so beware that without
# them, you won't be able to create the record.
self.attributes_whitelist += attributes + (fixed_attributes + internal_attributes)
end

def clone
# A shallow copy of DtoCollection, the copy will share @data of the original collection, otherwise we would be
# copying a lot of records in memory.
# A shallow copy of InventoryCollection, the copy will share @data of the original collection, otherwise we would
# be copying a lot of records in memory.
self.class.new(model_class,
:manager_ref => manager_ref,
:association => association,
Expand All @@ -215,15 +215,16 @@ def clone
:custom_save_block => custom_save_block,
:data => data,
:data_index => data_index,
# Dependency attributes need to be a hard copy, since those will differ for each DtoCollection
# Dependency attributes need to be a hard copy, since those will differ for each
# InventoryCollection
:dependency_attributes => dependency_attributes.clone)
end

def to_s
whitelist = ", whitelist: [#{attributes_whitelist.to_a.join(", ")}]" unless attributes_whitelist.blank?
blacklist = ", blacklist: [#{attributes_blacklist.to_a.join(", ")}]" unless attributes_blacklist.blank?

"DtoCollection:<#{@model_class}>#{whitelist}#{blacklist}"
"InventoryCollection:<#{@model_class}>#{whitelist}#{blacklist}"
end

def inspect
Expand All @@ -234,37 +235,38 @@ def inspect

attr_writer :attributes_blacklist, :attributes_whitelist

def actualize_dependencies(dto)
dto.data.each do |key, value|
def actualize_dependencies(inventory_object)
inventory_object.data.each do |key, value|
if dependency?(value)
(dependency_attributes[key] ||= Set.new) << value.dto_collection
(dependency_attributes[key] ||= Set.new) << value.inventory_collection
self.transitive_dependency_attributes << key if transitive_dependency?(value)
elsif value.kind_of?(Array) && value.any? { |x| dependency?(x) }
(dependency_attributes[key] ||= Set.new) << value.detect { |x| dependency?(x) }.dto_collection
(dependency_attributes[key] ||= Set.new) << value.detect { |x| dependency?(x) }.inventory_collection
self.transitive_dependency_attributes << key if value.any? { |x| transitive_dependency?(x) }
end
end
end

def dependency?(value)
(value.kind_of?(::ManagerRefresh::DtoLazy) && value.dependency?) || value.kind_of?(::ManagerRefresh::Dto)
(value.kind_of?(::ManagerRefresh::InventoryObjectLazy) && value.dependency?) ||
value.kind_of?(::ManagerRefresh::InventoryObject)
end

def transitive_dependency?(value)
# If the dependency is dto_collection.lazy_find(:ems_ref, :key => :stack)
# and a :stack is a relation to another object, in the Dto object,
# If the dependency is inventory_collection.lazy_find(:ems_ref, :key => :stack)
# and a :stack is a relation to another object, in the InventoryObject object,
# then this dependency is considered transitive.
(value.kind_of?(::ManagerRefresh::DtoLazy) && value.transitive_dependency?)
(value.kind_of?(::ManagerRefresh::InventoryObjectLazy) && value.transitive_dependency?)
end

def validate_dto_collection!
def validate_inventory_collection!
if @strategy == :local_db_cache_all
if (manager_ref & association_attributes).present?
# Our manager_ref unique key contains a reference, that means that index we get from the API and from the
# db will differ. We need a custom indexing method, so the indexing is correct.
raise "The unique key list manager_ref contains a reference, which can't be built automatically when loading"\
" the DtoCollection from the DB, you need to provide a custom_manager_uuid lambda, that builds the"\
" correct manager_uuid given a DB record" if custom_manager_uuid.nil?
" the InventoryCollection from the DB, you need to provide a custom_manager_uuid lambda, that builds"\
" the correct manager_uuid given a DB record" if custom_manager_uuid.nil?
end
end
end
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
module ManagerRefresh
class DtoCollection
class InventoryCollection
class Graph < ::ManagerRefresh::Graph
def initialize(nodes)
super(nodes)

assert_dto_collections(nodes)
assert_inventory_collections(nodes)
end

def build_directed_acyclic_graph!
Expand Down Expand Up @@ -53,76 +53,77 @@ def build_directed_acyclic_graph!

private

def assert_dto_collections(dto_collections)
dto_collections.each do |dto_collection|
unless dto_collection.kind_of? ::ManagerRefresh::DtoCollection
raise "A ManagerRefresh::SaveInventory needs a DtoCollection object, it got: #{dto_collection.inspect}"
def assert_inventory_collections(inventory_collections)
inventory_collections.each do |inventory_collection|
unless inventory_collection.kind_of? ::ManagerRefresh::InventoryCollection
raise "A ManagerRefresh::SaveInventory needs a InventoryCollection object, it got: #{inventory_collection.inspect}"
end
end
end

def convert_to_dag!(nodes, feedback_edge_set)
new_nodes = []
dto_collection_transformations = {}
nodes.each do |dto_collection|
feedback_dependencies = feedback_edge_set.select { |e| e.second == dto_collection }.map(&:first)
attrs = dto_collection.dependency_attributes_for(feedback_dependencies)
inventory_collection_transformations = {}
nodes.each do |inventory_collection|
feedback_dependencies = feedback_edge_set.select { |e| e.second == inventory_collection }.map(&:first)
attrs = inventory_collection.dependency_attributes_for(feedback_dependencies)

next if attrs.blank?

new_dto_collection = dto_collection.clone
new_inventory_collection = inventory_collection.clone

# Add dto_collection as a dependency of the new_dto_collection, so we make sure it runs after
# Add inventory_collection as a dependency of the new_inventory_collection, so we make sure it runs after
# TODO(lsmola) add a nice dependency_attributes setter? It's used also in actualize_dependencies method
new_dto_collection.dependency_attributes[:__feedback_edge_set_parent] = Set.new([dto_collection])
new_nodes << new_dto_collection
new_inventory_collection.dependency_attributes[:__feedback_edge_set_parent] = Set.new([inventory_collection])
new_nodes << new_inventory_collection

dto_collection.blacklist_attributes!(attrs)
new_dto_collection.whitelist_attributes!(attrs)
inventory_collection.blacklist_attributes!(attrs)
new_inventory_collection.whitelist_attributes!(attrs)

# Store a simple hash for transforming dto_collection to new_dto_collection
dto_collection_transformations[dto_collection] = new_dto_collection
# Store a simple hash for transforming inventory_collection to new_inventory_collection
inventory_collection_transformations[inventory_collection] = new_inventory_collection
end

all_nodes = nodes + new_nodes

# If we remove an attribute that was a dependency of another node, we need to move also the
# dependency. So e.g. floating_ip depends on network_port's attribute vm, but we move that attribute to new
# network_port dto_collection. We will need to move also the dependency to point to the new dto_collection.
# network_port inventory_collection. We will need to move also the dependency to point to the new
# inventory_collection.
#
# So we have to go through all dependencies that loads a key, which is the moved attribute. We can get a list
# of attributes that are using a key from transitive_dependency_attributes, from there we can get a list of
# dependencies. And from the list of dependencies, we can check which ones were moved just by looking into
# dto_collection_transformations.
all_nodes.each do |dto_collection|
dto_collection.transitive_dependency_attributes.each do |transitive_dependency_attribute|
transitive_dependencies = dto_collection.dependency_attributes[transitive_dependency_attribute]
# inventory_collection_transformations.
all_nodes.each do |inventory_collection|
inventory_collection.transitive_dependency_attributes.each do |transitive_dependency_attribute|
transitive_dependencies = inventory_collection.dependency_attributes[transitive_dependency_attribute]
next if transitive_dependencies.blank?

transitive_dependencies.map! do |dependency|
transformed_dependency = dto_collection_transformations[dependency]
transformed_dependency = inventory_collection_transformations[dependency]
transformed_dependency.blank? ? dependency : transformed_dependency
end
end
end

# Add the new DtoCollections to the list of nodes our our graph
# Add the new InventoryCollections to the list of nodes our our graph
construct_graph!(all_nodes)
end

def build_edges(dto_collections)
def build_edges(inventory_collections)
edges = []
transitive_edges = []
fixed_edges = []
dto_collections.each do |dto_collection|
dto_collection.dependencies.each do |dependency|
fixed_edges << [dependency, dto_collection] if dto_collection.fixed_dependencies.include?(dependency)
if dto_collection.dependency_attributes_for([dependency]).any? { |x| dto_collection.transitive_dependency_attributes.include?(x) }
# The condition checks if the dependency is a transitive dependency, in other words a DtoLazy with :key
inventory_collections.each do |inventory_collection|
inventory_collection.dependencies.each do |dependency|
fixed_edges << [dependency, inventory_collection] if inventory_collection.fixed_dependencies.include?(dependency)
if inventory_collection.dependency_attributes_for([dependency]).any? { |x| inventory_collection.transitive_dependency_attributes.include?(x) }
# The condition checks if the dependency is a transitive dependency, in other words a InventoryObjectLazy with :key
# pointing to another object.
transitive_edges << [dependency, dto_collection]
transitive_edges << [dependency, inventory_collection]
else
edges << [dependency, dto_collection]
edges << [dependency, inventory_collection]
end
end
end
Expand Down
Loading

0 comments on commit 21e0f8f

Please sign in to comment.