Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Changes needed for openshift graph refresh #57

Merged
merged 9 commits into from
Jul 21, 2017
Merged

Changes needed for openshift graph refresh #57

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cb01002
hook for openshift to parse more stuff before get_container_image*s_g…
cben Jul 12, 2017
558932d
link pods - build pods
cben Jul 13, 2017
9ed2d5d
collection fixes for openshift
cben Jul 12, 2017
77f6a9f
service/registry matching (BUG: last port only)
cben Jul 13, 2017
d9b7e1c
better error for missing custom attribute collections
cben Jul 16, 2017
2ebd841
inventory collections fix: project labels (TODO: needs tests)
cben Jul 16, 2017
d519de2
Unhide ems_refresh.kubernetes.inventory_object_refresh setting
cben Jul 19, 2017
6c24d7d
Implement archiving for graph refresh, run deletion tests too
cben Jul 19, 2017
c15d51f
Leave images labels & docker_labels collections up to openshift
cben Jul 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ def initialize_inventory_collections(ems)
:association => :container_projects,
:secondary_refs => {:by_name => [:name]},
)
initialize_custom_attributes_collections(ems.container_projects, %w(labels additional_attributes))

@inv_collections[:container_quotas] = ::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerQuota,
:parent => ems,
Expand Down Expand Up @@ -87,6 +89,7 @@ def initialize_inventory_collections(ems)
# TODO: should match on digest when available
:manager_ref => [:image_ref],
)
initialize_custom_attributes_collections(ems.container_images, %w(labels docker_labels))

@inv_collections[:container_groups] =
::ManagerRefresh::InventoryCollection.new(
Expand Down Expand Up @@ -172,13 +175,17 @@ def initialize_inventory_collections(ems)
:parent => ems,
:association => :container_service_port_configs,
)

@inv_collections[:container_routes] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerRoute,
:parent => ems,
:builder_params => {:ems_id => ems.id},
:association => :container_routes,
:model_class => ContainerRoute,
:parent => ems,
:builder_params => {:ems_id => ems.id},
:association => :container_routes,
:attributes_blacklist => [:namespace],
)
initialize_custom_attributes_collections(ems.container_routes, %w(labels))

@inv_collections[:container_component_statuses] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerComponentStatus,
Expand All @@ -187,6 +194,7 @@ def initialize_inventory_collections(ems)
:association => :container_component_statuses,
:manager_ref => [:name],
)

@inv_collections[:container_templates] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerTemplate,
Expand All @@ -195,31 +203,36 @@ def initialize_inventory_collections(ems)
:association => :container_templates,
:attributes_blacklist => [:namespace],
)
initialize_custom_attributes_collections(ems.container_templates, %w(labels))
@inv_collections[:container_template_parameters] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerTemplateParameter,
:parent => ems,
:association => :container_template_parameters,
:manager_ref => [:container_template, :name],
)

@inv_collections[:container_builds] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerBuild,
:parent => ems,
:builder_params => {:ems_id => ems.id},
:association => :container_builds,
:secondary_refs => {:by_namespace_and_name => [:namespace, :name]},
)
initialize_custom_attributes_collections(ems.container_builds, %w(labels))
@inv_collections[:container_build_pods] =
::ManagerRefresh::InventoryCollection.new(
:model_class => ContainerBuildPod,
:parent => ems,
:builder_params => {:ems_id => ems.id},
:association => :container_build_pods,
# TODO: is this unique? build pods do have uid that becomes ems_ref,
# but we need lazy_find by name for lookup from container_group
# TODO: rename namespace -> container_project column?
# TODO: convert namespace column -> container_project_id?
:manager_ref => [:namespace, :name],
:secondary_refs => {:by_namespace_and_name => [:namespace, :name]},
)
initialize_custom_attributes_collections(ems.container_build_pods, %w(labels))

@inv_collections[:persistent_volumes] =
::ManagerRefresh::InventoryCollection.new(
:model_class => PersistentVolume,
Expand Down Expand Up @@ -256,11 +269,12 @@ def initialize_custom_attributes_collections(relation, sections)
query = CustomAttribute.where(:resource_type => relation.model.name,
:resource_id => relation,
:section => section.to_s)
@inv_collections[[:custom_attributes_for, relation.model.name, section]] = ::ManagerRefresh::InventoryCollection.new(
:model_class => CustomAttribute,
:arel => query,
:manager_ref => [:resource, :section, :name],
)
@inv_collections[[:custom_attributes_for, relation.model.name, section.to_s]] =
::ManagerRefresh::InventoryCollection.new(
:model_class => CustomAttribute,
:arel => query,
:manager_ref => [:resource, :section, :name],
)
end
end
end
50 changes: 37 additions & 13 deletions app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,21 @@ def ems_inv_to_hashes(inventory, _options = Config::Options.new)
@data
end

def ems_inv_to_inv_collections(ems, inventory, _options = Config::Options.new)
def ems_inv_to_inv_collections(ems, inventory, options = Config::Options.new)
initialize_inventory_collections(ems)

ems_inv_populate_collections(inventory, options)

# The following take parsed hashes from @data_index, populated during
# parsing pods and possibly openshift images, so must be called at the end.
get_container_images_graph
get_container_image_registries_graph

# Returning an array triggers ManagerRefresh::SaveInventory code path.
@inv_collections.values
end

def ems_inv_populate_collections(inventory, _options)
get_additional_attributes_graph(inventory) # TODO: untested?
get_nodes_graph(inventory)
get_namespaces_graph(inventory)
Expand All @@ -54,13 +67,6 @@ def ems_inv_to_inv_collections(ems, inventory, _options = Config::Options.new)
get_pods_graph(inventory)
get_endpoints_and_services_graph(inventory)
get_component_statuses_graph(inventory)
# The following use images resulting from parsing pods, so must be called after.
# TODO: openshift images parsing will have to plug before this.
get_container_images_graph
get_container_image_registries_graph

# Returning an array triggers ManagerRefresh::SaveInventory code path.
@inv_collections.values
end

def get_nodes(inventory)
Expand Down Expand Up @@ -118,6 +124,8 @@ def get_pods(inventory)
path_for_entity("replication_controller"), :by_namespace_and_name,
replicator_ref[:namespace], replicator_ref[:name]
)
# Note: save_container_groups_inventory also links build_pod by :build_pod_name.

@data_index.store_path(key, :by_namespace_and_name,
cg[:namespace], cg[:name], cg)
end
Expand Down Expand Up @@ -277,7 +285,7 @@ def get_namespaces_graph(inv)

container_project = collection.build(h)

get_custom_attributes_graph(container_project, custom_attrs)
get_custom_attributes_graph(container_project, custom_attrs) # TODO: untested
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cben is this comment still relevant? When do you think this will be tested?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still relevant, none of our VCR recordings have project labels. (but I'm confident this works)
I didn't bother Erez with adding those while he re-recorded because there are a few more things we have 0 of, I'm still collecting the list...

end
end

Expand Down Expand Up @@ -386,7 +394,9 @@ def get_pods_graph(inv)
h[:container_project] = lazy_find_project(:name => h[:namespace])
h[:container_node] = lazy_find_node(:name => h.delete(:container_node_name))
h[:container_replicator] = lazy_find_replicator(h.delete(:container_replicator_ref))
_build_pod_name = h.delete(:build_pod_name)
h[:container_build_pod] = lazy_find_build_pod(:namespace => h[:namespace],
:name => h.delete(:build_pod_name))

custom_attrs = h.extract!(:labels, :node_selector_parts)
children = h.extract!(:container_definitions, :containers, :container_conditions, :container_volumes)

Expand Down Expand Up @@ -481,11 +491,18 @@ def get_endpoints_and_services_graph(inv)

h[:container_project] = lazy_find_project(:name => h[:namespace]) # TODO: untested?

# TODO: with multiple ports, how can I match any of them to known registries,
# like https://github.com/ManageIQ/manageiq-providers-kubernetes/pull/57 ?
if h[:container_service_port_configs].any?
registry_port = h[:container_service_port_configs].last[:port]
h[:container_image_registry] = lazy_find_image_registry(
:host => h[:portal_ip], :port => registry_port
)
end

custom_attrs = h.extract!(:labels, :selector_parts)
children = h.extract!(:container_service_port_configs)

_container_image_registry = h.delete(:container_image_registry) # TODO: derive from container_service_port_configs

h[:container_groups] = cgs_by_namespace_and_name.fetch_path(h[:namespace], h[:name])

h.except!(:tags)
Expand Down Expand Up @@ -542,7 +559,9 @@ def get_container_images_graph
def get_custom_attributes_graph(parent, hashes_by_section)
model_name = parent.inventory_collection.model_class.name
hashes_by_section.each do |section, hashes|
collection = @inv_collections[[:custom_attributes_for, model_name, section.to_s]]
key = [:custom_attributes_for, model_name, section.to_s]
collection = @inv_collections[key]
raise("can't save: missing @inv_collections[#{key}]") if collection.nil?
hashes.to_a.each do |h|
h = h.merge(:resource => parent)
if h[:section].to_s != section.to_s
Expand Down Expand Up @@ -1287,5 +1306,10 @@ def lazy_find_image_registry(hash)
return nil if hash.nil?
@inv_collections[:container_image_registries].lazy_find_by(hash)
end

def lazy_find_build_pod(hash)
return nil if hash.nil?
@inv_collections[:container_build_pods].lazy_find_by(hash, :ref => :by_namespace_and_name)
end
end
end