Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tag mapping in graph refresh #162

Merged
merged 1 commit into from
Dec 3, 2017
Merged

Tag mapping in graph refresh #162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions app/models/manageiq/providers/kubernetes/container_manager/inventory_collections.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_projects], %w(labels additional_attributes))
initialize_taggings_collection(@collections[:container_projects])

@collections[:container_quotas] = ::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
Expand Down Expand Up @@ -75,6 +76,7 @@ def initialize_inventory_collections
)
initialize_container_conditions_collection(manager, :container_nodes)
initialize_custom_attributes_collections(@collections[:container_nodes], %w(labels additional_attributes))
initialize_taggings_collection(@collections[:container_nodes])

# polymorphic child of ContainerNode & ContainerImage,
# but refresh only sets it on nodes.
Expand Down Expand Up @@ -147,6 +149,8 @@ def initialize_inventory_collections
)
initialize_container_conditions_collection(manager, :container_groups)
initialize_custom_attributes_collections(@collections[:container_groups], %w(labels node_selectors))
initialize_taggings_collection(@collections[:container_groups])

@collections[:container_volumes] =
::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
Expand Down Expand Up @@ -209,6 +213,7 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_replicators], %w(labels selectors))
initialize_taggings_collection(@collections[:container_replicators])

@collections[:container_services] =
::ManagerRefresh::InventoryCollection.new(
Expand All @@ -223,6 +228,8 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_services], %w(labels selectors))
initialize_taggings_collection(@collections[:container_services])

@collections[:container_service_port_configs] =
::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
Expand All @@ -244,6 +251,7 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_routes], %w(labels))
initialize_taggings_collection(@collections[:container_routes])

@collections[:container_templates] =
::ManagerRefresh::InventoryCollection.new(
Expand All @@ -256,6 +264,8 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_templates], %w(labels))
initialize_taggings_collection(@collections[:container_templates])
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I seem to be missing the refresh_parser.rb code that handles template as well as build_pods for that matter, where is it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As we discussed offline:

  • removed ContainerBuildPod (that model doesn't acts_as_miq_taggable, and don't have their own page in UI where one could see/edit tags).
  • indeed templates and images are also missing map_labels(), that's a bug I'll fix separately but I prefer to keep their taggings_for_ collections in these PR.

PTAL.


@collections[:container_template_parameters] =
::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
Expand All @@ -278,6 +288,8 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_builds], %w(labels))
initialize_taggings_collection(@collections[:container_builds])

@collections[:container_build_pods] =
::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
Expand All @@ -291,6 +303,7 @@ def initialize_inventory_collections
)
)
initialize_custom_attributes_collections(@collections[:container_build_pods], %w(labels))
# no taggings for build pods, they don't acts_as_miq_taggable.

@collections[:persistent_volumes] =
::ManagerRefresh::InventoryCollection.new(
Expand Down Expand Up @@ -355,6 +368,30 @@ def initialize_custom_attributes_collections(parent_collection, sections)
end
end

def initialize_taggings_collection(parent_collection)
type = parent_collection.model_class.base_class.name
relation = parent_collection.full_collection_for_comparison
query = Tagging.where(
:taggable_type => type,
:taggable_id => relation,
).joins(:tag).merge(Tag.controlled_by_mapping)

@collections[[:taggings_for, type]] =
::ManagerRefresh::InventoryCollection.new(
shared_options.merge(
:model_class => Tagging,
:name => "taggings_for_#{parent_collection.name}".to_sym,
:arel => query,
:manager_ref => [:taggable, :tag],
:parent_inventory_collections => [parent_collection.name],
)
)
end

def add_collection(collection)
@collections[collection.name] = collection
end

def custom_reconnect_block
# TODO(lsmola) once we have DB unique indexes, we can stop using manual reconnect, since it adds processing time
lambda do |inventory_collection, inventory_objects_index, attributes_index|
Expand Down
38 changes: 27 additions & 11 deletions app/models/manageiq/providers/kubernetes/container_manager/refresh_parser.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ def target_collection_inv_to_persister(ems, inventory, options = Config::Options
end

def persister_inv_to_persister(persister, inventory, options)
persister.add_collection(@tag_mapper.tags_to_resolve_collection)
# TODO(lsmola) expose persister and use that and use that instead of @inv_collections
@inv_collections = persister.collections

Expand Down Expand Up @@ -256,16 +257,18 @@ def get_nodes_graph(inv)
inv["node"].each do |data|
h = parse_node(data)

h.except!(:namespace, :tags)
h.except!(:namespace)

labels = h.delete(:labels)
tags = h.delete(:tags)
children = h.extract!(:container_conditions, :computer_system)

container_node = collection.build(h)

get_container_conditions_graph(container_node, children[:container_conditions])
get_node_computer_systems_graph(container_node, children[:computer_system])
get_custom_attributes_graph(container_node, :labels => labels)
get_taggings_graph(container_node, tags)
end
end

Expand Down Expand Up @@ -299,12 +302,13 @@ def get_namespaces_graph(inv)
inv["namespace"].each do |ns|
h = parse_namespace(ns)

h.except!(:tags)
custom_attrs = h.extract!(:labels)
tags = h.delete(:tags)

container_project = collection.build(h)

get_custom_attributes_graph(container_project, custom_attrs) # TODO: untested
get_taggings_graph(container_project, tags)
end
end

Expand Down Expand Up @@ -360,16 +364,17 @@ def get_replication_controllers_graph(inv)
inv["replication_controller"].each do |rc|
h = parse_replication_controllers(rc)

h.except!(:tags)

h[:container_project] = lazy_find_project(:name => h[:namespace])

custom_attrs = h.extract!(:labels, :selector_parts)
tags = h.delete(:tags)

container_replicator = collection.build(h)
get_custom_attributes_graph(container_replicator,
:labels => custom_attrs[:labels],
# The actual section is "selectors"
:selectors => custom_attrs[:selector_parts])
get_taggings_graph(container_replicator, tags)
end
end

Expand Down Expand Up @@ -404,16 +409,15 @@ def get_pods_graph(inv)
inv["pod"].each do |pod|
h = parse_pod(pod)

h.except!(:tags)

h[:container_project] = lazy_find_project(:name => h[:namespace])
h[:container_node] = lazy_find_node(:name => h.delete(:container_node_name))
h[:container_replicator] = lazy_find_replicator(h.delete(:container_replicator_ref))
h[:container_build_pod] = lazy_find_build_pod(:namespace => h[:namespace],
:name => h.delete(:build_pod_name))

custom_attrs = h.extract!(:labels, :node_selector_parts)
children = h.extract!(:containers, :container_conditions, :container_volumes)
custom_attrs = h.extract!(:labels, :node_selector_parts)
tags = h.delete(:tags)
children = h.extract!(:containers, :container_conditions, :container_volumes)

container_group = collection.build(h)

Expand All @@ -424,6 +428,7 @@ def get_pods_graph(inv)
:labels => custom_attrs[:labels],
# The actual section is "node_selectors"
:node_selectors => custom_attrs[:node_selector_parts])
get_taggings_graph(container_group, tags)
end
end

Expand Down Expand Up @@ -523,19 +528,19 @@ def get_endpoints_and_services_graph(inv)
end

custom_attrs = h.extract!(:labels, :selector_parts)
children = h.extract!(:container_service_port_configs)
tags = h.delete(:tags)
children = h.extract!(:container_service_port_configs)

h[:container_groups] = cgs_by_namespace_and_name.fetch_path(h[:namespace], h[:name])

h.except!(:tags)

container_service = collection.build(h)

get_container_service_port_configs_graph(container_service, children[:container_service_port_configs])
get_custom_attributes_graph(container_service,
:labels => custom_attrs[:labels],
# The actual section is "selectors"
:selectors => custom_attrs[:selector_parts])
get_taggings_graph(container_service, tags)
end
end

Expand Down Expand Up @@ -585,6 +590,17 @@ def get_custom_attributes_graph(parent, hashes_by_section)
end
end

# Conveniently, the tags map_labels emits are already in InventoryObject<Tag> form
def get_taggings_graph(parent, tags_inventory_objects)
model_name = parent.inventory_collection.model_class.base_class.name
key = [:taggings_for, model_name]
collection = @inv_collections[key]
raise("can't save: missing @inv_collections[#{key}]") if collection.nil?
tags_inventory_objects.each do |tag|
collection.build(:taggable => parent, :tag => tag)
end
end

## Helpers for @data / @data_index

def process_collection(collection, key, &block)
Expand Down
28 changes: 11 additions & 17 deletions spec/models/manageiq/providers/kubernetes/container_manager/refresher_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
# instantiated at the end, for both classical and graph refresh
shared_examples "kubernetes refresher VCR tests" do |check_tag_mapping: true|
let(:check_tag_mapping) { check_tag_mapping }

shared_examples "kubernetes refresher VCR tests" do
before(:each) do
allow(MiqServer).to receive(:my_zone).and_return("default")
auth = AuthToken.new(:name => "test", :auth_key => "valid-token")
Expand Down Expand Up @@ -155,12 +153,10 @@ def assert_specific_container_group(expected_extra_tags: [])
expect(@containergroup.labels).to contain_exactly(
label_with_name_value("name", "heapster")
)
if check_tag_mapping
expect(@containergroup.tags).to contain_exactly(
tag_in_category_with_description(@name_category, "heapster"),
*expected_extra_tags
)
end
expect(@containergroup.tags).to contain_exactly(
tag_in_category_with_description(@name_category, "heapster"),
*expected_extra_tags
)

# Check the relation to container node
expect(@containergroup.container_node).not_to be_nil
Expand Down Expand Up @@ -295,12 +291,10 @@ def assert_specific_container_replicator(expected_extra_tags: [])
expect(@replicator.labels).to contain_exactly(
label_with_name_value("name", "influxGrafana")
)
if check_tag_mapping
expect(@replicator.tags).to contain_exactly(
tag_in_category_with_description(@name_category, "influxGrafana"),
*expected_extra_tags
)
end
expect(@replicator.tags).to contain_exactly(
tag_in_category_with_description(@name_category, "influxGrafana"),
*expected_extra_tags
)
expect(@replicator.selector_parts.count).to eq(1)

@group = ContainerGroup.where(:name => "monitoring-influx-grafana-controller-22icy").first
Expand Down Expand Up @@ -587,7 +581,7 @@ def assert_disconnected(object)
end

# TODO: pending graph tag mapping implementation
include_examples "kubernetes refresher VCR tests", :check_tag_mapping => false
include_examples "kubernetes refresher VCR tests"
end

context "with :batch saver" do
Expand All @@ -598,7 +592,7 @@ def assert_disconnected(object)
end

# TODO: pending graph tag mapping implementation
include_examples "kubernetes refresher VCR tests", :check_tag_mapping => false
include_examples "kubernetes refresher VCR tests"
end
end
end