From 8ff07ca19b6a171b8ec6ac53ac16b161aea03f23 Mon Sep 17 00:00:00 2001 From: Bronagh Sorota Date: Tue, 28 Feb 2017 14:35:31 -0500 Subject: [PATCH 1/2] Gather AWS labels and create CustomAttributes --- .../amazon/cloud_manager/refresh_parser.rb | 19 ++++++++++++++++++- .../amazon/aws_refresher_spec_common.rb | 9 +++++---- .../refresher_other_region_spec.rb | 5 ++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb b/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb index 66716d510..bb546ba62 100644 --- a/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb +++ b/app/models/manageiq/providers/amazon/cloud_manager/refresh_parser.rb @@ -185,6 +185,7 @@ def parse_image(image, is_public) :vendor => "amazon", :raw_power_state => "never", :template => true, + :labels => parse_labels(image.tags), # the is_public flag here avoids having to make an additional API call # per image, since we already know whether it's a public image :publicly_available => is_public, @@ -231,7 +232,7 @@ def parse_instance(instance) :vendor => "amazon", :raw_power_state => status, :boot_time => instance.launch_time, - + :labels => parse_labels(instance.tags), :hardware => { :bitness => architecture_to_bitness(instance.architecture), :virtualization_type => instance.virtualization_type, @@ -387,4 +388,20 @@ def parse_stack_resource(resource) } return uid, new_result end + + def parse_labels(tags) + result = [] + return result if tags.empty? + tags.each do |tag| + custom_attr = { + :section => 'labels', + :name => tag[:key], + :value => tag[:value], + :source => 'amazon' + } + result << custom_attr + end + + result + end end diff --git a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb index ce149f188..72c6f0365 100644 --- a/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb +++ b/spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb @@ -31,7 +31,7 @@ def expected_table_counts :availability_zone => 5, :cloud_network => 5, :cloud_subnet => 10, - :custom_attribute => 0, + :custom_attribute => 43, :disk => 44, :ext_management_system => 4, :firewall_rule => 119, @@ -336,7 +336,7 @@ def assert_specific_vm_powered_on :cpu_reserve_expand => nil, :cpu_limit => nil, :cpu_shares => nil, - :cpu_shares_level => nil + :cpu_shares_level => nil, ) expect(v.ext_management_system).to eq(@ems) @@ -358,7 +358,7 @@ def assert_specific_vm_powered_on .to match_array [sg_2, @sg] expect(v.operating_system).to be_nil # TODO: This should probably not be nil - expect(v.custom_attributes.size).to eq(0) + expect(v.custom_attributes.size).to eq(1) expect(v.snapshots.size).to eq(0) expect(v.hardware).to have_attributes( @@ -403,6 +403,7 @@ def assert_specific_vm_powered_on v.with_relationship_type("genealogy") do expect(v.parent).to eq(@template) end + expect(v.custom_attributes.find_by(:name => "Name").value).to eq("EmsRefreshSpec-PoweredOn-Basic3") end def assert_specific_vm_powered_off @@ -443,7 +444,7 @@ def assert_specific_vm_powered_off expect(v.cloud_subnet).to be_nil expect(v.security_groups).to match_array([@sg]) expect(v.operating_system).to be_nil # TODO: This should probably not be nil - expect(v.custom_attributes.size).to eq(0) + expect(v.custom_attributes.size).to eq(1) expect(v.snapshots.size).to eq(0) expect(v.hardware).to have_attributes( diff --git a/spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb b/spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb index c0f40fdef..78f8f4cb1 100644 --- a/spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb +++ b/spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb @@ -44,8 +44,7 @@ def assert_table_counts expect(VmOrTemplate.count).to eq(3) expect(Vm.count).to eq(2) expect(MiqTemplate.count).to eq(1) - - expect(CustomAttribute.count).to eq(0) + expect(CustomAttribute.count).to eq(1) expect(Disk.count).to eq(3) expect(GuestDevice.count).to eq(0) expect(Hardware.count).to eq(3) @@ -228,7 +227,7 @@ def assert_specific_vm_powered_on expect(v.security_groups).to eq([@sg]) expect(v.key_pairs).to eq([@kp]) expect(v.operating_system).to be_nil # TODO: This should probably not be nil - expect(v.custom_attributes.size).to eq(0) + expect(v.custom_attributes.size).to eq(1) expect(v.snapshots.size).to eq(0) expect(v.hardware).to have_attributes( From f9123d84236a66348c4bd05d499c74f94f6d7594 Mon Sep 17 00:00:00 2001 From: Bronagh Sorota Date: Fri, 10 Mar 2017 15:23:17 -0500 Subject: [PATCH 2/2] Port saving AWS labels to the custom attributes table --- .../amazon/inventory/parser/cloud_manager.rb | 21 +++++++++++++++++++ .../inventory/persister/cloud_manager.rb | 2 +- .../cloud_manager.rb | 10 +++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb index b45068656..4ccee1bfa 100644 --- a/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/parser/cloud_manager.rb @@ -53,6 +53,9 @@ def get_images(images, is_public = false) process_inventory_collection(images, :miq_templates) do |image| get_image_hardware(image) + resource = persister.miq_templates.lazy_find(image['image_id']) + get_labels(resource, image["tags"]) + parse_image(image, is_public) end end @@ -102,6 +105,8 @@ def get_instances flavor = persister.flavors.find(instance['instance_type']) || persister.flavors.find("unknown") get_instance_hardware(instance, flavor) + resource = persister.vms.lazy_find(instance['instance_id']) + get_labels(resource, instance["tags"]) parse_instance(instance, flavor) end @@ -116,6 +121,12 @@ def get_instance_hardware(instance, flavor) end end + def get_labels(resource, tags) + process_inventory_collection(tags, :vm_and_template_labels) do |tag| + parse_label(resource, tag) + end + end + def get_hardware_networks(instance) process_inventory_collection([instance], :networks) { |i| parse_hardware_private_network(i) } process_inventory_collection([instance], :networks) { |i| parse_hardware_public_network(i) } @@ -368,6 +379,16 @@ def parse_stack_resource(resource, stack) } end + def parse_label(resource, tag) + { + :resource => resource, + :section => 'labels', + :name => tag["key"], + :value => tag["value"], + :source => 'amazon' + } + end + # Overridden helper methods, we should put them in helper once we get rid of old refresh def get_from_tags(resource, item) (resource['tags'] || []).detect { |tag, _| tag['key'].downcase == item.to_s.downcase }.try(:[], 'value') diff --git a/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb index 263f8a44d..4f5c98284 100644 --- a/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory/persister/cloud_manager.rb @@ -2,7 +2,7 @@ class ManageIQ::Providers::Amazon::Inventory::Persister::CloudManager < ManageIQ def initialize_inventory_collections add_inventory_collections( cloud, - %i(vms miq_templates hardwares networks disks availability_zones + %i(vms miq_templates hardwares networks disks availability_zones vm_and_template_labels flavors key_pairs orchestration_stacks orchestration_stacks_resources orchestration_stacks_outputs orchestration_stacks_parameters orchestration_templates) ) diff --git a/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb b/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb index 63ee3db4a..c6a31b809 100644 --- a/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb +++ b/app/models/manageiq/providers/amazon/inventory_collection_default/cloud_manager.rb @@ -39,6 +39,16 @@ def key_pairs(extra_attributes = {}) super(attributes.merge!(extra_attributes)) end + def vm_and_template_labels(extra_attributes = {}) + attributes = { + :model_class => CustomAttribute, + :association => :vm_and_template_labels, + :manager_ref => [:resource, :name] + } + + attributes.merge!(extra_attributes) + end + def orchestration_stacks(extra_attributes = {}) attributes = { :model_class => ::ManageIQ::Providers::Amazon::CloudManager::OrchestrationStack,