Merge pull request #86 from mansam/orchestration-stack-refresh

Orchestration Stack and Cloud Tenant targeted refresh

app/models/manageiq/providers/openstack/cloud_manager/event_target_parser.rb

@@ -22,6 +22,10 @@ def parse_ems_event_targets(ems_event)
     target_collection = ManagerRefresh::TargetCollection.new(:manager => ems_event.ext_management_system, :event => ems_event)
     if ems_event.event_type.start_with?("compute.instance")
       collect_compute_instance_references!(target_collection, ems_event)
+    elsif ems_event.event_type.start_with?("identity.project")
+      collect_identity_tenant_references!(target_collection, ems_event)
+    elsif ems_event.event_type.start_with?("orchestration.stack")
+      collect_orchestration_stack_references!(target_collection, ems_event)
     end
 
     target_collection.targets
@@ -39,4 +43,15 @@ def collect_compute_instance_references!(target_collection, ems_event)
     instance_id = ems_event.full_data.fetch_path(:payload, 'instance_id')
     add_target(target_collection, :vms, instance_id) if instance_id
   end
+
+  def collect_identity_tenant_references!(target_collection, ems_event)
+    tenant_id = ems_event.full_data.fetch_path(:payload, 'project_id') || ems_event.full_data.fetch_path(:payload, 'initiator', 'project_id')
+    add_target(target_collection, :cloud_tenants, tenant_id) if tenant_id
+  end
+
+  def collect_orchestration_stack_references!(target_collection, ems_event)
+    stack_id = ems_event.full_data.fetch_path(:payload, 'stack_id')
+    tenant_id = ems_event.full_data.fetch_path(:payload, 'tenant_id')
+    target_collection.add_target(:association => :orchestration_stacks, :manager_ref => {:ems_ref => stack_id}, :options => {:tenant_id => tenant_id})
+  end
 end

app/models/manageiq/providers/openstack/inventory/collector/target_collection.rb

@@ -15,6 +15,10 @@ def references(collection)
     target.manager_refs_by_association.try(:[], collection).try(:[], :ems_ref).try(:to_a) || []
   end
 
+  def orchestration_stack_references
+    @orchestration_stack_references ||= target.targets.select { |x| x.kind_of?(ManagerRefresh::Target) && x.association == :orchestration_stacks }
+  end
+
   def name_references(collection)
     target.manager_refs_by_association.try(:[], collection).try(:[], :name).try(:to_a) || []
   end
@@ -69,17 +73,22 @@ def floating_ips
   end
 
   def orchestration_stacks
-    return [] if references(:orchestration_stacks).blank?
+    return [] if orchestration_stack_references.blank?
     return @orchestration_stacks if @orchestration_stacks.any?
-    @orchestration_stacks = orchestration_service.handled_list(:stacks, :show_nested => true).collect(&:details).select do |s|
-      references(:orchestration_stacks).include?(s.id)
+    @orchestration_stacks = orchestration_stack_references.collect do |target|
+      get_orchestration_stack(target.manager_ref[:ems_ref], target.options[:tenant_id])
     end.compact
   rescue Excon::Errors::Forbidden
     # Orchestration service is detected but not open to the user
     $log.warn("Skip collecting stack references during targeted refresh because the user cannot access the orchestration service.")
     []
   end
 
+  def get_orchestration_stack(stack_id, tenant_id)
+    tenant = memoized_get_tenant(tenant_id)
+    safe_get { @os_handle.detect_orchestration_service(tenant.try(:name)).stacks.get(stack_id) }
+  end
+
   def vms
     return [] if references(:vms).blank?
     return @vms if @vms.any?
@@ -108,10 +117,18 @@ def tenants
     return [] if references(:cloud_tenants).blank?
     return @tenants if @tenants.any?
     @tenants = references(:cloud_tenants).collect do |cloud_tenant_id|
-      safe_get { identity_service.tenants.find_by_id(cloud_tenant_id) }
+      memoized_get_tenant(cloud_tenant_id)
     end.compact
   end
 
+  def memoized_get_tenant(tenant_id)
+    return nil if tenant_id.blank?
+    @tenant_memo ||= Hash.new do |h, key|
+      h[key] = safe_get { identity_service.tenants.find_by_id(key) }
+    end
+    @tenant_memo[tenant_id]
+  end
+
   def key_pairs
     return [] if references(:key_pairs).blank?
     return @key_pairs if @key_pairs.any?
@@ -180,19 +197,19 @@ def parse_targets!
     target.targets.each do |t|
       case t
       when Vm
-        parse_vm_target!(t)
+        add_simple_target!(:vms, t.ems_ref)
+      when CloudTenant
+        add_simple_target!(:cloud_tenants, t.ems_ref)
+      when OrchestrationStack
+        add_simple_target!(:orchestration_stacks, t.ems_ref)
       end
     end
   end
 
-  def parse_vm_target!(t)
-    add_simple_target!(:vms, t.ems_ref)
-  end
-
-  def add_simple_target!(association, ems_ref)
+  def add_simple_target!(association, ems_ref, options = {})
     return if ems_ref.blank?
 
-    target.add_target(:association => association, :manager_ref => {:ems_ref => ems_ref})
+    target.add_target(:association => association, :manager_ref => {:ems_ref => ems_ref}, :options => options)
   end
 
   def infer_related_ems_refs!
@@ -203,6 +220,42 @@ def infer_related_ems_refs!
       infer_related_vm_ems_refs_db!
       infer_related_vm_ems_refs_api!
     end
+    unless references(:cloud_tenants).blank?
+      infer_related_cloud_tenant_ems_refs_db!
+      infer_related_cloud_tenant_ems_refs_api!
+    end
+    unless references(:orchestration_stacks).blank?
+      infer_related_orchestration_stacks_ems_refs_db!
+      infer_related_orchestration_stacks_ems_refs_api!
+    end
+  end
+
+  def infer_related_orchestration_stacks_ems_refs_db!
+    changed_stacks = manager.orchestration_stacks.where(:ems_ref => references(:orchestration_stacks))
+    changed_stacks.each do |stack|
+      add_simple_target!(:cloud_tenants, stack.cloud_tenant.ems_ref) unless stack.cloud_tenant.nil?
+      add_simple_target!(:orchestration_stacks, stack.parent.ems_ref, :tenant_id => stack.parent.cloud_tenant.ems_ref) unless stack.parent.nil?
+    end
+  end
+
+  def infer_related_orchestration_stacks_ems_refs_api!
+    orchestration_stacks.each do |stack|
+      add_simple_target!(:orchestration_stacks, stack.parent, :tenant_id => stack.service.current_tenant["id"]) unless stack.parent.blank?
+      add_simple_target!(:cloud_tenants, stack.service.current_tenant["id"]) unless stack.service.current_tenant["id"].blank?
+    end
+  end
+
+  def infer_related_cloud_tenant_ems_refs_db!
+    changed_tenants = manager.cloud_tenants.where(:ems_ref => references(:cloud_tenants))
+    changed_tenants.each do |tenant|
+      add_simple_target!(:cloud_tenants, tenant.parent.ems_ref) unless tenant.parent.nil?
+    end
+  end
+
+  def infer_related_cloud_tenant_ems_refs_api!
+    tenants.each do |tenant|
+      add_simple_target(:cloud_tenants, tenant.try(:parent_id)) unless tenant.try(:parent_id).blank?
+    end
   end
 
   def infer_related_vm_ems_refs_db!
@@ -212,7 +265,7 @@ def infer_related_vm_ems_refs_db!
       stack      = vm.orchestration_stack
       all_stacks = ([stack] + (stack.try(:ancestors) || [])).compact
 
-      all_stacks.collect(&:ems_ref).compact.each { |ems_ref| add_simple_target!(:orchestration_stacks, ems_ref) }
+      all_stacks.each { |s| add_simple_target!(:orchestration_stacks, s.ems_ref, :tenant_id => s.cloud_tenant.id) }
       vm.cloud_networks.collect(&:ems_ref).compact.each { |ems_ref| add_simple_target!(:cloud_networks, ems_ref) }
       vm.floating_ips.collect(&:address).compact.each { |address| add_simple_target!(:floating_ips, address) }
       vm.network_ports.collect(&:ems_ref).compact.each do |ems_ref|

app/models/manageiq/providers/openstack/inventory/persister/target_collection.rb

@@ -4,7 +4,8 @@ def initialize_inventory_collections
     # Top level models with direct references for Cloud
     add_inventory_collections_with_references(
       cloud,
-      %i(vms miq_templates availability_zones orchestration_stacks cloud_tenants flavors)
+      %i(vms miq_templates availability_zones orchestration_stacks cloud_tenants flavors),
+      :builder_params => {:ext_management_system => manager}
     )
 
     add_inventory_collection_with_references(

lib/vcr_recorder.rb

@@ -54,6 +54,12 @@ def load_credentials
 
       file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_vm_targeted_refresh.yml")
       change_file(file_name, OBFUSCATED_PASSWORD, env["password"], OBFUSCATED_IP, env["ip"])
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_stack_targeted_refresh.yml")
+      change_file(file_name, OBFUSCATED_PASSWORD, env["password"], OBFUSCATED_IP, env["ip"])
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_tenant_targeted_refresh.yml")
+      change_file(file_name, OBFUSCATED_PASSWORD, env["password"], OBFUSCATED_IP, env["ip"])
     end
   end
 
@@ -81,6 +87,12 @@ def obfuscate_credentials
 
       file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_vm_targeted_refresh.yml")
       change_file(file_name, env["password"], OBFUSCATED_PASSWORD, env["ip"], OBFUSCATED_IP)
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_stack_targeted_refresh.yml")
+      change_file(file_name, env["password"], OBFUSCATED_PASSWORD, env["ip"], OBFUSCATED_IP)
+
+      file_name = File.join(vcr_base_dir, "refresher_rhos_#{env_name}_tenant_targeted_refresh.yml")
+      change_file(file_name, env["password"], OBFUSCATED_PASSWORD, env["ip"], OBFUSCATED_IP)
     end
   end
 

spec/models/manageiq/providers/openstack/cloud_manager/refresh_spec_common.rb

@@ -394,6 +394,23 @@ def assert_specific_tenant
       end
     end
 
+    def assert_targeted_tenant
+      assert_objects_with_hashes([CloudTenant.find_by(:name => "EmsRefreshSpec-Project")], [identity_data.projects[1]])
+
+      identity_data.projects.each do |project|
+        next unless project[:__parent_name]
+
+        parent_id = CloudTenant.find_by(:name => project[:__parent_name]).try(:id)
+        cloud_tenant = CloudTenant.find_by(:name => project[:name])
+        expect(cloud_tenant.parent_id).to eq(parent_id)
+      end
+
+      CloudTenant.all.each do |tenant|
+        expect(tenant).to be_kind_of(CloudTenant)
+        expect(tenant.ext_management_system).to eq(@ems)
+      end
+    end
+
     def assert_key_pairs
       assert_objects_with_hashes(ManageIQ::Providers::Openstack::CloudManager::AuthKeyPair.all, compute_data.key_pairs)
     end
@@ -679,6 +696,15 @@ def assert_specific_stacks
                                  [:template, :parameters])
     end
 
+    def assert_targeted_stack
+      stack_target = OrchestrationStack.find_by(:name => "stack1")
+      assert_objects_with_hashes([stack_target],
+                                 [orchestration_data.stacks[0]],
+                                 orchestration_data.stack_translate_table,
+                                 {},
+                                 [:template, :parameters])
+    end
+
     def assert_specific_vms
       all_vms = ManageIQ::Providers::Openstack::CloudManager::Vm.all
       if orchestration_supported?

spec/models/manageiq/providers/openstack/cloud_manager/refresher_rhos_liberty_spec.rb

@@ -22,7 +22,6 @@
   end
 
   context "when configured with skips" do
-
     it "will not parse the ignored items" do
       with_cassette(@environment, @ems) do
         EmsRefresh.refresh(@ems)
@@ -80,5 +79,33 @@
         end
       end
     end
+
+    it "will perform a targeted stack refresh against RHOS #{@environment}" do
+      # stack1
+      stack_target = ManagerRefresh::Target.new(:manager     => @ems,
+                                                :association => :orchestration_stacks,
+                                                :manager_ref => {:ems_ref => "091e1e54-e01c-4ec5-a0ab-b00bee4d425c"},
+                                                :options     => {:tenant_id => "69f8f7205ade4aa59084c32c83e60b5a"})
+      2.times do # Run twice to verify that a second run with existing data does not change anything
+        with_cassette("#{@environment}_stack_targeted_refresh", @ems) do
+          EmsRefresh.refresh(stack_target)
+          assert_targeted_stack
+        end
+      end
+    end
+
+    it "will perform a targeted tenant refresh against RHOS #{@environment}" do
+      # EmsRefreshSpec-Project
+      stack_target = ManagerRefresh::Target.new(:manager     => @ems,
+                                                :association => :cloud_tenants,
+                                                :manager_ref => {:ems_ref => "69f8f7205ade4aa59084c32c83e60b5a"})
+      2.times do # Run twice to verify that a second run with existing data does not change anything
+        with_cassette("#{@environment}_tenant_targeted_refresh", @ems) do
+          EmsRefresh.refresh(stack_target)
+          expect(CloudTenant.count).to eq(1)
+          assert_targeted_tenant
+        end
+      end
+    end
   end
 end