Merge pull request #104 from mansam/volume-snapshot-template-enable-p…

…rovisioning-from-volume-snapshots

Enable provisioning from Volumes and Volume Snapshots via a proxy type

app/models/manageiq/providers/openstack/cloud_manager/provision/cloning.rb

@@ -47,6 +47,25 @@ def log_clone_options(clone_options)
 
   def start_clone(clone_options)
     connection_options = {:tenant_name => options[:cloud_tenant][1]} if options[:cloud_tenant].kind_of?(Array)
+    if source.kind_of?(ManageIQ::Providers::Openstack::CloudManager::VolumeTemplate)
+      # remove the image_ref parameter from the options since it actually refers
+      # to a volume, and then overwrite the default root volume with the volume
+      # we are trying to boot the instance from
+      clone_options.delete(:image_ref)
+      clone_options[:block_device_mapping_v2][0][:source_type] = "volume"
+      clone_options[:block_device_mapping_v2][0].delete(:size)
+      clone_options[:block_device_mapping_v2][0][:delete_on_termination] = false
+      clone_options[:block_device_mapping_v2][0][:destination_type] = "volume"
+      # adjust the parameters to make booting from a volume work.
+    elsif source.kind_of?(ManageIQ::Providers::Openstack::CloudManager::VolumeSnapshotTemplate)
+      # remove the image_ref parameter from the options since it actually refers
+      # to a volume, and then overwrite the default root volume with the volume
+      # we are trying to boot the instance from
+      clone_options.delete(:image_ref)
+      clone_options[:block_device_mapping_v2][0][:source_type] = "snapshot"
+      clone_options[:block_device_mapping_v2][0].delete(:size)
+      clone_options[:block_device_mapping_v2][0][:destination_type] = "volume"
+    end
     source.with_provider_connection(connection_options) do |openstack|
       instance = openstack.servers.create(clone_options)
       return instance.id

app/models/manageiq/providers/openstack/cloud_manager/provision_workflow.rb

@@ -5,8 +5,14 @@ def allowed_instance_types(_options = {})
     source                  = load_ar_obj(get_source_vm)
     flavors                 = get_targets_for_ems(source, :cloud_filter, Flavor, 'flavors')
     return {} if flavors.blank?
-    minimum_disk_required   = [source.hardware.size_on_disk.to_i, source.hardware.disk_size_minimum.to_i].max
-    minimum_memory_required = source.hardware.memory_mb_minimum.to_i * 1.megabyte
+    if source.kind_of?(ManageIQ::Providers::Openstack::CloudManager::VolumeTemplate) || source.kind_of?(ManageIQ::Providers::Openstack::CloudManager::VolumeSnapshotTemplate)
+      # no flavor requirements for booting from a volume
+      minimum_disk_required = 0
+      minimum_memory_required = 0
+    else
+      minimum_disk_required   = [source.hardware.size_on_disk.to_i, source.hardware.disk_size_minimum.to_i].max
+      minimum_memory_required = source.hardware.memory_mb_minimum.to_i * 1.megabyte
+    end
     flavors.each_with_object({}) do |flavor, h|
       next if flavor.root_disk_size < minimum_disk_required
       next if flavor.memory         < minimum_memory_required

app/models/manageiq/providers/openstack/cloud_manager/volume_snapshot_template.rb

@@ -0,0 +1,12 @@
+class ManageIQ::Providers::Openstack::CloudManager::VolumeSnapshotTemplate < ManageIQ::Providers::CloudManager::Template
+  # VolumeSnapshotTemplates are proxies to allow provisioning instances from volumes
+  # without having to refactor the entire provisioning workflow to support types
+  # other than VmOrTemplate subtypes. VolumeSnapshotTemplates are created 1-to-1 during
+  # inventory refresh for each eligible snapshot.
+
+  belongs_to :cloud_tenant
+
+  def volume_snapshot_template?
+    true
+  end
+end

app/models/manageiq/providers/openstack/cloud_manager/volume_template.rb

@@ -0,0 +1,12 @@
+class ManageIQ::Providers::Openstack::CloudManager::VolumeTemplate < ManageIQ::Providers::CloudManager::Template
+  # VolumeTemplates are proxies to allow provisioning instances from volumes
+  # without having to refactor the entire provisioning workflow to support types
+  # other than VmOrTemplate subtypes. VolumeTemplates are created 1-to-1 during
+  # inventory refresh for each eligible bootable volume.
+
+  belongs_to :cloud_tenant
+
+  def volume_template?
+    true
+  end
+end

app/models/manageiq/providers/openstack/inventory/collector.rb

@@ -23,6 +23,8 @@ class ManageIQ::Providers::Openstack::Inventory::Collector < ManagerRefresh::Inv
   attr_reader :network_ports
   attr_reader :network_routers
   attr_reader :security_groups
+  attr_reader :volume_templates
+  attr_reader :volume_snapshot_templates
 
   def initialize(_manager, _target)
     super
@@ -32,24 +34,26 @@ def initialize(_manager, _target)
 
   def initialize_inventory_sources
     # cloud
-    @availability_zones   = []
-    @cloud_services       = []
-    @tenants              = []
-    @flavors              = []
-    @host_aggregates      = []
-    @key_pairs            = []
-    @images               = []
-    @orchestration_stacks = []
-    @quotas               = []
-    @vms                  = []
-    @vnfs                 = []
-    @vnfds                = []
+    @availability_zones        = []
+    @cloud_services            = []
+    @tenants                   = []
+    @flavors                   = []
+    @host_aggregates           = []
+    @key_pairs                 = []
+    @images                    = []
+    @orchestration_stacks      = []
+    @quotas                    = []
+    @vms                       = []
+    @vnfs                      = []
+    @vnfds                     = []
+    @volume_templates          = []
+    @volume_snapshot_templates = []
     # network
-    @cloud_networks       = []
-    @floating_ips         = []
-    @network_ports        = []
-    @network_routers      = []
-    @security_groups      = []
+    @cloud_networks            = []
+    @floating_ips              = []
+    @network_ports             = []
+    @network_routers           = []
+    @security_groups           = []
   end
 
   def connection

app/models/manageiq/providers/openstack/inventory/collector/cloud_manager.rb

@@ -82,7 +82,7 @@ def vms
   end
 
   def vms_by_id
-    @vms_by_id ||= Hash[vms.collect { |s| [s.id, s] }]
+    @vms_by_id ||= vms.index_by(&:id)
   end
 
   def tenants
@@ -144,4 +144,22 @@ def orchestration_resources(stack)
   def orchestration_template(stack)
     safe_call { stack.template }
   end
+
+  def volume_templates
+    return [] unless volume_service
+    return @volume_templates if @volume_templates.any?
+    @volume_templates = volume_service.handled_list(:volumes, {:status => "available"}, ::Settings.ems.ems_openstack.refresh.is_admin)
+  end
+
+  def volumes_by_id
+    @volumes_by_id ||= volume_templates.index_by(&:id)
+  end
+
+  def volume_snapshot_templates
+    return [] unless volume_service
+    return @volume_snapshot_templates if @volume_snapshot_templates.any?
+    @volume_snapshot_templates = volume_service.handled_list(:list_snapshots_detailed, :status => "available", :__request_body_index => "snapshots").select do |s|
+      volumes_by_id[s["volume_id"]] && (volumes_by_id[s["volume_id"]].attributes["bootable"].to_s == "true")
+    end
+  end
 end

app/models/manageiq/providers/openstack/inventory/parser/cloud_manager.rb

@@ -15,6 +15,32 @@ def parse
     vnfs
     vnfds
     host_aggregates
+    volume_templates
+    volume_snapshot_templates
+  end
+
+  def volume_templates
+    collector.volume_templates.each do |vt|
+      next if vt.attributes["bootable"].to_s != "true"
+      volume_template = persister.miq_templates.find_or_build(vt.id)
+      volume_template.type = "ManageIQ::Providers::Openstack::CloudManager::VolumeTemplate"
+      volume_template.name = vt.name
+      volume_template.cloud_tenant = persister.cloud_tenants.lazy_find(vt.tenant_id) if vt.tenant_id
+      volume_template.location = "N/A"
+      volume_template.vendor = "openstack"
+    end
+  end
+
+  def volume_snapshot_templates
+    collector.volume_snapshot_templates.each do |vt|
+      # next if vt["attributes"].["bootable"].to_s != "true"
+      volume_template = persister.miq_templates.find_or_build(vt["id"])
+      volume_template.type = "ManageIQ::Providers::Openstack::CloudManager::VolumeSnapshotTemplate"
+      volume_template.name = vt['display_name'] || vt['name']
+      volume_template.cloud_tenant = persister.cloud_tenants.lazy_find(vt["tenant_id"]) if vt["tenant_id"]
+      volume_template.location = "N/A"
+      volume_template.vendor = "openstack"
+    end
   end
 
   def availability_zones
@@ -144,6 +170,7 @@ def miq_templates
     collector.images.each do |i|
       parent_server_uid = parse_image_parent_id(i)
       image = persister.miq_templates.find_or_build(i.id)
+      image.type = "ManageIQ::Providers::Openstack::CloudManager::Template"
       image.uid_ems = i.id
       image.name = i.name || i.id.to_s
       image.vendor = "openstack"

app/models/manageiq/providers/openstack/inventory_collection_default/cloud_manager.rb

@@ -146,7 +146,6 @@ def key_pairs(extra_attributes = {})
 
     def miq_templates(extra_attributes = {})
       attributes = {
-        :model_class                 => ManageIQ::Providers::Openstack::CloudManager::Template,
         :inventory_object_attributes => [
           :type,
           :uid_ems,

spec/models/manageiq/providers/openstack/cloud_manager/refresh_spec_common.rb

@@ -76,8 +76,8 @@ def assert_with_errors
       expect(CloudSubnet.count).to                       eq network_data.subnets.count
       expect(NetworkRouter.count).to                     eq network_data.routers.count
       expect(CloudVolume.count).to                       eq volumes_count
-      expect(VmOrTemplate.count).to                      eq vms_count + images_count
-      expect(MiqTemplate.count).to                       eq images_count
+      expect(VmOrTemplate.count).to                      eq vms_count + images_count + volume_and_snapshot_templates_count
+      expect(MiqTemplate.count).to                       eq images_count + volume_and_snapshot_templates_count
       expect(Disk.count).to                              eq disks_count
       expect(Hardware.count).to                          eq vms_count + images_count
       expect(Vm.count).to                                eq vms_count
@@ -193,6 +193,11 @@ def stack_templates_count
       1
     end
 
+    def volume_and_snapshot_templates_count
+      # none of the snapshots on the VCR instances are bootable, so they are not included
+      CloudVolume.where(:status => "available", :bootable => true).count
+    end
+
     def all_vms_and_stacks
       all_vms = compute_data.servers + compute_data.servers_from_snapshot
       all_vms += orchestration_data.stacks if orchestration_supported?
@@ -243,9 +248,17 @@ def assert_table_counts
       expect(CloudSubnet.count).to                       eq network_data.subnets.count
       expect(NetworkRouter.count).to                     eq network_data.routers.count
       expect(CloudVolume.count).to                       eq volumes_count
-      expect(VmOrTemplate.count).to                      eq vms_count + images_count
+      if ::Settings.ems.ems_refresh.try(:openstack).try(:inventory_object_refresh)
+        expect(VmOrTemplate.count).to                    eq vms_count + images_count + volume_and_snapshot_templates_count
+      else
+        expect(VmOrTemplate.count).to                    eq vms_count + images_count
+      end
       expect(Vm.count).to                                eq vms_count
-      expect(MiqTemplate.count).to                       eq images_count
+      if ::Settings.ems.ems_refresh.try(:openstack).try(:inventory_object_refresh)
+        expect(MiqTemplate.count).to                     eq images_count + volume_and_snapshot_templates_count
+      else
+        expect(MiqTemplate.count).to                     eq images_count
+      end
       expect(Disk.count).to                              eq disks_count
       # One hardware per each VM
       expect(Hardware.count).to                          eq vms_count + images_count
@@ -290,16 +303,24 @@ def assert_ems
         :uid_ems     => identity_service == :v3 ? 'default' : nil
       )
 
-      expect(@ems.flavors.size).to            eq compute_data.flavors.count
-      expect(@ems.availability_zones.size).to eq availability_zones_count
-      expect(@ems.floating_ips.size).to       eq network_data.floating_ips.sum
-      expect(@ems.key_pairs.size).to          eq compute_data.key_pairs.count
+      expect(@ems.flavors.size).to             eq compute_data.flavors.count
+      expect(@ems.availability_zones.size).to  eq availability_zones_count
+      expect(@ems.floating_ips.size).to        eq network_data.floating_ips.sum
+      expect(@ems.key_pairs.size).to           eq compute_data.key_pairs.count
       security_groups_count = @ems.security_groups.count { |x| x.name != 'default' }
-      expect(security_groups_count).to        eq security_groups_count
-      expect(@ems.vms_and_templates.size).to  eq vms_count + images_count
-      expect(@ems.vms.size).to                eq vms_count
-      expect(@ems.miq_templates.size).to      eq images_count
-      expect(@ems.cloud_networks.size).to     eq network_data.networks.count
+      expect(security_groups_count).to         eq security_groups_count
+      if ::Settings.ems.ems_refresh.try(:openstack).try(:inventory_object_refresh)
+        expect(@ems.vms_and_templates.size).to eq vms_count + images_count + volume_and_snapshot_templates_count
+      else
+        expect(@ems.vms_and_templates.size).to eq vms_count + images_count
+      end
+      expect(@ems.vms.size).to                 eq vms_count
+      if ::Settings.ems.ems_refresh.try(:openstack).try(:inventory_object_refresh)
+        expect(@ems.miq_templates.size).to     eq images_count + volume_and_snapshot_templates_count
+      else
+        expect(@ems.miq_templates.size).to     eq images_count
+      end
+      expect(@ems.cloud_networks.size).to      eq network_data.networks.count
 
       if neutron_networking?
         expect(@ems.public_networks.first).to  be_kind_of(ManageIQ::Providers::Openstack::NetworkManager::CloudNetwork::Public)

spec/models/manageiq/providers/openstack/cloud_manager/stubbed_refresher_spec.rb

@@ -66,22 +66,24 @@ def refresh_spec
     def expected_table_counts(disconnect = nil)
       disconnect ||= disconnect_inv_factor
 
-      vm_count     = test_counts(@data_scaling)[:vms_count]
-      image_count  = test_counts(@data_scaling)[:miq_templates_count]
+      vm_count                    = test_counts(@data_scaling)[:vms_count]
+      image_count                 = test_counts(@data_scaling)[:miq_templates_count]
+      volumes_and_snapshots_count = test_counts(@data_scaling)[:volume_templates_count] + test_counts(@data_scaling)[:volume_snapshot_templates_count]
 
       # Disconnect_inv count, when these objects are not found in the API, they are not deleted in DB, but just marked
       # as disconnected
-      vm_count_plus_disconnect_inv    = vm_count + test_counts(disconnect)[:vms_count]
-      image_count_plus_disconnect_inv = image_count + test_counts(disconnect)[:miq_templates_count]
+      vm_count_plus_disconnect_inv              = vm_count + test_counts(disconnect)[:vms_count]
+      image_count_plus_disconnect_inv           = image_count + test_counts(disconnect)[:miq_templates_count]
+      volumes_and_snapshots_plus_disconnect_inv = volumes_and_snapshots_count + test_counts(disconnect)[:volume_templates_count] + test_counts(disconnect)[:volume_snapshot_templates_count]
       {
         :auth_private_key              => test_counts(@data_scaling)[:key_pairs_count],
         :ext_management_system         => 4,
         :flavor                        => test_counts(@data_scaling)[:flavors_count],
         :host_aggregate                => test_counts(@data_scaling)[:host_aggregates_count],
         :availability_zone             => 2,
-        :vm_or_template                => vm_count_plus_disconnect_inv + image_count_plus_disconnect_inv,
+        :vm_or_template                => vm_count_plus_disconnect_inv + image_count_plus_disconnect_inv + volumes_and_snapshots_plus_disconnect_inv,
         :vm                            => vm_count_plus_disconnect_inv,
-        :miq_template                  => image_count_plus_disconnect_inv,
+        :miq_template                  => image_count_plus_disconnect_inv + volumes_and_snapshots_plus_disconnect_inv,
         :disk                          => vm_count_plus_disconnect_inv,
         :hardware                      => vm_count_plus_disconnect_inv + image_count_plus_disconnect_inv,
         :network                       => vm_count_plus_disconnect_inv * 2,
@@ -132,6 +134,8 @@ def setup_mocked_collector
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:vnfds).and_return(mocked_vnfds)
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:tenants).and_return(mocked_cloud_tenants)
       allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:private_flavor).and_return(nil)
+      allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:volume_templates).and_return(mocked_volume_templates)
+      allow_any_instance_of(ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager).to receive(:volume_snapshot_templates).and_return(mocked_volume_snapshot_templates)
     end
 
     def assert_ems

spec/models/manageiq/providers/openstack/openstack_stubs.rb

@@ -11,17 +11,19 @@ def disconnect_inv_factor
   def test_counts(scaling = nil)
     scaling ||= scaling_factor
     {
-      :cloud_services_count       => scaling * 10,
-      :cloud_tenants_count        => scaling * 10,
-      :flavors_count              => scaling * 10,
-      :host_aggregates_count      => scaling * 10,
-      :key_pairs_count            => scaling * 5,
-      :quotas_count               => scaling * 10,
-      :miq_templates_count        => scaling * 10,
-      :orchestration_stacks_count => scaling * 10,
-      :vnfs_count                 => scaling * 10,
-      :vnfds_count                => scaling * 10,
-      :vms_count                  => scaling * 10
+      :cloud_services_count            => scaling * 10,
+      :cloud_tenants_count             => scaling * 10,
+      :flavors_count                   => scaling * 10,
+      :host_aggregates_count           => scaling * 10,
+      :key_pairs_count                 => scaling * 5,
+      :quotas_count                    => scaling * 10,
+      :miq_templates_count             => scaling * 10,
+      :orchestration_stacks_count      => scaling * 10,
+      :vnfs_count                      => scaling * 10,
+      :vnfds_count                     => scaling * 10,
+      :vms_count                       => scaling * 10,
+      :volume_templates_count          => scaling * 10,
+      :volume_snapshot_templates_count => scaling * 10,
     }
   end
 
@@ -236,4 +238,30 @@ def mocked_vms
     end
     mocked_vms
   end
+
+  def mocked_volume_templates
+    mocked_volume_templates = []
+    test_counts[:volume_templates_count].times do |i|
+      mocked_volume_templates << OpenStruct.new(
+        :id         => "volume_template_#{i}",
+        :name       => "volume_template_#{i}",
+        :status     => "available",
+        :attributes => {"bootable" => true}
+      )
+    end
+    mocked_volume_templates
+  end
+
+  def mocked_volume_snapshot_templates
+    mocked_volume_snapshot_templates = []
+    test_counts[:volume_snapshot_templates_count].times do |i|
+      mocked_volume_snapshot_templates << OpenStruct.new(
+        :id        => "volume_snapshot_template_#{i}",
+        :name      => "volume_snapshot_template_#{i}",
+        :status    => "available",
+        :volume_id => "volume_template_#{i}"
+      )
+    end
+    mocked_volume_snapshot_templates
+  end
 end