Merge branch 'master' into add_factory_ems_ph_with_invalid_credentials

.rubocop_local.yml

@@ -1,52 +1,33 @@
+AllCops:
+  Exclude:
+  - lib/generators/provider/templates/**/*
+  - db/schema.rb
 GlobalVars:
   AllowedVariables:
   # Loggers
-  - $audit_log
   - $api_log
+  - $audit_log
   - $aws_log
   - $azure_log
+  - $cn_monitoring_log
   - $container_log
   - $datawarehouse_log
   - $fog_log
+  - $kube_log
   - $lenovo_log
   - $log
   - $miq_ae_logger
+  - $mw_log
   - $policy_log
   - $rails_log
   - $rhevm_log
-  - $kube_log
-  - $mw_log
-  - $cn_monitoring_log
   - $scvmm_log
+  - $vcloud_log
   - $vim_log
   - $websocket_log
-  - $vcloud_log
-  # In Automate methods
-  - $evm
-#
-# Special Exclusions
-#
-AllCops:
-  TargetRubyVersion: 2.2
-  Exclude:
-  - lib/generators/provider/templates/**/*
-  - db/schema.rb
-  - vendor/**/*
-ClassAndModuleCamelCase:
-  Exclude:
-  - lib/miq_automation_engine/service_models/*.rb
-FileName:
-  Exclude:
-  - lib/miq_automation_engine/service_models/*.rb
-Metrics/LineLength:
-  Exclude:
-  - Gemfile
 Rails/Exit:
   Exclude:
   - lib/workers/bin/*
 Rails/Output:
   Exclude:
   - lib/workers/bin/*
-Style/ExtraSpacing:
-  Exclude:
-  - Gemfile

CHANGELOG.md

@@ -5,6 +5,61 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
 
+## Unreleased as of Sprint 75 ending 2017-12-11
+
+### Added
+- Authentication
+  - Handle OpenID-Connect authentication client protocol [(#16495)](https://github.com/ManageIQ/manageiq/pull/16495)
+- Control
+  - Default name to description for MiqAlertSet and MiqPolicySet [(#16529)](https://github.com/ManageIQ/manageiq/pull/16529)
+- Core
+  - Binary blob purging [(#16524)](https://github.com/ManageIQ/manageiq/pull/16524)
+  - GenericObjectDefinition: add pretty & translatable type names [(#16563)](https://github.com/ManageIQ/manageiq/pull/16563)
+- Providers
+  - Allow unassignment of specific tags/objects in assignment_mixin [(#16401)](https://github.com/ManageIQ/manageiq/pull/16401)
+  - Call super from all the EmbeddedAnsible subclasses' #initialize methods [(#16543)](https://github.com/ManageIQ/manageiq/pull/16543)
+  - Add middleware server to available fields in advanced search [(#16562)](https://github.com/ManageIQ/manageiq/pull/16562)
+- Provisioning
+  - Change provision daialog "vLan" to "Virtual Network" [(#16537)](https://github.com/ManageIQ/manageiq/pull/16537)
+- Services
+  - Parse default value in Service Dialog [(#16555)](https://github.com/ManageIQ/manageiq/pull/16555)
+
+### Fixed
+- Chargeback
+  - Fix max method on empty array in chargeback storage report [(#16575)](https://github.com/ManageIQ/manageiq/pull/16575)
+- Core
+  - Replace conditions with scope [(#16399)](https://github.com/ManageIQ/manageiq/pull/16399)
+  - Fixes virtual_totals when a default_scope exists [(#16518)](https://github.com/ManageIQ/manageiq/pull/16518)
+  - Add supported_consoles to VM [(#16519)](https://github.com/ManageIQ/manageiq/pull/16519)
+  - Looks like self is needed on PRODUCT_NAME and VERSION calls [(#16556)](https://github.com/ManageIQ/manageiq/pull/16556)
+  - Handle deprecated classes in arel [(#16560)](https://github.com/ManageIQ/manageiq/pull/16560)
+  - Skip seeding of categories if their creation is invalid [(#16568)](https://github.com/ManageIQ/manageiq/pull/16568)
+  - Don't return a duplicate object from MiqGroup#settings [(#16572)](https://github.com/ManageIQ/manageiq/pull/16572)
+  - Add an ENV variable to override the PG version check in production [(#16573)](https://github.com/ManageIQ/manageiq/pull/16573)
+  - Fix cached query for total unregistered vms [(#16577)](https://github.com/ManageIQ/manageiq/pull/16577)
+  - No need to call normalized_status [(#16578)](https://github.com/ManageIQ/manageiq/pull/16578)
+  - Cache invalidation... [(#16601)](https://github.com/ManageIQ/manageiq/pull/16601)
+  - Handle autoload error not caught by safe_constantize [(#16608)](https://github.com/ManageIQ/manageiq/pull/16608)
+  - Use the server IP for the AWX database host when the rails config is no good [(#16621)](https://github.com/ManageIQ/manageiq/pull/16621)
+  - Fix error in timeout checking for job without target [(#16627)](https://github.com/ManageIQ/manageiq/pull/16627)
+- i18n
+  - Dont store unnecessary i18n files in git [(#16590)](https://github.com/ManageIQ/manageiq/pull/16590)
+  - Add missing model names into locale/en.yml [(#16604)](https://github.com/ManageIQ/manageiq/pull/16604)
+- Providers
+  - Allow proxying WebMKS consoles using the WebsocketWorker [(#16490)](https://github.com/ManageIQ/manageiq/pull/16490)
+  - Set CloudVolume name to ID if empty [(#16554)](https://github.com/ManageIQ/manageiq/pull/16554)
+  - Catalog Item type list is dependent on installed providers [(#16559)](https://github.com/ManageIQ/manageiq/pull/16559)
+  - Destroy ems alert statuses when ems is destroyed [(#16592)](https://github.com/ManageIQ/manageiq/pull/16592)
+  - Add the missing container template openshift to the en.yml [(#16595)](https://github.com/ManageIQ/manageiq/pull/16595)
+  - Allowing OperatingSystem for CloudManager graph refresh [(#16605)](https://github.com/ManageIQ/manageiq/pull/16605)
+- Services
+  - Change the type name for the catalog Item from 'Container Template' to 'Openshift Template' [(#16639)](https://github.com/ManageIQ/manageiq/pull/16639)
+- Smart State
+  - Use Hostname or IP Address when connecting for SSA [(#16533)](https://github.com/ManageIQ/manageiq/pull/16533)
+- UI
+  - Update Catalog tab of Publish VM dialog [(#16599)](https://github.com/ManageIQ/manageiq/pull/16599)
+  - Set default values for help menu link target types in settings.yml [(#16549)](https://github.com/ManageIQ/manageiq/pull/16549)
+
 ## Unreleased as of Sprint 74 ending 2017-11-27
 
 ### Added
@@ -107,6 +162,72 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 - Providers
   - Revert "Replace ContainerLabelTagMapping.controls_tag? with Tag.controlled_by_mapping scope" [(#16462)](https://github.com/ManageIQ/manageiq/pull/16462)
 
+## Gaprindashvili Beta2
+
+### Added
+- Alerts
+  - Seed MiqAlerts used for Prometheus Alerts [(#16479)](https://github.com/ManageIQ/manageiq/pull/16479)
+- Ansible
+  - Add log_output option for embedded ansible service [(#16414)](https://github.com/ManageIQ/manageiq/pull/16414)
+- Automate
+  - Add #raw_stdout_via_worker method [(#16441)](https://github.com/ManageIQ/manageiq/pull/16441)
+  - Imports old associations [(#16471)](https://github.com/ManageIQ/manageiq/pull/16471)
+- Core
+  - Remove the column reordering tool and the schema structure validations [(#16488)](https://github.com/ManageIQ/manageiq/pull/16488)
+  - Added Product name to VMDB::Appliance [(#16409)](https://github.com/ManageIQ/manageiq/pull/16409)
+  - Added User Agent to VMDB::Appliance [(#16410)](https://github.com/ManageIQ/manageiq/pull/16410)
+- Providers
+  - Add tag categories for VM migration [(#16402)](https://github.com/ManageIQ/manageiq/pull/16402)
+  - Add requests and limits to Persistent Volume Claim [(#16026)](https://github.com/ManageIQ/manageiq/pull/16026)
+  - Add ems_infra_admin_ui feature [(#16403)](https://github.com/ManageIQ/manageiq/pull/16403)
+  - Update ems_infra_admin_ui feature to role assignment [(#16484)](https://github.com/ManageIQ/manageiq/pull/16484)
+  - Move graph refresh internals logging to debug [(#16442)](https://github.com/ManageIQ/manageiq/pull/16442)
+- Reporting
+  - Add scope :without_configuration_profile_id needed by Foreman explorer UI [(#16439)](https://github.com/ManageIQ/manageiq/pull/16439)
+- Services
+  - Name service during provisioning from dialog input [(#16338)](https://github.com/ManageIQ/manageiq/pull/16338)
+
+### Fixed
+- Automate
+  - Adds field unique validator check to dialog [(#16487)](https://github.com/ManageIQ/manageiq/pull/16487)
+- Authentication
+  - Fixes a stack trace issue (500) during API authentication. [(#16520)](https://github.com/ManageIQ/manageiq/pull/16520)
+- Core
+  - Add purging for vim_performance_tag_values with disabled tags [(#16425)](https://github.com/ManageIQ/manageiq/pull/16425)
+  - Add a connection to the pool if there is only one for embedded ansible [(#16477)](https://github.com/ManageIQ/manageiq/pull/16477)
+  - Fix Expression builder argument error by reverting #5506 [(#16255)](https://github.com/ManageIQ/manageiq/pull/16255)
+  - Move the notifications to include more of the setup [(#16508)](https://github.com/ManageIQ/manageiq/pull/16508)
+  - Finish simplifying NTP configuration using Settings [(#16393)](https://github.com/ManageIQ/manageiq/pull/16393)
+  - Fix issue where plugin settings had higher precedence than manageiq [(#16535)](https://github.com/ManageIQ/manageiq/pull/16535)
+- i18n
+  - Fix string interpolations [(#16468)](https://github.com/ManageIQ/manageiq/pull/16468)
+  - Add Middleware Server EAP/Wildfly translation [(#16492)](https://github.com/ManageIQ/manageiq/pull/16492)
+- Providers
+  - Fix has_required_role? for InventoryCollectorWorker [(#16415)](https://github.com/ManageIQ/manageiq/pull/16415)
+  - Missing cascade delete for host_storages [(#16440)](https://github.com/ManageIQ/manageiq/pull/16440)
+  - Truncate name of refresh task to 255 [(#16444)](https://github.com/ManageIQ/manageiq/pull/16444)
+  - Datastores duplicated after a refresh [(#16408)](https://github.com/ManageIQ/manageiq/pull/16408)
+  - Added a Maintenance key to the hash struct [(#16464)](https://github.com/ManageIQ/manageiq/pull/16464)
+  - Refresh new target do not run post_refresh [(#16436)](https://github.com/ManageIQ/manageiq/pull/16436)
+  - Add two new types to MW server factories [(#16478)](https://github.com/ManageIQ/manageiq/pull/16478)
+  - Container ssa annotate success [(#15031)](https://github.com/ManageIQ/manageiq/pull/15031)
+  - Require handsoap in VimConnectMixin [(#16450)](https://github.com/ManageIQ/manageiq/pull/16450)
+  - Require miq_fault_tolerant_vim in raw_connect [(#16500)](https://github.com/ManageIQ/manageiq/pull/16500)
+  - Unique EmsRefresh.refresh targets if there are over 1,000 targets [(#16432)](https://github.com/ManageIQ/manageiq/pull/16432)
+- RBAC
+  - Standalone ServiceUI product features require, updating affected roles [(#16329)](https://github.com/ManageIQ/manageiq/pull/16329)
+  - Fix nil cases of allocated disk types in chargeback reporting [(#16434)](https://github.com/ManageIQ/manageiq/pull/16434)
+  - Add Missing button features to miq_features.yaml [(#16027)](https://github.com/ManageIQ/manageiq/pull/16027)
+- Replication
+  - Prevent replication subscription to the same region as the current region [(#16446)](https://github.com/ManageIQ/manageiq/pull/16446)
+  - Scope Tenant#name validation to the current region [(#16506)](https://github.com/ManageIQ/manageiq/pull/16506)
+- Reporting
+  - Fixed syntax with `orderby` in RSS Feed YAML files [(#16493)](https://github.com/ManageIQ/manageiq/pull/16493)
+- Services
+  - ServiceTemplate: add scope for unassigned items. [(#16445)](https://github.com/ManageIQ/manageiq/pull/16445)
+- UI
+  - Fix resize approval to work for editing requests [(#16381)](https://github.com/ManageIQ/manageiq/pull/16381)
+
 ## Unreleased as of Sprint 72 ending 2017-10-30
 
 ### Added

Gemfile

@@ -47,7 +47,7 @@ gem "manageiq-messaging",                              :require => false, :git =
 gem "manageiq-network_discovery",     "~>0.1.2",       :require => false
 gem "memoist",                        "~>0.15.0",      :require => false
 gem "mime-types",                     "~>2.6.1",       :path => File.expand_path("mime-types-redirector", __dir__)
-gem "more_core_extensions",           "~>3.3"
+gem "more_core_extensions",           "~>3.5"
 gem "nakayoshi_fork",                 "~>0.0.3"  # provides a more CoW friendly fork (GC a few times before fork)
 gem "net-ldap",                       "~>0.14.0",      :require => false
 gem "net-ping",                       "~>1.7.4",       :require => false
@@ -214,7 +214,7 @@ unless ENV["APPLIANCE"]
   group :development do
     gem "foreman"
     gem "haml_lint",        "~>0.20.0", :require => false
-    gem "rubocop",          "~>0.49.0", :require => false
+    gem "rubocop",          "~>0.52.1", :require => false
     # ruby_parser is required for i18n string extraction
     gem "ruby_parser",                  :require => false
     gem "scss_lint",        "~>0.48.0", :require => false

app/models/dialog.rb

@@ -124,6 +124,8 @@ def content(target = nil, resource_action = nil, all_attributes = false)
   # Creates a new item without an ID,
   # Removes any items not passed in the content.
   def update_tabs(tabs)
+    association_list = dialog_import_service.build_association_list("dialog_tabs" => tabs)
+
     transaction do
       updated_tabs = []
       tabs.each do |dialog_tab|
@@ -134,11 +136,15 @@ def update_tabs(tabs)
             updated_tabs << tab
           end
         else
-          updated_tabs << DialogImportService.new.build_dialog_tabs('dialog_tabs' => [dialog_tab]).first
+          updated_tabs << dialog_import_service.build_dialog_tabs('dialog_tabs' => [dialog_tab]).first
         end
       end
       self.dialog_tabs = updated_tabs
     end
+
+    transaction do
+      dialog_import_service.build_associations(self, association_list)
+    end
   end
 
   def deep_copy(new_attributes = {})
@@ -162,4 +168,8 @@ def reject_if_has_resource_actions
       raise _("Dialog cannot be deleted because it is connected to other components.")
     end
   end
+
+  def dialog_import_service
+    @dialog_import_service ||= DialogImportService.new
+  end
 end

app/models/dialog_group.rb

@@ -23,7 +23,6 @@ def update_dialog_fields(fields)
           resource_action_fields = field.delete('resource_action') || {}
           update_resource_fields(resource_action_fields, dialog_field)
           dialog_field.update_attributes(field.except('id', 'href', 'dialog_group_id', 'dialog_field_responders'))
-          dialog_field.update_dialog_field_responders(field['dialog_field_responders'])
           updated_fields << dialog_field
         end
       else

app/models/ext_management_system.rb

@@ -30,7 +30,6 @@ def self.supported_types_and_descriptions_hash
   belongs_to :provider
   has_many :child_managers, :class_name => 'ExtManagementSystem', :foreign_key => 'parent_ems_id'
 
-  include CustomAttributeMixin
   belongs_to :tenant
   has_many :container_deployments, :foreign_key => :deployed_on_ems_id, :inverse_of => :deployed_on_ems
   has_many :endpoints, :as => :resource, :dependent => :destroy, :autosave => true
@@ -76,7 +75,7 @@ def self.supported_types_and_descriptions_hash
 
   validates :name,     :presence => true, :uniqueness => {:scope => [:tenant_id]}
   validates :hostname, :presence => true, :if => :hostname_required?
-  validate :hostname_uniqueness_valid?, :if => :hostname_required?
+  validate :hostname_uniqueness_valid?, :hostname_format_valid?, :if => :hostname_required?
 
   scope :with_eligible_manager_types, ->(eligible_types) { where(:type => eligible_types) }
 
@@ -92,6 +91,11 @@ def hostname_uniqueness_valid?
     errors.add(:hostname, N_("has to be unique per provider type")) if existing_hostnames.include?(hostname.downcase)
   end
 
+  def hostname_format_valid?
+    return if hostname.ipaddress? || hostname.hostname?
+    errors.add(:hostname, _("format is invalid."))
+  end
+
   include NewWithTypeStiMixin
   include UuidMixin
   include EmsRefresh::Manager

app/models/manageiq/providers/cloud_manager/provision_workflow.rb

@@ -22,7 +22,9 @@ def allowed_cloud_subnets(_options = {})
 
     az_id = src[:availability_zone_id].to_i
     if (cn = CloudNetwork.find_by(:id => src[:cloud_network_id]))
-      cn.cloud_subnets.each_with_object({}) do |cs, hash|
+
+      targets = get_targets_for_source(cn, :cloud_filter, CloudNetwork, 'cloud_subnets')
+      targets.each_with_object({}) do |cs, hash|
         next if !az_id.zero? && az_id != cs.availability_zone_id
         hash[cs.id] = "#{cs.name} (#{cs.cidr}) | #{cs.availability_zone.try(:name)}"
       end
@@ -33,10 +35,8 @@ def allowed_cloud_subnets(_options = {})
 
   def allowed_guest_access_key_pairs(_options = {})
     source = load_ar_obj(get_source_vm)
-    ems = source.try(:ext_management_system)
-
-    return {} if ems.nil?
-    ems.key_pairs.each_with_object({}) { |kp, h| h[kp.id] = kp.name }
+    targets = get_targets_for_ems(source, :cloud_filter, ManageIQ::Providers, 'key_pairs')
+    targets.each_with_object({}) { |kp, h| h[kp.id] = kp.name }
   end
 
   def allowed_security_groups(_options = {})
@@ -51,8 +51,8 @@ def allowed_security_groups(_options = {})
 
   def allowed_floating_ip_addresses(_options = {})
     return {} unless (src_obj = provider_or_tenant_object)
-
-    src_obj.floating_ips.available.each_with_object({}) do |ip, h|
+    targets = get_targets_for_source(src_obj, :cloud_filter, FloatingIp, 'floating_ips.available')
+    targets.each_with_object({}) do |ip, h|
       h[ip.id] = ip.address
     end
   end

app/models/manager_refresh/inventory_collection_default/infra_manager.rb

@@ -281,11 +281,66 @@ def hosts(extra_attributes = {})
         :builder_params              => {
           :ems_id => ->(persister) { persister.manager.id },
         },
+        :custom_reconnect_block      => lambda do |inventory_collection, inventory_objects_index, attributes_index|
+          relation = inventory_collection.model_class.where(:ems_id => nil)
+
+          return if relation.count <= 0
+
+          inventory_objects_index.each_slice(100) do |batch|
+            relation.where(inventory_collection.manager_ref.first => batch.map(&:first)).each do |record|
+              index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record)
+
+              # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it
+              # would be sent for create.
+              inventory_object = inventory_objects_index.delete(index)
+              hash             = attributes_index.delete(index)
+
+              record.assign_attributes(hash.except(:id, :type))
+              if !inventory_collection.check_changed? || record.changed?
+                record.save!
+                inventory_collection.store_updated_records(record)
+              end
+
+              inventory_object.id = record.id
+            end
+          end
+        end
       }
 
       attributes.merge!(extra_attributes)
     end
 
+    def vms(extra_attributes = {})
+      attributes = {
+        :custom_reconnect_block => lambda do |inventory_collection, inventory_objects_index, attributes_index|
+          relation = inventory_collection.model_class.where(:ems_id => nil)
+
+          return if relation.count <= 0
+
+          inventory_objects_index.each_slice(100) do |batch|
+            relation.where(inventory_collection.manager_ref.first => batch.map(&:first)).each do |record|
+              index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record)
+
+              # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it
+              # would be sent for create.
+              inventory_object = inventory_objects_index.delete(index)
+              hash             = attributes_index.delete(index)
+
+              record.assign_attributes(hash.except(:id, :type))
+              if !inventory_collection.check_changed? || record.changed?
+                record.save!
+                inventory_collection.store_updated_records(record)
+              end
+
+              inventory_object.id = record.id
+            end
+          end
+        end
+      }
+
+      super(attributes.merge!(extra_attributes))
+    end
+
     def host_storages(extra_attributes = {})
       attributes = {
         :model_class                 => ::HostStorage,