Rescue worker class sync_workers exceptions and move on

Related to #13958

In the above issue, if ManageIQ::Providers::StorageManager::CinderManager::EventCatcher.sync_workers
raises an exception, the server process exits fatally and
all workers exit `Error heartbeating to MiqServer because DRb::DRbConnError: Connection reset by peer Worker exiting.`

We now rescue any exceptions here, log it and move on to other worker
classes.

app/models/miq_server/worker_management/monitor.rb

@@ -48,9 +48,15 @@ def worker_not_responding(w)
   def sync_workers
     result = {}
     self.class.monitor_class_names.each do |class_name|
-      c = class_name.constantize
-      result[c.name] = c.sync_workers
-      result[c.name][:adds].each { |pid| worker_add(pid) unless pid.nil? }
+      begin
+        c = class_name.constantize
+        result[c.name] = c.sync_workers
+        result[c.name][:adds].each { |pid| worker_add(pid) unless pid.nil? }
+      rescue => error
+        _log.error("Failed to sync_workers for class: #{class_name}")
+        _log.log_backtrace(error)
+        next
+      end
     end
     result
   end

spec/models/miq_server/worker_management/monitor_spec.rb

@@ -30,5 +30,15 @@
       server.reload
       expect(server.miq_workers.first.id).to eq(worker.id)
     end
+
+    context "#sync_workers" do
+      let(:server) { EvmSpecHelper.local_miq_server }
+      it "rescues exceptions and moves on" do
+        allow(MiqServer).to receive(:monitor_class_names).and_return(%w(MiqGenericWorker MiqPriorityWorker))
+        allow(MiqGenericWorker).to receive(:sync_workers).and_raise
+        expect(MiqPriorityWorker).to receive(:sync_workers).and_return(:adds => [123])
+        expect(server.sync_workers).to eq("MiqPriorityWorker"=>{:adds=>[123]})
+      end
+    end
   end
 end