Skip to content

Commit

Permalink
job proxy dispatcher should use all container image classes
Browse files Browse the repository at this point in the history
  • Loading branch information
Erez Freiberger committed Jul 6, 2017
1 parent d0cf206 commit c8a16c6
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 17 deletions.
15 changes: 9 additions & 6 deletions app/models/job_proxy_dispatcher.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
class JobProxyDispatcher
include Vmdb::Logging

def self.dispatch
new.dispatch
end
Expand Down Expand Up @@ -93,6 +94,10 @@ def dispatch
_log.info "Complete - Timings: #{t.inspect}"
end

def container_image_classes
ContainerImage.descendants.append(ContainerImage)
end

def dispatch_container_scan_jobs
jobs_by_ems, = Benchmark.realtime_block(:pending_container_jobs) { pending_container_jobs }
Benchmark.current_realtime[:container_jobs_to_dispatch_count] = jobs_by_ems.values.reduce(0) { |m, o| m + o.length }
Expand Down Expand Up @@ -193,12 +198,11 @@ def self.waiting?
end

def pending_jobs(target_class = VmOrTemplate)
class_name = target_class.base_class.name
@zone = MiqServer.my_zone
Job.order(:id)
.where(:state => "waiting_to_start")
.where(:dispatch_status => "pending")
.where(:target_class => class_name)
.where(:target_class => target_class)
.where("zone is null or zone = ?", @zone)
.where("sync_key is NULL or
sync_key not in (
Expand All @@ -210,7 +214,7 @@ def pending_jobs(target_class = VmOrTemplate)
end

def pending_container_jobs
pending_jobs(ContainerImage).each_with_object(Hash.new { |h, k| h[k] = [] }) do |job, h|
pending_jobs(container_image_classes).each_with_object(Hash.new { |h, k| h[k] = [] }) do |job, h|
h[job.options[:ems_id]] << job
end
end
Expand Down Expand Up @@ -254,9 +258,8 @@ def busy_proxies
end

def active_scans_by_zone(target_class, count = true)
class_name = target_class.base_class.name
actives = Hash.new(0)
jobs = Job.where(:zone => @zone, :dispatch_status => "active", :target_class => class_name)
jobs = Job.where(:zone => @zone, :dispatch_status => "active", :target_class => target_class)
.where.not(:state => "finished")
actives[@zone] = count ? jobs.count : jobs
actives
Expand All @@ -271,7 +274,7 @@ def active_container_scans_by_zone_and_ems
memo = Hash.new do |h, k|
h[k] = Hash.new(0)
end
active_scans_by_zone(ContainerImage, false)[@zone].each do |job|
active_scans_by_zone(container_image_classes, false)[@zone].each do |job|
memo[@zone][job.options[:ems_id]] += 1
end
memo
Expand Down
17 changes: 9 additions & 8 deletions spec/models/job_proxy_dispatcher_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@
end

context "with container and vms jobs" do
let (:container_image_classes) { ContainerImage.descendants.collect(&:name).append('ContainerImage') }
before(:each) do
@jobs = (@vms + @repo_vms).collect(&:raw_scan)
User.current_user = FactoryGirl.create(:user)
Expand All @@ -191,10 +192,10 @@
end

it "returns only container images jobs when requested" do
jobs = dispatcher.pending_jobs(ContainerImage)
jobs = dispatcher.pending_jobs(dispatcher.container_image_classes)
expect(jobs.count).to eq(@container_images.count)
jobs.each do |x|
expect(x.target_class).to eq 'ContainerImage'
expect(container_image_classes).to include x.target_class
end
expect(jobs.count).to be > 0 # in case something unexpected goes wrong
end
Expand All @@ -212,7 +213,7 @@

describe "#active_container_scans_by_zone_and_ems" do
it "returns active container acans for zone" do
job = @jobs.find { |j| j.target_class == ContainerImage.name }
job = @jobs.find { |j| container_image_classes.include?(j.target_class) }
job.update(:dispatch_status => "active")
provider = ExtManagementSystem.find(job.options[:ems_id])
dispatcher.instance_variable_set(:@zone, MiqServer.my_zone) # memoized during pending_jobs call
Expand All @@ -226,29 +227,29 @@
it "dispatches jobs until reaching limit" do
stub_settings(:container_scanning => {:concurrent_per_ems => 1})
dispatcher.dispatch_container_scan_jobs
expect(Job.where(:target_class => ContainerImage, :dispatch_status => "pending").count).to eq(1)
expect(Job.where(:target_class => container_image_classes, :dispatch_status => "pending").count).to eq(1)
# 1 per ems, one ems has 1 job and the other 2
end

it "does not dispach if limit is already reached" do
stub_settings(:container_scanning => {:concurrent_per_ems => 1})
dispatcher.dispatch_container_scan_jobs
expect(Job.where(:target_class => ContainerImage, :dispatch_status => "pending").count).to eq(1)
expect(Job.where(:target_class => container_image_classes, :dispatch_status => "pending").count).to eq(1)
dispatcher.dispatch_container_scan_jobs
expect(Job.where(:target_class => ContainerImage, :dispatch_status => "pending").count).to eq(1)
expect(Job.where(:target_class => container_image_classes, :dispatch_status => "pending").count).to eq(1)
end

it "does not apply limit when concurrent_per_ems is 0" do
stub_settings(:container_scanning => {:concurrent_per_ems => 0})
dispatcher.dispatch_container_scan_jobs
expect(Job.where(:target_class => ContainerImage, :dispatch_status => "pending").count).to eq(0)
expect(Job.where(:target_class => container_image_classes, :dispatch_status => "pending").count).to eq(0)
# 1 per ems, one ems has 1 job and the other 2
end

it "does not apply limit when concurrent_per_ems is -1" do
stub_settings(:container_scanning => {:concurrent_per_ems => -1})
dispatcher.dispatch_container_scan_jobs
expect(Job.where(:target_class => ContainerImage, :dispatch_status => "pending").count).to eq(0)
expect(Job.where(:target_class => container_image_classes, :dispatch_status => "pending").count).to eq(0)
# 1 per ems, one ems has 1 job and the other 2
end
end
Expand Down
9 changes: 6 additions & 3 deletions spec/support/job_proxy_dispatcher_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,13 @@ def build_entities(options = {})

container_providers = []
options[:container_providers].each_with_index do |images_count, i|
ems_kubernetes = FactoryGirl.create(:ems_kubernetes, :name => "test_container_provider_#{i}")
container_providers << ems_kubernetes
ems_openshift = FactoryGirl.create(:ems_openshift, :name => "test_container_provider_#{i}")
container_providers << ems_openshift
images_count.times do |idx|
FactoryGirl.create(:container_image, :name => "test_container_images_#{idx}", :ems_id => ems_kubernetes.id)
FactoryGirl.create(:container_image,
:name => "test_container_images_#{idx}",
:ems_id => ems_openshift.id,
:type => 'ManageIQ::Providers::Openshift::ContainerManager::ContainerImage')
end
end
return hosts, proxies, storages, vms, repo_vms, container_providers
Expand Down

0 comments on commit c8a16c6

Please sign in to comment.