Skip to content

Commit

Permalink
Let ansible worker gracefully stop
Browse files Browse the repository at this point in the history
https://bugzilla.redhat.com/show_bug.cgi?id=1474508

When the ansible worker is not responding, let it try to gracefuly exit
by marking it as stopping.  Allow, the stopping worker code to eventually
kill the worker if it doesn't exit on it's own.

While we're allowing the worker to gracefully exit, don't allow multiple
ansible monitor workers to run at once. Normally, a replacement worker
is started the moment we ask a prior worker to exit.
  • Loading branch information
jrafanie committed Jul 24, 2017
1 parent 71d244b commit bc8c26f
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 24 deletions.
6 changes: 3 additions & 3 deletions app/models/embedded_ansible_worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ class EmbeddedAnsibleWorker < MiqWorker
require_nested :Runner
include_concern 'ObjectManagement'

# Don't allow multiple ansible monitor workers to run at once
self.include_stopping_workers_on_synchronize = true

self.required_roles = ['embedded_ansible']

def start_runner
Expand Down Expand Up @@ -48,9 +51,6 @@ def find_worker_thread_object
end
end

alias terminate kill
alias stop kill

def status_update
# don't monitor the memory/cpu usage of this process yet
# If we don't have a pid of a process we want to monitor,super will catch an Errno::ESRCH and abort the worker
Expand Down
40 changes: 19 additions & 21 deletions spec/models/embedded_ansible_worker_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -161,27 +161,25 @@
end
end

%w(kill stop terminate).each do |stop_method|
describe "##{stop_method}" do
it "exits the monitoring thread and destroys the worker row" do
thread_double = double
expect(thread_double).to receive(:exit)
allow(subject).to receive(:find_worker_thread_object).and_return(thread_double)
subject.public_send(stop_method)
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end

it "destroys the worker row but refuses to kill the main thread" do
allow(subject).to receive(:find_worker_thread_object).and_return(Thread.main)
subject.public_send(stop_method)
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end

it "destroys the worker row if the monitor thread is not found" do
allow(subject).to receive(:find_worker_thread_object).and_return(nil)
subject.public_send(stop_method)
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end
describe "#kill" do
it "exits the monitoring thread and destroys the worker row" do
thread_double = double
expect(thread_double).to receive(:exit)
allow(subject).to receive(:find_worker_thread_object).and_return(thread_double)
subject.kill
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end

it "destroys the worker row but refuses to kill the main thread" do
allow(subject).to receive(:find_worker_thread_object).and_return(Thread.main)
subject.kill
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end

it "destroys the worker row if the monitor thread is not found" do
allow(subject).to receive(:find_worker_thread_object).and_return(nil)
subject.kill
expect { subject.reload }.to raise_error(ActiveRecord::RecordNotFound)
end
end
end
Expand Down

0 comments on commit bc8c26f

Please sign in to comment.