Terminate playbooks running longer than timeout

app/models/manageiq/providers/ansible_operation_workflow.rb

@@ -21,9 +21,11 @@ def run_playbook
 
     uuid = Ansible::Runner.run_async(env_vars, extra_vars, playbook_path)
     if uuid.nil?
-      queue_signal(:error)
+      queue_signal(:abort, "Failed to run ansible playbook", "error")
     else
-      context[:ansible_runner_uuid] = uuid
+      context[:ansible_runner_uuid]       = uuid
+      context[:ansible_runner_started_on] = Time.now.utc
+
       update_attributes!(:context => context)
 
       queue_signal(:poll_runner)
@@ -32,7 +34,11 @@ def run_playbook
 
   def poll_runner
     if Ansible::Runner.running?(context[:ansible_runner_uuid])
-      queue_signal(:poll_runner, :deliver_on => deliver_on)
+      if context[:ansible_runner_started_on] + options[:timeout] < Time.now.utc
+        queue_signal(:abort, "Playbook has been running longer than timeout", "error")
+      else
+        queue_signal(:poll_runner, :deliver_on => deliver_on)
+      end
     else
       queue_signal(:post_playbook)
     end

spec/models/manageiq/providers/ansible_operation_workflow_spec.rb

@@ -111,6 +111,7 @@
 
     before do
       job.context[:ansible_runner_uuid] = uuid
+      job.context[:ansible_runner_started_on] = Time.now.utc
       job.save!
     end
 
@@ -130,6 +131,17 @@
       job.signal(:poll_runner)
     end
 
+    it "fails if the playbook has been running too long" do
+      time = job.context[:ansible_runner_started_on] + job.options[:timeout] + 5.minutes
+
+      Timecop.travel(time) do
+        expect(Ansible::Runner).to receive(:running?).with(uuid).and_return(true)
+        expect(job).to receive(:queue_signal).with(:error)
+
+        job.signal(:poll_runner)
+      end
+    end
+
     context ".deliver_on" do
       let(:options) { [{"ENV" => "VAR"}, %w(arg1 arg2), "/path/to/playbook", :poll_interval => 5.minutes] }