Merge pull request #1 from talis/working_multiple_queues

90%: working multiple queues

Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 gemspec
 gem 'rails', '~> 4.0.3'
-gem 'resque', :git => 'https://github.com/resque/resque.git', :branch => "1-x-stable"
+gem 'resque', github: 'talis/resque', branch: 'rel-1-25-2'
 
 gem 'sqlite3', :platforms => :ruby
 gem 'activerecord-jdbcsqlite3-adapter', :platforms => :jruby

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
-  remote: https://github.com/resque/resque.git
-  revision: 68b43a3a01006abb0aabf0d564967f546ec1e134
-  branch: 1-x-stable
+  remote: git://github.com/talis/resque.git
+  revision: 466782724cfa9e190d63e04bfe3abc171d6784de
+  branch: rel-1-25-2
   specs:
     resque (1.25.2)
       mono_logger (~> 1.0)
@@ -13,7 +13,7 @@ GIT
 PATH
   remote: .
   specs:
-    resque-web (0.0.6)
+    resque-web (0.0.7)
       coffee-rails
       jquery-rails
       resque
@@ -60,7 +60,8 @@ GEM
     coffee-script (2.4.1)
       coffee-script-source
       execjs
-    coffee-script-source (1.9.1.1)
+    coffee-script-source (1.10.0)
+    commonjs (0.2.7)
     coveralls (0.8.1)
       json (~> 1.8)
       rest-client (>= 1.6.8, < 2)
@@ -80,11 +81,18 @@ GEM
       domain_name (~> 0.5)
     i18n (0.7.0)
     jdbc-sqlite3 (3.8.10.1)
-    jquery-rails (3.1.2)
+    jquery-rails (3.1.4)
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
     json (1.8.2)
     json (1.8.2-java)
+    less (2.6.0)
+      commonjs (~> 0.2.7)
+    less-rails (2.7.0)
+      actionpack (>= 4.0)
+      less (~> 2.6.0)
+      sprockets (> 2, < 4)
+      tilt
     libv8 (3.16.14.7)
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
@@ -121,7 +129,7 @@ GEM
     rake (10.4.2)
     rdoc (4.2.0)
     redcard (1.1.0)
-    redis (3.2.1)
+    redis (3.2.2)
     redis-namespace (1.5.2)
       redis (~> 3.0, >= 3.0.4)
     ref (1.0.5)
@@ -330,13 +338,13 @@ GEM
     rubysl-xmlrpc (2.0.0)
     rubysl-yaml (2.1.0)
     rubysl-zlib (2.0.1)
-    sass (3.4.14)
-    sass-rails (5.0.3)
+    sass (3.4.19)
+    sass-rails (5.0.4)
       railties (>= 4.0.0, < 5.0)
       sass (~> 3.1)
       sprockets (>= 2.8, < 4.0)
       sprockets-rails (>= 2.0, < 4.0)
-      tilt (~> 1.1)
+      tilt (>= 1.1, < 3)
     simplecov (0.10.0)
       docile (~> 1.1.0)
       json (~> 1.8)
@@ -366,10 +374,10 @@ GEM
     thread_safe (0.3.5-java)
     tilt (1.4.1)
     tins (1.5.1)
-    twitter-bootstrap-rails (2.2.8)
+    twitter-bootstrap-rails (3.2.2)
       actionpack (>= 3.1)
-      execjs
-      rails (>= 3.1)
+      execjs (>= 2.2.2, >= 2.2)
+      less-rails (>= 2.5.0)
       railties (>= 3.1)
     tzinfo (0.3.44)
     uglifier (2.7.1)
@@ -406,3 +414,6 @@ DEPENDENCIES
   therubyracer (~> 0.12.1)
   therubyrhino (= 2.0.2)
   uglifier (>= 1.0.3)
+
+BUNDLED WITH
+   1.10.6

app/controllers/resque_web/failures_controller.rb

@@ -1,5 +1,6 @@
 module ResqueWeb
   class FailuresController < ResqueWeb::ApplicationController
+    include ResqueWeb::FailureQueueNameHelper
 
     # Display all jobs in the failure queue
     #
@@ -17,36 +18,43 @@ def destroy
 
     # destroy all jobs from the failure queue
     def destroy_all
-      queue = params[:queue] || 'failed'
-      Resque::Failure.clear(queue)
+      # This needs to stay as params[:queue] as if nil it will delete
+      # all queues and this is the behaviour that we want
+      Resque::Failure.clear(params[:queue])
       redirect_to failures_path(redirect_params)
     end
 
     # retry an individual job from the failure queue
     def retry
-      reque_single_job(params[:id])
+      requeue_single_job(params[:id], failure_queue)
       redirect_to failures_path(redirect_params)
     end
 
     # retry all jobs from the failure queue
     def retry_all
-      if params[:queue].present? && params[:queue]!="failed"
-        Resque::Failure.requeue_queue(params[:queue])
+      if params[:queue].present? || !multiple_failure_queues?
+        requeue_queue(failure_queue)
       else
-        (Resque::Failure.count-1).downto(0).each { |id| reque_single_job(id) }
+        Resque::Failure.queues.each { |queue| requeue_queue(queue) }
       end
       redirect_to failures_path(redirect_params)
     end
 
     private
 
+    def requeue_queue(queue)
+      (Resque::Failure.count(queue)-1).downto(0).each { |id| requeue_single_job(id, queue) }
+    end
+
     #API agnostic for Resque 2 with duck typing on requeue_and_remove
-    def reque_single_job(id)
+    def requeue_single_job(id, queue)
       if Resque::Failure.respond_to?(:requeue_and_remove)
+        # The API for Resque 2 does not support providing a queue name
+        # to requeue_and_remove
         Resque::Failure.requeue_and_remove(id)
       else
-        Resque::Failure.requeue(id)
-        Resque::Failure.remove(id)
+        Resque::Failure.requeue(id, queue)
+        Resque::Failure.remove(id, queue)
       end
     end
 

app/helpers/resque_web/failure_queue_name_helper.rb

@@ -0,0 +1,11 @@
+module ResqueWeb
+  module FailureQueueNameHelper
+    def failure_queue
+      multiple_failure_queues? ? params[:queue] : 'failed'
+    end
+
+    def multiple_failure_queues?
+      Resque::Failure.backend == Resque::Failure::RedisMultiQueue
+    end
+  end
+end

app/helpers/resque_web/failures_helper.rb

@@ -1,5 +1,11 @@
 module ResqueWeb
   module FailuresHelper
+    def self.included klass
+      klass.class_eval do
+        include FailureQueueNameHelper
+      end
+    end
+
     def each_failure(&block)
       Resque::Failure.each(failure_start_at, failure_per_page, params[:queue], params[:class], &block)
     end
@@ -8,20 +14,12 @@ def failure_date_format
       "%Y/%m/%d %T %z"
     end
 
-    def multiple_failure_queues?
-      @multiple_failure_queues ||= Resque::Failure.queues.size > 1
-    end
-
-    def failure_queue
-      multiple_failure_queues? ? params[:id] : 'failed'
-    end
-
     def failure_queue_name
       @failure_queue_name ||= params[:queue] ? params[:queue] : 'Failed'
     end
 
     def failure_size
-      @failure_size ||= Resque::Failure.count(params[:id], params[:class])
+      @failure_size ||= Resque::Failure.count(params[:queue], params[:class])
     end
 
     def failure_per_page
@@ -40,7 +38,7 @@ def failure_end_at
       end
     end
 
-    def failure_class_counts(queue = params[:id])
+    def failure_class_counts(queue = params[:queue])
       classes = Hash.new(0)
       Resque::Failure.each(0, Resque::Failure.count(queue), queue) do |_, item|
         class_name = item['payload']['class'] if item['payload']

app/helpers/resque_web/queues_helper.rb

@@ -60,7 +60,7 @@ def failed_queue_info(queue_name)
 
       if size > 0
         css_class = "badge badge-important"
-        badge = link_to(size, failure_path(failed_queue))
+        badge = link_to(size, failures_path(:queue => failed_queue))
       else
         css_class = "badge"
         badge = size.to_s

app/views/resque_web/failures/_overview.html.erb

@@ -7,18 +7,18 @@
 
   <% Resque::Failure.queues.sort.each do |queue| %>
     <tr>
-      <th><%= link_to queue, failure_path(queue), :class => 'label label-info' %></th>
+      <th><%= link_to queue, failures_path(:queue => queue), :class => 'label label-info' %></th>
       <th style="width:75px;" class="center"><%= Resque::Failure.count(queue) %></th>
     </tr>
 
     <% failure_class_counts(queue).each do |klass, count| %>
     <tr id="<%= klass %>">
       <td>
-        <%= link_to klass, failure_path(queue, :class => klass), :class => "failed failed_class" %>
+        <%= link_to klass, failures_path(:queue => queue, :class => klass), :class => "failed failed_class" %>
       </td>
       <td style="text-align: center;" class="failed<%= (count.to_i > 1000) ? '_many' : '' %>"><%= count %></td>
     </tr>
     <% end %>
   <% end %>
 </tbody>
-</table>
+</table>

config/initializers/resque_config.rb

@@ -1,2 +1,4 @@
+require 'resque/failure/redis_multi_queue'
+
 config = ENV.fetch("RAILS_RESQUE_REDIS", "127.0.0.1:6379")
 Resque.redis = config

test/functional/failures_controller_test.rb

@@ -1,4 +1,5 @@
 require 'test_helper'
+require 'resque/failure/redis_multi_queue'
 
 module ResqueWeb
   class FailuresControllerTest < ActionController::TestCase
@@ -38,8 +39,8 @@ class FailuresControllerTest < ActionController::TestCase
         assert_redirected_to failures_path
       end
       it "retries should work also in case of pre 2.0 Resque" do
-        Resque::Failure.expects(:requeue).with('123')
-        Resque::Failure.expects(:remove).with('123')
+        Resque::Failure.expects(:requeue).with('123', 'failed')
+        Resque::Failure.expects(:remove).with('123', 'failed')
         visit(:retry, {:id => 123}, :method => :put)
         assert_redirected_to failures_path
       end
@@ -57,17 +58,60 @@ class FailuresControllerTest < ActionController::TestCase
       it "retries all failures should also work case of pre 2.0 Resque" do
         Resque::Failure.stubs(:count).returns(2)
         Resque::Failure.stubs(:requeue).returns(true)
-        Resque::Failure.expects(:requeue).with(0)
-        Resque::Failure.expects(:remove).with(0)
-        Resque::Failure.expects(:requeue).with(1)
-        Resque::Failure.expects(:remove).with(1)
+        Resque::Failure.expects(:requeue).with(0, 'failed')
+        Resque::Failure.expects(:remove).with(0, 'failed')
+        Resque::Failure.expects(:requeue).with(1, 'failed')
+        Resque::Failure.expects(:remove).with(1, 'failed')
         visit(:retry_all, nil, :method => :put)
         assert_redirected_to failures_path
       end
-      it "retries all failures using requeue_queue if queue specified" do
-        Resque::Failure.expects(:requeue_queue).with('myqueue')
-        visit(:retry_all, {:queue=>"myqueue"}, :method => :put)
-        assert_redirected_to failures_path(:queue=>'myqueue')
+
+    end
+
+    describe "With Multiple Failed Queues" do
+      setup do
+        @routes = Engine.routes
+        Resque::Failure.backend = Resque::Failure::RedisMultiQueue
+      end
+
+      describe "GET /failures" do
+        it "renders the index page when not supplied a queue name" do
+          visit(:index)
+          assert_template :index
+        end
+        it "renders the index page when supplied a queue name" do
+          visit(:index, {:queue => "my_queue"})
+          assert_template :index
+        end
+      end
+
+      describe "PUT /failures/retry_all" do
+        it "retries all failures on all queues if no queue specified" do
+          Resque::Failure.stubs(:queues).returns(%w(foo bar))
+          Resque::Failure.stubs(:count).with('foo').returns(2)
+          Resque::Failure.stubs(:count).with('bar').returns(2)
+          Resque::Failure.expects(:requeue).with(0, 'foo')
+          Resque::Failure.expects(:remove).with(0, 'foo')
+          Resque::Failure.expects(:requeue).with(1, 'foo')
+          Resque::Failure.expects(:remove).with(1, 'foo')
+          Resque::Failure.expects(:requeue).with(0, 'bar')
+          Resque::Failure.expects(:remove).with(0, 'bar')
+          Resque::Failure.expects(:requeue).with(1, 'bar')
+          Resque::Failure.expects(:remove).with(1, 'bar')
+          visit(:retry_all, nil, :method => :put)
+          assert_redirected_to failures_path
+        end
+
+        it "retries all failures using the queue name if queue specified" do
+          Resque::Failure.stubs(:count).returns(2)
+          Resque::Failure.stubs(:requeue).returns(true)
+          Resque::Failure.expects(:requeue).with(0, 'my_queue')
+          Resque::Failure.expects(:remove).with(0, 'my_queue')
+          Resque::Failure.expects(:requeue).with(1, 'my_queue')
+          Resque::Failure.expects(:remove).with(1, 'my_queue')
+          visit(:retry_all, {:queue=>"my_queue"}, :method => :put)
+          assert_redirected_to failures_path(:queue=>'my_queue')
+        end
       end
     end
   end