Abort Ruby Reaper when sidekiq queues are full

.reek.yml

@@ -61,6 +61,7 @@ detectors:
       - SidekiqUniqueJobs::OnConflict::Reject#push_to_deadset
       - SidekiqUniqueJobs::Orphans::RubyReaper#active?
       - SidekiqUniqueJobs::Orphans::RubyReaper#entries
+      - SidekiqUniqueJobs::Orphans::RubyReaper#queues_very_full?
       - SidekiqUniqueJobs::Redis::Entity#exist?
       - SidekiqUniqueJobs::Web::Helpers#cparams
       - SidekiqUniqueJobs::Web::Helpers#display_lock_args
@@ -115,6 +116,7 @@ detectors:
       - SidekiqUniqueJobs::OnConflict::Reject#push_to_deadset
       - SidekiqUniqueJobs::Orphans::RubyReaper#active?
       - SidekiqUniqueJobs::Orphans::RubyReaper#enqueued?
+      - SidekiqUniqueJobs::Orphans::RubyReaper#queues_very_full?
       - SidekiqUniqueJobs::UpgradeLocks#keys_for_digest
   RepeatedConditional:
     exclude:
@@ -156,6 +158,7 @@ detectors:
       - SidekiqUniqueJobs::Orphans::RubyReaper#enqueued?
       - SidekiqUniqueJobs::Orphans::RubyReaper#entries
       - SidekiqUniqueJobs::Orphans::RubyReaper#orphans
+      - SidekiqUniqueJobs::Orphans::RubyReaper#queues_very_full?
       - SidekiqUniqueJobs::Profiler#self.stop
       - SidekiqUniqueJobs::Script::Caller#call_script
       - SidekiqUniqueJobs::Script::Caller#extract_args

lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -15,6 +15,9 @@ class RubyReaper < Reaper
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
       #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
       # @!attribute [r] digests
       #   @return [SidekiqUniqueJobs::Digests] digest collection
       attr_reader :digests
@@ -46,6 +49,8 @@ def initialize(conn)
       # @return [Integer] the number of reaped locks
       #
       def call
+        return if queues_very_full?
+
         BatchDelete.call(orphans, conn)
       end
 
@@ -212,6 +217,22 @@ def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
         end
       end
 
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
+        end
+        false
+      end
+
       #
       # Checks a sorted set for the existance of this digest
       #

spec/sidekiq_unique_jobs/orphans/ruby_reaper_spec.rb

@@ -95,4 +95,16 @@
       end
     end
   end
+
+  describe "#call" do
+    before do
+      stub_const("SidekiqUniqueJobs::Orphans::RubyReaper::MAX_QUEUE_LENGTH", 3)
+      4.times { push_item(item) }
+    end
+    it 'quits early if sidekiq queues are very full' do
+      expect(service).not_to receive(:orphans)
+
+      service.call
+    end
+  end
 end