chore: Add benchmarks for judoscale-sidekiq (#188)

Benchmarks are run automatically via GH Actions, using a matrix of multiple Ruby, Sidekiq, and Redis versions. This adds a benchmark for varying sizes of Sidekiq queues.

.github/workflows/judoscale-sidekiq-benchmarks.yml

@@ -0,0 +1,47 @@
+name: judoscale-sidekiq benchmarks
+defaults:
+  run:
+    working-directory: judoscale-sidekiq
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  benchmarks:
+    strategy:
+      fail-fast: false
+      matrix:
+        gemfile:
+          - Gemfile
+          - Gemfile-sidekiq-5
+        ruby:
+          - "2.7"
+          - "3.1"
+        redis:
+          - "5.0"
+          - "6.0"
+          - "7.0"
+        exclude:
+          # Recent redis-client requires Redis 6+
+          - gemfile: Gemfile
+            redis: "5.0"
+
+    runs-on: ubuntu-latest
+
+    env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps
+      BUNDLE_GEMFILE: ${{ github.workspace }}/judoscale-sidekiq/${{ matrix.gemfile }}
+
+    services:
+      redis:
+        image: redis:${{ matrix.redis }}
+        ports:
+          - 6379:6379
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # runs bundle install and caches installed gems automatically
+      - run: bundle exec rake bench

judoscale-sidekiq/Rakefile

@@ -3,9 +3,13 @@
 require "rake/testtask"
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "lib"
-  t.libs << "test"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.libs = %w[lib test]
+  t.pattern = "test/**/*_test.rb"
+end
+
+Rake::TestTask.new(:bench) do |t|
+  t.libs = %w[lib test]
+  t.pattern = "test/benchmarks/**/*_benchmark.rb"
 end
 
 task default: :test

judoscale-sidekiq/test/benchmarks/collect_with_large_queues_benchmark.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "minitest/benchmark"
+require "judoscale/sidekiq/metrics_collector"
+
+class CollectWithLargeQueuesBenchmark < Minitest::Benchmark
+  BATCH_SIZE = 1_000
+  MAX_RETRIES = 3
+
+  # performance assertions will iterate over `bench_range`.
+  # We'll use it to define the number Sidekiq jobs we enqueue in Redis.
+  def self.bench_range
+    bench_exp 10, 1_000_000 #=> [10, 100, 1,000, 10,000, 100,000, 1,000,000]
+  end
+
+  def setup
+    # Override ConfigHelpers and log to STDOUT for debugging
+    Judoscale::Config.instance.reset
+
+    @collector = Judoscale::Sidekiq::MetricsCollector.new
+    sidekiq_args = BATCH_SIZE.times.map { [] }
+
+    puts "Sidekiq verison: #{Sidekiq::VERSION}"
+    puts "Redis version: #{Sidekiq.redis(&:info)["redis_version"]}"
+
+    # We need to prepare data for all benchmarks in advance. Each benchmark
+    # will target an isolated Redis DB with a different number of jobs.
+    self.class.bench_range.each do |n|
+      with_isolated_redis(n) do
+        Sidekiq.redis(&:flushdb)
+
+        (n / BATCH_SIZE).times do |i|
+          attempts = 0
+
+          begin
+            Sidekiq::Client.push_bulk "class" => "Foo", "args" => sidekiq_args
+          rescue => e
+            # Redis sometimes fails locally when enqueueing a million jobs, so we need
+            # to retry a few times.
+            attempts += 1
+            puts "RESCUED batch #{i}, attempt #{attempts}: #{e.class}, #{e.message}"
+
+            # Give the connection a moment to recover
+            sleep(1)
+
+            retry if attempts < MAX_RETRIES
+            raise e
+          end
+        end
+      end
+    end
+  end
+
+  def bench_collect
+    # assert_performance_constant needs a VERY high threshold to ever fail.
+    assert_performance_constant 0.9999999 do |n|
+      with_isolated_redis(n) do
+        @collector.collect
+      end
+    end
+  end
+
+  private
+
+  def with_isolated_redis(n, &block)
+    # n is in powers of 10, but we want to use a database number in the range 0-9
+    db_number = Math.log10(n).to_i
+
+    if Sidekiq.respond_to?(:default_configuration)
+      # `new_redis_pool` will use the configuration from Sidekiq.default_configuration
+      Sidekiq.default_configuration.redis = {db: db_number}
+      pool = Sidekiq.default_configuration.new_redis_pool 10, "bench-#{n}"
+      Sidekiq::Client.via(pool, &block)
+    else
+      # For older (pre-capsule) versions of Sidekiq
+      Sidekiq.redis = {db: db_number}
+      block.call
+    end
+  end
+end