diff --git a/.github/workflows/judoscale-sidekiq-benchmarks.yml b/.github/workflows/judoscale-sidekiq-benchmarks.yml new file mode 100644 index 00000000..5cc57a0b --- /dev/null +++ b/.github/workflows/judoscale-sidekiq-benchmarks.yml @@ -0,0 +1,47 @@ +name: judoscale-sidekiq benchmarks +defaults: + run: + working-directory: judoscale-sidekiq +on: + push: + branches: + - main + pull_request: +jobs: + benchmarks: + strategy: + fail-fast: false + matrix: + gemfile: + - Gemfile + - Gemfile-sidekiq-5 + ruby: + - "2.7" + - "3.1" + redis: + - "5.0" + - "6.0" + - "7.0" + exclude: + # Recent redis-client requires Redis 6+ + - gemfile: Gemfile + redis: "5.0" + + runs-on: ubuntu-latest + + env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps + BUNDLE_GEMFILE: ${{ github.workspace }}/judoscale-sidekiq/${{ matrix.gemfile }} + + services: + redis: + image: redis:${{ matrix.redis }} + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true # runs bundle install and caches installed gems automatically + - run: bundle exec rake bench diff --git a/judoscale-sidekiq/Rakefile b/judoscale-sidekiq/Rakefile index b4a24c8f..a171694e 100644 --- a/judoscale-sidekiq/Rakefile +++ b/judoscale-sidekiq/Rakefile @@ -3,9 +3,13 @@ require "rake/testtask" Rake::TestTask.new(:test) do |t| - t.libs << "lib" - t.libs << "test" - t.test_files = FileList["test/**/*_test.rb"] + t.libs = %w[lib test] + t.pattern = "test/**/*_test.rb" +end + +Rake::TestTask.new(:bench) do |t| + t.libs = %w[lib test] + t.pattern = "test/benchmarks/**/*_benchmark.rb" end task default: :test diff --git a/judoscale-sidekiq/test/benchmarks/collect_with_large_queues_benchmark.rb b/judoscale-sidekiq/test/benchmarks/collect_with_large_queues_benchmark.rb new file mode 100644 index 00000000..72a3fd91 --- /dev/null +++ b/judoscale-sidekiq/test/benchmarks/collect_with_large_queues_benchmark.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require "test_helper" +require "minitest/benchmark" +require "judoscale/sidekiq/metrics_collector" + +class CollectWithLargeQueuesBenchmark < Minitest::Benchmark + BATCH_SIZE = 1_000 + MAX_RETRIES = 3 + + # performance assertions will iterate over `bench_range`. + # We'll use it to define the number Sidekiq jobs we enqueue in Redis. + def self.bench_range + bench_exp 10, 1_000_000 #=> [10, 100, 1,000, 10,000, 100,000, 1,000,000] + end + + def setup + # Override ConfigHelpers and log to STDOUT for debugging + Judoscale::Config.instance.reset + + @collector = Judoscale::Sidekiq::MetricsCollector.new + sidekiq_args = BATCH_SIZE.times.map { [] } + + puts "Sidekiq verison: #{Sidekiq::VERSION}" + puts "Redis version: #{Sidekiq.redis(&:info)["redis_version"]}" + + # We need to prepare data for all benchmarks in advance. Each benchmark + # will target an isolated Redis DB with a different number of jobs. + self.class.bench_range.each do |n| + with_isolated_redis(n) do + Sidekiq.redis(&:flushdb) + + (n / BATCH_SIZE).times do |i| + attempts = 0 + + begin + Sidekiq::Client.push_bulk "class" => "Foo", "args" => sidekiq_args + rescue => e + # Redis sometimes fails locally when enqueueing a million jobs, so we need + # to retry a few times. + attempts += 1 + puts "RESCUED batch #{i}, attempt #{attempts}: #{e.class}, #{e.message}" + + # Give the connection a moment to recover + sleep(1) + + retry if attempts < MAX_RETRIES + raise e + end + end + end + end + end + + def bench_collect + # assert_performance_constant needs a VERY high threshold to ever fail. + assert_performance_constant 0.9999999 do |n| + with_isolated_redis(n) do + @collector.collect + end + end + end + + private + + def with_isolated_redis(n, &block) + # n is in powers of 10, but we want to use a database number in the range 0-9 + db_number = Math.log10(n).to_i + + if Sidekiq.respond_to?(:default_configuration) + # `new_redis_pool` will use the configuration from Sidekiq.default_configuration + Sidekiq.default_configuration.redis = {db: db_number} + pool = Sidekiq.default_configuration.new_redis_pool 10, "bench-#{n}" + Sidekiq::Client.via(pool, &block) + else + # For older (pre-capsule) versions of Sidekiq + Sidekiq.redis = {db: db_number} + block.call + end + end +end